commit bb4fd7557639d6e2ebe1c25a4b5cbed66db1d7bc parent 8586eed635afa595a6cc9143e633706e6a0f9742 Author: William Casarin <jb55@jb55.com> Date: Mon, 28 Aug 2023 07:52:59 -0700 nostrdb: add profiles to nostrdb This adds profiles to nostrdb - Remove in-memory Profiles caches, nostrdb is as fast as an in-memory cache - Remove ProfileDatabase and just use nostrdb directly Changelog-Changed: Use nostrdb for profiles Diffstat:
42 files changed, 362 insertions(+), 705 deletions(-)
diff --git a/damus.xcodeproj/project.pbxproj b/damus.xcodeproj/project.pbxproj @@ -355,6 +355,7 @@ 4CEE2AF5280B29E600AB5EEF /* TimeAgo.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CEE2AF4280B29E600AB5EEF /* TimeAgo.swift */; }; 4CEE2AF7280B2DEA00AB5EEF /* ProfileName.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CEE2AF6280B2DEA00AB5EEF /* ProfileName.swift */; }; 4CEE2B02280B39E800AB5EEF /* EventActionBar.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CEE2B01280B39E800AB5EEF /* EventActionBar.swift */; }; + 4CEF958D2A9CE650000F901B /* verifier.c in Sources */ = {isa = PBXBuildFile; fileRef = 4C4792D42A9939BD00489948 /* verifier.c */; }; 4CF0ABD42980996B00D66079 /* Report.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CF0ABD32980996B00D66079 /* Report.swift */; }; 4CF0ABD629817F5B00D66079 /* ReportView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CF0ABD529817F5B00D66079 /* ReportView.swift */; }; 4CF0ABD82981980C00D66079 /* Lists.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CF0ABD72981980C00D66079 /* Lists.swift */; }; @@ -377,10 +378,6 @@ 4CFF8F6D29CD022E008DB934 /* WideEventView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4CFF8F6C29CD022E008DB934 /* WideEventView.swift */; }; 4FE60CDD295E1C5E00105A1F /* Wallet.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4FE60CDC295E1C5E00105A1F /* Wallet.swift */; }; 50088DA129E8271A008A1FDF /* WebSocket.swift in Sources */ = {isa = PBXBuildFile; fileRef = 50088DA029E8271A008A1FDF /* WebSocket.swift */; }; - 5019CADD2A0FB0A9000069E1 /* ProfileDatabaseTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 5019CADC2A0FB0A9000069E1 /* ProfileDatabaseTests.swift */; }; - 501F8C5529FF5EF6001AFC1D /* PersistedProfile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 501F8C5429FF5EF6001AFC1D /* PersistedProfile.swift */; }; - 501F8C5829FF5FC5001AFC1D /* Damus.xcdatamodeld in Sources */ = {isa = PBXBuildFile; fileRef = 501F8C5629FF5FC5001AFC1D /* Damus.xcdatamodeld */; }; - 501F8C5A29FF70F5001AFC1D /* ProfileDatabase.swift in Sources */ = {isa = PBXBuildFile; fileRef = 501F8C5929FF70F5001AFC1D /* ProfileDatabase.swift */; }; 501F8C802A0220E1001AFC1D /* KeychainStorage.swift in Sources */ = {isa = PBXBuildFile; fileRef = 501F8C7F2A0220E1001AFC1D /* KeychainStorage.swift */; }; 501F8C822A0224EB001AFC1D /* KeychainStorageTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 501F8C812A0224EB001AFC1D /* KeychainStorageTests.swift */; }; 504323A72A34915F006AE6DC /* RelayModel.swift in Sources */ = {isa = PBXBuildFile; fileRef = 504323A62A34915F006AE6DC /* RelayModel.swift */; }; @@ -1059,10 +1056,6 @@ 4CFF8F6C29CD022E008DB934 /* WideEventView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WideEventView.swift; sourceTree = "<group>"; }; 4FE60CDC295E1C5E00105A1F /* Wallet.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Wallet.swift; sourceTree = "<group>"; }; 50088DA029E8271A008A1FDF /* WebSocket.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WebSocket.swift; sourceTree = "<group>"; }; - 5019CADC2A0FB0A9000069E1 /* ProfileDatabaseTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProfileDatabaseTests.swift; sourceTree = "<group>"; }; - 501F8C5429FF5EF6001AFC1D /* PersistedProfile.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = PersistedProfile.swift; sourceTree = "<group>"; }; - 501F8C5729FF5FC5001AFC1D /* Damus.xcdatamodel */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcdatamodel; path = Damus.xcdatamodel; sourceTree = "<group>"; }; - 501F8C5929FF70F5001AFC1D /* ProfileDatabase.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ProfileDatabase.swift; sourceTree = "<group>"; }; 501F8C7F2A0220E1001AFC1D /* KeychainStorage.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = KeychainStorage.swift; sourceTree = "<group>"; }; 501F8C812A0224EB001AFC1D /* KeychainStorageTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = KeychainStorageTests.swift; sourceTree = "<group>"; }; 504323A62A34915F006AE6DC /* RelayModel.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RelayModel.swift; sourceTree = "<group>"; }; @@ -1691,7 +1684,6 @@ 4C75EFAB28049CC80006080F /* Nostr */ = { isa = PBXGroup; children = ( - 501F8C5329FF5EE2001AFC1D /* CoreData */, 4CE6DF1527F8DEBF00C66700 /* RelayConnection.swift */, 50A60D132A28BEEE00186190 /* RelayLog.swift */, 4C75EFA527FF87A20006080F /* Nostr.swift */, @@ -1704,7 +1696,6 @@ 4C75EFBA2804A34C0006080F /* ProofOfWork.swift */, 4CEE2AEC2805B22500AB5EEF /* NostrRequest.swift */, 4CACA9DB280C38C000D9BBE8 /* Profiles.swift */, - 501F8C5929FF70F5001AFC1D /* ProfileDatabase.swift */, 4C3BEFD32819DE8F00B3DE84 /* NostrKind.swift */, 4C363A8F28247A1D006E126D /* NostrLink.swift */, 50088DA029E8271A008A1FDF /* WebSocket.swift */, @@ -2137,7 +2128,6 @@ 4CB883A9297612FF00DC99E7 /* ZapTests.swift */, 4CB883AD2976FA9300DC99E7 /* FormatTests.swift */, 3A3040EC29A5CB86008A0F29 /* ReplyDescriptionTests.swift */, - 5019CADC2A0FB0A9000069E1 /* ProfileDatabaseTests.swift */, 3A3040F229A91366008A0F29 /* ProfileViewTests.swift */, 3A30410029AB12AA008A0F29 /* EventGroupViewTests.swift */, 4C8D00D329E3C5D40036AF10 /* NIP19Tests.swift */, @@ -2249,15 +2239,6 @@ path = Images; sourceTree = "<group>"; }; - 501F8C5329FF5EE2001AFC1D /* CoreData */ = { - isa = PBXGroup; - children = ( - 501F8C5429FF5EF6001AFC1D /* PersistedProfile.swift */, - 501F8C5629FF5FC5001AFC1D /* Damus.xcdatamodeld */, - ); - path = CoreData; - sourceTree = "<group>"; - }; 7C0F392D29B57C8F0039859C /* Extensions */ = { isa = PBXGroup; children = ( @@ -2497,6 +2478,7 @@ isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( + 4CEF958D2A9CE650000F901B /* verifier.c in Sources */, 4C32B9342A9AD01A00DC3548 /* NdbProfile.swift in Sources */, 4C32B9332A99845B00DC3548 /* Ndb.swift in Sources */, 4C4793082A993E8900489948 /* refmap.c in Sources */, @@ -2624,7 +2606,6 @@ 4CD348EF29C3659D00497EB2 /* ImageUploadModel.swift in Sources */, 4C7D096E2A0AEA0400943473 /* ScannerCoordinator.swift in Sources */, 4C3BEFDC281DCE6100B3DE84 /* Liked.swift in Sources */, - 501F8C5A29FF70F5001AFC1D /* ProfileDatabase.swift in Sources */, 4CF0ABE7298444FD00D66079 /* MutedEventView.swift in Sources */, 9C83F89329A937B900136C08 /* TextViewWrapper.swift in Sources */, 4C1253502A76C5B20004F4B8 /* UnfollowedNotify.swift in Sources */, @@ -2757,7 +2738,6 @@ 4CB8838629656C8B00DC99E7 /* NIP05.swift in Sources */, 4CF0ABD82981980C00D66079 /* Lists.swift in Sources */, F71694EA2A662232001F4053 /* SuggestedUsersView.swift in Sources */, - 501F8C5829FF5FC5001AFC1D /* Damus.xcdatamodeld in Sources */, 4C12536A2A76D3850004F4B8 /* RelaysChangedNotify.swift in Sources */, 4C30AC8029A6A53F00E2BD5A /* ProfilePicturesView.swift in Sources */, 5C6E1DAD2A193EC2008FC15A /* GradientButtonStyle.swift in Sources */, @@ -2848,7 +2828,6 @@ 4CE879552996BAB900F758CC /* RelayPaidDetail.swift in Sources */, 4C1253602A76CF890004F4B8 /* ScrollToTopNotify.swift in Sources */, 4CA3529E2A76AE67003BB08B /* FollowNotify.swift in Sources */, - 501F8C5529FF5EF6001AFC1D /* PersistedProfile.swift in Sources */, 4CF0ABD42980996B00D66079 /* Report.swift in Sources */, 4C06670B28FDE64700038D2A /* damus.c in Sources */, 4C1253642A76D08F0004F4B8 /* ReportNotify.swift in Sources */, @@ -2893,7 +2872,6 @@ buildActionMask = 2147483647; files = ( 4C684A572A7FFAE6005E6031 /* UrlTests.swift in Sources */, - 5019CADD2A0FB0A9000069E1 /* ProfileDatabaseTests.swift in Sources */, 3A90B1832A4EA3C600000D94 /* UserSearchCacheTests.swift in Sources */, 4C9B0DEE2A65A75F00CBDA21 /* AttrStringTestExtensions.swift in Sources */, 4C19AE552A5D977400C90DB7 /* HashtagTests.swift in Sources */, @@ -3451,19 +3429,6 @@ productName = secp256k1; }; /* End XCSwiftPackageProductDependency section */ - -/* Begin XCVersionGroup section */ - 501F8C5629FF5FC5001AFC1D /* Damus.xcdatamodeld */ = { - isa = XCVersionGroup; - children = ( - 501F8C5729FF5FC5001AFC1D /* Damus.xcdatamodel */, - ); - currentVersion = 501F8C5729FF5FC5001AFC1D /* Damus.xcdatamodel */; - path = Damus.xcdatamodeld; - sourceTree = "<group>"; - versionGroupType = wrapper.xcdatamodel; - }; -/* End XCVersionGroup section */ }; rootObject = 4CE6DEDB27F7A08100C66700 /* Project object */; } diff --git a/damus/Components/Status/UserStatusSheet.swift b/damus/Components/Status/UserStatusSheet.swift @@ -136,6 +136,6 @@ struct UserStatusSheet: View { struct UserStatusSheet_Previews: PreviewProvider { static var previews: some View { - UserStatusSheet(postbox: PostBox(pool: RelayPool()), keypair: Keypair(pubkey: .empty, privkey: nil), status: .init()) + UserStatusSheet(postbox: PostBox(pool: RelayPool(ndb: .empty)), keypair: Keypair(pubkey: .empty, privkey: nil), status: .init()) } } diff --git a/damus/ContentView.swift b/damus/ContentView.swift @@ -9,12 +9,6 @@ import SwiftUI import AVKit import MediaPlayer -struct TimestampedProfile { - let profile: Profile - let timestamp: UInt32 - let event: NostrEvent -} - struct ZapSheet { let target: ZapTarget let lnurl: String @@ -378,10 +372,9 @@ struct ContentView: View { invalidate_zapper_cache(pubkey: keypair.pubkey, profiles: ds.profiles, lnurl: ds.lnurls) } - profile.lud16 = lud16 - guard let ev = make_metadata_event(keypair: keypair, metadata: profile) else { - return - } + let prof = Profile(name: profile.name, display_name: profile.display_name, about: profile.about, picture: profile.picture, banner: profile.banner, website: profile.website, lud06: profile.lud06, lud16: lud16, nip05: profile.nip05, damus_donation: profile.damus_donation, reactions: profile.reactions) + + guard let ev = make_metadata_event(keypair: keypair, metadata: prof) else { return } ds.postbox.send(ev) } .onReceive(handle_notify(.broadcast)) { ev in @@ -389,8 +382,10 @@ struct ContentView: View { return } ds.postbox.send(ev) - if let profile = ds.profiles.lookup_with_timestamp(id: ev.pubkey) { - ds.postbox.send(profile.event) + if let record = ds.profiles.lookup_with_timestamp(ev.pubkey), + let event = ds.events.lookup_by_key(record.noteKey) + { + ds.postbox.send(event) } } .onReceive(handle_notify(.unfollow)) { target in @@ -500,11 +495,10 @@ struct ContentView: View { else { return } - - profile.reactions = !hide - guard let profile_ev = make_metadata_event(keypair: keypair, metadata: profile) else { - return - } + + let prof = Profile(name: profile.name, display_name: profile.display_name, about: profile.about, picture: profile.picture, banner: profile.banner, website: profile.website, lud06: profile.lud06, lud16: profile.lud16, nip05: profile.nip05, damus_donation: profile.damus_donation, reactions: !hide) + + guard let profile_ev = make_metadata_event(keypair: keypair, metadata: prof) else { return } damus_state.postbox.send(profile_ev) } .alert(NSLocalizedString("User muted", comment: "Alert message to indicate the user has been muted"), isPresented: $user_muted_confirm, actions: { @@ -597,7 +591,10 @@ struct ContentView: View { } func connect() { - let pool = RelayPool() + // nostrdb + let ndb = Ndb()! + + let pool = RelayPool(ndb: ndb) let model_cache = RelayModelCache() let relay_filters = RelayFilters(our_pubkey: pubkey) let bootstrap_relays = load_bootstrap_relays(pubkey: pubkey) @@ -622,13 +619,14 @@ struct ContentView: View { try? pool.add_relay(.nwc(url: nwc.relay)) } + let user_search_cache = UserSearchCache() self.damus_state = DamusState(pool: pool, keypair: keypair, likes: EventCounter(our_pubkey: pubkey), boosts: EventCounter(our_pubkey: pubkey), contacts: Contacts(our_pubkey: pubkey), - profiles: Profiles(user_search_cache: user_search_cache), + profiles: Profiles(user_search_cache: user_search_cache, ndb: ndb), dms: home.dms, previews: PreviewCache(), zaps: Zaps(our_pubkey: pubkey), @@ -637,7 +635,7 @@ struct ContentView: View { relay_filters: relay_filters, relay_model_cache: model_cache, drafts: Drafts(), - events: EventCache(), + events: EventCache(ndb: ndb), bookmarks: BookmarksManager(pubkey: pubkey), postbox: PostBox(pool: pool), bootstrap_relays: bootstrap_relays, @@ -647,7 +645,8 @@ struct ContentView: View { nav: self.navigationCoordinator, user_search_cache: user_search_cache, music: MusicController(onChange: music_changed), - video: VideoController() + video: VideoController(), + ndb: ndb ) home.damus_state = self.damus_state! @@ -817,8 +816,11 @@ func find_event_with_subid(state: DamusState, query query_: FindEvent, subid: St switch query { case .profile(let pubkey): - if let profile = state.profiles.lookup_with_timestamp(id: pubkey) { - callback(.profile(profile.profile, profile.event)) + if let record = state.profiles.lookup_with_timestamp(pubkey), + let profile = record.profile, + let event = state.events.lookup_by_key(record.noteKey) + { + callback(.profile(profile, event)) return } filter = NostrFilter(kinds: [.metadata], limit: 1, authors: [pubkey]) @@ -855,14 +857,11 @@ func find_event_with_subid(state: DamusState, query query_: FindEvent, subid: St switch query { case .profile: if ev.known_kind == .metadata { - process_metadata_event(events: state.events, our_pubkey: state.pubkey, profiles: state.profiles, ev: ev) { profile in - guard let profile else { - callback(.invalid_profile(ev)) - return - } - callback(.profile(profile, ev)) + guard let profile = state.profiles.lookup(id: ev.pubkey) else { + callback(.invalid_profile(ev)) return } + callback(.profile(profile, ev)) } case .event: callback(.event(ev)) diff --git a/damus/Models/DamusState.swift b/damus/Models/DamusState.swift @@ -34,6 +34,7 @@ struct DamusState { let user_search_cache: UserSearchCache let music: MusicController? let video: VideoController + let ndb: Ndb @discardableResult func add_zap(zap: Zapping) -> Bool { @@ -67,12 +68,12 @@ struct DamusState { let kp = Keypair(pubkey: empty_pub, privkey: nil) return DamusState.init( - pool: RelayPool(), + pool: RelayPool(ndb: .empty), keypair: Keypair(pubkey: empty_pub, privkey: empty_sec), likes: EventCounter(our_pubkey: empty_pub), boosts: EventCounter(our_pubkey: empty_pub), contacts: Contacts(our_pubkey: empty_pub), - profiles: Profiles(user_search_cache: user_search_cache), + profiles: Profiles(user_search_cache: user_search_cache, ndb: .empty), dms: DirectMessagesModel(our_pubkey: empty_pub), previews: PreviewCache(), zaps: Zaps(our_pubkey: empty_pub), @@ -81,9 +82,9 @@ struct DamusState { relay_filters: RelayFilters(our_pubkey: empty_pub), relay_model_cache: RelayModelCache(), drafts: Drafts(), - events: EventCache(), + events: EventCache(ndb: .empty), bookmarks: BookmarksManager(pubkey: empty_pub), - postbox: PostBox(pool: RelayPool()), + postbox: PostBox(pool: RelayPool(ndb: .empty)), bootstrap_relays: [], replies: ReplyCounter(our_pubkey: empty_pub), muted_threads: MutedThreadsManager(keypair: kp), @@ -91,7 +92,8 @@ struct DamusState { nav: NavigationCoordinator(), user_search_cache: user_search_cache, music: nil, - video: VideoController() + video: VideoController(), + ndb: .empty ) } } diff --git a/damus/Models/FollowersModel.swift b/damus/Models/FollowersModel.swift @@ -77,10 +77,7 @@ class FollowersModel: ObservableObject { if ev.known_kind == .contacts { handle_contact_event(ev) - } else if ev.known_kind == .metadata { - process_metadata_event(events: damus_state.events, our_pubkey: damus_state.pubkey, profiles: damus_state.profiles, ev: ev) } - case .notice(let msg): print("followingmodel notice: \(msg)") diff --git a/damus/Models/FollowingModel.swift b/damus/Models/FollowingModel.swift @@ -54,22 +54,6 @@ class FollowingModel { } func handle_event(relay_id: String, ev: NostrConnectionEvent) { - switch ev { - case .ws_event: - break - case .nostr_event(let nev): - switch nev { - case .ok: - break - case .event(_, let ev): - if ev.kind == 0 { - process_metadata_event(events: damus_state.events, our_pubkey: damus_state.pubkey, profiles: damus_state.profiles, ev: ev) - } - case .notice(let msg): - print("followingmodel notice: \(msg)") - case .eose: - break - } - } + // don't need to do anything here really } } diff --git a/damus/Models/HomeModel.swift b/damus/Models/HomeModel.swift @@ -149,6 +149,7 @@ class HomeModel { } } + @MainActor func process_event(sub_id: String, relay_id: String, ev: NostrEvent) { if has_sub_id_event(sub_id: sub_id, ev_id: ev.id) { return @@ -169,7 +170,8 @@ class HomeModel { case .contacts: handle_contact_event(sub_id: sub_id, relay_id: relay_id, ev: ev) case .metadata: - handle_metadata_event(ev) + // profile metadata processing is handled by nostrdb + break case .list: handle_list_event(ev) case .boost: @@ -195,6 +197,7 @@ class HomeModel { } } + @MainActor func handle_status_event(_ ev: NostrEvent) { guard let st = UserStatus(ev: ev) else { return @@ -248,7 +251,8 @@ class HomeModel { nwc_success(state: self.damus_state, resp: resp) } } - + + @MainActor func handle_zap_event(_ ev: NostrEvent) { process_zap_event(damus_state: damus_state, ev: ev) { zapres in guard case .done(let zap) = zapres, @@ -373,7 +377,7 @@ class HomeModel { } } - + @MainActor func handle_event(relay_id: String, conn_event: NostrConnectionEvent) { switch conn_event { case .ws_event(let ev): @@ -582,10 +586,6 @@ class HomeModel { damus_state.contacts.set_mutelist(ev) } - func handle_metadata_event(_ ev: NostrEvent) { - process_metadata_event(events: damus_state.events, our_pubkey: damus_state.pubkey, profiles: damus_state.profiles, ev: ev) - } - func get_last_event_of_kind(relay_id: String, kind: UInt32) -> NostrEvent? { guard let m = last_event_of_kind[relay_id] else { last_event_of_kind[relay_id] = [:] @@ -791,45 +791,6 @@ func print_filters(relay_id: String?, filters groups: [[NostrFilter]]) { } */ -func process_metadata_profile(our_pubkey: Pubkey, profiles: Profiles, profile: Profile, ev: NostrEvent) { - var old_nip05: String? = nil - let mprof = profiles.lookup_with_timestamp(id: ev.pubkey) - - if let mprof { - old_nip05 = mprof.profile.nip05 - if mprof.event.created_at > ev.created_at { - // skip if we already have an newer profile - return - } - } - - if old_nip05 != profile.nip05 { - // if it's been validated before, invalidate it now - profiles.invalidate_nip05(ev.pubkey) - } - - let tprof = TimestampedProfile(profile: profile, timestamp: ev.created_at, event: ev) - profiles.add(id: ev.pubkey, profile: tprof) - - // load pfps asap - - var changed = false - - let picture = tprof.profile.picture ?? robohash(ev.pubkey) - if URL(string: picture) != nil { - changed = true - } - - let banner = tprof.profile.banner ?? "" - if URL(string: banner) != nil { - changed = true - } - - if changed { - notify(.profile_updated(pubkey: ev.pubkey, profile: profile)) - } -} - // TODO: remove this, let nostrdb handle all validation func guard_valid_event(events: EventCache, ev: NostrEvent, callback: @escaping () -> Void) { let validated = events.is_event_valid(ev.id) @@ -856,24 +817,6 @@ func guard_valid_event(events: EventCache, ev: NostrEvent, callback: @escaping ( } } -func process_metadata_event(events: EventCache, our_pubkey: Pubkey, profiles: Profiles, ev: NostrEvent, completion: ((Profile?) -> Void)? = nil) { - guard_valid_event(events: events, ev: ev) { - DispatchQueue.global(qos: .background).async { - guard let profile: Profile = decode_data(Data(ev.content.utf8)) else { - completion?(nil) - return - } - - profile.cache_lnurl() - - DispatchQueue.main.async { - process_metadata_profile(our_pubkey: our_pubkey, profiles: profiles, profile: profile, ev: ev) - completion?(profile) - } - } - } -} - func robohash(_ pk: Pubkey) -> String { return "https://robohash.org/" + pk.hex() } @@ -1394,6 +1337,7 @@ func get_zap_target_pubkey(ev: NostrEvent, events: EventCache) -> Pubkey? { return pk } +@MainActor func process_zap_event(damus_state: DamusState, ev: NostrEvent, completion: @escaping (ProcessZapResult) -> Void) { // These are zap notifications guard let ptag = get_zap_target_pubkey(ev: ev, events: damus_state.events) else { @@ -1417,17 +1361,13 @@ func process_zap_event(damus_state: DamusState, ev: NostrEvent, completion: @esc return } - guard let profile = damus_state.profiles.lookup(id: ptag) else { - completion(.failed) - return - } - - guard let lnurl = profile.lnurl else { + guard let record = damus_state.profiles.lookup_with_timestamp(ptag), + let lnurl = record.lnurl else { completion(.failed) return } - Task { + Task { [lnurl] in guard let zapper = await fetch_zapper_from_lnurl(lnurls: damus_state.lnurls, pubkey: ptag, lnurl: lnurl) else { completion(.failed) return diff --git a/damus/Models/ProfileModel.swift b/damus/Models/ProfileModel.swift @@ -102,8 +102,6 @@ class ProfileModel: ObservableObject, Equatable { } } else if ev.known_kind == .contacts { handle_profile_contact_event(ev) - } else if ev.known_kind == .metadata { - process_metadata_event(events: damus.events, our_pubkey: damus.pubkey, profiles: damus.profiles, ev: ev) } seen_event.insert(ev.id) } diff --git a/damus/Models/SearchHomeModel.swift b/damus/Models/SearchHomeModel.swift @@ -139,21 +139,13 @@ func load_profiles(profiles_subid: String, relay_id: String, load: PubkeysToLoad authors: authors) damus_state.pool.subscribe_to(sub_id: profiles_subid, filters: [filter], to: [relay_id]) { sub_id, conn_ev in - let (sid, done) = handle_subid_event(pool: damus_state.pool, relay_id: relay_id, ev: conn_ev) { sub_id, ev in - guard sub_id == profiles_subid else { - return - } - - if ev.known_kind == .metadata { - process_metadata_event(events: damus_state.events, our_pubkey: damus_state.pubkey, profiles: damus_state.profiles, ev: ev) - } - - } - - guard done && sid == profiles_subid else { + guard case .nostr_event(let ev) = conn_ev, + case .eose = ev, + sub_id == profiles_subid + else { return } - + print("done loading \(authors.count) profiles from \(relay_id)") damus_state.pool.unsubscribe(sub_id: profiles_subid, to: [relay_id]) } diff --git a/damus/Models/ThreadModel.swift b/damus/Models/ThreadModel.swift @@ -97,7 +97,8 @@ class ThreadModel: ObservableObject { event_map.insert(ev) objectWillChange.send() } - + + @MainActor func handle_event(relay_id: String, ev: NostrConnectionEvent) { let (sub_id, done) = handle_subid_event(pool: damus_state.pool, relay_id: relay_id, ev: ev) { sid, ev in @@ -105,9 +106,7 @@ class ThreadModel: ObservableObject { return } - if ev.known_kind == .metadata { - process_metadata_event(events: damus_state.events, our_pubkey: damus_state.pubkey, profiles: damus_state.profiles, ev: ev) - } else if ev.known_kind == .zap { + if ev.known_kind == .zap { process_zap_event(damus_state: damus_state, ev: ev) { zap in } diff --git a/damus/Models/UserSearchCache.swift b/damus/Models/UserSearchCache.swift @@ -10,6 +10,9 @@ import Foundation /// Cache of searchable users by name, display_name, NIP-05 identifier, or own contact list petname. /// Optimized for fast searches of substrings by using a Trie. /// Optimal for performing user searches that could be initiated by typing quickly on a keyboard into a text input field. + +// TODO: replace with lmdb (the b tree should handle this just fine ?) +// we just need a name to profile index class UserSearchCache { private let trie = Trie<Pubkey>() @@ -19,6 +22,7 @@ class UserSearchCache { } /// Computes the differences between an old profile, if it exists, and a new profile, and updates the user search cache accordingly. + @MainActor func updateProfile(id: Pubkey, profiles: Profiles, oldProfile: Profile?, newProfile: Profile) { // Remove searchable keys tied to the old profile if they differ from the new profile // to keep the trie clean without empty nodes while avoiding excessive graph searching. @@ -38,6 +42,7 @@ class UserSearchCache { } /// Adds a profile to the user search cache. + @MainActor private func addProfile(id: Pubkey, profiles: Profiles, profile: Profile) { // Searchable by name. if let name = profile.name { diff --git a/damus/Models/ZapsModel.swift b/damus/Models/ZapsModel.swift @@ -37,7 +37,8 @@ class ZapsModel: ObservableObject { func unsubscribe() { state.pool.unsubscribe(sub_id: zaps_subid) } - + + @MainActor func handle_event(relay_id: String, conn_ev: NostrConnectionEvent) { guard case .nostr_event(let resp) = conn_ev else { return diff --git a/damus/Nostr/CoreData/PersistedProfile.swift b/damus/Nostr/CoreData/PersistedProfile.swift @@ -1,39 +0,0 @@ -// -// PersistedProfile.swift -// damus -// -// Created by Bryan Montz on 4/30/23. -// - -import Foundation -import CoreData - -@objc(PersistedProfile) -final class PersistedProfile: NSManagedObject { - @NSManaged var id: String? - @NSManaged var name: String? - @NSManaged var display_name: String? - @NSManaged var about: String? - @NSManaged var picture: String? - @NSManaged var banner: String? - @NSManaged var website: String? - @NSManaged var lud06: String? - @NSManaged var lud16: String? - @NSManaged var nip05: String? - @NSManaged var damus_donation: Int16 - @NSManaged var last_update: Date? // The date that the profile was last updated by the user - @NSManaged var network_pull_date: Date? // The date we got this profile from a relay (for staleness checking) - - func copyValues(from profile: Profile) { - name = profile.name - display_name = profile.display_name - about = profile.about - picture = profile.picture - banner = profile.banner - website = profile.website - lud06 = profile.lud06 - lud16 = profile.lud16 - nip05 = profile.nip05 - damus_donation = profile.damus_donation != nil ? Int16(profile.damus_donation!) : 0 - } -} diff --git a/damus/Nostr/Nostr.swift b/damus/Nostr/Nostr.swift @@ -7,6 +7,114 @@ import Foundation +typealias Profile = NdbProfile +//typealias ProfileRecord = NdbProfileRecord + +class ProfileRecord { + let data: NdbProfileRecord + + init(data: NdbProfileRecord) { + self.data = data + } + + var profile: Profile? { return data.profile } + var receivedAt: UInt64 { data.receivedAt } + var noteKey: UInt64 { data.noteKey } + + private var _lnurl: String? = nil + var lnurl: String? { + if let _lnurl { + return _lnurl + } + + guard let profile = data.profile, + let addr = profile.lud16 ?? profile.lud06 else { + return nil; + } + + if addr.contains("@") { + // this is a heavy op and is used a lot in views, cache it! + let addr = lnaddress_to_lnurl(addr); + self._lnurl = addr + return addr + } + + if !addr.lowercased().hasPrefix("lnurl") { + return nil + } + + return addr; + } + +} + +extension NdbProfile { + var display_name: String? { + return displayName + } + + static func displayName(profile: Profile?, pubkey: Pubkey) -> DisplayName { + return parse_display_name(profile: profile, pubkey: pubkey) + } + + var damus_donation: Int? { + return Int(damusDonation) + } + + var damus_donation_v2: Int { + return Int(damusDonationV2) + } + + var website_url: URL? { + if self.website?.trimmingCharacters(in: .whitespacesAndNewlines) == "" { + return nil + } + return self.website.flatMap { url in + let trim = url.trimmingCharacters(in: .whitespacesAndNewlines) + if !(trim.hasPrefix("http://") || trim.hasPrefix("https://")) { + return URL(string: "https://" + trim) + } + return URL(string: trim) + } + } + + init(name: String? = nil, display_name: String? = nil, about: String? = nil, picture: String? = nil, banner: String? = nil, website: String? = nil, lud06: String? = nil, lud16: String? = nil, nip05: String? = nil, damus_donation: Int? = nil, reactions: Bool = true) { + + var fbb = FlatBufferBuilder() + + let name_off = fbb.create(string: name) + let display_name_off = fbb.create(string: display_name) + let about_off = fbb.create(string: about) + let picture_off = fbb.create(string: picture) + let banner_off = fbb.create(string: banner) + let website_off = fbb.create(string: website) + let lud06_off = fbb.create(string: lud06) + let lud16_off = fbb.create(string: lud16) + let nip05_off = fbb.create(string: nip05) + + let profile_data = NdbProfile.createNdbProfile(&fbb, + nameOffset: name_off, + websiteOffset: website_off, + aboutOffset: about_off, + lud16Offset: lud16_off, + bannerOffset: banner_off, + displayNameOffset: display_name_off, + reactions: reactions, + pictureOffset: picture_off, + nip05Offset: nip05_off, + damusDonation: 0, + damusDonationV2: damus_donation.map({ Int32($0) }) ?? 0, + lud06Offset: lud06_off) + + fbb.finish(offset: profile_data) + + var buf = ByteBuffer(bytes: fbb.sizedByteArray) + let profile: Profile = try! getCheckedRoot(byteBuffer: &buf) + self = profile + } +} + +/* class Profile: Codable { var value: [String: AnyCodable] @@ -24,19 +132,6 @@ class Profile: Codable { self.damus_donation = damus_donation } - convenience init(persisted_profile: PersistedProfile) { - self.init(name: persisted_profile.name, - display_name: persisted_profile.display_name, - about: persisted_profile.about, - picture: persisted_profile.picture, - banner: persisted_profile.banner, - website: persisted_profile.website, - lud06: persisted_profile.lud06, - lud16: persisted_profile.lud16, - nip05: persisted_profile.nip05, - damus_donation: Int(persisted_profile.damus_donation)) - } - private func str(_ str: String) -> String? { return get_val(str) } @@ -200,6 +295,7 @@ class Profile: Codable { return parse_display_name(profile: profile, pubkey: pubkey) } } +*/ func make_test_profile() -> Profile { return Profile(name: "jb55", display_name: "Will", about: "Its a me", picture: "https://cdn.jb55.com/img/red-me.jpg", banner: "https://pbs.twimg.com/profile_banners/9918032/1531711830/600x200", website: "jb55.com", lud06: "jb55@jb55.com", lud16: nil, nip05: "jb55@jb55.com", damus_donation: 1) @@ -222,3 +318,4 @@ func lnaddress_to_lnurl(_ lnaddr: String) -> String? { return bech32_encode(hrp: "lnurl", Array(dat)) } + diff --git a/damus/Nostr/ProfileDatabase.swift b/damus/Nostr/ProfileDatabase.swift @@ -1,181 +0,0 @@ -// -// ProfileDatabase.swift -// damus -// -// Created by Bryan Montz on 4/30/23. -// - -import Foundation -import CoreData - -enum ProfileDatabaseError: Error { - case missing_context - case outdated_input -} - -final class ProfileDatabase { - - private let entity_name = "PersistedProfile" - private var persistent_container: NSPersistentContainer? - private var background_context: NSManagedObjectContext? - private let cache_url: URL - - /// This queue is used to synchronize access to the network_pull_date_cache dictionary, which - /// prevents data races from crashing the app. - private var queue = DispatchQueue(label: "io.damus.profile_db", - qos: .userInteractive, - attributes: .concurrent) - private var network_pull_date_cache = [Pubkey: Date]() - - init(cache_url: URL = ProfileDatabase.profile_cache_url) { - self.cache_url = cache_url - set_up() - } - - private static var profile_cache_url: URL { - (FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first?.appendingPathComponent("profiles"))! - } - - private var persistent_store_description: NSPersistentStoreDescription { - let description = NSPersistentStoreDescription(url: cache_url) - description.type = NSSQLiteStoreType - description.setOption(true as NSNumber, forKey: NSMigratePersistentStoresAutomaticallyOption) - description.setOption(true as NSNumber, forKey: NSInferMappingModelAutomaticallyOption) - description.setOption(true as NSNumber, forKey: NSSQLiteManualVacuumOption) - return description - } - - private var object_model: NSManagedObjectModel? { - guard let url = Bundle.main.url(forResource: "Damus", withExtension: "momd") else { - return nil - } - return NSManagedObjectModel(contentsOf: url) - } - - private func set_up() { - guard let object_model else { - print("⚠️ Warning: ProfileDatabase failed to load its object model") - return - } - - persistent_container = NSPersistentContainer(name: "Damus", managedObjectModel: object_model) - persistent_container?.persistentStoreDescriptions = [persistent_store_description] - persistent_container?.loadPersistentStores { _, error in - if let error { - print("WARNING: ProfileDatabase failed to load: \(error)") - } - } - - persistent_container?.viewContext.automaticallyMergesChangesFromParent = true - persistent_container?.viewContext.mergePolicy = NSMergePolicy(merge: .mergeByPropertyObjectTrumpMergePolicyType) - - background_context = persistent_container?.newBackgroundContext() - background_context?.mergePolicy = NSMergePolicy(merge: .mergeByPropertyObjectTrumpMergePolicyType) - } - - private func get_persisted(id: Pubkey, context: NSManagedObjectContext) -> PersistedProfile? { - let request = NSFetchRequest<PersistedProfile>(entityName: entity_name) - request.predicate = NSPredicate(format: "id == %@", id.hex()) - request.fetchLimit = 1 - return try? context.fetch(request).first - } - - func get_network_pull_date(id: Pubkey) -> Date? { - var pull_date: Date? - queue.sync { - pull_date = network_pull_date_cache[id] - } - if let pull_date { - return pull_date - } - - let request = NSFetchRequest<PersistedProfile>(entityName: entity_name) - request.predicate = NSPredicate(format: "id == %@", id.hex()) - request.fetchLimit = 1 - request.propertiesToFetch = ["network_pull_date"] - guard let profile = try? persistent_container?.viewContext.fetch(request).first else { - return nil - } - - queue.async(flags: .barrier) { - self.network_pull_date_cache[id] = profile.network_pull_date - } - return profile.network_pull_date - } - - // MARK: - Public - - /// Updates or inserts a new Profile into the local database. Rejects profiles whose update date - /// is older than one we already have. Database writes occur on a background context for best performance. - /// - Parameters: - /// - id: Profile id (pubkey) - /// - profile: Profile object to be stored - /// - last_update: Date that the Profile was updated - func upsert(id: Pubkey, profile: Profile, last_update: Date) async throws { - guard let context = background_context else { - throw ProfileDatabaseError.missing_context - } - - try await context.perform { - var persisted_profile: PersistedProfile? - if let profile = self.get_persisted(id: id, context: context) { - if let existing_last_update = profile.last_update, last_update < existing_last_update { - throw ProfileDatabaseError.outdated_input - } else { - persisted_profile = profile - } - } else { - persisted_profile = NSEntityDescription.insertNewObject(forEntityName: self.entity_name, into: context) as? PersistedProfile - persisted_profile?.id = id.hex() - } - persisted_profile?.copyValues(from: profile) - persisted_profile?.last_update = last_update - - let pull_date = Date.now - persisted_profile?.network_pull_date = pull_date - self.queue.async(flags: .barrier) { - self.network_pull_date_cache[id] = pull_date - } - - try context.save() - } - } - - func get(id: Pubkey) -> Profile? { - guard let container = persistent_container, - let profile = get_persisted(id: id, context: container.viewContext) else { - return nil - } - return Profile(persisted_profile: profile) - } - - var count: Int { - let request = NSFetchRequest<PersistedProfile>(entityName: entity_name) - let count = try? persistent_container?.viewContext.count(for: request) - return count ?? 0 - } - - func remove_all_profiles() throws { - guard let context = background_context, let container = persistent_container else { - throw ProfileDatabaseError.missing_context - } - - queue.async(flags: .barrier) { - self.network_pull_date_cache.removeAll() - } - - let request = NSFetchRequest<NSFetchRequestResult>(entityName: entity_name) - let batch_delete_request = NSBatchDeleteRequest(fetchRequest: request) - batch_delete_request.resultType = .resultTypeObjectIDs - - let result = try container.persistentStoreCoordinator.execute(batch_delete_request, with: context) as! NSBatchDeleteResult - - // NSBatchDeleteRequest is an NSPersistentStoreRequest, which operates on disk. So now we'll manually update our in-memory context. - if let object_ids = result.result as? [NSManagedObjectID] { - let changes: [AnyHashable: Any] = [ - NSDeletedObjectsKey: object_ids - ] - NSManagedObjectContext.mergeChanges(fromRemoteContextSave: changes, into: [context]) - } - } -} diff --git a/damus/Nostr/Profiles.swift b/damus/Nostr/Profiles.swift @@ -15,72 +15,52 @@ class ValidationModel: ObservableObject { } } -class ProfileDataModel: ObservableObject { - @Published var profile: TimestampedProfile? - - init() { - self.profile = nil - } -} - class ProfileData { var status: UserStatusModel - var profile_model: ProfileDataModel var validation_model: ValidationModel var zapper: Pubkey? init() { status = .init() - profile_model = .init() validation_model = .init() zapper = nil } } class Profiles { - + private var ndb: Ndb + static let db_freshness_threshold: TimeInterval = 24 * 60 * 60 - - /// This queue is used to synchronize access to the profiles dictionary, which - /// prevents data races from crashing the app. - private var profiles_queue = DispatchQueue(label: "io.damus.profiles", - qos: .userInteractive, - attributes: .concurrent) - - private var validated_queue = DispatchQueue(label: "io.damus.profiles.validated", - qos: .userInteractive, - attributes: .concurrent) - + + @MainActor private var profiles: [Pubkey: ProfileData] = [:] + @MainActor var nip05_pubkey: [String: Pubkey] = [:] - private let database = ProfileDatabase() - let user_search_cache: UserSearchCache - init(user_search_cache: UserSearchCache) { + init(user_search_cache: UserSearchCache, ndb: Ndb) { self.user_search_cache = user_search_cache + self.ndb = ndb } - + + @MainActor func is_validated(_ pk: Pubkey) -> NIP05? { - validated_queue.sync { - self.profile_data(pk).validation_model.validated - } + self.profile_data(pk).validation_model.validated } + @MainActor func invalidate_nip05(_ pk: Pubkey) { - validated_queue.async(flags: .barrier) { - self.profile_data(pk).validation_model.validated = nil - } + self.profile_data(pk).validation_model.validated = nil } + @MainActor func set_validated(_ pk: Pubkey, nip05: NIP05?) { - validated_queue.async(flags: .barrier) { - self.profile_data(pk).validation_model.validated = nip05 - } + self.profile_data(pk).validation_model.validated = nip05 } - + + @MainActor func profile_data(_ pubkey: Pubkey) -> ProfileData { guard let data = profiles[pubkey] else { let data = ProfileData() @@ -91,60 +71,28 @@ class Profiles { return data } + @MainActor func lookup_zapper(pubkey: Pubkey) -> Pubkey? { profile_data(pubkey).zapper } - - func add(id: Pubkey, profile: TimestampedProfile) { - profiles_queue.async(flags: .barrier) { - let old_timestamped_profile = self.profile_data(id).profile_model.profile - self.profile_data(id).profile_model.profile = profile - self.user_search_cache.updateProfile(id: id, profiles: self, oldProfile: old_timestamped_profile?.profile, newProfile: profile.profile) - } - - Task { - do { - try await database.upsert(id: id, profile: profile.profile, last_update: Date(timeIntervalSince1970: TimeInterval(profile.timestamp))) - } catch { - print("⚠️ Warning: Profiles failed to save a profile: \(error)") - } - } + + func lookup_with_timestamp(_ pubkey: Pubkey) -> ProfileRecord? { + return ndb.lookup_profile(pubkey) } - + func lookup(id: Pubkey) -> Profile? { - var profile: Profile? - profiles_queue.sync { - profile = self.profile_data(id).profile_model.profile?.profile - } - return profile ?? database.get(id: id) + return ndb.lookup_profile(id)?.profile } - - func lookup_with_timestamp(id: Pubkey) -> TimestampedProfile? { - profiles_queue.sync { - return self.profile_data(id).profile_model.profile - } - } - + func has_fresh_profile(id: Pubkey) -> Bool { var profile: Profile? - profiles_queue.sync { - profile = self.profile_data(id).profile_model.profile?.profile - } - if profile != nil { - return true - } - // check memory first - return false - - // then disk - guard let pull_date = database.get_network_pull_date(id: id) else { - return false - } - return Date.now.timeIntervalSince(pull_date) < Profiles.db_freshness_threshold + guard let profile = lookup_with_timestamp(id) else { return false } + return Date.now.timeIntervalSince(Date(timeIntervalSince1970: Double(profile.receivedAt))) < Profiles.db_freshness_threshold } } +@MainActor func invalidate_zapper_cache(pubkey: Pubkey, profiles: Profiles, lnurl: LNUrls) { profiles.profile_data(pubkey).zapper = nil lnurl.endpoints.removeValue(forKey: pubkey) diff --git a/damus/Nostr/RelayConnection.swift b/damus/Nostr/RelayConnection.swift @@ -56,12 +56,17 @@ final class RelayConnection: ObservableObject { private var subscriptionToken: AnyCancellable? private var handleEvent: (NostrConnectionEvent) -> () + private var processEvent: (WebSocketEvent) -> () private let url: RelayURL var log: RelayLog? - init(url: RelayURL, handleEvent: @escaping (NostrConnectionEvent) -> ()) { + init(url: RelayURL, + handleEvent: @escaping (NostrConnectionEvent) -> (), + processEvent: @escaping (WebSocketEvent) -> ()) + { self.url = url self.handleEvent = handleEvent + self.processEvent = processEvent } func ping() { @@ -138,6 +143,7 @@ final class RelayConnection: ObservableObject { } private func receive(event: WebSocketEvent) { + processEvent(event) switch event { case .connected: DispatchQueue.main.async { diff --git a/damus/Nostr/RelayPool.swift b/damus/Nostr/RelayPool.swift @@ -30,12 +30,15 @@ class RelayPool { var request_queue: [QueuedRequest] = [] var seen: Set<SeenEvent> = Set() var counts: [String: UInt64] = [:] - + var ndb: Ndb + private let network_monitor = NWPathMonitor() private let network_monitor_queue = DispatchQueue(label: "io.damus.network_monitor") private var last_network_status: NWPath.Status = .unsatisfied - init() { + init(ndb: Ndb) { + self.ndb = ndb + network_monitor.pathUpdateHandler = { [weak self] path in if (path.status == .satisfied || path.status == .requiresConnection) && self?.last_network_status != path.status { DispatchQueue.main.async { @@ -110,9 +113,15 @@ class RelayPool { if get_relay(relay_id) != nil { throw RelayError.RelayAlreadyExists } - let conn = RelayConnection(url: url) { event in + let conn = RelayConnection(url: url, handleEvent: { event in self.handle_event(relay_id: relay_id, event: event) - } + }, processEvent: { wsev in + guard case .message(let msg) = wsev, + case .string(let str) = msg + else { return } + + self.ndb.process_event(str) + }) let relay = Relay(descriptor: desc, connection: conn) self.relays.append(relay) } diff --git a/damus/TestData.swift b/damus/TestData.swift @@ -54,9 +54,11 @@ let test_following_model = FollowingModel(damus_state: test_damus_state(), conta func test_damus_state() -> DamusState { let damus = DamusState.empty + /* let prof = Profile(name: "damus", display_name: "damus", about: "iOS app!", picture: "https://damus.io/img/logo.png", banner: "", website: "https://damus.io", lud06: nil, lud16: "jb55@sendsats.lol", nip05: "damus.io", damus_donation: nil) let tsprof = TimestampedProfile(profile: prof, timestamp: 0, event: test_note) damus.profiles.add(id: test_pubkey, profile: tsprof) + */ return damus } diff --git a/damus/Util/AccountDeletion.swift b/damus/Util/AccountDeletion.swift @@ -9,12 +9,11 @@ import Foundation func created_deleted_account_profile(keypair: FullKeypair) -> NostrEvent? { - let profile = Profile() - profile.about = "account deleted" - profile.name = "nobody" - - guard let content = encode_json(profile) else { - return nil - } + let about = "account deleted" + let name = "nobody" + + let profile = Profile(name: name, about: about) + + guard let content = encode_json(profile) else { return nil } return NostrEvent(content: content, keypair: keypair.to_keypair(), kind: 0) } diff --git a/damus/Util/EventCache.swift b/damus/Util/EventCache.swift @@ -137,6 +137,7 @@ class EventData { } class EventCache { + private let ndb: Ndb private var events: [NoteId: NostrEvent] = [:] private var replies = ReplyMap() private var cancellable: AnyCancellable? @@ -145,7 +146,8 @@ class EventCache { //private var thread_latest: [String: Int64] - init() { + init(ndb: Ndb) { + self.ndb = ndb cancellable = NotificationCenter.default.publisher( for: UIApplication.didReceiveMemoryWarningNotification ).sink { [weak self] _ in @@ -250,7 +252,11 @@ class EventCache { insert(ev) return ev } - + + func lookup_by_key(_ key: UInt64) -> NostrEvent? { + ndb.lookup_note_by_key(key) + } + func lookup(_ evid: NoteId) -> NostrEvent? { return events[evid] } diff --git a/damus/Views/ActionBar/EventActionBar.swift b/damus/Views/ActionBar/EventActionBar.swift @@ -28,7 +28,7 @@ struct EventActionBar: View { } var lnurl: String? { - damus_state.profiles.lookup(id: event.pubkey)?.lnurl + damus_state.profiles.lookup_with_timestamp(event.pubkey)?.lnurl } var show_like: Bool { diff --git a/damus/Views/Events/Components/EventTop.swift b/damus/Views/Events/Components/EventTop.swift @@ -7,6 +7,7 @@ import SwiftUI +@MainActor struct EventTop: View { let state: DamusState let event: NostrEvent diff --git a/damus/Views/Events/EventShell.swift b/damus/Views/Events/EventShell.swift @@ -7,6 +7,7 @@ import SwiftUI +@MainActor struct EventShell<Content: View>: View { let state: DamusState let event: NostrEvent diff --git a/damus/Views/Onboarding/SuggestedUserView.swift b/damus/Views/Onboarding/SuggestedUserView.swift @@ -7,7 +7,7 @@ import SwiftUI -struct SuggestedUser: Codable { +struct SuggestedUser { let pubkey: Pubkey let name: String let about: String diff --git a/damus/Views/Onboarding/SuggestedUsersViewModel.swift b/damus/Views/Onboarding/SuggestedUsersViewModel.swift @@ -86,13 +86,7 @@ class SuggestedUsersViewModel: ObservableObject { switch nev { case .event(let sub_id, let ev): - guard sub_id == self.sub_id else { - return - } - - if ev.known_kind == .metadata { - process_metadata_event(events: damus_state.events, our_pubkey: damus_state.pubkey, profiles: damus_state.profiles, ev: ev) - } + break case .notice(let msg): print("suggested user profiles notice: \(msg)") diff --git a/damus/Views/Profile/EditMetadataView.swift b/damus/Views/Profile/EditMetadataView.swift @@ -45,18 +45,14 @@ struct EditMetadataView: View { } func to_profile() -> Profile { - let profile = self.profile ?? Profile() - - profile.name = name - profile.display_name = display_name - profile.about = about - profile.website = website - profile.nip05 = nip05.isEmpty ? nil : nip05 - profile.picture = picture.isEmpty ? nil : picture - profile.banner = banner.isEmpty ? nil : banner - profile.lud06 = ln.contains("@") ? nil : ln - profile.lud16 = ln.contains("@") ? ln : nil - + let new_nip05 = nip05.isEmpty ? nil : nip05 + let new_picture = picture.isEmpty ? nil : picture + let new_banner = banner.isEmpty ? nil : banner + let new_lud06 = ln.contains("@") ? nil : ln + let new_lud16 = ln.contains("@") ? ln : nil + + let profile = Profile(name: name, display_name: display_name, about: about, picture: new_picture, banner: new_banner, website: website, lud06: new_lud06, lud16: new_lud16, nip05: new_nip05, damus_donation: nil) + return profile } diff --git a/damus/Views/Profile/EventProfileName.swift b/damus/Views/Profile/EventProfileName.swift @@ -8,6 +8,7 @@ import SwiftUI /// Profile Name used when displaying an event in the timeline +@MainActor struct EventProfileName: View { let damus_state: DamusState let pubkey: Pubkey diff --git a/damus/Views/Profile/ProfileName.swift b/damus/Views/Profile/ProfileName.swift @@ -47,7 +47,8 @@ struct ProfileName: View { var friend_type: FriendType? { return get_friend_type(contacts: damus_state.contacts, pubkey: self.pubkey) } - + + @MainActor var current_nip05: NIP05? { nip05 ?? damus_state.profiles.is_validated(pubkey) } diff --git a/damus/Views/Profile/ProfilePicView.swift b/damus/Views/Profile/ProfilePicView.swift @@ -105,11 +105,11 @@ func get_profile_url(picture: String?, pubkey: Pubkey, profiles: Profiles) -> UR func make_preview_profiles(_ pubkey: Pubkey) -> Profiles { let user_search_cache = UserSearchCache() - let profiles = Profiles(user_search_cache: user_search_cache) + let profiles = Profiles(user_search_cache: user_search_cache, ndb: .empty) let picture = "http://cdn.jb55.com/img/red-me.jpg" let profile = Profile(name: "jb55", display_name: "William Casarin", about: "It's me", picture: picture, banner: "", website: "https://jb55.com", lud06: nil, lud16: nil, nip05: "jb55.com", damus_donation: nil) - let ts_profile = TimestampedProfile(profile: profile, timestamp: 0, event: test_note) - profiles.add(id: pubkey, profile: ts_profile) + //let ts_profile = TimestampedProfile(profile: profile, timestamp: 0, event: test_note) + //profiles.add(id: pubkey, profile: ts_profile) return profiles } diff --git a/damus/Views/Profile/ProfileView.swift b/damus/Views/Profile/ProfileView.swift @@ -216,7 +216,8 @@ struct ProfileView: View { .accentColor(DamusColors.white) } - func lnButton(lnurl: String, profile: Profile) -> some View { + func lnButton(lnurl: String, record: ProfileRecord, profile: Profile) -> some View { + let profile = record.profile! let button_img = profile.reactions == false ? "zap.fill" : "zap" return Button(action: { present_sheet(.zap(target: .profile(self.profile.pubkey), lnurl: lnurl)) @@ -235,7 +236,7 @@ struct ProfileView: View { } label: { Label(addr, image: "copy2") } - } else if let lnurl = profile.lnurl { + } else if let lnurl = record.lnurl { Button { UIPasteboard.general.string = lnurl } label: { @@ -268,14 +269,14 @@ struct ProfileView: View { .font(.footnote) } - func actionSection(profile_data: Profile?) -> some View { + func actionSection(record: ProfileRecord?) -> some View { return Group { - - if let profile = profile_data, - let lnurl = profile.lnurl, + if let record, + let profile = record.profile, + let lnurl = record.lnurl, lnurl != "" { - lnButton(lnurl: lnurl, profile: profile) + lnButton(lnurl: lnurl, record: record, profile: profile) } dmButton @@ -307,8 +308,9 @@ struct ProfileView: View { return scale < 1 ? scale : 1 } - func nameSection(profile_data: Profile?) -> some View { + func nameSection(profile_data: ProfileRecord?) -> some View { return Group { + let follows_you = profile.pubkey != damus_state.pubkey && profile.follows(pubkey: damus_state.pubkey) HStack(alignment: .center) { ProfilePicView(pubkey: profile.pubkey, size: pfp_size, highlight: .custom(imageBorderColor(), 4.0), profiles: damus_state.profiles, disable_animation: damus_state.settings.disable_animation) .padding(.top, -(pfp_size / 2.0)) @@ -322,48 +324,46 @@ struct ProfileView: View { } Spacer() - - let follows_you = profile.pubkey != damus_state.pubkey && profile.follows(pubkey: damus_state.pubkey) + if follows_you { followsYouBadge } - actionSection(profile_data: profile_data) + actionSection(record: profile_data) } - - ProfileNameView(pubkey: profile.pubkey, profile: profile_data, damus: damus_state) + + ProfileNameView(pubkey: profile.pubkey, profile: profile_data?.profile, damus: damus_state) } } var followersCount: some View { HStack { - if followers.count == nil { + if let followerCount = followers.count { + let nounString = pluralizedString(key: "followers_count", count: followerCount) + let nounText = Text(verbatim: nounString).font(.subheadline).foregroundColor(.gray) + Text("\(Text(verbatim: followerCount.formatted()).font(.subheadline.weight(.medium))) \(nounText)", comment: "Sentence composed of 2 variables to describe how many people are following a user. In source English, the first variable is the number of followers, and the second variable is 'Follower' or 'Followers'.") + } else { Image("download") .resizable() .frame(width: 20, height: 20) Text("Followers", comment: "Label describing followers of a user.") .font(.subheadline) .foregroundColor(.gray) - } else { - let followerCount = followers.count! - let nounString = pluralizedString(key: "followers_count", count: followerCount) - let nounText = Text(verbatim: nounString).font(.subheadline).foregroundColor(.gray) - Text("\(Text(verbatim: followerCount.formatted()).font(.subheadline.weight(.medium))) \(nounText)", comment: "Sentence composed of 2 variables to describe how many people are following a user. In source English, the first variable is the number of followers, and the second variable is 'Follower' or 'Followers'.") } } } var aboutSection: some View { VStack(alignment: .leading, spacing: 8.0) { - let profile_data = damus_state.profiles.lookup(id: profile.pubkey) + let profile_data = damus_state.profiles.lookup_with_timestamp(profile.pubkey) nameSection(profile_data: profile_data) - if let about = profile_data?.about { + if let about = profile_data?.profile?.about { AboutView(state: damus_state, about: about) } - if let url = profile_data?.website_url { + if let url = profile_data?.profile?.website_url { WebsiteLink(url: url) } @@ -514,6 +514,7 @@ extension View { } } +@MainActor func check_nip05_validity(pubkey: Pubkey, profiles: Profiles) { guard let profile = profiles.lookup(id: pubkey), let nip05 = profile.nip05, diff --git a/damus/Views/SaveKeysView.swift b/damus/Views/SaveKeysView.swift @@ -10,7 +10,7 @@ import Security struct SaveKeysView: View { let account: CreateAccountModel - let pool: RelayPool = RelayPool() + let pool: RelayPool = RelayPool(ndb: Ndb()!) @State var pub_copied: Bool = false @State var priv_copied: Bool = false @State var loading: Bool = false diff --git a/damus/Views/Search/SearchingEventView.swift b/damus/Views/Search/SearchingEventView.swift @@ -20,6 +20,7 @@ enum SearchType: Equatable { case nip05(String) } +@MainActor struct SearchingEventView: View { let state: DamusState let search_type: SearchType diff --git a/damus/Views/SideMenuView.swift b/damus/Views/SideMenuView.swift @@ -7,6 +7,7 @@ import SwiftUI +@MainActor struct SideMenuView: View { let damus_state: DamusState @Binding var isSidebarVisible: Bool diff --git a/damus/Views/Wallet/WalletView.swift b/damus/Views/Wallet/WalletView.swift @@ -168,9 +168,9 @@ struct WalletView: View { return } - profile.damus_donation = p + let prof = Profile(name: profile.name, display_name: profile.display_name, about: profile.about, picture: profile.picture, banner: profile.banner, website: profile.website, lud06: profile.lud06, lud16: profile.lud16, nip05: profile.nip05, damus_donation: p, reactions: profile.reactions) - notify(.profile_updated(pubkey: damus_state.pubkey, profile: profile)) + notify(.profile_updated(pubkey: damus_state.pubkey, profile: prof)) } .onDisappear { guard let keypair = damus_state.keypair.to_full(), @@ -180,12 +180,11 @@ struct WalletView: View { return } - profile.damus_donation = settings.donation_percent - guard let meta = make_metadata_event(keypair: keypair, metadata: profile) else { + let prof = Profile(name: profile.name, display_name: profile.display_name, about: profile.about, picture: profile.picture, banner: profile.banner, website: profile.website, lud06: profile.lud06, lud16: profile.lud16, nip05: profile.nip05, damus_donation: settings.donation_percent, reactions: profile.reactions) + + guard let meta = make_metadata_event(keypair: keypair, metadata: prof) else { return } - let tsprofile = TimestampedProfile(profile: profile, timestamp: meta.created_at, event: meta) - damus_state.profiles.add(id: damus_state.pubkey, profile: tsprofile) damus_state.postbox.send(meta) } } diff --git a/damusTests/NostrScriptTests.swift b/damusTests/NostrScriptTests.swift @@ -37,7 +37,7 @@ final class NostrScriptTests: XCTestCase { func test_bool_set() throws { let data = try load_bool_set_test_wasm().bytes - let pool = RelayPool() + let pool = RelayPool(ndb: .empty) let script = NostrScript(pool: pool, data: data) let pk = Pubkey(hex: "32e1827635450ebb3c5a7d12c1f8e7b2b514439ac10a67eef3d9fd9c5c68e245")! UserSettingsStore.pubkey = pk diff --git a/damusTests/ProfileDatabaseTests.swift b/damusTests/ProfileDatabaseTests.swift @@ -1,124 +0,0 @@ -// -// ProfileDatabaseTests.swift -// damusTests -// -// Created by Bryan Montz on 5/13/23. -// - -import XCTest -@testable import damus - -class ProfileDatabaseTests: XCTestCase { - - static let cache_url = (FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first?.appendingPathComponent("test-profiles"))! - let database = ProfileDatabase(cache_url: ProfileDatabaseTests.cache_url) - - override func tearDownWithError() throws { - // This method is called after the invocation of each test method in the class. - try database.remove_all_profiles() - } - - var test_profile: Profile { - Profile(name: "test-name", - display_name: "test-display-name", - about: "test-about", - picture: "test-picture", - banner: "test-banner", - website: "test-website", - lud06: "test-lud06", - lud16: "test-lud16", - nip05: "test-nip05", - damus_donation: 100) - } - - func testStoreAndRetrieveProfile() async throws { - let id = test_pubkey - - let profile = test_profile - - // make sure it's not there yet - XCTAssertNil(database.get(id: id)) - - // store the profile - try await database.upsert(id: id, profile: profile, last_update: .now) - - // read the profile out of the database - let retrievedProfile = try XCTUnwrap(database.get(id: id)) - - XCTAssertEqual(profile.name, retrievedProfile.name) - XCTAssertEqual(profile.display_name, retrievedProfile.display_name) - XCTAssertEqual(profile.about, retrievedProfile.about) - XCTAssertEqual(profile.picture, retrievedProfile.picture) - XCTAssertEqual(profile.banner, retrievedProfile.banner) - XCTAssertEqual(profile.website, retrievedProfile.website) - XCTAssertEqual(profile.lud06, retrievedProfile.lud06) - XCTAssertEqual(profile.lud16, retrievedProfile.lud16) - XCTAssertEqual(profile.nip05, retrievedProfile.nip05) - XCTAssertEqual(profile.damus_donation, retrievedProfile.damus_donation) - } - - func testRejectOutdatedProfile() async throws { - let id = test_pubkey - - // store a profile - let profile = test_profile - let profile_last_updated = Date.now - try await database.upsert(id: id, profile: profile, last_update: profile_last_updated) - - // try to store a profile with the same id but the last_update date is older than the previously stored profile - let outdatedProfile = test_profile - let outdated_last_updated = profile_last_updated.addingTimeInterval(-60) - - do { - try await database.upsert(id: id, profile: outdatedProfile, last_update: outdated_last_updated) - XCTFail("expected to throw error") - } catch let error as ProfileDatabaseError { - XCTAssertEqual(error, ProfileDatabaseError.outdated_input) - } catch { - XCTFail("not the expected error") - } - } - - func testUpdateExistingProfile() async throws { - let id = test_pubkey - - // store a profile - let profile = test_profile - let profile_last_update = Date.now - try await database.upsert(id: id, profile: profile, last_update: profile_last_update) - - // update the same profile - let updated_profile = test_profile - updated_profile.nip05 = "updated-nip05" - let updated_profile_last_update = profile_last_update.addingTimeInterval(60) - try await database.upsert(id: id, profile: updated_profile, last_update: updated_profile_last_update) - - // retrieve the profile and make sure it was updated - let retrieved_profile = database.get(id: id) - XCTAssertEqual(retrieved_profile?.nip05, "updated-nip05") - } - - func testStoreMultipleAndRemoveAllProfiles() async throws { - XCTAssertEqual(database.count, 0) - - // store a profile - let id = test_pubkey - let profile = test_profile - let profile_last_update = Date.now - try await database.upsert(id: id, profile: profile, last_update: profile_last_update) - - XCTAssertEqual(database.count, 1) - - // store another profile - let id2 = test_pubkey_2 - let profile2 = test_profile - let profile_last_update2 = Date.now - try await database.upsert(id: id2, profile: profile2, last_update: profile_last_update2) - - XCTAssertEqual(database.count, 2) - - try database.remove_all_profiles() - - XCTAssertEqual(database.count, 0) - } -} diff --git a/damusTests/UserSearchCacheTests.swift b/damusTests/UserSearchCacheTests.swift @@ -14,6 +14,7 @@ final class UserSearchCacheTests: XCTestCase { let damusState = DamusState.empty let nip05 = "_@somedomain.com" + @MainActor override func setUpWithError() throws { keypair = try XCTUnwrap(generate_new_keypair()) @@ -24,8 +25,6 @@ final class UserSearchCacheTests: XCTestCase { damusState.profiles.set_validated(pubkey, nip05: validatedNip05) let profile = Profile(name: "tyiu", display_name: "Terry Yiu", about: nil, picture: nil, banner: nil, website: nil, lud06: nil, lud16: nil, nip05: nip05, damus_donation: nil) - let timestampedProfile = TimestampedProfile(profile: profile, timestamp: 0, event: test_note) - damusState.profiles.add(id: pubkey, profile: timestampedProfile) // Lookup to synchronize access on profiles dictionary to avoid race conditions. let _ = damusState.profiles.lookup(id: pubkey) @@ -47,6 +46,7 @@ final class UserSearchCacheTests: XCTestCase { XCTAssertEqual(damusState.user_search_cache.search(key: "i"), [keypair.pubkey]) } + @MainActor func testUpdateProfile() throws { let keypair = try XCTUnwrap(keypair) @@ -56,8 +56,6 @@ final class UserSearchCacheTests: XCTestCase { damusState.profiles.set_validated(keypair.pubkey, nip05: NIP05.parse(newNip05)) let newProfile = Profile(name: "whoami", display_name: "T-DAWG", about: nil, picture: nil, banner: nil, website: nil, lud06: nil, lud16: nil, nip05: newNip05, damus_donation: nil) - let newTimestampedProfile = TimestampedProfile(profile: newProfile, timestamp: 1000, event: test_note) - damusState.profiles.add(id: keypair.pubkey, profile: newTimestampedProfile) // Lookup to synchronize access on profiles dictionary to avoid race conditions. let _ = damusState.profiles.lookup(id: keypair.pubkey) diff --git a/damusTests/WalletConnectTests.swift b/damusTests/WalletConnectTests.swift @@ -84,7 +84,7 @@ final class WalletConnectTests: XCTestCase { let pk = "89446b900c70d62438dcf66756405eea6225ad94dc61f3856f62f9699111a9a6" let nwc = WalletConnectURL(str: "nostrwalletconnect://\(pk)?relay=ws://127.0.0.1&secret=\(sec)&lud16=jb55@jb55.com")! - let pool = RelayPool() + let pool = RelayPool(ndb: .empty) let box = PostBox(pool: pool) nwc_pay(url: nwc, pool: pool, post: box, invoice: "invoice") diff --git a/damusTests/ZapTests.swift b/damusTests/ZapTests.swift @@ -11,8 +11,6 @@ import XCTest final class ZapTests: XCTestCase { override func setUpWithError() throws { - let db = ProfileDatabase() - try db.remove_all_profiles() } override func tearDownWithError() throws { @@ -71,7 +69,7 @@ final class ZapTests: XCTestCase { XCTAssertEqual(zap.target, ZapTarget.profile(profile)) XCTAssertEqual(zap_notification_title(zap), "Zap") - XCTAssertEqual(zap_notification_body(profiles: Profiles(user_search_cache: UserSearchCache()), zap: zap), "You received 1k sats from 107jk7ht:2quqncxg") + XCTAssertEqual(zap_notification_body(profiles: Profiles(user_search_cache: UserSearchCache(), ndb: .empty), zap: zap), "You received 1k sats from 107jk7ht:2quqncxg") } } diff --git a/nostrdb/Ndb.swift b/nostrdb/Ndb.swift @@ -14,11 +14,28 @@ class Ndb { (FileManager.default.urls(for: .documentDirectory, in: .userDomainMask).first?.absoluteString.replacingOccurrences(of: "file://", with: ""))! } + static var empty: Ndb { + Ndb(ndb: ndb_t(ndb: nil)) + } + init?() { + //try? FileManager.default.removeItem(atPath: Ndb.db_path + "/lock.mdb") + //try? FileManager.default.removeItem(atPath: Ndb.db_path + "/data.mdb") + var ndb_p: OpaquePointer? = nil + let ingest_threads: Int32 = 4 + var mapsize: Int = 1024 * 1024 * 1024 * 32 + let ok = Ndb.db_path.withCString { testdir in - return ndb_init(&ndb_p, testdir, 1024 * 1024 * 1024 * 32, 4) != 0 + var ok = false + while !ok && mapsize > 1024 * 1024 * 700 { + ok = ndb_init(&ndb_p, testdir, mapsize, ingest_threads) != 0 + if !ok { + mapsize /= 2 + } + } + return ok } if !ok { @@ -28,24 +45,51 @@ class Ndb { self.ndb = ndb_t(ndb: ndb_p) } + init(ndb: ndb_t) { + self.ndb = ndb + } + + func lookup_note_by_key(_ key: UInt64) -> NdbNote? { + guard let note_p = ndb_get_note_by_key(ndb.ndb, key, nil) else { + return nil + } + return NdbNote(note: note_p, owned_size: nil) + } + func lookup_note(_ id: NoteId) -> NdbNote? { - id.id.withUnsafeBytes { bs in - guard let note_p = ndb_get_note_by_id(ndb.ndb, bs, nil) else { + id.id.withUnsafeBytes { (ptr: UnsafeRawBufferPointer) -> NdbNote? in + guard let baseAddress = ptr.baseAddress, + let note_p = ndb_get_note_by_id(ndb.ndb, baseAddress, nil) else { return nil } return NdbNote(note: note_p, owned_size: nil) } } - func lookup_profile(_ pubkey: Pubkey) -> NdbProfile? { - return pubkey.id.withUnsafeBytes { pk_bytes in + func lookup_profile(_ pubkey: Pubkey) -> ProfileRecord? { + return pubkey.id.withUnsafeBytes { (ptr: UnsafeRawBufferPointer) -> ProfileRecord? in var size: Int = 0 - guard let profile_p = ndb_get_profile_by_pubkey(ndb.ndb, pk_bytes, &size) else { + + guard let baseAddress = ptr.baseAddress, + let profile_p = ndb_get_profile_by_pubkey(ndb.ndb, baseAddress, &size) + else { return nil } - let buf = ByteBuffer(assumingMemoryBound: profile_p, capacity: size) - return NdbProfile(buf, o: 0) + do { + var buf = ByteBuffer(assumingMemoryBound: profile_p, capacity: size) + let rec: NdbProfileRecord = try getDebugCheckedRoot(byteBuffer: &buf) + return ProfileRecord(data: rec) + } catch { + // Handle error appropriately + print("UNUSUAL: \(error)") + return nil + } + } + } + func process_event(_ str: String) -> Bool { + return str.withCString { cstr in + return ndb_process_event(ndb.ndb, cstr, Int32(str.utf8.count)) != 0 } } @@ -59,3 +103,13 @@ class Ndb { ndb_destroy(ndb.ndb) } } + +#if DEBUG +func getDebugCheckedRoot<T: FlatBufferObject & Verifiable>(byteBuffer: inout ByteBuffer) throws -> T { + return try getCheckedRoot(byteBuffer: &byteBuffer) +} +#else +func getDebugCheckedRoot<T: FlatBufferObject>(byteBuffer: inout ByteBuffer) throws -> T { + return try getRoot(byteBuffer: &byteBuffer) +} +#endif diff --git a/nostrdb/Test/NdbTests.swift b/nostrdb/Test/NdbTests.swift @@ -33,6 +33,11 @@ final class NdbTests: XCTestCase { } + func test_profile_creation() { + let profile = make_test_profile() + XCTAssertEqual(profile.name, "jb55") + } + func test_ndb_init() { do { @@ -54,7 +59,8 @@ final class NdbTests: XCTestCase { XCTAssertNotNil(profile) guard let profile else { return } - XCTAssertEqual(profile.name, "jb55") + XCTAssertEqual(profile.profile?.name, "jb55") + XCTAssertEqual(profile.lnurl, "fixme") } @@ -71,7 +77,7 @@ final class NdbTests: XCTestCase { XCTAssertEqual(note.id, id) XCTAssertEqual(note.pubkey, pubkey) - XCTAssertEqual(note.count, 34322) + XCTAssertEqual(note.count, 34328) XCTAssertEqual(note.kind, 3) XCTAssertEqual(note.created_at, 1689904312)