damus

nostr ios client
git clone git://jb55.com/damus
Log | Files | Refs | README | LICENSE

commit 8f32c81b6cd0c386bdb9d8c20644fd0fd2d517fa
parent f8185d0ca5740d3b9c7ddac777cd5e7a3d3b98fb
Author: Daniel D’Aquino <daniel@daquino.me>
Date:   Wed,  9 Apr 2025 22:41:50 -0700

Create NostrDB streaming and async lookup interfaces

This commit introduces new interfaces for working with NostrDB from
Swift, including `NostrFilter` conversion, subscription streaming via
AsyncStreams and lookup/wait functions.

No user-facing changes.

Changelog-None
Signed-off-by: Daniel D’Aquino <daniel@daquino.me>

Diffstat:
Mdamus.xcodeproj/project.pbxproj | 24++++++++++++++++++++++++
Mdamus/Core/Types/Ids/NoteId.swift | 11+++++++++++
Mdamus/Core/Types/Ids/Pubkey.swift | 10++++++++++
Mdamus/Shared/Utilities/Log.swift | 1+
Anostrdb/Ndb+.swift | 30++++++++++++++++++++++++++++++
Mnostrdb/Ndb.swift | 323+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++--
Anostrdb/NdbFilter.swift | 356+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Mnostrdb/NdbNote.swift | 31++++++-------------------------
8 files changed, 755 insertions(+), 31 deletions(-)

diff --git a/damus.xcodeproj/project.pbxproj b/damus.xcodeproj/project.pbxproj @@ -1561,6 +1561,12 @@ D74E64132DC95CC7004C7892 /* HumanReadableErrors.swift in Sources */ = {isa = PBXBuildFile; fileRef = D74E64112DC95CBE004C7892 /* HumanReadableErrors.swift */; }; D74E64142DC95CC7004C7892 /* HumanReadableErrors.swift in Sources */ = {isa = PBXBuildFile; fileRef = D74E64112DC95CBE004C7892 /* HumanReadableErrors.swift */; }; D74E64152DC95CC7004C7892 /* HumanReadableErrors.swift in Sources */ = {isa = PBXBuildFile; fileRef = D74E64112DC95CBE004C7892 /* HumanReadableErrors.swift */; }; + D74DEC8A2DA0A19B00E69FA6 /* Ndb+.swift in Sources */ = {isa = PBXBuildFile; fileRef = D74DEC892DA0A19800E69FA6 /* Ndb+.swift */; }; + D74DEC8B2DA0A19B00E69FA6 /* Ndb+.swift in Sources */ = {isa = PBXBuildFile; fileRef = D74DEC892DA0A19800E69FA6 /* Ndb+.swift */; }; + D74DEC8C2DA0A19B00E69FA6 /* Ndb+.swift in Sources */ = {isa = PBXBuildFile; fileRef = D74DEC892DA0A19800E69FA6 /* Ndb+.swift */; }; + D74DEC8F2DA0C65F00E69FA6 /* Ndb+.swift in Sources */ = {isa = PBXBuildFile; fileRef = D74DEC892DA0A19800E69FA6 /* Ndb+.swift */; }; + D74DEC902DA0C6B500E69FA6 /* NostrFilter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C75EFAE28049D340006080F /* NostrFilter.swift */; }; + D74DEC912DA0CA2400E69FA6 /* Array.swift in Sources */ = {isa = PBXBuildFile; fileRef = D72E12772BEED22400F4F781 /* Array.swift */; }; D74EA08A2D2BF2A7002290DD /* URLHandler.swift in Sources */ = {isa = PBXBuildFile; fileRef = D767066E2C8BB3CE00F09726 /* URLHandler.swift */; }; D74EA08E2D2E271E002290DD /* ErrorView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D74EA08D2D2E271E002290DD /* ErrorView.swift */; }; D74EA08F2D2E271E002290DD /* ErrorView.swift in Sources */ = {isa = PBXBuildFile; fileRef = D74EA08D2D2E271E002290DD /* ErrorView.swift */; }; @@ -1779,6 +1785,10 @@ D7FA46E52DBDAA7E002C9BB0 /* ImageCacheMigrations.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7FA46E42DBDAA75002C9BB0 /* ImageCacheMigrations.swift */; }; D7FA46E62DBDAA7E002C9BB0 /* ImageCacheMigrations.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7FA46E42DBDAA75002C9BB0 /* ImageCacheMigrations.swift */; }; D7FA46E72DBDAA7E002C9BB0 /* ImageCacheMigrations.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7FA46E42DBDAA75002C9BB0 /* ImageCacheMigrations.swift */; }; + D7F563102DEE71C0008509DE /* NdbFilter.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7F5630F2DEE71BB008509DE /* NdbFilter.swift */; }; + D7F563112DEE71C0008509DE /* NdbFilter.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7F5630F2DEE71BB008509DE /* NdbFilter.swift */; }; + D7F563122DEE71C0008509DE /* NdbFilter.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7F5630F2DEE71BB008509DE /* NdbFilter.swift */; }; + D7F563132DEE71C0008509DE /* NdbFilter.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7F5630F2DEE71BB008509DE /* NdbFilter.swift */; }; D7FB10A72B0C371A00FA8D42 /* Log.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C2B10272A7B0F5C008AA43E /* Log.swift */; }; D7FB14222BE5970000398331 /* PrivacyInfo.xcprivacy in Resources */ = {isa = PBXBuildFile; fileRef = D7FB14212BE5970000398331 /* PrivacyInfo.xcprivacy */; }; D7FB14252BE5A9A800398331 /* PrivacyInfo.xcprivacy in Resources */ = {isa = PBXBuildFile; fileRef = D7FB14242BE5A9A800398331 /* PrivacyInfo.xcprivacy */; }; @@ -2619,6 +2629,7 @@ D74AAFD32B155ECB006CF0F4 /* Zaps+.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Zaps+.swift"; sourceTree = "<group>"; }; D74AAFD52B155F0C006CF0F4 /* WalletConnect+.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "WalletConnect+.swift"; sourceTree = "<group>"; }; D74E64112DC95CBE004C7892 /* HumanReadableErrors.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HumanReadableErrors.swift; sourceTree = "<group>"; }; + D74DEC892DA0A19800E69FA6 /* Ndb+.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Ndb+.swift"; sourceTree = "<group>"; }; D74EA08D2D2E271E002290DD /* ErrorView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ErrorView.swift; sourceTree = "<group>"; }; D74EA0922D2E77B9002290DD /* LoadableNostrEventView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LoadableNostrEventView.swift; sourceTree = "<group>"; }; D74F43092B23F0BE00425B75 /* DamusPurple.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DamusPurple.swift; sourceTree = "<group>"; }; @@ -2684,6 +2695,7 @@ D7EDED322B12ACAE0018B19C /* DamusUserDefaults.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DamusUserDefaults.swift; sourceTree = "<group>"; }; D7EFBA362CC322F300F45588 /* DamusVideoControlsView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DamusVideoControlsView.swift; sourceTree = "<group>"; }; D7FA46E42DBDAA75002C9BB0 /* ImageCacheMigrations.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ImageCacheMigrations.swift; sourceTree = "<group>"; }; + D7F5630F2DEE71BB008509DE /* NdbFilter.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NdbFilter.swift; sourceTree = "<group>"; }; D7FB14212BE5970000398331 /* PrivacyInfo.xcprivacy */ = {isa = PBXFileReference; lastKnownFileType = text.xml; path = PrivacyInfo.xcprivacy; sourceTree = "<group>"; }; D7FB14242BE5A9A800398331 /* PrivacyInfo.xcprivacy */ = {isa = PBXFileReference; lastKnownFileType = text.xml; path = PrivacyInfo.xcprivacy; sourceTree = "<group>"; }; D7FD12252BD345A700CF195B /* FirstAidSettingsView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FirstAidSettingsView.swift; sourceTree = "<group>"; }; @@ -3148,6 +3160,8 @@ isa = PBXGroup; children = ( D733F9E42D92C75C00317B11 /* UnownedNdbNote.swift */, + D7F5630F2DEE71BB008509DE /* NdbFilter.swift */, + D74DEC892DA0A19800E69FA6 /* Ndb+.swift */, 4CC6A9F92CAB688500989CEF /* ccan */, 4C15224A2B8D499F007CDC17 /* parser.h */, 4CF47FDC2B631C0100F2B2C0 /* src */, @@ -5395,6 +5409,7 @@ 4CDD1AE22A6B3074001CD4DF /* NdbTagsIterator.swift in Sources */, 4C216F34286F5ACD00040376 /* DMView.swift in Sources */, D7CB5D512B1174D100AD4105 /* FriendFilter.swift in Sources */, + D74DEC8C2DA0A19B00E69FA6 /* Ndb+.swift in Sources */, D7CBD1D42B8D21DC00BFD889 /* DamusPurpleNotificationManagement.swift in Sources */, 4C32B9572A9AD44700DC3548 /* Root.swift in Sources */, 5C8498032D5D150000F74FEB /* ZapExplainer.swift in Sources */, @@ -5691,6 +5706,7 @@ 4C5E54032A9522F600FF6E60 /* UserStatus.swift in Sources */, 4C7D095F2A098C5D00943473 /* ConnectWalletView.swift in Sources */, 3AA24802297E3DC20090C62D /* RepostView.swift in Sources */, + D7F563122DEE71C0008509DE /* NdbFilter.swift in Sources */, 5C6E1DAF2A194075008FC15A /* PinkGradient.swift in Sources */, 4CD7641B28A1641400B6928F /* EndBlock.swift in Sources */, 4C198DEF29F88C6B004C165C /* BlurHashEncode.swift in Sources */, @@ -6204,6 +6220,7 @@ 82D6FB7E2CD99F7900C925F4 /* Mentions.swift in Sources */, 82D6FB7F2CD99F7900C925F4 /* ProfileUpdate.swift in Sources */, 82D6FB802CD99F7900C925F4 /* Post.swift in Sources */, + D7F563132DEE71C0008509DE /* NdbFilter.swift in Sources */, 82D6FB822CD99F7900C925F4 /* Reply.swift in Sources */, 82D6FB832CD99F7900C925F4 /* SearchModel.swift in Sources */, 82D6FB842CD99F7900C925F4 /* NostrFilter+Hashable.swift in Sources */, @@ -6321,6 +6338,7 @@ 3A2BAC602DE02E8600EBB4CC /* NIP05DomainPubkeysView.swift in Sources */, 82D6FBF02CD99F7900C925F4 /* LogoView.swift in Sources */, 82D6FBF12CD99F7900C925F4 /* IAPProductStateView.swift in Sources */, + D74DEC8B2DA0A19B00E69FA6 /* Ndb+.swift in Sources */, 82D6FBF22CD99F7900C925F4 /* PurpleBackdrop.swift in Sources */, 82D6FBF32CD99F7900C925F4 /* DamusPurpleView.swift in Sources */, 82D6FBF42CD99F7900C925F4 /* DamusPurpleWelcomeView.swift in Sources */, @@ -6622,6 +6640,7 @@ D73E5E952C6A97F4007EB227 /* ThreadModel.swift in Sources */, D73E5E962C6A97F4007EB227 /* ReplyMap.swift in Sources */, D73E5E972C6A97F4007EB227 /* ProfileModel.swift in Sources */, + D74DEC8A2DA0A19B00E69FA6 /* Ndb+.swift in Sources */, D73E5E982C6A97F4007EB227 /* ActionBarModel.swift in Sources */, D73E5E992C6A97F4007EB227 /* Liked.swift in Sources */, D73E5E9A2C6A97F4007EB227 /* ProfileUpdate.swift in Sources */, @@ -6647,6 +6666,7 @@ 4CC6A9FA2CAB688500989CEF /* str.c in Sources */, 4CC6A9FB2CAB688500989CEF /* tal.c in Sources */, 4CC6A9FD2CAB688500989CEF /* mem.c in Sources */, + D7F563102DEE71C0008509DE /* NdbFilter.swift in Sources */, 4CC6A9FE2CAB688500989CEF /* sha256.c in Sources */, 4CC6AA002CAB688500989CEF /* likely.c in Sources */, 4CC6AA042CAB688500989CEF /* htable.c in Sources */, @@ -7016,6 +7036,7 @@ 4CBB6F682B72B5F0000477A4 /* NdbProfile.swift in Sources */, 4CBB6F672B72B5E8000477A4 /* NdbBlock.swift in Sources */, 4CBB6F662B72B5DD000477A4 /* NdbBlocksIterator.swift in Sources */, + D74DEC8F2DA0C65F00E69FA6 /* Ndb+.swift in Sources */, D798D21F2B0858D600234419 /* MigratedTypes.swift in Sources */, D7CE1B472B0BE719002EDAD4 /* NativeObject.swift in Sources */, D71AD9002CEC176A002E2C3C /* AppAccessibilityIdentifiers.swift in Sources */, @@ -7040,6 +7061,7 @@ D7B76C902C825042003A16CB /* PushNotificationClient.swift in Sources */, D7CE1B432B0BE719002EDAD4 /* String+extension.swift in Sources */, D7CB5D3F2B116DAD00AD4105 /* NotificationsManager.swift in Sources */, + D74DEC912DA0CA2400E69FA6 /* Array.swift in Sources */, D7CB5D602B11770C00AD4105 /* FollowState.swift in Sources */, D7CB5D402B116E8A00AD4105 /* UserSettingsStore.swift in Sources */, D7CE1B1C2B0BE147002EDAD4 /* refmap.c in Sources */, @@ -7059,6 +7081,7 @@ D7CB5D522B1174D100AD4105 /* FriendFilter.swift in Sources */, D7CE1B362B0BE702002EDAD4 /* FbConstants.swift in Sources */, D74AAFD12B155DA4006CF0F4 /* RelayURL.swift in Sources */, + D74DEC902DA0C6B500E69FA6 /* NostrFilter.swift in Sources */, D7EDED272B117FF10018B19C /* CompatibleAttribute.swift in Sources */, D74AAFCD2B155D07006CF0F4 /* MakeZapRequest.swift in Sources */, D7CCFC072B05833200323D86 /* NdbNote.swift in Sources */, @@ -7068,6 +7091,7 @@ D7EDED222B117DCA0018B19C /* SequenceUtils.swift in Sources */, D7CE1B422B0BE719002EDAD4 /* Offset.swift in Sources */, D7FB10A72B0C371A00FA8D42 /* Log.swift in Sources */, + D7F563112DEE71C0008509DE /* NdbFilter.swift in Sources */, D7CE1B182B0BDFDD002EDAD4 /* mdb.c in Sources */, D7CCFC162B05894300323D86 /* Pubkey.swift in Sources */, D7EDED2C2B128CFA0018B19C /* DamusColors.swift in Sources */, diff --git a/damus/Core/Types/Ids/NoteId.swift b/damus/Core/Types/Ids/NoteId.swift @@ -51,4 +51,15 @@ struct NoteId: IdType, TagKey, TagConvertible { return note_id } + + func withUnsafePointer<T>(_ body: (UnsafePointer<UInt8>) throws -> T) rethrows -> T { + return try self.id.withUnsafeBytes { (bytes: UnsafeRawBufferPointer) in + guard let baseAddress = bytes.baseAddress else { + fatalError("Cannot get base address") + } + return try baseAddress.withMemoryRebound(to: UInt8.self, capacity: bytes.count) { ptr in + return try body(ptr) + } + } + } } diff --git a/damus/Core/Types/Ids/Pubkey.swift b/damus/Core/Types/Ids/Pubkey.swift @@ -44,4 +44,14 @@ struct Pubkey: IdType, TagKey, TagConvertible, Identifiable { return pubkey } + func withUnsafePointer<T>(_ body: (UnsafePointer<UInt8>) throws -> T) rethrows -> T { + return try self.id.withUnsafeBytes { (bytes: UnsafeRawBufferPointer) in + guard let baseAddress = bytes.baseAddress else { + fatalError("Cannot get base address") + } + return try baseAddress.withMemoryRebound(to: UInt8.self, capacity: bytes.count) { ptr in + return try body(ptr) + } + } + } } diff --git a/damus/Shared/Utilities/Log.swift b/damus/Shared/Utilities/Log.swift @@ -22,6 +22,7 @@ enum LogCategory: String { case image_uploading case video_coordination case tips + case ndb } /// Damus structured logger diff --git a/nostrdb/Ndb+.swift b/nostrdb/Ndb+.swift @@ -0,0 +1,30 @@ +// +// Ndb+.swift +// damus +// +// Created by Daniel D’Aquino on 2025-04-04. +// + +/// ## Implementation notes +/// +/// 1. This was created as a separate file because it contains dependencies to damus-specific structures such as `NostrFilter`, which is not yet available inside the NostrDB codebase. + +import Foundation + +extension Ndb { + /// Subscribe to events matching the provided NostrFilters + /// - Parameters: + /// - filters: Array of NostrFilter objects + /// - maxSimultaneousResults: Maximum number of initial results to return + /// - Returns: AsyncStream of StreamItem events + /// - Throws: NdbStreamError if subscription fails + func subscribe(filters: [NostrFilter], maxSimultaneousResults: Int = 1000) throws(NdbStreamError) -> AsyncStream<StreamItem> { + let ndbFilters: [NdbFilter] + do { + ndbFilters = try filters.toNdbFilters() + } catch { + throw .cannotConvertFilter(error) + } + return try self.subscribe(filters: ndbFilters, maxSimultaneousResults: maxSimultaneousResults) + } +} diff --git a/nostrdb/Ndb.swift b/nostrdb/Ndb.swift @@ -33,11 +33,12 @@ class Ndb { let owns_db: Bool var generation: Int private var closed: Bool + private var callbackHandler: Ndb.CallbackHandler var is_closed: Bool { self.closed || self.ndb.ndb == nil } - + static func safemode() -> Ndb? { guard let path = db_path ?? old_db_path else { return nil } @@ -80,7 +81,7 @@ class Ndb { return Ndb(ndb: ndb_t(ndb: nil)) } - static func open(path: String? = nil, owns_db_file: Bool = true) -> ndb_t? { + static func open(path: String? = nil, owns_db_file: Bool = true, callbackHandler: Ndb.CallbackHandler) -> ndb_t? { var ndb_p: OpaquePointer? = nil let ingest_threads: Int32 = 4 @@ -111,6 +112,19 @@ class Ndb { var ok = false while !ok && mapsize > 1024 * 1024 * 700 { var cfg = ndb_config(flags: 0, ingester_threads: ingest_threads, mapsize: mapsize, filter_context: nil, ingest_filter: nil, sub_cb_ctx: nil, sub_cb: nil) + + // Here we hook up the global callback function for subscription callbacks. + // We do an "unretained" pass here because the lifetime of the callback handler is larger than the lifetime of the nostrdb monitor in the C code. + // The NostrDB monitor that makes the callbacks should in theory _never_ outlive the callback handler. + // + // This means that: + // - for as long as nostrdb is running, its parent Ndb instance will be alive, keeping the callback handler alive. + // - when the Ndb instance is deinitialized — and the callback handler comes down with it — the `deinit` function will destroy the nostrdb monitor, preventing it from accessing freed memory. + // + // Therefore, we do not need to increase reference count to callbackHandler. The tightly coupled lifetimes will ensure that it is always alive when the ndb_monitor is alive. + let ctx: UnsafeMutableRawPointer = Unmanaged.passUnretained(callbackHandler).toOpaque() + ndb_config_set_subscription_callback(&cfg, subscription_callback, ctx) + let res = ndb_init(&ndb_p, testdir, &cfg); ok = res != 0; if !ok { @@ -124,12 +138,15 @@ class Ndb { if !ok { return nil } - - return ndb_t(ndb: ndb_p) + + let ndb_instance = ndb_t(ndb: ndb_p) + Task { await callbackHandler.set(ndb: ndb_instance) } + return ndb_instance } init?(path: String? = nil, owns_db_file: Bool = true) { - guard let db = Self.open(path: path, owns_db_file: owns_db_file) else { + let callbackHandler = Ndb.CallbackHandler() + guard let db = Self.open(path: path, owns_db_file: owns_db_file, callbackHandler: callbackHandler) else { return nil } @@ -138,6 +155,7 @@ class Ndb { self.owns_db = owns_db_file self.ndb = db self.closed = false + self.callbackHandler = callbackHandler } private static func migrate_db_location_if_needed() throws { @@ -183,6 +201,8 @@ class Ndb { self.path = nil self.owns_db = true self.closed = false + // This simple initialization will cause subscriptions not to be ever called. Probably fine because this initializer is used only for empty example ndb instances. + self.callbackHandler = Ndb.CallbackHandler() } func close() { @@ -196,7 +216,7 @@ class Ndb { func reopen() -> Bool { guard self.is_closed, - let db = Self.open(path: self.path, owns_db_file: self.owns_db) else { + let db = Self.open(path: self.path, owns_db_file: self.owns_db, callbackHandler: self.callbackHandler) else { return false } @@ -581,10 +601,220 @@ class Ndb { } } + // MARK: NdbFilter queries and subscriptions + + /// Safe wrapper around the `ndb_query` C function + /// - Parameters: + /// - txn: Database transaction + /// - filters: Array of NdbFilter objects + /// - maxResults: Maximum number of results to return + /// - Returns: Array of note keys matching the filters + /// - Throws: NdbStreamError if the query fails + func query<Y>(with txn: NdbTxn<Y>, filters: [NdbFilter], maxResults: Int) throws(NdbStreamError) -> [NoteKey] { + let filtersPointer = UnsafeMutablePointer<ndb_filter>.allocate(capacity: filters.count) + defer { filtersPointer.deallocate() } + + for (index, ndbFilter) in filters.enumerated() { + filtersPointer.advanced(by: index).pointee = ndbFilter.ndbFilter + } + + let count = UnsafeMutablePointer<Int32>.allocate(capacity: 1) + defer { count.deallocate() } + + let results = UnsafeMutablePointer<ndb_query_result>.allocate(capacity: maxResults) + defer { results.deallocate() } + + guard ndb_query(&txn.txn, filtersPointer, Int32(filters.count), results, Int32(maxResults), count) == 1 else { + throw NdbStreamError.initialQueryFailed + } + + var noteIds: [NoteKey] = [] + for i in 0..<count.pointee { + noteIds.append(results.advanced(by: Int(i)).pointee.note_id) + } + + return noteIds + } + + /// Safe wrapper around `ndb_subscribe` that handles all pointer management + /// - Parameters: + /// - filters: Array of NdbFilter objects + /// - Returns: AsyncStream of StreamItem events for new matches only + private func ndbSubscribe(filters: [NdbFilter]) -> AsyncStream<StreamItem> { + return AsyncStream<StreamItem> { continuation in + // Allocate filters pointer - will be deallocated when subscription ends + // Cannot use `defer` to deallocate `filtersPointer` because it needs to remain valid for the lifetime of the subscription, which extends beyond this block's scope. + let filtersPointer = UnsafeMutablePointer<ndb_filter>.allocate(capacity: filters.count) + for (index, ndbFilter) in filters.enumerated() { + filtersPointer.advanced(by: index).pointee = ndbFilter.ndbFilter + } + + var streaming = true + var subid: UInt64 = 0 + var terminationStarted = false + + // Set up termination handler + continuation.onTermination = { @Sendable _ in + guard !terminationStarted else { return } // Avoid race conditions between two termination closures + terminationStarted = true + Log.debug("ndb_wait: stream: Terminated early", for: .ndb) + streaming = false + // Clean up resources on early termination + if subid != 0 { + ndb_unsubscribe(self.ndb.ndb, subid) + Task { await self.unsetCallback(subscriptionId: subid) } + } + filtersPointer.deallocate() + } + + if !streaming { + return + } + + // Set up subscription + subid = ndb_subscribe(self.ndb.ndb, filtersPointer, Int32(filters.count)) + + // Set the subscription callback + Task { + await self.setCallback(for: subid, callback: { noteKey in + continuation.yield(.event(noteKey)) + }) + } + + // Update termination handler to include subscription cleanup + continuation.onTermination = { @Sendable _ in + guard !terminationStarted else { return } // Avoid race conditions between two termination closures + terminationStarted = true + Log.debug("ndb_wait: stream: Terminated early", for: .ndb) + streaming = false + ndb_unsubscribe(self.ndb.ndb, subid) + Task { await self.unsetCallback(subscriptionId: subid) } + filtersPointer.deallocate() + } + } + } + + func subscribe(filters: [NdbFilter], maxSimultaneousResults: Int = 1000) throws(NdbStreamError) -> AsyncStream<StreamItem> { + // Fetch initial results + guard let txn = NdbTxn(ndb: self) else { throw .cannotOpenTransaction } + + // Use our safe wrapper instead of direct C function call + let noteIds = try query(with: txn, filters: filters, maxResults: maxSimultaneousResults) + + // Create a subscription for new events + let newEventsStream = ndbSubscribe(filters: filters) + + // Create a cascading stream that combines initial results with new events + return AsyncStream<StreamItem> { continuation in + // Stream all results already present in the database + for noteId in noteIds { + continuation.yield(.event(noteId)) + } + + // Indicate this is the end of the results currently present in the database + continuation.yield(.eose) + + // Create a task to forward events from the subscription stream + let forwardingTask = Task { + for await item in newEventsStream { + continuation.yield(item) + } + continuation.finish() + } + + // Handle termination by canceling the forwarding task + continuation.onTermination = { @Sendable _ in + forwardingTask.cancel() + } + } + } + + private func waitWithoutTimeout(for noteId: NoteId) async throws(NdbLookupError) -> NdbTxn<NdbNote>? { + do { + for try await item in try self.subscribe(filters: [NostrFilter(ids: [noteId])]) { + switch item { + case .eose: + continue + case .event(let noteKey): + guard let txn = NdbTxn(ndb: self) else { throw NdbLookupError.cannotOpenTransaction } + guard let note = self.lookup_note_by_key_with_txn(noteKey, txn: txn) else { throw NdbLookupError.internalInconsistency } + if note.id == noteId { + Log.debug("ndb wait: %d has matching id %s. Returning transaction", for: .ndb, noteKey, noteId.hex()) + return NdbTxn<NdbNote>.pure(ndb: self, val: note) + } + } + } + } + catch { + if let error = error as? NdbStreamError { throw NdbLookupError.streamError(error) } + else if let error = error as? NdbLookupError { throw error } + else { throw .internalInconsistency } + } + return nil + } + + func waitFor(noteId: NoteId, timeout: TimeInterval = 10) async throws(NdbLookupError) -> NdbTxn<NdbNote>? { + do { + return try await withCheckedThrowingContinuation({ continuation in + var done = false + let waitTask = Task { + do { + Log.debug("ndb_wait: Waiting for %s", for: .ndb, noteId.hex()) + let result = try await self.waitWithoutTimeout(for: noteId) + if !done { + Log.debug("ndb_wait: Found %s", for: .ndb, noteId.hex()) + continuation.resume(returning: result) + done = true + } + } + catch { + if Task.isCancelled { + return // the timeout task will handle throwing the timeout error + } + if !done { + Log.debug("ndb_wait: Error on %s: %s", for: .ndb, noteId.hex(), error.localizedDescription) + continuation.resume(throwing: error) + done = true + } + } + } + + let timeoutTask = Task { + try await Task.sleep(for: .seconds(Int(timeout))) + if !done { + Log.debug("ndb_wait: Timeout on %s. Cancelling wait task…", for: .ndb, noteId.hex()) + done = true + print("ndb_wait: throwing timeout error") + continuation.resume(throwing: NdbLookupError.timeout) + } + waitTask.cancel() + } + }) + } + catch { + if let error = error as? NdbLookupError { throw error } + else { throw .internalInconsistency } + } + } + + // MARK: Internal ndb callback interfaces + + internal func setCallback(for subscriptionId: UInt64, callback: @escaping (NoteKey) -> Void) async { + await self.callbackHandler.set(callback: callback, for: subscriptionId) + } + + internal func unsetCallback(subscriptionId: UInt64) async { + await self.callbackHandler.unset(subid: subscriptionId) + } + + // MARK: Helpers + enum Errors: Error { case cannot_find_db_path case db_file_migration_error } + + // MARK: Deinitialization deinit { print("txn: Ndb de-init") @@ -592,6 +822,87 @@ class Ndb { } } + +// MARK: - Extensions and helper structures and functions + +extension Ndb { + /// A class that is used to handle callbacks from nostrdb + /// + /// This is a separate class from `Ndb` because it simplifies the initialization logic + actor CallbackHandler { + /// Holds the ndb instance in the C codebase. Should be shared with `Ndb` + var ndb: ndb_t? = nil + /// A map from nostrdb subscription ids to callbacks + var subscriptionCallbackMap: [UInt64: (NoteKey) -> Void] = [:] + + func set(callback: @escaping (NoteKey) -> Void, for subid: UInt64) { + subscriptionCallbackMap[subid] = callback + } + + func unset(subid: UInt64) { + subscriptionCallbackMap[subid] = nil + } + + func set(ndb: ndb_t?) { + self.ndb = ndb + } + + /// Handles callbacks from nostrdb subscriptions, and routes them to the correct callback + func handleSubscriptionCallback(subId: UInt64, maxCapacity: Int32 = 1000) { + if let callback = subscriptionCallbackMap[subId] { + let result = UnsafeMutablePointer<UInt64>.allocate(capacity: Int(maxCapacity)) + defer { result.deallocate() } // Ensure we deallocate memory before leaving the function to avoid memory leaks + if let ndb { + let numberOfNotes = ndb_poll_for_notes(ndb.ndb, subId, result, maxCapacity) + for i in 0..<numberOfNotes { + callback(result.advanced(by: Int(i)).pointee) + } + } + } + } + } + + /// An item that comes out of a subscription stream + enum StreamItem { + /// End of currently stored events + case eose + /// An event in NostrDB available at the given note key + case event(NoteKey) + } + + /// An error that may happen during nostrdb streaming + enum NdbStreamError: Error { + case cannotOpenTransaction + case cannotConvertFilter(any Error) + case initialQueryFailed + case timeout + } + + /// An error that may happen when looking something up + enum NdbLookupError: Error { + case cannotOpenTransaction + case streamError(NdbStreamError) + case internalInconsistency + case timeout + } +} + +/// This callback "trampoline" function will be called when new notes arrive for NostrDB subscriptions. +/// +/// This is needed as a separate global function in order to allow us to pass it to the C code as a callback (We can't pass native Swift fuctions directly as callbacks). +/// +/// - Parameters: +/// - ctx: A pointer to a context object setup during initialization. This allows this function to "find" the correct place to call. MUST be a pointer to a `CallbackHandler`, otherwise this will trigger a crash +/// - subid: The NostrDB subscription ID, which identifies the subscription that is being called back +@_cdecl("subscription_callback") +public func subscription_callback(ctx: UnsafeMutableRawPointer?, subid: UInt64) { + guard let ctx else { return } + let handler = Unmanaged<Ndb.CallbackHandler>.fromOpaque(ctx).takeUnretainedValue() + Task { + await handler.handleSubscriptionCallback(subId: subid) + } +} + #if DEBUG func getDebugCheckedRoot<T: FlatBufferObject>(byteBuffer: inout ByteBuffer) throws -> T { return getRoot(byteBuffer: &byteBuffer) diff --git a/nostrdb/NdbFilter.swift b/nostrdb/NdbFilter.swift @@ -0,0 +1,356 @@ +// +// NdbFilter.swift +// damus +// +// Created by Daniel D'Aquino on 2025-06-02. +// + +import Foundation + +/// A safe Swift wrapper around `UnsafeMutablePointer<ndb_filter>` that manages memory automatically. +/// +/// This class provides a safe interface to the underlying C `ndb_filter` structure, handling +/// memory allocation and deallocation automatically. It eliminates the need for manual memory +/// management when working with NostrDB filters. +/// +/// ## Usage +/// ```swift +/// let nostrFilter = NostrFilter(kinds: [.text_note]) +/// let ndbFilter = try NdbFilter(from: nostrFilter) +/// // Use ndbFilter.ndbFilter or ndbFilter.unsafePointer as needed +/// // Memory is automatically cleaned up when ndbFilter goes out of scope +/// ``` +class NdbFilter { + private let filterPointer: UnsafeMutablePointer<ndb_filter> + + /// Creates a new NdbFilter from a NostrFilter. + /// - Parameter nostrFilter: The NostrFilter to convert + /// - Throws: `NdbFilterError.conversionFailed` if the underlying conversion fails + init(from nostrFilter: NostrFilter) throws { + do { + self.filterPointer = try Self.from(nostrFilter: nostrFilter) + } catch { + throw NdbFilterError.conversionFailed(error) + } + } + + /// Provides access to the underlying `ndb_filter` structure. + /// - Returns: The underlying `ndb_filter` value (not a pointer) + var ndbFilter: ndb_filter { + return filterPointer.pointee + } + + /// Provides access to the underlying unsafe pointer when needed for C interop. + /// - Warning: The caller must not deallocate this pointer. It will be automatically + /// deallocated when this NdbFilter is destroyed. + /// - Returns: The unsafe mutable pointer to the underlying ndb_filter + var unsafePointer: UnsafeMutablePointer<ndb_filter> { + return filterPointer + } + + /// Creates multiple NdbFilter instances from an array of NostrFilters. + /// - Parameter nostrFilters: Array of NostrFilter instances to convert + /// - Returns: Array of NdbFilter instances + /// - Throws: `NdbFilterError.conversionFailed` if any conversion fails + static func create(from nostrFilters: [NostrFilter]) throws -> [NdbFilter] { + return try nostrFilters.map { try NdbFilter(from: $0) } + } + + // MARK: - Conversion to/from ndb_filter + + // TODO: This function is long and repetitive, refactor it into something cleaner. + private static func from(nostrFilter: NostrFilter) throws(NdbFilterConversionError) -> UnsafeMutablePointer<ndb_filter> { + let filterPointer = UnsafeMutablePointer<ndb_filter>.allocate(capacity: 1) + + guard ndb_filter_init(filterPointer) == 1 else { + filterPointer.deallocate() + throw NdbFilterConversionError.failedToInitialize + } + + // Handle `ids` field + if let ids = nostrFilter.ids { + guard ndb_filter_start_field(filterPointer, NDB_FILTER_IDS) == 1 else { + ndb_filter_destroy(filterPointer) + filterPointer.deallocate() + throw NdbFilterConversionError.failedToStartField + } + + for noteId in ids { + do { + try noteId.withUnsafePointer({ idPointer in + if ndb_filter_add_id_element(filterPointer, idPointer) != 1 { + ndb_filter_destroy(filterPointer) + filterPointer.deallocate() + throw NdbFilterConversionError.failedToAddElement + } + }) + } + catch { + ndb_filter_destroy(filterPointer) + filterPointer.deallocate() + throw NdbFilterConversionError.failedToAddElement + } + } + + ndb_filter_end_field(filterPointer) + } + + // Handle `kinds` field + if let kinds = nostrFilter.kinds { + guard ndb_filter_start_field(filterPointer, NDB_FILTER_KINDS) == 1 else { + ndb_filter_destroy(filterPointer) + filterPointer.deallocate() + throw NdbFilterConversionError.failedToStartField + } + + for kind in kinds { + if ndb_filter_add_int_element(filterPointer, UInt64(kind.rawValue)) != 1 { + ndb_filter_destroy(filterPointer) + filterPointer.deallocate() + throw NdbFilterConversionError.failedToAddElement + } + } + + ndb_filter_end_field(filterPointer) + } + + // Handle `referenced_ids` field + if let referencedIds = nostrFilter.referenced_ids { + guard ndb_filter_start_tag_field(filterPointer, CChar(UnicodeScalar("e").value)) == 1 else { + ndb_filter_destroy(filterPointer) + filterPointer.deallocate() + throw NdbFilterConversionError.failedToStartField + } + + for refId in referencedIds { + do { + try refId.withUnsafePointer({ refPointer in + if ndb_filter_add_id_element(filterPointer, refPointer) != 1 { + ndb_filter_destroy(filterPointer) + filterPointer.deallocate() + throw NdbFilterConversionError.failedToAddElement + } + }) + } + catch { + ndb_filter_destroy(filterPointer) + filterPointer.deallocate() + throw NdbFilterConversionError.failedToAddElement + } + } + + ndb_filter_end_field(filterPointer) + } + + // Handle `pubkeys` + if let pubkeys = nostrFilter.pubkeys { + guard ndb_filter_start_tag_field(filterPointer, CChar(UnicodeScalar("p").value)) == 1 else { + ndb_filter_destroy(filterPointer) + filterPointer.deallocate() + throw NdbFilterConversionError.failedToStartField + } + + for pubkey in pubkeys { + do { + try pubkey.withUnsafePointer({ pubkeyPointer in + if ndb_filter_add_id_element(filterPointer, pubkeyPointer) != 1 { + ndb_filter_destroy(filterPointer) + filterPointer.deallocate() + throw NdbFilterConversionError.failedToAddElement + } + }) + } + catch { + ndb_filter_destroy(filterPointer) + filterPointer.deallocate() + throw NdbFilterConversionError.failedToAddElement + } + } + + ndb_filter_end_field(filterPointer) + } + + // Handle `since` + if let since = nostrFilter.since { + if ndb_filter_start_field(filterPointer, NDB_FILTER_SINCE) != 1 { + ndb_filter_destroy(filterPointer) + filterPointer.deallocate() + throw NdbFilterConversionError.failedToAddElement + } + + if ndb_filter_add_int_element(filterPointer, UInt64(since)) != 1 { + ndb_filter_destroy(filterPointer) + filterPointer.deallocate() + throw NdbFilterConversionError.failedToAddElement + } + + ndb_filter_end_field(filterPointer) + } + + // Handle `until` + if let until = nostrFilter.until { + if ndb_filter_start_field(filterPointer, NDB_FILTER_UNTIL) != 1 { + ndb_filter_destroy(filterPointer) + filterPointer.deallocate() + throw NdbFilterConversionError.failedToAddElement + } + + if ndb_filter_add_int_element(filterPointer, UInt64(until)) != 1 { + ndb_filter_destroy(filterPointer) + filterPointer.deallocate() + throw NdbFilterConversionError.failedToAddElement + } + + ndb_filter_end_field(filterPointer) + } + + // Handle `limit` + if let limit = nostrFilter.limit { + if ndb_filter_start_field(filterPointer, NDB_FILTER_LIMIT) != 1 { + ndb_filter_destroy(filterPointer) + filterPointer.deallocate() + throw NdbFilterConversionError.failedToAddElement + } + + if ndb_filter_add_int_element(filterPointer, UInt64(limit)) != 1 { + ndb_filter_destroy(filterPointer) + filterPointer.deallocate() + throw NdbFilterConversionError.failedToAddElement + } + + ndb_filter_end_field(filterPointer) + } + + // Handle `authors` + if let authors = nostrFilter.authors { + guard ndb_filter_start_field(filterPointer, NDB_FILTER_AUTHORS) == 1 else { + ndb_filter_destroy(filterPointer) + filterPointer.deallocate() + throw NdbFilterConversionError.failedToStartField + } + + for author in authors { + do { + try author.withUnsafePointer({ authorPointer in + if ndb_filter_add_id_element(filterPointer, authorPointer) != 1 { + ndb_filter_destroy(filterPointer) + filterPointer.deallocate() + throw NdbFilterConversionError.failedToAddElement + } + }) + } + catch { + ndb_filter_destroy(filterPointer) + filterPointer.deallocate() + throw NdbFilterConversionError.failedToAddElement + } + + } + + ndb_filter_end_field(filterPointer) + } + + // Handle `hashtag` + if let hashtags = nostrFilter.hashtag { + guard ndb_filter_start_tag_field(filterPointer, CChar(UnicodeScalar("t").value)) == 1 else { + ndb_filter_destroy(filterPointer) + filterPointer.deallocate() + throw NdbFilterConversionError.failedToStartField + } + + for tag in hashtags { + if ndb_filter_add_str_element(filterPointer, tag.cString(using: .utf8)) != 1 { + ndb_filter_destroy(filterPointer) + filterPointer.deallocate() + throw NdbFilterConversionError.failedToAddElement + } + } + ndb_filter_end_field(filterPointer) + } + + // Handle `parameter` + if let parameters = nostrFilter.parameter { + guard ndb_filter_start_tag_field(filterPointer, CChar(UnicodeScalar("d").value)) == 1 else { + ndb_filter_destroy(filterPointer) + filterPointer.deallocate() + throw NdbFilterConversionError.failedToStartField + } + + for parameter in parameters { + if ndb_filter_add_str_element(filterPointer, parameter.cString(using: .utf8)) != 1 { + ndb_filter_destroy(filterPointer) + filterPointer.deallocate() + throw NdbFilterConversionError.failedToAddElement + } + } + ndb_filter_end_field(filterPointer) + } + + // Handle `quotes` + if let quotes = nostrFilter.quotes { + guard ndb_filter_start_tag_field(filterPointer, CChar(UnicodeScalar("q").value)) == 1 else { + ndb_filter_destroy(filterPointer) + filterPointer.deallocate() + throw NdbFilterConversionError.failedToStartField + } + + for quote in quotes { + do { + try quote.withUnsafePointer({ quotePointer in + if ndb_filter_add_id_element(filterPointer, quotePointer) != 1 { + ndb_filter_destroy(filterPointer) + filterPointer.deallocate() + throw NdbFilterConversionError.failedToAddElement + } + }) + } + catch { + ndb_filter_destroy(filterPointer) + filterPointer.deallocate() + throw NdbFilterConversionError.failedToAddElement + } + + } + + ndb_filter_end_field(filterPointer) + } + + // Finalize the filter + guard ndb_filter_end(filterPointer) == 1 else { + ndb_filter_destroy(filterPointer) + filterPointer.deallocate() + throw NdbFilterConversionError.failedToFinalize + } + + return filterPointer + } + + enum NdbFilterConversionError: Error { + case failedToInitialize + case failedToStartField + case failedToAddElement + case failedToFinalize + } + + deinit { + ndb_filter_destroy(filterPointer) + filterPointer.deallocate() + } +} + +/// Errors that can occur when working with NdbFilter. +enum NdbFilterError: Error { + /// Thrown when conversion from NostrFilter to NdbFilter fails. + /// - Parameter Error: The underlying error that caused the conversion to fail + case conversionFailed(Error) +} + +/// Extension to create multiple NdbFilters safely from an array of NostrFilters. +extension Array where Element == NostrFilter { + /// Converts an array of NostrFilters to NdbFilters. + /// - Returns: Array of NdbFilter instances + /// - Throws: `NdbFilterError.conversionFailed` if any conversion fails + func toNdbFilters() throws -> [NdbFilter] { + return try self.map { try NdbFilter(from: $0) } + } +} diff --git a/nostrdb/NdbNote.swift b/nostrdb/NdbNote.swift @@ -263,27 +263,8 @@ class NdbNote: Codable, Equatable, Hashable { } var n = ndb_note_ptr() - - var the_kp: ndb_keypair? = nil - - if let sec = keypair.privkey { - var kp = ndb_keypair() - memcpy(&kp.secret.0, sec.id.bytes, 32); - - if ndb_create_keypair(&kp) <= 0 { - print("bad keypair") - } else { - the_kp = kp - } - } - var len: Int32 = 0 - if var the_kp { - len = ndb_builder_finalize(&builder, &n.ptr, &the_kp) - } else { - len = ndb_builder_finalize(&builder, &n.ptr, nil) - } - + switch noteConstructionMaterial { case .keypair(let keypair): var the_kp: ndb_keypair? = nil @@ -300,9 +281,9 @@ class NdbNote: Codable, Equatable, Hashable { } if var the_kp { - len = ndb_builder_finalize(&builder, &n, &the_kp) + len = ndb_builder_finalize(&builder, &n.ptr, &the_kp) } else { - len = ndb_builder_finalize(&builder, &n, nil) + len = ndb_builder_finalize(&builder, &n.ptr, nil) } if len <= 0 { @@ -315,7 +296,7 @@ class NdbNote: Codable, Equatable, Hashable { do { // Finalize note, save length, and ensure it is higher than zero (which signals finalization has succeeded) - len = ndb_builder_finalize(&builder, &n, nil) + len = ndb_builder_finalize(&builder, &n.ptr, nil) guard len > 0 else { throw InitError.generic } let scratch_buf_len = MAX_NOTE_SIZE @@ -323,11 +304,11 @@ class NdbNote: Codable, Equatable, Hashable { defer { free(scratch_buf) } // Ensure we deallocate as soon as we leave this scope, regardless of the outcome // Calculate the ID based on the content - guard ndb_calculate_id(n, scratch_buf, Int32(scratch_buf_len)) == 1 else { throw InitError.generic } + guard ndb_calculate_id(n.ptr, scratch_buf, Int32(scratch_buf_len)) == 1 else { throw InitError.generic } // Verify the signature against the pubkey and the computed ID, to verify the validity of the whole note var ctx = secp256k1_context_create(UInt32(SECP256K1_CONTEXT_VERIFY)) - guard ndb_note_verify(&ctx, ndb_note_pubkey(n), ndb_note_id(n), ndb_note_sig(n)) == 1 else { throw InitError.generic } + guard ndb_note_verify(&ctx, ndb_note_pubkey(n.ptr), ndb_note_id(n.ptr), ndb_note_sig(n.ptr)) == 1 else { throw InitError.generic } } catch { free(buf)