commit a1e6be214e413f2d3c020b4bd401cefd7b3c60a2
parent 87860a71516dd9bcb9617e85c33d86e5ab1e1cbe
Author: Daniel D’Aquino <daniel@daquino.me>
Date: Mon, 20 Nov 2023 17:31:35 -0800
Migrate NostrDB files to shared app group file container
This change was made so that NostrDB data can be accessed from different build targets such as the notification service extension.
Upon initialization of NostrDB, it will check both DB file locations (the old documents directory, and the new shared app group container). If it sees the DB is present on the old location, and not on the new location, it will move the files to the new location. In any other condition it will keep the files intact to prevent data loss.
In order to avoid any conflicts between the damusApp's Ndb instance and the extension's Ndb instance when writing or moving the file, a new parameter called "owns_db_file" was added, and set to "false" for the extension. This ensures that the extension will not attempt to move DB files or create a new DB file on its own. Only the main app can move or create the DB file.
Testing
-------
PASS
Device: iPhone 15 Pro simulator
iOS: 17.0.1
Damus: This commit
Steps:
1. Run with the debugger attached to the extension target.
2. Using Apple's push notification testing dashboard, send a test push notification with a real payload (that includes the nostr event under `nostr_event`. Payload generated by strfry-push-notify).
3. Watch logs. It should show a message like "Got push notification from <DISPLAY_NAME>", where `DISPLAY_NAME` is the correct profile name of the user who generated the event. PASS
Regression testing
------------------
Device: iPhone 13 Mini (Real device)
iOS: 17.1.1
Damus: This commit
Other preconditions:
- Damus is at 1.6 (29) at the start of the test
- NostrDB filled with real data on the old location
Steps:
1. Flash (upgrade) the new Damus version (this commit) (This will be the first time upgrading, shared file container is empty)
2. Try to use the app normally. Scroll and navigate to several locations. Interact with some notes. App should be stable, work, and appear to have profile names already (i.e. It shouldn't start with a bunch of npubs in the place of profile names on known contacts). PASS
3. Downgrade back to the App store version (v1.6 (29))
4. Try to use the app normally. Scroll and navigate, interact, etc. App should work and be stable, but profile name cache is expected to be lost (i.e. shows npubs for a bit until profile is reloaded into NostrDB). PASS
5. Upgrade app again to the version in this commit.
6. Repeat step 2. Everything should work as normal and all profiles should be preloaded from the start. PASS
Closes: https://github.com/damus-io/damus/issues/1744
Diffstat:
10 files changed, 104 insertions(+), 15 deletions(-)
diff --git a/DamusNotificationService/NotificationService.swift b/DamusNotificationService/NotificationService.swift
@@ -16,6 +16,8 @@ class NotificationService: UNNotificationServiceExtension {
override func didReceive(_ request: UNNotificationRequest, withContentHandler contentHandler: @escaping (UNNotificationContent) -> Void) {
self.contentHandler = contentHandler
+ let ndb: Ndb? = try? Ndb(owns_db_file: false)
+
// Modify the notification content here...
guard let nostrEventInfoDictionary = request.content.userInfo["nostr_event"] as? [AnyHashable: Any],
let nostrEventInfo = NostrEventInfoFromPushNotification.from(dictionary: nostrEventInfoDictionary) else {
@@ -23,6 +25,12 @@ class NotificationService: UNNotificationServiceExtension {
return;
}
+ // Log that we got a push notification
+ if let pubkey = Pubkey(hex: nostrEventInfo.pubkey),
+ let txn = ndb?.lookup_profile(pubkey) {
+ Log.debug("Got push notification from %s (%s)", for: .push_notifications, (txn.unsafeUnownedValue?.profile?.display_name ?? "Unknown"), nostrEventInfo.pubkey)
+ }
+
if let improvedContent = NotificationFormatter.shared.formatMessage(event: nostrEventInfo) {
contentHandler(improvedContent)
}
diff --git a/damus.xcodeproj/project.pbxproj b/damus.xcodeproj/project.pbxproj
@@ -531,6 +531,7 @@
D7DBD41F2B02F15E002A6197 /* NostrKind.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C3BEFD32819DE8F00B3DE84 /* NostrKind.swift */; };
D7DBD4202B0307C7002A6197 /* NostrEventInfoFromPushNotification.swift in Sources */ = {isa = PBXBuildFile; fileRef = D70A3B182B02DD2D008BD568 /* NostrEventInfoFromPushNotification.swift */; };
D7DEEF2F2A8C021E00E0C99F /* NostrEventTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7DEEF2E2A8C021E00E0C99F /* NostrEventTests.swift */; };
+ D7FB10A72B0C371A00FA8D42 /* Log.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4C2B10272A7B0F5C008AA43E /* Log.swift */; };
D7FF94002AC7AC5300FD969D /* RelayURL.swift in Sources */ = {isa = PBXBuildFile; fileRef = D7FF93FF2AC7AC5200FD969D /* RelayURL.swift */; };
E4FA1C032A24BB7F00482697 /* SearchSettingsView.swift in Sources */ = {isa = PBXBuildFile; fileRef = E4FA1C022A24BB7F00482697 /* SearchSettingsView.swift */; };
E990020F2955F837003BBC5A /* EditMetadataView.swift in Sources */ = {isa = PBXBuildFile; fileRef = E990020E2955F837003BBC5A /* EditMetadataView.swift */; };
@@ -3236,6 +3237,7 @@
D7CCFC072B05833200323D86 /* NdbNote.swift in Sources */,
D7CE1B3F2B0BE719002EDAD4 /* Enum.swift in Sources */,
D7CE1B422B0BE719002EDAD4 /* Offset.swift in Sources */,
+ D7FB10A72B0C371A00FA8D42 /* Log.swift in Sources */,
D7CE1B232B0BE1EE002EDAD4 /* bolt11.c in Sources */,
D7CE1B182B0BDFDD002EDAD4 /* mdb.c in Sources */,
D7CCFC162B05894300323D86 /* Pubkey.swift in Sources */,
diff --git a/damus/ContentView.swift b/damus/ContentView.swift
@@ -608,7 +608,7 @@ struct ContentView: View {
func connect() {
// nostrdb
- let ndb = Ndb()!
+ let ndb = try! Ndb()!
let pool = RelayPool(ndb: ndb)
let model_cache = RelayModelCache()
diff --git a/damus/TestData.swift b/damus/TestData.swift
@@ -64,7 +64,7 @@ var test_damus_state: DamusState = ({
}
print("opening \(tempDir!)")
- let ndb = Ndb(path: tempDir)!
+ let ndb = try! Ndb(path: tempDir)!
let our_pubkey = test_pubkey
let pool = RelayPool(ndb: ndb)
let settings = UserSettingsStore()
diff --git a/damus/Util/Log.swift b/damus/Util/Log.swift
@@ -13,6 +13,7 @@ enum LogCategory: String {
case nav
case render
case storage
+ case push_notifications
}
/// Damus structured logger
diff --git a/damus/Views/SaveKeysView.swift b/damus/Views/SaveKeysView.swift
@@ -10,7 +10,7 @@ import Security
struct SaveKeysView: View {
let account: CreateAccountModel
- let pool: RelayPool = RelayPool(ndb: Ndb()!)
+ let pool: RelayPool = RelayPool(ndb: try! Ndb()!)
@State var pub_copied: Bool = false
@State var priv_copied: Bool = false
@State var loading: Bool = false
diff --git a/damusTests/Mocking/MockDamusState.swift b/damusTests/Mocking/MockDamusState.swift
@@ -24,7 +24,7 @@ func generate_test_damus_state(
}
print("opening \(tempDir!)")
- let ndb = Ndb(path: tempDir)!
+ let ndb = try! Ndb(path: tempDir)!
let our_pubkey = test_pubkey
let pool = RelayPool(ndb: ndb)
let settings = UserSettingsStore()
diff --git a/damusTests/ProfileViewTests.swift b/damusTests/ProfileViewTests.swift
@@ -27,7 +27,7 @@ final class ProfileViewTests: XCTestCase {
let pk4 = Pubkey(hex: "cc590e46363d0fa66bb27081368d01f169b8ffc7c614629d4e9eef6c88b38670")!
let pk5 = Pubkey(hex: "f2aa579bb998627e04a8f553842a09446360c9d708c6141dd119c479f6ab9d29")!
- let ndb = Ndb(path: Ndb.db_path)!
+ let ndb = try! Ndb(path: Ndb.db_path)!
let txn = NdbTxn(ndb: ndb)
let damus_name = "17ldvg64:nq5mhr77"
diff --git a/nostrdb/Ndb.swift b/nostrdb/Ndb.swift
@@ -6,29 +6,63 @@
//
import Foundation
+import OSLog
+
+fileprivate let APPLICATION_GROUP_IDENTIFIER = "group.com.damus"
class Ndb {
let ndb: ndb_t
+ let owns_db_file: Bool // Determines whether this class should be allowed to create or move the db file.
+
+ // NostrDB used to be stored on the app container's document directory
+ static private var old_db_path: String? {
+ guard let path = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask).first?.absoluteString else {
+ return nil
+ }
+ return remove_file_prefix(path)
+ }
- static var db_path: String {
- let path = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask).first?.absoluteString
- return remove_file_prefix(path!)
+ static var db_path: String? {
+ // Use the `group.com.damus` container, so that it can be accessible from other targets
+ // e.g. The notification service extension needs to access Ndb data, which is done through this shared file container.
+ guard let containerURL = FileManager.default.containerURL(forSecurityApplicationGroupIdentifier: APPLICATION_GROUP_IDENTIFIER) else {
+ return nil
+ }
+ return remove_file_prefix(containerURL.absoluteString)
}
+
+ static private var db_files: [String] = ["data.mdb", "lock.mdb"]
static var empty: Ndb {
Ndb(ndb: ndb_t(ndb: nil))
}
- init?(path: String? = nil) {
- //try? FileManager.default.removeItem(atPath: Ndb.db_path + "/lock.mdb")
- //try? FileManager.default.removeItem(atPath: Ndb.db_path + "/data.mdb")
-
+ init?(path: String? = nil, owns_db_file: Bool = true) throws {
var ndb_p: OpaquePointer? = nil
let ingest_threads: Int32 = 4
var mapsize: Int = 1024 * 1024 * 1024 * 32
+
+ if path == nil && owns_db_file {
+ // `nil` path indicates the default path will be used.
+ // The default path changed over time, so migrate the database to the new location if needed
+ do {
+ try Self.migrate_db_location_if_needed()
+ }
+ catch {
+ // If it fails to migrate, the app can still run without serious consequences. Log instead.
+ Log.error("Error migrating NostrDB to new file container", for: .storage)
+ }
+ }
+
+ guard let db_path = Self.db_path,
+ owns_db_file || Self.db_files_exist(path: db_path) else {
+ return nil // If the caller claims to not own the DB file, and the DB files do not exist, then we should not initialize Ndb
+ }
- let path = path.map(remove_file_prefix) ?? Ndb.db_path
+ guard let path = path.map(remove_file_prefix) ?? Ndb.db_path else {
+ throw Errors.cannot_find_db_path
+ }
let ok = path.withCString { testdir in
var ok = false
@@ -45,10 +79,49 @@ class Ndb {
return nil
}
+ self.owns_db_file = owns_db_file
self.ndb = ndb_t(ndb: ndb_p)
}
+
+ private static func migrate_db_location_if_needed() throws {
+ guard let old_db_path, let db_path else {
+ throw Errors.cannot_find_db_path
+ }
+
+ let file_manager = FileManager.default
+
+ let old_db_files_exist = Self.db_files_exist(path: old_db_path)
+ let new_db_files_exist = Self.db_files_exist(path: db_path)
+
+ // Migration rules:
+ // 1. If DB files exist in the old path but not the new one, move files to the new path
+ // 2. If files do not exist anywhere, do nothing (let new DB be initialized)
+ // 3. If files exist in the new path, but not the old one, nothing needs to be done
+ // 4. If files exist on both, do nothing.
+ // Scenario 4 likely means that user has downgraded and re-upgraded.
+ // Although it might make sense to get the most recent DB, it might lead to data loss.
+ // If we leave both intact, it makes it easier to fix later, as no data loss would occur.
+ if old_db_files_exist && !new_db_files_exist {
+ Log.info("Migrating NostrDB to new file location…", for: .storage)
+ do {
+ try db_files.forEach { db_file in
+ let old_path = "\(old_db_path)/\(db_file)"
+ let new_path = "\(db_path)/\(db_file)"
+ try file_manager.moveItem(atPath: old_path, toPath: new_path)
+ }
+ Log.info("NostrDB files successfully migrated to the new location", for: .storage)
+ } catch {
+ throw Errors.db_file_migration_error
+ }
+ }
+ }
+
+ private static func db_files_exist(path: String) -> Bool {
+ return db_files.allSatisfy { FileManager.default.fileExists(atPath: "\(path)/\($0)") }
+ }
init(ndb: ndb_t) {
+ self.owns_db_file = true
self.ndb = ndb
}
@@ -238,6 +311,11 @@ class Ndb {
return pks
}
}
+
+ enum Errors: Error {
+ case cannot_find_db_path
+ case db_file_migration_error
+ }
deinit {
ndb_destroy(ndb.ndb)
diff --git a/nostrdb/Test/NdbTests.swift b/nostrdb/Test/NdbTests.swift
@@ -55,13 +55,13 @@ final class NdbTests: XCTestCase {
func test_ndb_init() {
do {
- let ndb = Ndb(path: db_dir)!
+ let ndb = try! Ndb(path: db_dir)!
let ok = ndb.process_events(test_wire_events)
XCTAssertTrue(ok)
}
do {
- let ndb = Ndb(path: db_dir)!
+ let ndb = try! Ndb(path: db_dir)!
let id = NoteId(hex: "d12c17bde3094ad32f4ab862a6cc6f5c289cfe7d5802270bdf34904df585f349")!
let txn = NdbTxn(ndb: ndb)
let note = ndb.lookup_note_with_txn(id: id, txn: txn)
