Ndb.swift (24208B)
1 // 2 // Ndb.swift 3 // damus 4 // 5 // Created by William Casarin on 2023-08-25. 6 // 7 8 import Foundation 9 import OSLog 10 11 fileprivate let APPLICATION_GROUP_IDENTIFIER = "group.com.damus" 12 13 enum NdbSearchOrder { 14 case oldest_first 15 case newest_first 16 } 17 18 19 enum DatabaseError: Error { 20 case failed_open 21 22 var errorDescription: String? { 23 switch self { 24 case .failed_open: 25 return "Failed to open database" 26 } 27 } 28 } 29 30 class Ndb { 31 var ndb: ndb_t 32 let path: String? 33 let owns_db: Bool 34 var generation: Int 35 private var closed: Bool 36 37 var is_closed: Bool { 38 self.closed || self.ndb.ndb == nil 39 } 40 41 static func safemode() -> Ndb? { 42 guard let path = db_path ?? old_db_path else { return nil } 43 44 // delete the database and start fresh 45 if Self.db_files_exist(path: path) { 46 let file_manager = FileManager.default 47 for db_file in db_files { 48 try? file_manager.removeItem(atPath: "\(path)/\(db_file)") 49 } 50 } 51 52 guard let ndb = Ndb(path: path) else { 53 return nil 54 } 55 56 return ndb 57 } 58 59 // NostrDB used to be stored on the app container's document directory 60 static private var old_db_path: String? { 61 guard let path = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask).first?.absoluteString else { 62 return nil 63 } 64 return remove_file_prefix(path) 65 } 66 67 static var db_path: String? { 68 // Use the `group.com.damus` container, so that it can be accessible from other targets 69 // e.g. The notification service extension needs to access Ndb data, which is done through this shared file container. 70 guard let containerURL = FileManager.default.containerURL(forSecurityApplicationGroupIdentifier: APPLICATION_GROUP_IDENTIFIER) else { 71 return nil 72 } 73 return remove_file_prefix(containerURL.absoluteString) 74 } 75 76 static private var db_files: [String] = ["data.mdb", "lock.mdb"] 77 78 static var empty: Ndb { 79 print("txn: NOSTRDB EMPTY") 80 return Ndb(ndb: ndb_t(ndb: nil)) 81 } 82 83 static func open(path: String? = nil, owns_db_file: Bool = true) -> ndb_t? { 84 var ndb_p: OpaquePointer? = nil 85 86 let ingest_threads: Int32 = 4 87 var mapsize: Int = 1024 * 1024 * 1024 * 32 88 89 if path == nil && owns_db_file { 90 // `nil` path indicates the default path will be used. 91 // The default path changed over time, so migrate the database to the new location if needed 92 do { 93 try Self.migrate_db_location_if_needed() 94 } 95 catch { 96 // If it fails to migrate, the app can still run without serious consequences. Log instead. 97 Log.error("Error migrating NostrDB to new file container", for: .storage) 98 } 99 } 100 101 guard let db_path = Self.db_path, 102 owns_db_file || Self.db_files_exist(path: db_path) else { 103 return nil // If the caller claims to not own the DB file, and the DB files do not exist, then we should not initialize Ndb 104 } 105 106 guard let path = path.map(remove_file_prefix) ?? Ndb.db_path else { 107 return nil 108 } 109 110 let ok = path.withCString { testdir in 111 var ok = false 112 while !ok && mapsize > 1024 * 1024 * 700 { 113 var cfg = ndb_config(flags: 0, ingester_threads: ingest_threads, mapsize: mapsize, filter_context: nil, ingest_filter: nil) 114 let res = ndb_init(&ndb_p, testdir, &cfg) 115 ok = res != 0; 116 if !ok { 117 Log.error("ndb_init failed: %d, reducing mapsize from %d to %d", for: .storage, res, mapsize, mapsize / 2) 118 mapsize /= 2 119 } 120 } 121 return ok 122 } 123 124 if !ok { 125 return nil 126 } 127 128 return ndb_t(ndb: ndb_p) 129 } 130 131 init?(path: String? = nil, owns_db_file: Bool = true) { 132 guard let db = Self.open(path: path, owns_db_file: owns_db_file) else { 133 return nil 134 } 135 136 self.generation = 0 137 self.path = path 138 self.owns_db = owns_db_file 139 self.ndb = db 140 self.closed = false 141 } 142 143 private static func migrate_db_location_if_needed() throws { 144 guard let old_db_path, let db_path else { 145 throw Errors.cannot_find_db_path 146 } 147 148 let file_manager = FileManager.default 149 150 let old_db_files_exist = Self.db_files_exist(path: old_db_path) 151 let new_db_files_exist = Self.db_files_exist(path: db_path) 152 153 // Migration rules: 154 // 1. If DB files exist in the old path but not the new one, move files to the new path 155 // 2. If files do not exist anywhere, do nothing (let new DB be initialized) 156 // 3. If files exist in the new path, but not the old one, nothing needs to be done 157 // 4. If files exist on both, do nothing. 158 // Scenario 4 likely means that user has downgraded and re-upgraded. 159 // Although it might make sense to get the most recent DB, it might lead to data loss. 160 // If we leave both intact, it makes it easier to fix later, as no data loss would occur. 161 if old_db_files_exist && !new_db_files_exist { 162 Log.info("Migrating NostrDB to new file location…", for: .storage) 163 do { 164 try db_files.forEach { db_file in 165 let old_path = "\(old_db_path)/\(db_file)" 166 let new_path = "\(db_path)/\(db_file)" 167 try file_manager.moveItem(atPath: old_path, toPath: new_path) 168 } 169 Log.info("NostrDB files successfully migrated to the new location", for: .storage) 170 } catch { 171 throw Errors.db_file_migration_error 172 } 173 } 174 } 175 176 private static func db_files_exist(path: String) -> Bool { 177 return db_files.allSatisfy { FileManager.default.fileExists(atPath: "\(path)/\($0)") } 178 } 179 180 init(ndb: ndb_t) { 181 self.ndb = ndb 182 self.generation = 0 183 self.path = nil 184 self.owns_db = true 185 self.closed = false 186 } 187 188 func close() { 189 guard !self.is_closed else { return } 190 self.closed = true 191 print("txn: CLOSING NOSTRDB") 192 ndb_destroy(self.ndb.ndb) 193 self.generation += 1 194 print("txn: NOSTRDB CLOSED") 195 } 196 197 func reopen() -> Bool { 198 guard self.is_closed, 199 let db = Self.open(path: self.path, owns_db_file: self.owns_db) else { 200 return false 201 } 202 203 print("txn: NOSTRDB REOPENED (gen \(generation))") 204 205 self.closed = false 206 self.ndb = db 207 return true 208 } 209 210 func lookup_note_by_key_with_txn<Y>(_ key: NoteKey, txn: NdbTxn<Y>) -> NdbNote? { 211 var size: Int = 0 212 guard let note_p = ndb_get_note_by_key(&txn.txn, key, &size) else { 213 return nil 214 } 215 return NdbNote(note: note_p, size: size, owned: false, key: key) 216 } 217 218 func text_search(query: String, limit: Int = 128, order: NdbSearchOrder = .newest_first) -> [NoteKey] { 219 guard let txn = NdbTxn(ndb: self) else { return [] } 220 var results = ndb_text_search_results() 221 let res = query.withCString { q in 222 let order = order == .newest_first ? NDB_ORDER_DESCENDING : NDB_ORDER_ASCENDING 223 var config = ndb_text_search_config(order: order, limit: Int32(limit)) 224 return ndb_text_search(&txn.txn, q, &results, &config) 225 } 226 227 if res == 0 { 228 return [] 229 } 230 231 var note_ids = [NoteKey]() 232 for i in 0..<results.num_results { 233 // seriously wtf 234 switch i { 235 case 0: note_ids.append(results.results.0.key.note_id) 236 case 1: note_ids.append(results.results.1.key.note_id) 237 case 2: note_ids.append(results.results.2.key.note_id) 238 case 3: note_ids.append(results.results.3.key.note_id) 239 case 4: note_ids.append(results.results.4.key.note_id) 240 case 5: note_ids.append(results.results.5.key.note_id) 241 case 6: note_ids.append(results.results.6.key.note_id) 242 case 7: note_ids.append(results.results.7.key.note_id) 243 case 8: note_ids.append(results.results.8.key.note_id) 244 case 9: note_ids.append(results.results.9.key.note_id) 245 case 10: note_ids.append(results.results.10.key.note_id) 246 case 11: note_ids.append(results.results.11.key.note_id) 247 case 12: note_ids.append(results.results.12.key.note_id) 248 case 13: note_ids.append(results.results.13.key.note_id) 249 case 14: note_ids.append(results.results.14.key.note_id) 250 case 15: note_ids.append(results.results.15.key.note_id) 251 case 16: note_ids.append(results.results.16.key.note_id) 252 case 17: note_ids.append(results.results.17.key.note_id) 253 case 18: note_ids.append(results.results.18.key.note_id) 254 case 19: note_ids.append(results.results.19.key.note_id) 255 case 20: note_ids.append(results.results.20.key.note_id) 256 case 21: note_ids.append(results.results.21.key.note_id) 257 case 22: note_ids.append(results.results.22.key.note_id) 258 case 23: note_ids.append(results.results.23.key.note_id) 259 case 24: note_ids.append(results.results.24.key.note_id) 260 case 25: note_ids.append(results.results.25.key.note_id) 261 case 26: note_ids.append(results.results.26.key.note_id) 262 case 27: note_ids.append(results.results.27.key.note_id) 263 case 28: note_ids.append(results.results.28.key.note_id) 264 case 29: note_ids.append(results.results.29.key.note_id) 265 case 30: note_ids.append(results.results.30.key.note_id) 266 case 31: note_ids.append(results.results.31.key.note_id) 267 case 32: note_ids.append(results.results.32.key.note_id) 268 case 33: note_ids.append(results.results.33.key.note_id) 269 case 34: note_ids.append(results.results.34.key.note_id) 270 case 35: note_ids.append(results.results.35.key.note_id) 271 case 36: note_ids.append(results.results.36.key.note_id) 272 case 37: note_ids.append(results.results.37.key.note_id) 273 case 38: note_ids.append(results.results.38.key.note_id) 274 case 39: note_ids.append(results.results.39.key.note_id) 275 case 40: note_ids.append(results.results.40.key.note_id) 276 case 41: note_ids.append(results.results.41.key.note_id) 277 case 42: note_ids.append(results.results.42.key.note_id) 278 case 43: note_ids.append(results.results.43.key.note_id) 279 case 44: note_ids.append(results.results.44.key.note_id) 280 case 45: note_ids.append(results.results.45.key.note_id) 281 case 46: note_ids.append(results.results.46.key.note_id) 282 case 47: note_ids.append(results.results.47.key.note_id) 283 case 48: note_ids.append(results.results.48.key.note_id) 284 case 49: note_ids.append(results.results.49.key.note_id) 285 case 50: note_ids.append(results.results.50.key.note_id) 286 case 51: note_ids.append(results.results.51.key.note_id) 287 case 52: note_ids.append(results.results.52.key.note_id) 288 case 53: note_ids.append(results.results.53.key.note_id) 289 case 54: note_ids.append(results.results.54.key.note_id) 290 case 55: note_ids.append(results.results.55.key.note_id) 291 case 56: note_ids.append(results.results.56.key.note_id) 292 case 57: note_ids.append(results.results.57.key.note_id) 293 case 58: note_ids.append(results.results.58.key.note_id) 294 case 59: note_ids.append(results.results.59.key.note_id) 295 case 60: note_ids.append(results.results.60.key.note_id) 296 case 61: note_ids.append(results.results.61.key.note_id) 297 case 62: note_ids.append(results.results.62.key.note_id) 298 case 63: note_ids.append(results.results.63.key.note_id) 299 case 64: note_ids.append(results.results.64.key.note_id) 300 case 65: note_ids.append(results.results.65.key.note_id) 301 case 66: note_ids.append(results.results.66.key.note_id) 302 case 67: note_ids.append(results.results.67.key.note_id) 303 case 68: note_ids.append(results.results.68.key.note_id) 304 case 69: note_ids.append(results.results.69.key.note_id) 305 case 70: note_ids.append(results.results.70.key.note_id) 306 case 71: note_ids.append(results.results.71.key.note_id) 307 case 72: note_ids.append(results.results.72.key.note_id) 308 case 73: note_ids.append(results.results.73.key.note_id) 309 case 74: note_ids.append(results.results.74.key.note_id) 310 case 75: note_ids.append(results.results.75.key.note_id) 311 case 76: note_ids.append(results.results.76.key.note_id) 312 case 77: note_ids.append(results.results.77.key.note_id) 313 case 78: note_ids.append(results.results.78.key.note_id) 314 case 79: note_ids.append(results.results.79.key.note_id) 315 case 80: note_ids.append(results.results.80.key.note_id) 316 case 81: note_ids.append(results.results.81.key.note_id) 317 case 82: note_ids.append(results.results.82.key.note_id) 318 case 83: note_ids.append(results.results.83.key.note_id) 319 case 84: note_ids.append(results.results.84.key.note_id) 320 case 85: note_ids.append(results.results.85.key.note_id) 321 case 86: note_ids.append(results.results.86.key.note_id) 322 case 87: note_ids.append(results.results.87.key.note_id) 323 case 88: note_ids.append(results.results.88.key.note_id) 324 case 89: note_ids.append(results.results.89.key.note_id) 325 case 90: note_ids.append(results.results.90.key.note_id) 326 case 91: note_ids.append(results.results.91.key.note_id) 327 case 92: note_ids.append(results.results.92.key.note_id) 328 case 93: note_ids.append(results.results.93.key.note_id) 329 case 94: note_ids.append(results.results.94.key.note_id) 330 case 95: note_ids.append(results.results.95.key.note_id) 331 case 96: note_ids.append(results.results.96.key.note_id) 332 case 97: note_ids.append(results.results.97.key.note_id) 333 case 98: note_ids.append(results.results.98.key.note_id) 334 case 99: note_ids.append(results.results.99.key.note_id) 335 case 100: note_ids.append(results.results.100.key.note_id) 336 case 101: note_ids.append(results.results.101.key.note_id) 337 case 102: note_ids.append(results.results.102.key.note_id) 338 case 103: note_ids.append(results.results.103.key.note_id) 339 case 104: note_ids.append(results.results.104.key.note_id) 340 case 105: note_ids.append(results.results.105.key.note_id) 341 case 106: note_ids.append(results.results.106.key.note_id) 342 case 107: note_ids.append(results.results.107.key.note_id) 343 case 108: note_ids.append(results.results.108.key.note_id) 344 case 109: note_ids.append(results.results.109.key.note_id) 345 case 110: note_ids.append(results.results.110.key.note_id) 346 case 111: note_ids.append(results.results.111.key.note_id) 347 case 112: note_ids.append(results.results.112.key.note_id) 348 case 113: note_ids.append(results.results.113.key.note_id) 349 case 114: note_ids.append(results.results.114.key.note_id) 350 case 115: note_ids.append(results.results.115.key.note_id) 351 case 116: note_ids.append(results.results.116.key.note_id) 352 case 117: note_ids.append(results.results.117.key.note_id) 353 case 118: note_ids.append(results.results.118.key.note_id) 354 case 119: note_ids.append(results.results.119.key.note_id) 355 case 120: note_ids.append(results.results.120.key.note_id) 356 case 121: note_ids.append(results.results.121.key.note_id) 357 case 122: note_ids.append(results.results.122.key.note_id) 358 case 123: note_ids.append(results.results.123.key.note_id) 359 case 124: note_ids.append(results.results.124.key.note_id) 360 case 125: note_ids.append(results.results.125.key.note_id) 361 case 126: note_ids.append(results.results.126.key.note_id) 362 case 127: note_ids.append(results.results.127.key.note_id) 363 default: 364 break 365 } 366 } 367 368 return note_ids 369 } 370 371 func lookup_note_by_key(_ key: NoteKey) -> NdbTxn<NdbNote?>? { 372 return NdbTxn(ndb: self) { txn in 373 lookup_note_by_key_with_txn(key, txn: txn) 374 } 375 } 376 377 private func lookup_profile_by_key_inner<Y>(_ key: ProfileKey, txn: NdbTxn<Y>) -> ProfileRecord? { 378 var size: Int = 0 379 guard let profile_p = ndb_get_profile_by_key(&txn.txn, key, &size) else { 380 return nil 381 } 382 383 return profile_flatbuf_to_record(ptr: profile_p, size: size, key: key) 384 } 385 386 private func profile_flatbuf_to_record(ptr: UnsafeMutableRawPointer, size: Int, key: UInt64) -> ProfileRecord? { 387 do { 388 var buf = ByteBuffer(assumingMemoryBound: ptr, capacity: size) 389 let rec: NdbProfileRecord = try getDebugCheckedRoot(byteBuffer: &buf) 390 return ProfileRecord(data: rec, key: key) 391 } catch { 392 // Handle error appropriately 393 print("UNUSUAL: \(error)") 394 return nil 395 } 396 } 397 398 private func lookup_note_with_txn_inner<Y>(id: NoteId, txn: NdbTxn<Y>) -> NdbNote? { 399 return id.id.withUnsafeBytes { (ptr: UnsafeRawBufferPointer) -> NdbNote? in 400 var key: UInt64 = 0 401 var size: Int = 0 402 guard let baseAddress = ptr.baseAddress, 403 let note_p = ndb_get_note_by_id(&txn.txn, baseAddress, &size, &key) else { 404 return nil 405 } 406 return NdbNote(note: note_p, size: size, owned: false, key: key) 407 } 408 } 409 410 private func lookup_profile_with_txn_inner<Y>(pubkey: Pubkey, txn: NdbTxn<Y>) -> ProfileRecord? { 411 return pubkey.id.withUnsafeBytes { (ptr: UnsafeRawBufferPointer) -> ProfileRecord? in 412 var size: Int = 0 413 var key: UInt64 = 0 414 415 guard let baseAddress = ptr.baseAddress, 416 let profile_p = ndb_get_profile_by_pubkey(&txn.txn, baseAddress, &size, &key) 417 else { 418 return nil 419 } 420 421 return profile_flatbuf_to_record(ptr: profile_p, size: size, key: key) 422 } 423 } 424 425 func lookup_profile_by_key_with_txn<Y>(key: ProfileKey, txn: NdbTxn<Y>) -> ProfileRecord? { 426 lookup_profile_by_key_inner(key, txn: txn) 427 } 428 429 func lookup_profile_by_key(key: ProfileKey) -> NdbTxn<ProfileRecord?>? { 430 return NdbTxn(ndb: self) { txn in 431 lookup_profile_by_key_inner(key, txn: txn) 432 } 433 } 434 435 func lookup_note_with_txn<Y>(id: NoteId, txn: NdbTxn<Y>) -> NdbNote? { 436 lookup_note_with_txn_inner(id: id, txn: txn) 437 } 438 439 func lookup_profile_key(_ pubkey: Pubkey) -> ProfileKey? { 440 guard let txn = NdbTxn(ndb: self, with: { txn in 441 lookup_profile_key_with_txn(pubkey, txn: txn) 442 }) else { 443 return nil 444 } 445 446 return txn.value 447 } 448 449 func lookup_profile_key_with_txn<Y>(_ pubkey: Pubkey, txn: NdbTxn<Y>) -> ProfileKey? { 450 return pubkey.id.withUnsafeBytes { (ptr: UnsafeRawBufferPointer) -> NoteKey? in 451 guard let p = ptr.baseAddress else { return nil } 452 let r = ndb_get_profilekey_by_pubkey(&txn.txn, p) 453 if r == 0 { 454 return nil 455 } 456 return r 457 } 458 } 459 460 func lookup_note_key_with_txn<Y>(_ id: NoteId, txn: NdbTxn<Y>) -> NoteKey? { 461 guard !closed else { return nil } 462 return id.id.withUnsafeBytes { (ptr: UnsafeRawBufferPointer) -> NoteKey? in 463 guard let p = ptr.baseAddress else { 464 return nil 465 } 466 let r = ndb_get_notekey_by_id(&txn.txn, p) 467 if r == 0 { 468 return nil 469 } 470 return r 471 } 472 } 473 474 func lookup_note_key(_ id: NoteId) -> NoteKey? { 475 guard let txn = NdbTxn(ndb: self, with: { txn in 476 lookup_note_key_with_txn(id, txn: txn) 477 }) else { 478 return nil 479 } 480 481 return txn.value 482 } 483 484 func lookup_note(_ id: NoteId, txn_name: String? = nil) -> NdbTxn<NdbNote?>? { 485 NdbTxn(ndb: self, name: txn_name) { txn in 486 lookup_note_with_txn_inner(id: id, txn: txn) 487 } 488 } 489 490 func lookup_profile(_ pubkey: Pubkey, txn_name: String? = nil) -> NdbTxn<ProfileRecord?>? { 491 NdbTxn(ndb: self, name: txn_name) { txn in 492 lookup_profile_with_txn_inner(pubkey: pubkey, txn: txn) 493 } 494 } 495 496 func lookup_profile_with_txn<Y>(_ pubkey: Pubkey, txn: NdbTxn<Y>) -> ProfileRecord? { 497 lookup_profile_with_txn_inner(pubkey: pubkey, txn: txn) 498 } 499 500 func process_client_event(_ str: String) -> Bool { 501 guard !self.is_closed else { return false } 502 return str.withCString { cstr in 503 return ndb_process_client_event(ndb.ndb, cstr, Int32(str.utf8.count)) != 0 504 } 505 } 506 507 func write_profile_last_fetched(pubkey: Pubkey, fetched_at: UInt64) { 508 guard !closed else { return } 509 let _ = pubkey.id.withUnsafeBytes { (ptr: UnsafeRawBufferPointer) -> () in 510 guard let p = ptr.baseAddress else { return } 511 ndb_write_last_profile_fetch(ndb.ndb, p, fetched_at) 512 } 513 } 514 515 func read_profile_last_fetched<Y>(txn: NdbTxn<Y>, pubkey: Pubkey) -> UInt64? { 516 guard !closed else { return nil } 517 return pubkey.id.withUnsafeBytes { (ptr: UnsafeRawBufferPointer) -> UInt64? in 518 guard let p = ptr.baseAddress else { return nil } 519 let res = ndb_read_last_profile_fetch(&txn.txn, p) 520 if res == 0 { 521 return nil 522 } 523 524 return res 525 } 526 } 527 528 func process_event(_ str: String) -> Bool { 529 guard !is_closed else { return false } 530 return str.withCString { cstr in 531 return ndb_process_event(ndb.ndb, cstr, Int32(str.utf8.count)) != 0 532 } 533 } 534 535 func process_events(_ str: String) -> Bool { 536 guard !is_closed else { return false } 537 return str.withCString { cstr in 538 return ndb_process_events(ndb.ndb, cstr, str.utf8.count) != 0 539 } 540 } 541 542 func search_profile<Y>(_ search: String, limit: Int, txn: NdbTxn<Y>) -> [Pubkey] { 543 var pks = Array<Pubkey>() 544 545 return search.withCString { q in 546 var s = ndb_search() 547 guard ndb_search_profile(&txn.txn, &s, q) != 0 else { 548 return pks 549 } 550 551 defer { ndb_search_profile_end(&s) } 552 pks.append(Pubkey(Data(bytes: &s.key.pointee.id.0, count: 32))) 553 554 var n = limit 555 while n > 0 { 556 guard ndb_search_profile_next(&s) != 0 else { 557 return pks 558 } 559 pks.append(Pubkey(Data(bytes: &s.key.pointee.id.0, count: 32))) 560 561 n -= 1 562 } 563 564 return pks 565 } 566 } 567 568 enum Errors: Error { 569 case cannot_find_db_path 570 case db_file_migration_error 571 } 572 573 deinit { 574 print("txn: Ndb de-init") 575 self.close() 576 } 577 } 578 579 #if DEBUG 580 func getDebugCheckedRoot<T: FlatBufferObject>(byteBuffer: inout ByteBuffer) throws -> T { 581 return getRoot(byteBuffer: &byteBuffer) 582 } 583 #else 584 func getDebugCheckedRoot<T: FlatBufferObject>(byteBuffer: inout ByteBuffer) throws -> T { 585 return getRoot(byteBuffer: &byteBuffer) 586 } 587 #endif 588 589 func remove_file_prefix(_ str: String) -> String { 590 return str.replacingOccurrences(of: "file://", with: "") 591 } 592