unknowns.rs (12164B)
1 use crate::{ 2 column::Columns, 3 note::NoteRef, 4 notecache::{CachedNote, NoteCache}, 5 timeline::ViewFilter, 6 Result, 7 }; 8 9 use enostr::{Filter, NoteId, Pubkey}; 10 use nostrdb::{BlockType, Mention, Ndb, Note, NoteKey, Transaction}; 11 use std::collections::HashSet; 12 use std::time::{Duration, Instant}; 13 use tracing::error; 14 15 #[must_use = "process_action should be used on this result"] 16 pub enum SingleUnkIdAction { 17 NoAction, 18 NeedsProcess(UnknownId), 19 } 20 21 #[must_use = "process_action should be used on this result"] 22 pub enum NoteRefsUnkIdAction { 23 NoAction, 24 NeedsProcess(Vec<NoteRef>), 25 } 26 27 impl NoteRefsUnkIdAction { 28 pub fn new(refs: Vec<NoteRef>) -> Self { 29 NoteRefsUnkIdAction::NeedsProcess(refs) 30 } 31 32 pub fn no_action() -> Self { 33 Self::NoAction 34 } 35 36 pub fn process_action( 37 &self, 38 txn: &Transaction, 39 ndb: &Ndb, 40 unk_ids: &mut UnknownIds, 41 note_cache: &mut NoteCache, 42 ) { 43 match self { 44 Self::NoAction => {} 45 Self::NeedsProcess(refs) => { 46 UnknownIds::update_from_note_refs(txn, ndb, unk_ids, note_cache, refs); 47 } 48 } 49 } 50 } 51 52 impl SingleUnkIdAction { 53 pub fn new(id: UnknownId) -> Self { 54 SingleUnkIdAction::NeedsProcess(id) 55 } 56 57 pub fn no_action() -> Self { 58 Self::NoAction 59 } 60 61 pub fn pubkey(pubkey: Pubkey) -> Self { 62 SingleUnkIdAction::new(UnknownId::Pubkey(pubkey)) 63 } 64 65 pub fn note_id(note_id: NoteId) -> Self { 66 SingleUnkIdAction::new(UnknownId::Id(note_id)) 67 } 68 69 /// Some functions may return unknown id actions that need to be processed. 70 /// For example, when we add a new account we need to make sure we have the 71 /// profile for that account. This function ensures we add this to the 72 /// unknown id tracker without adding side effects to functions. 73 pub fn process_action(&self, ids: &mut UnknownIds, ndb: &Ndb, txn: &Transaction) { 74 match self { 75 Self::NeedsProcess(id) => { 76 ids.add_unknown_id_if_missing(ndb, txn, id); 77 } 78 Self::NoAction => {} 79 } 80 } 81 } 82 83 /// Unknown Id searcher 84 #[derive(Default)] 85 pub struct UnknownIds { 86 ids: HashSet<UnknownId>, 87 first_updated: Option<Instant>, 88 last_updated: Option<Instant>, 89 } 90 91 impl UnknownIds { 92 /// Simple debouncer 93 pub fn ready_to_send(&self) -> bool { 94 if self.ids.is_empty() { 95 return false; 96 } 97 98 // we trigger on first set 99 if self.first_updated == self.last_updated { 100 return true; 101 } 102 103 let last_updated = if let Some(last) = self.last_updated { 104 last 105 } else { 106 // if we've 107 return true; 108 }; 109 110 Instant::now() - last_updated >= Duration::from_secs(2) 111 } 112 113 pub fn ids(&self) -> &HashSet<UnknownId> { 114 &self.ids 115 } 116 117 pub fn ids_mut(&mut self) -> &mut HashSet<UnknownId> { 118 &mut self.ids 119 } 120 121 pub fn clear(&mut self) { 122 self.ids = HashSet::default(); 123 } 124 125 pub fn filter(&self) -> Option<Vec<Filter>> { 126 let ids: Vec<&UnknownId> = self.ids.iter().collect(); 127 get_unknown_ids_filter(&ids) 128 } 129 130 /// We've updated some unknown ids, update the last_updated time to now 131 pub fn mark_updated(&mut self) { 132 let now = Instant::now(); 133 if self.first_updated.is_none() { 134 self.first_updated = Some(now); 135 } 136 self.last_updated = Some(now); 137 } 138 139 pub fn update_from_note_key( 140 txn: &Transaction, 141 ndb: &Ndb, 142 unknown_ids: &mut UnknownIds, 143 note_cache: &mut NoteCache, 144 key: NoteKey, 145 ) -> bool { 146 let note = if let Ok(note) = ndb.get_note_by_key(txn, key) { 147 note 148 } else { 149 return false; 150 }; 151 152 UnknownIds::update_from_note(txn, ndb, unknown_ids, note_cache, ¬e) 153 } 154 155 /// Should be called on freshly polled notes from subscriptions 156 pub fn update_from_note_refs( 157 txn: &Transaction, 158 ndb: &Ndb, 159 unknown_ids: &mut UnknownIds, 160 note_cache: &mut NoteCache, 161 note_refs: &[NoteRef], 162 ) { 163 for note_ref in note_refs { 164 Self::update_from_note_key(txn, ndb, unknown_ids, note_cache, note_ref.key); 165 } 166 } 167 168 pub fn update_from_note( 169 txn: &Transaction, 170 ndb: &Ndb, 171 unknown_ids: &mut UnknownIds, 172 note_cache: &mut NoteCache, 173 note: &Note, 174 ) -> bool { 175 let before = unknown_ids.ids().len(); 176 let key = note.key().expect("note key"); 177 //let cached_note = note_cache.cached_note_or_insert(key, note).clone(); 178 let cached_note = note_cache.cached_note_or_insert(key, note); 179 if let Err(e) = get_unknown_note_ids(ndb, cached_note, txn, note, unknown_ids.ids_mut()) { 180 error!("UnknownIds::update_from_note {e}"); 181 } 182 let after = unknown_ids.ids().len(); 183 184 if before != after { 185 unknown_ids.mark_updated(); 186 true 187 } else { 188 false 189 } 190 } 191 192 pub fn add_unknown_id_if_missing(&mut self, ndb: &Ndb, txn: &Transaction, unk_id: &UnknownId) { 193 match unk_id { 194 UnknownId::Pubkey(pk) => self.add_pubkey_if_missing(ndb, txn, pk), 195 UnknownId::Id(note_id) => self.add_note_id_if_missing(ndb, txn, note_id), 196 } 197 } 198 199 pub fn add_pubkey_if_missing(&mut self, ndb: &Ndb, txn: &Transaction, pubkey: &Pubkey) { 200 // we already have this profile, skip 201 if ndb.get_profile_by_pubkey(txn, pubkey).is_ok() { 202 return; 203 } 204 205 self.ids.insert(UnknownId::Pubkey(*pubkey)); 206 self.mark_updated(); 207 } 208 209 pub fn add_note_id_if_missing(&mut self, ndb: &Ndb, txn: &Transaction, note_id: &NoteId) { 210 // we already have this note, skip 211 if ndb.get_note_by_id(txn, note_id.bytes()).is_ok() { 212 return; 213 } 214 215 self.ids.insert(UnknownId::Id(*note_id)); 216 self.mark_updated(); 217 } 218 219 pub fn update( 220 txn: &Transaction, 221 unknown_ids: &mut UnknownIds, 222 columns: &Columns, 223 ndb: &Ndb, 224 note_cache: &mut NoteCache, 225 ) -> bool { 226 let before = unknown_ids.ids().len(); 227 if let Err(e) = get_unknown_ids(txn, unknown_ids, columns, ndb, note_cache) { 228 error!("UnknownIds::update {e}"); 229 } 230 let after = unknown_ids.ids().len(); 231 232 if before != after { 233 unknown_ids.mark_updated(); 234 true 235 } else { 236 false 237 } 238 } 239 } 240 241 #[derive(Hash, Clone, Copy, PartialEq, Eq)] 242 pub enum UnknownId { 243 Pubkey(Pubkey), 244 Id(NoteId), 245 } 246 247 impl UnknownId { 248 pub fn is_pubkey(&self) -> Option<&Pubkey> { 249 match self { 250 UnknownId::Pubkey(pk) => Some(pk), 251 _ => None, 252 } 253 } 254 255 pub fn is_id(&self) -> Option<&NoteId> { 256 match self { 257 UnknownId::Id(id) => Some(id), 258 _ => None, 259 } 260 } 261 } 262 263 /// Look for missing notes in various parts of notes that we see: 264 /// 265 /// - pubkeys and notes mentioned inside the note 266 /// - notes being replied to 267 /// 268 /// We return all of this in a HashSet so that we can fetch these from 269 /// remote relays. 270 /// 271 pub fn get_unknown_note_ids<'a>( 272 ndb: &Ndb, 273 cached_note: &CachedNote, 274 txn: &'a Transaction, 275 note: &Note<'a>, 276 ids: &mut HashSet<UnknownId>, 277 ) -> Result<()> { 278 // the author pubkey 279 280 if ndb.get_profile_by_pubkey(txn, note.pubkey()).is_err() { 281 ids.insert(UnknownId::Pubkey(Pubkey::new(*note.pubkey()))); 282 } 283 284 // pull notes that notes are replying to 285 if cached_note.reply.root.is_some() { 286 let note_reply = cached_note.reply.borrow(note.tags()); 287 if let Some(root) = note_reply.root() { 288 if ndb.get_note_by_id(txn, root.id).is_err() { 289 ids.insert(UnknownId::Id(NoteId::new(*root.id))); 290 } 291 } 292 293 if !note_reply.is_reply_to_root() { 294 if let Some(reply) = note_reply.reply() { 295 if ndb.get_note_by_id(txn, reply.id).is_err() { 296 ids.insert(UnknownId::Id(NoteId::new(*reply.id))); 297 } 298 } 299 } 300 } 301 302 let blocks = ndb.get_blocks_by_key(txn, note.key().expect("note key"))?; 303 for block in blocks.iter(note) { 304 if block.blocktype() != BlockType::MentionBech32 { 305 continue; 306 } 307 308 match block.as_mention().unwrap() { 309 Mention::Pubkey(npub) => { 310 if ndb.get_profile_by_pubkey(txn, npub.pubkey()).is_err() { 311 ids.insert(UnknownId::Pubkey(Pubkey::new(*npub.pubkey()))); 312 } 313 } 314 Mention::Profile(nprofile) => { 315 if ndb.get_profile_by_pubkey(txn, nprofile.pubkey()).is_err() { 316 ids.insert(UnknownId::Pubkey(Pubkey::new(*nprofile.pubkey()))); 317 } 318 } 319 Mention::Event(ev) => match ndb.get_note_by_id(txn, ev.id()) { 320 Err(_) => { 321 ids.insert(UnknownId::Id(NoteId::new(*ev.id()))); 322 if let Some(pk) = ev.pubkey() { 323 if ndb.get_profile_by_pubkey(txn, pk).is_err() { 324 ids.insert(UnknownId::Pubkey(Pubkey::new(*pk))); 325 } 326 } 327 } 328 Ok(note) => { 329 if ndb.get_profile_by_pubkey(txn, note.pubkey()).is_err() { 330 ids.insert(UnknownId::Pubkey(Pubkey::new(*note.pubkey()))); 331 } 332 } 333 }, 334 Mention::Note(note) => match ndb.get_note_by_id(txn, note.id()) { 335 Err(_) => { 336 ids.insert(UnknownId::Id(NoteId::new(*note.id()))); 337 } 338 Ok(note) => { 339 if ndb.get_profile_by_pubkey(txn, note.pubkey()).is_err() { 340 ids.insert(UnknownId::Pubkey(Pubkey::new(*note.pubkey()))); 341 } 342 } 343 }, 344 _ => {} 345 } 346 } 347 348 Ok(()) 349 } 350 351 fn get_unknown_ids( 352 txn: &Transaction, 353 unknown_ids: &mut UnknownIds, 354 columns: &Columns, 355 ndb: &Ndb, 356 note_cache: &mut NoteCache, 357 ) -> Result<()> { 358 #[cfg(feature = "profiling")] 359 puffin::profile_function!(); 360 361 let mut new_cached_notes: Vec<(NoteKey, CachedNote)> = vec![]; 362 363 for timeline in columns.timelines() { 364 for noteref in timeline.notes(ViewFilter::NotesAndReplies) { 365 let note = ndb.get_note_by_key(txn, noteref.key)?; 366 let note_key = note.key().unwrap(); 367 let cached_note = note_cache.cached_note(noteref.key); 368 let cached_note = if let Some(cn) = cached_note { 369 cn.clone() 370 } else { 371 let new_cached_note = CachedNote::new(¬e); 372 new_cached_notes.push((note_key, new_cached_note.clone())); 373 new_cached_note 374 }; 375 376 let _ = get_unknown_note_ids(ndb, &cached_note, txn, ¬e, unknown_ids.ids_mut()); 377 } 378 } 379 380 // This is mainly done to avoid the double mutable borrow that would happen 381 // if we tried to update the note_cache mutably in the loop above 382 for (note_key, note) in new_cached_notes { 383 note_cache.cache_mut().insert(note_key, note); 384 } 385 386 Ok(()) 387 } 388 389 fn get_unknown_ids_filter(ids: &[&UnknownId]) -> Option<Vec<Filter>> { 390 if ids.is_empty() { 391 return None; 392 } 393 394 let ids = &ids[0..500.min(ids.len())]; 395 let mut filters: Vec<Filter> = vec![]; 396 397 let pks: Vec<&[u8; 32]> = ids 398 .iter() 399 .flat_map(|id| id.is_pubkey().map(|pk| pk.bytes())) 400 .collect(); 401 if !pks.is_empty() { 402 let pk_filter = Filter::new().authors(pks).kinds([0]).build(); 403 filters.push(pk_filter); 404 } 405 406 let note_ids: Vec<&[u8; 32]> = ids 407 .iter() 408 .flat_map(|id| id.is_id().map(|id| id.bytes())) 409 .collect(); 410 if !note_ids.is_empty() { 411 filters.push(Filter::new().ids(note_ids).build()); 412 } 413 414 Some(filters) 415 }