notedeck

One damus client to rule them all
git clone git://jb55.com/notedeck
Log | Files | Refs | README | LICENSE

unknowns.rs (12164B)


      1 use crate::{
      2     column::Columns,
      3     note::NoteRef,
      4     notecache::{CachedNote, NoteCache},
      5     timeline::ViewFilter,
      6     Result,
      7 };
      8 
      9 use enostr::{Filter, NoteId, Pubkey};
     10 use nostrdb::{BlockType, Mention, Ndb, Note, NoteKey, Transaction};
     11 use std::collections::HashSet;
     12 use std::time::{Duration, Instant};
     13 use tracing::error;
     14 
     15 #[must_use = "process_action should be used on this result"]
     16 pub enum SingleUnkIdAction {
     17     NoAction,
     18     NeedsProcess(UnknownId),
     19 }
     20 
     21 #[must_use = "process_action should be used on this result"]
     22 pub enum NoteRefsUnkIdAction {
     23     NoAction,
     24     NeedsProcess(Vec<NoteRef>),
     25 }
     26 
     27 impl NoteRefsUnkIdAction {
     28     pub fn new(refs: Vec<NoteRef>) -> Self {
     29         NoteRefsUnkIdAction::NeedsProcess(refs)
     30     }
     31 
     32     pub fn no_action() -> Self {
     33         Self::NoAction
     34     }
     35 
     36     pub fn process_action(
     37         &self,
     38         txn: &Transaction,
     39         ndb: &Ndb,
     40         unk_ids: &mut UnknownIds,
     41         note_cache: &mut NoteCache,
     42     ) {
     43         match self {
     44             Self::NoAction => {}
     45             Self::NeedsProcess(refs) => {
     46                 UnknownIds::update_from_note_refs(txn, ndb, unk_ids, note_cache, refs);
     47             }
     48         }
     49     }
     50 }
     51 
     52 impl SingleUnkIdAction {
     53     pub fn new(id: UnknownId) -> Self {
     54         SingleUnkIdAction::NeedsProcess(id)
     55     }
     56 
     57     pub fn no_action() -> Self {
     58         Self::NoAction
     59     }
     60 
     61     pub fn pubkey(pubkey: Pubkey) -> Self {
     62         SingleUnkIdAction::new(UnknownId::Pubkey(pubkey))
     63     }
     64 
     65     pub fn note_id(note_id: NoteId) -> Self {
     66         SingleUnkIdAction::new(UnknownId::Id(note_id))
     67     }
     68 
     69     /// Some functions may return unknown id actions that need to be processed.
     70     /// For example, when we add a new account we need to make sure we have the
     71     /// profile for that account. This function ensures we add this to the
     72     /// unknown id tracker without adding side effects to functions.
     73     pub fn process_action(&self, ids: &mut UnknownIds, ndb: &Ndb, txn: &Transaction) {
     74         match self {
     75             Self::NeedsProcess(id) => {
     76                 ids.add_unknown_id_if_missing(ndb, txn, id);
     77             }
     78             Self::NoAction => {}
     79         }
     80     }
     81 }
     82 
     83 /// Unknown Id searcher
     84 #[derive(Default)]
     85 pub struct UnknownIds {
     86     ids: HashSet<UnknownId>,
     87     first_updated: Option<Instant>,
     88     last_updated: Option<Instant>,
     89 }
     90 
     91 impl UnknownIds {
     92     /// Simple debouncer
     93     pub fn ready_to_send(&self) -> bool {
     94         if self.ids.is_empty() {
     95             return false;
     96         }
     97 
     98         // we trigger on first set
     99         if self.first_updated == self.last_updated {
    100             return true;
    101         }
    102 
    103         let last_updated = if let Some(last) = self.last_updated {
    104             last
    105         } else {
    106             // if we've
    107             return true;
    108         };
    109 
    110         Instant::now() - last_updated >= Duration::from_secs(2)
    111     }
    112 
    113     pub fn ids(&self) -> &HashSet<UnknownId> {
    114         &self.ids
    115     }
    116 
    117     pub fn ids_mut(&mut self) -> &mut HashSet<UnknownId> {
    118         &mut self.ids
    119     }
    120 
    121     pub fn clear(&mut self) {
    122         self.ids = HashSet::default();
    123     }
    124 
    125     pub fn filter(&self) -> Option<Vec<Filter>> {
    126         let ids: Vec<&UnknownId> = self.ids.iter().collect();
    127         get_unknown_ids_filter(&ids)
    128     }
    129 
    130     /// We've updated some unknown ids, update the last_updated time to now
    131     pub fn mark_updated(&mut self) {
    132         let now = Instant::now();
    133         if self.first_updated.is_none() {
    134             self.first_updated = Some(now);
    135         }
    136         self.last_updated = Some(now);
    137     }
    138 
    139     pub fn update_from_note_key(
    140         txn: &Transaction,
    141         ndb: &Ndb,
    142         unknown_ids: &mut UnknownIds,
    143         note_cache: &mut NoteCache,
    144         key: NoteKey,
    145     ) -> bool {
    146         let note = if let Ok(note) = ndb.get_note_by_key(txn, key) {
    147             note
    148         } else {
    149             return false;
    150         };
    151 
    152         UnknownIds::update_from_note(txn, ndb, unknown_ids, note_cache, &note)
    153     }
    154 
    155     /// Should be called on freshly polled notes from subscriptions
    156     pub fn update_from_note_refs(
    157         txn: &Transaction,
    158         ndb: &Ndb,
    159         unknown_ids: &mut UnknownIds,
    160         note_cache: &mut NoteCache,
    161         note_refs: &[NoteRef],
    162     ) {
    163         for note_ref in note_refs {
    164             Self::update_from_note_key(txn, ndb, unknown_ids, note_cache, note_ref.key);
    165         }
    166     }
    167 
    168     pub fn update_from_note(
    169         txn: &Transaction,
    170         ndb: &Ndb,
    171         unknown_ids: &mut UnknownIds,
    172         note_cache: &mut NoteCache,
    173         note: &Note,
    174     ) -> bool {
    175         let before = unknown_ids.ids().len();
    176         let key = note.key().expect("note key");
    177         //let cached_note = note_cache.cached_note_or_insert(key, note).clone();
    178         let cached_note = note_cache.cached_note_or_insert(key, note);
    179         if let Err(e) = get_unknown_note_ids(ndb, cached_note, txn, note, unknown_ids.ids_mut()) {
    180             error!("UnknownIds::update_from_note {e}");
    181         }
    182         let after = unknown_ids.ids().len();
    183 
    184         if before != after {
    185             unknown_ids.mark_updated();
    186             true
    187         } else {
    188             false
    189         }
    190     }
    191 
    192     pub fn add_unknown_id_if_missing(&mut self, ndb: &Ndb, txn: &Transaction, unk_id: &UnknownId) {
    193         match unk_id {
    194             UnknownId::Pubkey(pk) => self.add_pubkey_if_missing(ndb, txn, pk),
    195             UnknownId::Id(note_id) => self.add_note_id_if_missing(ndb, txn, note_id),
    196         }
    197     }
    198 
    199     pub fn add_pubkey_if_missing(&mut self, ndb: &Ndb, txn: &Transaction, pubkey: &Pubkey) {
    200         // we already have this profile, skip
    201         if ndb.get_profile_by_pubkey(txn, pubkey).is_ok() {
    202             return;
    203         }
    204 
    205         self.ids.insert(UnknownId::Pubkey(*pubkey));
    206         self.mark_updated();
    207     }
    208 
    209     pub fn add_note_id_if_missing(&mut self, ndb: &Ndb, txn: &Transaction, note_id: &NoteId) {
    210         // we already have this note, skip
    211         if ndb.get_note_by_id(txn, note_id.bytes()).is_ok() {
    212             return;
    213         }
    214 
    215         self.ids.insert(UnknownId::Id(*note_id));
    216         self.mark_updated();
    217     }
    218 
    219     pub fn update(
    220         txn: &Transaction,
    221         unknown_ids: &mut UnknownIds,
    222         columns: &Columns,
    223         ndb: &Ndb,
    224         note_cache: &mut NoteCache,
    225     ) -> bool {
    226         let before = unknown_ids.ids().len();
    227         if let Err(e) = get_unknown_ids(txn, unknown_ids, columns, ndb, note_cache) {
    228             error!("UnknownIds::update {e}");
    229         }
    230         let after = unknown_ids.ids().len();
    231 
    232         if before != after {
    233             unknown_ids.mark_updated();
    234             true
    235         } else {
    236             false
    237         }
    238     }
    239 }
    240 
    241 #[derive(Hash, Clone, Copy, PartialEq, Eq)]
    242 pub enum UnknownId {
    243     Pubkey(Pubkey),
    244     Id(NoteId),
    245 }
    246 
    247 impl UnknownId {
    248     pub fn is_pubkey(&self) -> Option<&Pubkey> {
    249         match self {
    250             UnknownId::Pubkey(pk) => Some(pk),
    251             _ => None,
    252         }
    253     }
    254 
    255     pub fn is_id(&self) -> Option<&NoteId> {
    256         match self {
    257             UnknownId::Id(id) => Some(id),
    258             _ => None,
    259         }
    260     }
    261 }
    262 
    263 /// Look for missing notes in various parts of notes that we see:
    264 ///
    265 /// - pubkeys and notes mentioned inside the note
    266 /// - notes being replied to
    267 ///
    268 /// We return all of this in a HashSet so that we can fetch these from
    269 /// remote relays.
    270 ///
    271 pub fn get_unknown_note_ids<'a>(
    272     ndb: &Ndb,
    273     cached_note: &CachedNote,
    274     txn: &'a Transaction,
    275     note: &Note<'a>,
    276     ids: &mut HashSet<UnknownId>,
    277 ) -> Result<()> {
    278     // the author pubkey
    279 
    280     if ndb.get_profile_by_pubkey(txn, note.pubkey()).is_err() {
    281         ids.insert(UnknownId::Pubkey(Pubkey::new(*note.pubkey())));
    282     }
    283 
    284     // pull notes that notes are replying to
    285     if cached_note.reply.root.is_some() {
    286         let note_reply = cached_note.reply.borrow(note.tags());
    287         if let Some(root) = note_reply.root() {
    288             if ndb.get_note_by_id(txn, root.id).is_err() {
    289                 ids.insert(UnknownId::Id(NoteId::new(*root.id)));
    290             }
    291         }
    292 
    293         if !note_reply.is_reply_to_root() {
    294             if let Some(reply) = note_reply.reply() {
    295                 if ndb.get_note_by_id(txn, reply.id).is_err() {
    296                     ids.insert(UnknownId::Id(NoteId::new(*reply.id)));
    297                 }
    298             }
    299         }
    300     }
    301 
    302     let blocks = ndb.get_blocks_by_key(txn, note.key().expect("note key"))?;
    303     for block in blocks.iter(note) {
    304         if block.blocktype() != BlockType::MentionBech32 {
    305             continue;
    306         }
    307 
    308         match block.as_mention().unwrap() {
    309             Mention::Pubkey(npub) => {
    310                 if ndb.get_profile_by_pubkey(txn, npub.pubkey()).is_err() {
    311                     ids.insert(UnknownId::Pubkey(Pubkey::new(*npub.pubkey())));
    312                 }
    313             }
    314             Mention::Profile(nprofile) => {
    315                 if ndb.get_profile_by_pubkey(txn, nprofile.pubkey()).is_err() {
    316                     ids.insert(UnknownId::Pubkey(Pubkey::new(*nprofile.pubkey())));
    317                 }
    318             }
    319             Mention::Event(ev) => match ndb.get_note_by_id(txn, ev.id()) {
    320                 Err(_) => {
    321                     ids.insert(UnknownId::Id(NoteId::new(*ev.id())));
    322                     if let Some(pk) = ev.pubkey() {
    323                         if ndb.get_profile_by_pubkey(txn, pk).is_err() {
    324                             ids.insert(UnknownId::Pubkey(Pubkey::new(*pk)));
    325                         }
    326                     }
    327                 }
    328                 Ok(note) => {
    329                     if ndb.get_profile_by_pubkey(txn, note.pubkey()).is_err() {
    330                         ids.insert(UnknownId::Pubkey(Pubkey::new(*note.pubkey())));
    331                     }
    332                 }
    333             },
    334             Mention::Note(note) => match ndb.get_note_by_id(txn, note.id()) {
    335                 Err(_) => {
    336                     ids.insert(UnknownId::Id(NoteId::new(*note.id())));
    337                 }
    338                 Ok(note) => {
    339                     if ndb.get_profile_by_pubkey(txn, note.pubkey()).is_err() {
    340                         ids.insert(UnknownId::Pubkey(Pubkey::new(*note.pubkey())));
    341                     }
    342                 }
    343             },
    344             _ => {}
    345         }
    346     }
    347 
    348     Ok(())
    349 }
    350 
    351 fn get_unknown_ids(
    352     txn: &Transaction,
    353     unknown_ids: &mut UnknownIds,
    354     columns: &Columns,
    355     ndb: &Ndb,
    356     note_cache: &mut NoteCache,
    357 ) -> Result<()> {
    358     #[cfg(feature = "profiling")]
    359     puffin::profile_function!();
    360 
    361     let mut new_cached_notes: Vec<(NoteKey, CachedNote)> = vec![];
    362 
    363     for timeline in columns.timelines() {
    364         for noteref in timeline.notes(ViewFilter::NotesAndReplies) {
    365             let note = ndb.get_note_by_key(txn, noteref.key)?;
    366             let note_key = note.key().unwrap();
    367             let cached_note = note_cache.cached_note(noteref.key);
    368             let cached_note = if let Some(cn) = cached_note {
    369                 cn.clone()
    370             } else {
    371                 let new_cached_note = CachedNote::new(&note);
    372                 new_cached_notes.push((note_key, new_cached_note.clone()));
    373                 new_cached_note
    374             };
    375 
    376             let _ = get_unknown_note_ids(ndb, &cached_note, txn, &note, unknown_ids.ids_mut());
    377         }
    378     }
    379 
    380     // This is mainly done to avoid the double mutable borrow that would happen
    381     // if we tried to update the note_cache mutably in the loop above
    382     for (note_key, note) in new_cached_notes {
    383         note_cache.cache_mut().insert(note_key, note);
    384     }
    385 
    386     Ok(())
    387 }
    388 
    389 fn get_unknown_ids_filter(ids: &[&UnknownId]) -> Option<Vec<Filter>> {
    390     if ids.is_empty() {
    391         return None;
    392     }
    393 
    394     let ids = &ids[0..500.min(ids.len())];
    395     let mut filters: Vec<Filter> = vec![];
    396 
    397     let pks: Vec<&[u8; 32]> = ids
    398         .iter()
    399         .flat_map(|id| id.is_pubkey().map(|pk| pk.bytes()))
    400         .collect();
    401     if !pks.is_empty() {
    402         let pk_filter = Filter::new().authors(pks).kinds([0]).build();
    403         filters.push(pk_filter);
    404     }
    405 
    406     let note_ids: Vec<&[u8; 32]> = ids
    407         .iter()
    408         .flat_map(|id| id.is_id().map(|id| id.bytes()))
    409         .collect();
    410     if !note_ids.is_empty() {
    411         filters.push(Filter::new().ids(note_ids).build());
    412     }
    413 
    414     Some(filters)
    415 }