commit 539cb0651e197a5f0670762413c087b5e121df69
parent d5aeed4abd44291021834e7b08569b8205266cd0
Author: William Casarin <jb55@jb55.com>
Date: Mon, 4 Dec 2023 14:13:14 -0800
writer: make sure we don't write a note if we already have it
I'm noticing duplicate notes in the database, which might happen when
the ingester and writer get spammed with the same note rapidly. Add a
sanity check during the write so that we only ever write a note once.
Fixes: 866dc0a15817 ("stats: add reaction counter")
Changelog-Fixed: Fix duplicate notes getting written to nostrdb
Diffstat:
2 files changed, 7 insertions(+), 10 deletions(-)
diff --git a/nostrdb.c b/nostrdb.c
@@ -1998,16 +1998,6 @@ void *ndb_get_note_meta(struct ndb_txn *txn, const unsigned char *id, size_t *le
// When receiving a reaction note, look for the liked id and increase the
// reaction counter in the note metadata database
-//
-// TODO: I found some bugs when implementing this feature. If the same note id
-// is processed multiple times in the same ingestion block, then it will count
-// the like twice. This is because it hasn't been written to the DB yet and the
-// ingestor doesn't know about notes that are being processed at the same time.
-// One fix for this is to maintain a hashtable in the ingestor and make sure
-// the same note is not processed twice.
-//
-// I'm not sure how common this would be, so I'm not going to worry about it
-// for now, but it's something to keep in mind.
static int ndb_write_reaction_stats(struct ndb_txn *txn, struct ndb_note *note)
{
size_t len;
@@ -2803,6 +2793,10 @@ static uint64_t ndb_write_note(struct ndb_txn *txn,
uint64_t note_key;
MDB_dbi note_db;
MDB_val key, val;
+
+ // let's quickly sanity check if we already have this note
+ if (ndb_get_notekey_by_id(txn, note->note->id))
+ return 0;
// get dbs
note_db = txn->lmdb->dbs[NDB_DB_NOTE];
diff --git a/testdata/reactions.json b/testdata/reactions.json
@@ -1,3 +1,6 @@
["EVENT", {"id": "1a4156303109bb4a660a6a9004b0cdce8d83c3991de7864f1876eb0f622c68e8","pubkey": "c511ed64e93f3aa053f85c82ee5f1ef9be6b61254606b88b8656f47091dd6e52","created_at": 1696738537,"kind": 1,"tags": [],"content": "hello!","sig": "8f215d3a1673c7e857fa3a23b4e9c93c0770e03c2c90dc2d0f00ed29ed7e0b54f99eacbdd2abb5ade3c98e13f2218f746b51d88df7617552f59f276e50ba020a"}]
["EVENT", {"id": "028a90d81a1379ec07141e4cef36f0c993140c807f8bc179bea213c80ef8f807","pubkey": "5b1affd872a5b42fa613e96d7a898211bd8a284e246e1f06713b8ecc1d123d83","created_at": 1696738664,"kind": 7,"tags": [["e","2a4156303109bb4a660a6a9004b0cdce8d83c3991de7864f1876eb0f622c68e9"],["p","c511ed64e93f3aa053f85c82ee5f1ef9be6b61254606b88b8656f47091dd6e52"],["e","1a4156303109bb4a660a6a9004b0cdce8d83c3991de7864f1876eb0f622c68e8"]],"content": "+","sig": "1556911a9208f315d63deb994dcdbc9e73ad5cbb7c357d43e35946610f413c9d83c6304af624bb8c71cd254c2a89ccdca1fe74d4b1f8dbab69a9e2805a0738e8"}]
+["EVENT", {"id": "028a90d81a1379ec07141e4cef36f0c993140c807f8bc179bea213c80ef8f807","pubkey": "5b1affd872a5b42fa613e96d7a898211bd8a284e246e1f06713b8ecc1d123d83","created_at": 1696738664,"kind": 7,"tags": [["e","2a4156303109bb4a660a6a9004b0cdce8d83c3991de7864f1876eb0f622c68e9"],["p","c511ed64e93f3aa053f85c82ee5f1ef9be6b61254606b88b8656f47091dd6e52"],["e","1a4156303109bb4a660a6a9004b0cdce8d83c3991de7864f1876eb0f622c68e8"]],"content": "+","sig": "1556911a9208f315d63deb994dcdbc9e73ad5cbb7c357d43e35946610f413c9d83c6304af624bb8c71cd254c2a89ccdca1fe74d4b1f8dbab69a9e2805a0738e8"}]
+["EVENT", {"id": "028a90d81a1379ec07141e4cef36f0c993140c807f8bc179bea213c80ef8f807","pubkey": "5b1affd872a5b42fa613e96d7a898211bd8a284e246e1f06713b8ecc1d123d83","created_at": 1696738664,"kind": 7,"tags": [["e","2a4156303109bb4a660a6a9004b0cdce8d83c3991de7864f1876eb0f622c68e9"],["p","c511ed64e93f3aa053f85c82ee5f1ef9be6b61254606b88b8656f47091dd6e52"],["e","1a4156303109bb4a660a6a9004b0cdce8d83c3991de7864f1876eb0f622c68e8"]],"content": "+","sig": "1556911a9208f315d63deb994dcdbc9e73ad5cbb7c357d43e35946610f413c9d83c6304af624bb8c71cd254c2a89ccdca1fe74d4b1f8dbab69a9e2805a0738e8"}]
+["EVENT", {"id": "028a90d81a1379ec07141e4cef36f0c993140c807f8bc179bea213c80ef8f807","pubkey": "5b1affd872a5b42fa613e96d7a898211bd8a284e246e1f06713b8ecc1d123d83","created_at": 1696738664,"kind": 7,"tags": [["e","2a4156303109bb4a660a6a9004b0cdce8d83c3991de7864f1876eb0f622c68e9"],["p","c511ed64e93f3aa053f85c82ee5f1ef9be6b61254606b88b8656f47091dd6e52"],["e","1a4156303109bb4a660a6a9004b0cdce8d83c3991de7864f1876eb0f622c68e8"]],"content": "+","sig": "1556911a9208f315d63deb994dcdbc9e73ad5cbb7c357d43e35946610f413c9d83c6304af624bb8c71cd254c2a89ccdca1fe74d4b1f8dbab69a9e2805a0738e8"}]
["EVENT", {"id": "9c350d1f3822be358abbd5654721bcf45e5919c95a3835517a9290c45b5278ab","pubkey": "0e78f7d0620f726f41eb1a1e97df02312710fcf59acf6c303bfe2e00fe50be0e","created_at": 1696738688,"kind": 7,"tags": [["p","c511ed64e93f3aa053f85c82ee5f1ef9be6b61254606b88b8656f47091dd6e52"],["e","1a4156303109bb4a660a6a9004b0cdce8d83c3991de7864f1876eb0f622c68e8"]],"content": "-","sig": "08e72aa3d2f1d6a2b7cb18c4ddff06e46e895670d1c9821c1b0b1ba3d53dca9de07cfc1829b6c3bc5e8d733b2dcc239395a2826af45245aca96722f01196960a"}]
