commit 866dc0a15817b977f43e1104cd4a7ff7ed111ac5
parent a1acd807aa7abbe4aabdeffc895c0a5f1481e29d
Author: William Casarin <jb55@jb55.com>
Date: Sat, 7 Oct 2023 22:38:12 -0700
stats: add reaction counter
Implement the initial nostrdb stats counter. We start by creating a
simple reaction counter that increases the reaction count in the note
metadata whenever it sees a new like for a note. It counts any like,
even - atm which is probably wrong. In the future we can fix these with
migrations.
There is a race condition if there are duplicate notes in the same
ingestor block, so we should fix that some time in the future. I added a
note on the ndb_write_reaction_stats function for future bug fixers.
This will be used to replace the EventCounter logic in Damus iOS. No
longer will we need to maintain an in-memory counter which is always out
of date. Now we will have proper stats when scrolling through the feed,
at least after the likes have been fetched...
Changelog-Added: Add reaction counts to note metadata
Diffstat:
5 files changed, 181 insertions(+), 6 deletions(-)
diff --git a/nostrdb.c b/nostrdb.c
@@ -16,6 +16,8 @@
#include "bindings/c/profile_json_parser.h"
#include "bindings/c/profile_builder.h"
+#include "bindings/c/meta_builder.h"
+#include "bindings/c/meta_reader.h"
#include "bindings/c/profile_verifier.h"
#include "secp256k1.h"
#include "secp256k1_ecdh.h"
@@ -807,8 +809,12 @@ static int ndb_ingester_process_note(secp256k1_context *ctx,
size_t note_size,
struct ndb_writer_msg *out)
{
+ //printf("ndb_ingester_process_note ");
+ //print_hex(note->id, 32);
+ //printf("\n");
+
// Verify! If it's an invalid note we don't need to
- // bothter writing it to the database
+ // bother writing it to the database
if (!ndb_note_verify(ctx, note->pubkey, note->id, note->sig)) {
ndb_debug("signature verification failed\n");
return 0;
@@ -1135,6 +1141,126 @@ static int ndb_write_profile(struct ndb_txn *txn,
return 1;
}
+// find the last id tag in a note (e, p, etc)
+static unsigned char *ndb_note_last_id_tag(struct ndb_note *note, char type)
+{
+ unsigned char *last = NULL;
+ struct ndb_iterator iter;
+ struct ndb_str str;
+
+ // get the liked event id (last id)
+ ndb_tags_iterate_start(note, &iter);
+
+ while (ndb_tags_iterate_next(&iter)) {
+ if (iter.tag->count < 2)
+ continue;
+
+ str = ndb_note_str(note, &iter.tag->strs[0]);
+
+ // assign liked to the last e tag
+ if (str.flag == NDB_PACKED_STR && str.str[0] == type) {
+ str = ndb_note_str(note, &iter.tag->strs[1]);
+ if (str.flag == NDB_PACKED_ID)
+ last = str.id;
+ }
+ }
+
+ return last;
+}
+
+void *ndb_get_note_meta(struct ndb_txn *txn, const unsigned char *id, size_t *len)
+{
+ MDB_val k, v;
+
+ k.mv_data = (unsigned char*)id;
+ k.mv_size = 32;
+
+ if (mdb_get(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_META], &k, &v)) {
+ ndb_debug("ndb_get_note_meta: mdb_get note failed\n");
+ return NULL;
+ }
+
+ if (len)
+ *len = v.mv_size;
+
+ return v.mv_data;
+}
+
+// When receiving a reaction note, look for the liked id and increase the
+// reaction counter in the note metadata database
+//
+// TODO: I found some bugs when implementing this feature. If the same note id
+// is processed multiple times in the same ingestion block, then it will count
+// the like twice. This is because it hasn't been written to the DB yet and the
+// ingestor doesn't know about notes that are being processed at the same time.
+// One fix for this is to maintain a hashtable in the ingestor and make sure
+// the same note is not processed twice.
+//
+// I'm not sure how common this would be, so I'm not going to worry about it
+// for now, but it's something to keep in mind.
+static int ndb_write_reaction_stats(struct ndb_txn *txn, struct ndb_note *note)
+{
+ size_t len;
+ void *root;
+ int reactions, rc;
+ MDB_val key, val;
+ NdbEventMeta_table_t meta;
+ unsigned char *liked = ndb_note_last_id_tag(note, 'e');
+
+ if (liked == NULL)
+ return 0;
+
+ root = ndb_get_note_meta(txn, liked, &len);
+
+ flatcc_builder_t builder;
+ flatcc_builder_init(&builder);
+ NdbEventMeta_start_as_root(&builder);
+
+ // no meta record, let's make one
+ if (root == NULL) {
+ NdbEventMeta_reactions_add(&builder, 1);
+ } else {
+ // clone existing and add to it
+ meta = NdbEventMeta_as_root(root);
+
+ reactions = NdbEventMeta_reactions_get(meta);
+ NdbEventMeta_clone(&builder, meta);
+ NdbEventMeta_reactions_add(&builder, reactions + 1);
+ }
+
+ NdbProfileRecord_end_as_root(&builder);
+ root = flatcc_builder_finalize_aligned_buffer(&builder, &len);
+ assert(((uint64_t)root % 8) == 0);
+
+ if (root == NULL) {
+ ndb_debug("failed to create note metadata record\n");
+ return 0;
+ }
+
+ // metadata is keyed on id because we want to collect stats regardless
+ // if we have the note yet or not
+ key.mv_data = liked;
+ key.mv_size = 32;
+
+ val.mv_data = root;
+ val.mv_size = len;
+
+ // write the new meta record
+ //ndb_debug("writing stats record for ");
+ //print_hex(liked, 32);
+ //ndb_debug("\n");
+
+ if ((rc = mdb_put(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_META], &key, &val, 0))) {
+ ndb_debug("write reaction stats to db failed: %s\n", mdb_strerror(rc));
+ return 0;
+ }
+
+ free(root);
+
+ return 1;
+}
+
+
static uint64_t ndb_write_note(struct ndb_txn *txn,
struct ndb_writer_note *note)
{
@@ -1177,7 +1303,7 @@ static uint64_t ndb_write_note(struct ndb_txn *txn,
}
if (note->note->kind == 7) {
- ndb_write_reaction_stats(txn, note->note, note_key);
+ ndb_write_reaction_stats(txn, note->note);
}
return note_key;
@@ -1261,6 +1387,9 @@ static void *ndb_writer_thread(void *data)
break;
case NDB_WRITER_NOTE:
ndb_write_note(&txn, &msg->note);
+ //printf("wrote note ");
+ //print_hex(msg->note.note->id, 32);
+ //printf("\n");
break;
case NDB_WRITER_DBMETA:
ndb_write_version(&txn, msg->ndb_meta.version);
@@ -1277,7 +1406,6 @@ static void *ndb_writer_thread(void *data)
assert(false);
}
-
// free notes
for (i = 0; i < popped; i++) {
msg = &msgs[i];
@@ -1486,7 +1614,7 @@ static int ndb_init_lmdb(const char *filename, struct ndb_lmdb *lmdb, size_t map
}
// note metadata db
- if ((rc = mdb_dbi_open(txn, "meta", MDB_CREATE | MDB_INTEGERKEY, &lmdb->dbs[NDB_DB_META]))) {
+ if ((rc = mdb_dbi_open(txn, "meta", MDB_CREATE, &lmdb->dbs[NDB_DB_META]))) {
fprintf(stderr, "mdb_dbi_open meta failed, error %d\n", rc);
return 0;
}
diff --git a/nostrdb.h b/nostrdb.h
@@ -205,6 +205,7 @@ uint64_t ndb_get_notekey_by_id(struct ndb_txn *txn, const unsigned char *id);
uint64_t ndb_get_profilekey_by_pubkey(struct ndb_txn *txn, const unsigned char *id);
struct ndb_note *ndb_get_note_by_id(struct ndb_txn *txn, const unsigned char *id, size_t *len, uint64_t *primkey);
struct ndb_note *ndb_get_note_by_key(struct ndb_txn *txn, uint64_t key, size_t *len);
+void *ndb_get_note_meta(struct ndb_txn *txn, const unsigned char *id, size_t *len);
void ndb_destroy(struct ndb *);
// BUILDER
diff --git a/random.h b/random.h
@@ -66,7 +66,6 @@ static int fill_random(unsigned char* data, size_t size) {
static void print_hex(unsigned char* data, size_t size) {
size_t i;
for (i = 0; i < size; i++) {
- fprintf(stderr, "%02x", data[i]);
+ printf("%02x", data[i]);
}
- fprintf(stderr, "\n");
}
diff --git a/test.c b/test.c
@@ -6,6 +6,8 @@
#include "memchr.h"
#include "bindings/c/profile_reader.h"
#include "bindings/c/profile_verifier.h"
+#include "bindings/c/meta_reader.h"
+#include "bindings/c/meta_verifier.h"
#include <stdio.h>
#include <assert.h>
@@ -45,6 +47,47 @@ static void print_search(struct ndb_txn *txn, struct ndb_search *search)
}
+static void test_reaction_counter()
+{
+ static const int alloc_size = 1024 * 1024;
+ char *json = malloc(alloc_size);
+ struct ndb *ndb;
+ size_t mapsize, len;
+ void *root;
+ int written, ingester_threads, reactions;
+ NdbEventMeta_table_t meta;
+ struct ndb_txn txn;
+
+ mapsize = 1024 * 1024 * 100;
+ ingester_threads = 1;
+
+ assert(ndb_init(&ndb, test_dir, mapsize, ingester_threads, 0));
+
+ read_file("testdata/reactions.json", (unsigned char*)json, alloc_size, &written);
+ assert(ndb_process_client_events(ndb, json, written));
+ ndb_destroy(ndb);
+
+ assert(ndb_init(&ndb, test_dir, mapsize, ingester_threads, 0));
+
+ assert(ndb_begin_query(ndb, &txn));
+
+ const unsigned char id[32] = {
+ 0x1a, 0x41, 0x56, 0x30, 0x31, 0x09, 0xbb, 0x4a, 0x66, 0x0a, 0x6a, 0x90,
+ 0x04, 0xb0, 0xcd, 0xce, 0x8d, 0x83, 0xc3, 0x99, 0x1d, 0xe7, 0x86, 0x4f,
+ 0x18, 0x76, 0xeb, 0x0f, 0x62, 0x2c, 0x68, 0xe8
+ };
+
+ assert((root = ndb_get_note_meta(&txn, id, &len)));
+ assert(0 == NdbEventMeta_verify_as_root(root, len));
+ assert((meta = NdbEventMeta_as_root(root)));
+
+ reactions = NdbEventMeta_reactions_get(meta);
+ //printf("counted reactions: %d\n", reactions);
+ assert(reactions == 2);
+ ndb_end_query(&txn);
+ ndb_destroy(ndb);
+}
+
static void test_profile_search(struct ndb *ndb)
{
struct ndb_txn txn;
@@ -779,6 +822,7 @@ static void test_fast_strchr()
int main(int argc, const char *argv[]) {
test_migrate();
test_profile_updates();
+ test_reaction_counter();
test_load_profiles();
test_basic_event();
test_empty_tags();
diff --git a/testdata/reactions.json b/testdata/reactions.json
@@ -0,0 +1,3 @@
+["EVENT", {"id": "1a4156303109bb4a660a6a9004b0cdce8d83c3991de7864f1876eb0f622c68e8","pubkey": "c511ed64e93f3aa053f85c82ee5f1ef9be6b61254606b88b8656f47091dd6e52","created_at": 1696738537,"kind": 1,"tags": [],"content": "hello!","sig": "8f215d3a1673c7e857fa3a23b4e9c93c0770e03c2c90dc2d0f00ed29ed7e0b54f99eacbdd2abb5ade3c98e13f2218f746b51d88df7617552f59f276e50ba020a"}]
+["EVENT", {"id": "028a90d81a1379ec07141e4cef36f0c993140c807f8bc179bea213c80ef8f807","pubkey": "5b1affd872a5b42fa613e96d7a898211bd8a284e246e1f06713b8ecc1d123d83","created_at": 1696738664,"kind": 7,"tags": [["e","2a4156303109bb4a660a6a9004b0cdce8d83c3991de7864f1876eb0f622c68e9"],["p","c511ed64e93f3aa053f85c82ee5f1ef9be6b61254606b88b8656f47091dd6e52"],["e","1a4156303109bb4a660a6a9004b0cdce8d83c3991de7864f1876eb0f622c68e8"]],"content": "+","sig": "1556911a9208f315d63deb994dcdbc9e73ad5cbb7c357d43e35946610f413c9d83c6304af624bb8c71cd254c2a89ccdca1fe74d4b1f8dbab69a9e2805a0738e8"}]
+["EVENT", {"id": "9c350d1f3822be358abbd5654721bcf45e5919c95a3835517a9290c45b5278ab","pubkey": "0e78f7d0620f726f41eb1a1e97df02312710fcf59acf6c303bfe2e00fe50be0e","created_at": 1696738688,"kind": 7,"tags": [["p","c511ed64e93f3aa053f85c82ee5f1ef9be6b61254606b88b8656f47091dd6e52"],["e","1a4156303109bb4a660a6a9004b0cdce8d83c3991de7864f1876eb0f622c68e8"]],"content": "-","sig": "08e72aa3d2f1d6a2b7cb18c4ddff06e46e895670d1c9821c1b0b1ba3d53dca9de07cfc1829b6c3bc5e8d733b2dcc239395a2826af45245aca96722f01196960a"}]
