commit f0edc1eb11e58a87e6338cb92973b523c19d3af4
parent b81490cda1c6c18459e515c130e88b331dc9fea9
Author: William Casarin <jb55@jb55.com>
Date: Thu, 30 Oct 2025 14:39:03 -0700
migration: migrate full reaction stats for all kind1 notes
Signed-off-by: William Casarin <jb55@jb55.com>
Diffstat:
6 files changed, 150 insertions(+), 37 deletions(-)
diff --git a/TODO b/TODO
@@ -1,4 +1 @@
test write +metadata
-(B) migrate reply stats +metadata
-(B) migrate quote stats +metadata
-(B) migrate reaction stats +metadata
diff --git a/ndb.c b/ndb.c
@@ -28,6 +28,7 @@ static int usage()
printf(" print-tag-keys\n");
printf(" print-relay-kind-index-keys\n");
printf(" print-author-kind-index-keys\n");
+ printf(" print-note-metadata\n");
printf(" import <line-delimited json file>\n\n");
printf("settings\n\n");
@@ -108,6 +109,7 @@ int ndb_print_kind_keys(struct ndb_txn *txn);
int ndb_print_tag_index(struct ndb_txn *txn);
int ndb_print_relay_kind_index(struct ndb_txn *txn);
int ndb_print_author_kind_index(struct ndb_txn *txn);
+int ndb_print_note_metadata(struct ndb_txn *txn);
static void print_note(struct ndb_note *note)
{
@@ -443,6 +445,10 @@ int main(int argc, char *argv[])
ndb_begin_query(ndb, &txn);
ndb_print_author_kind_index(&txn);
ndb_end_query(&txn);
+ } else if (argc == 2 && !strcmp(argv[1], "print-note-metadata")) {
+ ndb_begin_query(ndb, &txn);
+ ndb_print_note_metadata(&txn);
+ ndb_end_query(&txn);
} else if (argc == 3 && !strcmp(argv[1], "note-relays")) {
struct ndb_note_relay_iterator iter;
const char *relay;
diff --git a/src/metadata.c b/src/metadata.c
@@ -221,7 +221,7 @@ uint16_t *ndb_note_meta_entry_type(struct ndb_note_meta_entry *entry)
}
/* find a metadata entry, optionally matching a payload */
-struct ndb_note_meta_entry *ndb_note_meta_find_entry(struct ndb_note_meta *meta, uint16_t type, uint64_t *payload)
+static struct ndb_note_meta_entry *ndb_note_meta_find_entry_impl(struct ndb_note_meta *meta, uint16_t type, uint64_t *payload, int sorted)
{
struct ndb_note_meta_entry *entries, *entry;
int i;
@@ -232,6 +232,8 @@ struct ndb_note_meta_entry *ndb_note_meta_find_entry(struct ndb_note_meta *meta,
entries = ndb_note_meta_entries(meta);
assert(((intptr_t)entries - (intptr_t)meta) == 16);
+ /* TODO(jb55): do bsearch for large sorted entries */
+
for (i = 0; i < meta->count; i++) {
entry = &entries[i];
assert(((uintptr_t)entry % 8) == 0);
@@ -250,6 +252,22 @@ struct ndb_note_meta_entry *ndb_note_meta_find_entry(struct ndb_note_meta *meta,
return NULL;
}
+struct ndb_note_meta_entry *ndb_note_meta_find_entry(struct ndb_note_meta *meta, uint16_t type, uint64_t *payload)
+{
+ int sorted = 1;
+ return ndb_note_meta_find_entry_impl(meta, type, payload, sorted);
+}
+
+struct ndb_note_meta_entry *ndb_note_meta_builder_find_entry(
+ struct ndb_note_meta_builder *builder,
+ uint16_t type,
+ uint64_t *payload)
+{
+ /* meta building in progress is not necessarily sorted */
+ int sorted = 0;
+ return ndb_note_meta_find_entry_impl((struct ndb_note_meta *)builder->cursor.start, type, payload, sorted);
+}
+
void ndb_note_meta_reaction_set(struct ndb_note_meta_entry *entry, uint32_t count, union ndb_reaction_str str)
{
entry->type = NDB_NOTE_META_REACTION;
@@ -389,3 +407,35 @@ union ndb_reaction_str ndb_note_meta_reaction_str(struct ndb_note_meta_entry *en
{
return entry->payload.reaction_str;
}
+
+void print_note_meta(struct ndb_note_meta *meta)
+{
+ int count, i;
+ struct ndb_note_meta_entry *entries, *entry;
+ union ndb_reaction_str reaction;
+ char strbuf[128];
+
+ count = ndb_note_meta_entries_count(meta);
+ entries = ndb_note_meta_entries(meta);
+
+ for (i = 0; i < count; i++) {
+ entry = &entries[i];
+ switch (entry->type) {
+ case NDB_NOTE_META_REACTION:
+ reaction = ndb_note_meta_reaction_str(entry);
+
+ ndb_reaction_to_str(&reaction, strbuf);
+ printf("%s%d ", strbuf, *ndb_note_meta_reaction_count(entry));
+ break;
+ case NDB_NOTE_META_COUNTS:
+ printf("quotes %d\treplies %d\tall_replies %d\treactions %d\t",
+ *ndb_note_meta_counts_quotes(entry),
+ *ndb_note_meta_counts_direct_replies(entry),
+ *ndb_note_meta_counts_thread_replies(entry),
+ *ndb_note_meta_counts_total_reactions(entry));
+ break;
+ }
+ }
+
+ printf("\n");
+}
diff --git a/src/nostrdb.c b/src/nostrdb.c
@@ -2178,7 +2178,7 @@ cleanup:
}
/* count all of the reactions for a note */
-int ndb_count_reactions(struct ndb_txn *txn, const unsigned char *note_id, uint32_t *count)
+int ndb_rebuild_reaction_metadata(struct ndb_txn *txn, const unsigned char *note_id, struct ndb_note_meta_builder *builder, uint32_t *count)
{
MDB_val k, v;
MDB_cursor *cur;
@@ -2189,6 +2189,8 @@ int ndb_count_reactions(struct ndb_txn *txn, const unsigned char *note_id, uint3
size_t size;
struct ndb_note *note;
unsigned char *keybuf, *last_id;
+ struct ndb_note_meta_entry *entry;
+ union ndb_reaction_str reaction_str;
char buffer[41]; /* 1 + 32 + 8 */
*count = 0;
@@ -2227,6 +2229,21 @@ int ndb_count_reactions(struct ndb_txn *txn, const unsigned char *note_id, uint3
continue;
if (memcmp(last_id, note_id, 32))
continue;
+
+ if (builder) {
+ if (!ndb_reaction_set(&reaction_str, ndb_note_content(note)))
+ ndb_reaction_set(&reaction_str, "+");
+
+ if ((entry = ndb_note_meta_builder_find_entry(builder, NDB_NOTE_META_REACTION, &reaction_str.binmoji))) {
+ (*ndb_note_meta_reaction_count(entry))++;
+ } else if ((entry = ndb_note_meta_add_entry(builder))) {
+ ndb_note_meta_reaction_set(entry, 1, reaction_str);
+ } else {
+ /* couldn't add reaction entry ? */
+ ndb_debug("ndb_rebuild_note_indices: couldn't add reaction count entry to metadata builder\n");
+ }
+ }
+
(*count)++;
} while (mdb_cursor_get(cur, &k, &v, MDB_NEXT) == 0);
@@ -2301,19 +2318,22 @@ static int ndb_note_meta_builder_counts(struct ndb_txn *txn,
thread_replies = 0;
total_reactions = 0;
- if (!(entry = ndb_note_meta_add_entry(builder)))
- return 0;
-
- rcs[0] = ndb_count_reactions(txn, note_id, &total_reactions);
+ rcs[0] = ndb_rebuild_reaction_metadata(txn, note_id, builder, &total_reactions);
rcs[1] = ndb_count_quotes(txn, note_id, "es);
rcs[2] = ndb_count_replies(txn, note_id, &direct_replies, &thread_replies);
- if (!rcs[0] && !rcs[1] && !rcs[2])
+ if (!rcs[0] && !rcs[1] && !rcs[2]) {
return 0;
+ }
/* no entry needed */
- if (quotes == 0 && direct_replies == 0 && thread_replies == 0 && quotes == 0)
- return 1;
+ if (quotes == 0 && direct_replies == 0 && thread_replies == 0 && quotes == 0) {
+ return 0;
+ }
+
+ if (!(entry = ndb_note_meta_add_entry(builder))) {
+ return 0;
+ }
ndb_note_meta_counts_set(entry, total_reactions, quotes, direct_replies, thread_replies);
@@ -2334,49 +2354,64 @@ static int ndb_migrate_metadata(struct ndb_txn *txn)
{
MDB_val k, k2, v, v2;
MDB_cursor *cur;
- MDB_dbi db;
+ MDB_dbi note_db, meta_db;
unsigned char *id;
- unsigned char buffer[4096];
- int rc;
+ size_t scratch_size = 1024 * 1024;
+ unsigned char *buffer = malloc(scratch_size);
+ int rc, count;
struct ndb_note_meta_builder builder;
+ struct ndb_note *note;
struct ndb_note_meta *meta;
- db = txn->lmdb->dbs[NDB_DB_META];
+ meta_db = txn->lmdb->dbs[NDB_DB_META];
+ note_db = txn->lmdb->dbs[NDB_DB_NOTE];
- if ((rc = mdb_cursor_open(txn->mdb_txn, db, &cur))) {
- fprintf(stderr, "ndb_migrate_reaction_stats: mdb_cursor_open failed, error %d\n", rc);
+ /* drop metadata table to avoid issues */
+ if (mdb_drop(txn->mdb_txn, meta_db, 0)) {
+ fprintf(stderr, "ndb_migrate_metadata: mdb_drop failed\n");
return -1;
}
+ if ((rc = mdb_cursor_open(txn->mdb_txn, note_db, &cur))) {
+ fprintf(stderr, "ndb_migrate_metadata: mdb_cursor_open failed, error %d\n", rc);
+ return -1;
+ }
+
+ count = 0;
+
/* loop through every metadata entry */
while (mdb_cursor_get(cur, &k, &v, MDB_NEXT) == 0) {
- ndb_note_meta_builder_init(&builder, buffer, sizeof(buffer));
+ ndb_note_meta_builder_init(&builder, buffer, scratch_size);
- id = (unsigned char *)k.mv_data;
- ndb_note_meta_builder_count_reactions(txn, &builder);
- ndb_note_meta_builder_counts(txn, id, &builder);
- ndb_note_meta_build(&builder, &meta);
+ note = (struct ndb_note *)v.mv_data;
+ id = ndb_note_id(note);
+ k2.mv_data = (unsigned char *)id;
+ k2.mv_size = 32;
- /* no counts found, just delete this entry */
- if (ndb_note_meta_entries_count(meta) == 0) {
- if ((rc = mdb_del(txn->mdb_txn, db, &k, &v))) {
- ndb_debug("delete old metadata entry failed: %s\n", mdb_strerror(rc));
- return -1;
- }
+ rc = ndb_note_meta_builder_counts(txn, id, &builder);
+ if (!rc) {
+ mdb_del(txn->mdb_txn, meta_db, &k2, NULL);
continue;
}
- k2.mv_data = (unsigned char *)id;
- k2.mv_size = 32;
+ ndb_note_meta_build(&builder, &meta);
+ assert(ndb_note_meta_entries(meta)->type != 0);
+
v2.mv_data = meta;
v2.mv_size = ndb_note_meta_total_size(meta);
/* set entry */
- if ((rc = mdb_put(txn->mdb_txn, db, &k2, &v2, 0))) {
+ if ((rc = mdb_put(txn->mdb_txn, meta_db, &k2, &v2, 0))) {
ndb_debug("migrate metadata entry failed on write: %s\n", mdb_strerror(rc));
}
+
+ count++;
}
+ fprintf(stderr, "nostrdb: migrated %d metadata entries\n", count);
+
+ free(buffer);
+ mdb_cursor_close(cur);
return 1;
}
@@ -2713,7 +2748,7 @@ static struct ndb_migration MIGRATIONS[] = {
{ .fn = ndb_migrate_lower_user_search_indices },
{ .fn = ndb_migrate_utf8_profile_names },
{ .fn = ndb_migrate_profile_indices },
- //{ .fn = ndb_migrate_metadata },
+ { .fn = ndb_migrate_metadata },
};
@@ -8515,6 +8550,29 @@ void ndb_config_set_ingest_filter(struct ndb_config *config,
config->filter_context = filter_ctx;
}
+int ndb_print_note_metadata(struct ndb_txn *txn)
+{
+ MDB_cursor *cur;
+ MDB_val k, v;
+ int i;
+
+ if (mdb_cursor_open(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_META], &cur))
+ return 0;
+
+ i = 1;
+ while (mdb_cursor_get(cur, &k, &v, MDB_NEXT) == 0) {
+ print_hex(k.mv_data, 32);
+ printf("\t");
+ print_note_meta((struct ndb_note_meta*)v.mv_data);
+ i++;
+ }
+
+ mdb_cursor_close(cur);
+
+ return i;
+}
+
+
int ndb_print_author_kind_index(struct ndb_txn *txn)
{
MDB_cursor *cur;
diff --git a/src/nostrdb.h b/src/nostrdb.h
@@ -49,8 +49,8 @@ struct bolt11;
*/
enum ndb_metadata_type {
NDB_NOTE_META_RESERVED = 0, /* not used */
- NDB_NOTE_META_COUNTS = 2, /* replies, quotes, etc */
- NDB_NOTE_META_REACTION = 4, /* count of all the reactions on a post, grouped by different reaction strings */
+ NDB_NOTE_META_COUNTS = 100, /* replies, quotes, etc */
+ NDB_NOTE_META_REACTION = 200, /* count of all the reactions on a post, grouped by different reaction strings */
};
// some bindings like swift needs help with forward declared pointers
@@ -658,6 +658,7 @@ struct ndb_note_meta *ndb_get_note_meta(struct ndb_txn *txn, const unsigned char
int ndb_set_note_meta(struct ndb *ndb, const unsigned char *id, struct ndb_note_meta *meta);
size_t ndb_note_meta_total_size(struct ndb_note_meta *header);
int ndb_note_meta_builder_init(struct ndb_note_meta_builder *builder, unsigned char *, size_t);
+struct ndb_note_meta_entry *ndb_note_meta_builder_find_entry(struct ndb_note_meta_builder *builder, uint16_t type, uint64_t *payload);
void ndb_note_meta_build(struct ndb_note_meta_builder *builder, struct ndb_note_meta **meta);
uint16_t ndb_note_meta_entries_count(struct ndb_note_meta *meta);
struct ndb_note_meta_entry *ndb_note_meta_entries(struct ndb_note_meta *meta);
@@ -673,6 +674,7 @@ uint16_t *ndb_note_meta_counts_direct_replies(struct ndb_note_meta_entry *entry)
uint32_t *ndb_note_meta_counts_thread_replies(struct ndb_note_meta_entry *entry);
uint16_t *ndb_note_meta_entry_type(struct ndb_note_meta_entry *entry);
struct ndb_note_meta_entry *ndb_note_meta_entry_at(struct ndb_note_meta *meta, int ind);
+void print_note_meta(struct ndb_note_meta *meta);
// META STRINGS
int ndb_reaction_set(union ndb_reaction_str *reaction, const char *str);
diff --git a/test.c b/test.c
@@ -30,7 +30,7 @@ static void delete_test_db() {
unlink(TEST_DIR "/data.lock");
}
-int ndb_count_reactions(struct ndb_txn *txn, const unsigned char *note_id, uint32_t *count);
+int ndb_rebuild_reaction_metadata(struct ndb_txn *txn, const unsigned char *note_id, struct ndb_note_meta_builder *builder, uint32_t *count);
int ndb_count_replies(struct ndb_txn *txn, const unsigned char *note_id, uint16_t *direct_replies, uint32_t *thread_replies);
static void db_load_events(struct ndb *ndb, const char *filename)
@@ -141,7 +141,7 @@ static void test_count_metadata()
ndb_begin_query(ndb, &txn);
/* this is used in the migration code,
* let's make sure it matches the online logic */
- ndb_count_reactions(&txn, id, &reactions);
+ ndb_rebuild_reaction_metadata(&txn, id, NULL, &reactions);
printf("\t# after-counted reactions %d\n", reactions);
assert(reactions == total_reactions);
