nostrdb

an unfairly fast embedded nostr database backed by lmdb
git clone git://jb55.com/nostrdb
Log | Files | Refs | Submodules | README | LICENSE

commit 201ab6fa16b5ec2445f64e29b76d0e4e94d6a5f1
parent fad218c1bdda74168e4d265b40e176d00590fa39
Author: William Casarin <jb55@jb55.com>
Date:   Thu,  2 Nov 2023 14:12:13 +0900

ndb: migration fix for japanese profile names

Diffstat:
Mnostrdb.c | 119+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++--------------
Mtest.c | 2+-
2 files changed, 99 insertions(+), 22 deletions(-)

diff --git a/nostrdb.c b/nostrdb.c @@ -64,6 +64,14 @@ struct ndb_ingest_controller struct ndb_lmdb *lmdb; }; +enum ndb_writer_msgtype { + NDB_WRITER_QUIT, // kill thread immediately + NDB_WRITER_NOTE, // write a note to the db + NDB_WRITER_PROFILE, // write a profile to the db + NDB_WRITER_DBMETA, // write ndb metadata + NDB_WRITER_PROFILE_LAST_FETCH, // when profiles were last fetched +}; + enum ndb_dbs { NDB_DB_NOTE, NDB_DB_META, @@ -121,6 +129,7 @@ struct ndb { // lmdb environ handles, etc }; + // A clustered key with an id and a timestamp struct ndb_tsid { unsigned char id[32]; @@ -147,7 +156,6 @@ static void lowercase_strncpy(char *dst, const char *src, int n) { } } - static void ndb_make_search_key(struct ndb_search_key *key, unsigned char *id, uint64_t timestamp, const char *search) { @@ -319,10 +327,7 @@ static int ndb_migrate_lower_user_search_indices(struct ndb *ndb) return ndb_migrate_user_search_indices(ndb); } -static struct ndb_migration MIGRATIONS[] = { - { .fn = ndb_migrate_user_search_indices }, - { .fn = ndb_migrate_lower_user_search_indices } -}; +int ndb_process_profile_note(struct ndb_note *note, struct ndb_profile_record_builder *profile); int ndb_db_version(struct ndb *ndb) @@ -418,14 +423,6 @@ enum ndb_ingester_msgtype { NDB_INGEST_QUIT, // kill ingester thread immediately }; -enum ndb_writer_msgtype { - NDB_WRITER_QUIT, // kill thread immediately - NDB_WRITER_NOTE, // write a note to the db - NDB_WRITER_PROFILE, // write a profile to the db - NDB_WRITER_DBMETA, // write ndb metadata - NDB_WRITER_PROFILE_LAST_FETCH, // when profiles were last fetched -}; - struct ndb_ingester_event { char *json; unsigned client : 1; // ["EVENT", {...}] messages @@ -473,6 +470,92 @@ struct ndb_writer_msg { }; }; +static inline int ndb_writer_queue_msg(struct ndb_writer *writer, + struct ndb_writer_msg *msg) +{ + return prot_queue_push(&writer->inbox, msg); +} + +static int ndb_migrate_utf8_profile_names(struct ndb *ndb) +{ + int rc; + MDB_cursor *cur; + MDB_val k, v; + void *profile_root; + NdbProfileRecord_table_t record; + struct ndb_txn txn; + struct ndb_note *note, *copied_note; + uint64_t note_key; + size_t len; + int count, failed; + struct ndb_writer_msg out; + + if (!ndb_begin_rw_query(ndb, &txn)) { + fprintf(stderr, "ndb_migrate_utf8_profile_names: ndb_begin_rw_query failed\n"); + return 0; + } + + if ((rc = mdb_cursor_open(txn.mdb_txn, ndb->lmdb.dbs[NDB_DB_PROFILE], &cur))) { + fprintf(stderr, "ndb_migrate_utf8_profile_names: mdb_cursor_open failed, error %d\n", rc); + return 0; + } + + count = 0; + failed = 0; + + // loop through all profiles and write search indices + while (mdb_cursor_get(cur, &k, &v, MDB_NEXT) == 0) { + profile_root = v.mv_data; + record = NdbProfileRecord_as_root(profile_root); + note_key = NdbProfileRecord_note_key(record); + note = ndb_get_note_by_key(&txn, note_key, &len); + + if (note == NULL) { + fprintf(stderr, "ndb_migrate_utf8_profile_names: note lookup failed\n"); + return 0; + } + + struct ndb_profile_record_builder *b = &out.profile.record; + + // reprocess profile + if (!ndb_process_profile_note(note, b)) { + failed++; + continue; + } + + // the writer needs to own this note, and its expected to free it + copied_note = malloc(len); + memcpy(copied_note, note, len); + + out.type = NDB_WRITER_PROFILE; + out.profile.note.note = copied_note; + out.profile.note.note_len = len; + + ndb_writer_queue_msg(&ndb->writer, &out); + + count++; + } + + fprintf(stderr, "migrated %d profiles to fix utf8 profile names\n", count); + + if (failed != 0) { + fprintf(stderr, "failed to migrate %d profiles to fix utf8 profile names\n", failed); + } + + mdb_cursor_close(cur); + + ndb_end_query(&txn); + + return 1; +} + +static struct ndb_migration MIGRATIONS[] = { + { .fn = ndb_migrate_user_search_indices }, + { .fn = ndb_migrate_lower_user_search_indices }, + { .fn = ndb_migrate_utf8_profile_names } +}; + + int ndb_end_query(struct ndb_txn *txn) { // this works on read or write queries. @@ -496,12 +579,6 @@ int ndb_note_verify(void *ctx, unsigned char pubkey[32], unsigned char id[32], return 1; } -static inline int ndb_writer_queue_msg(struct ndb_writer *writer, - struct ndb_writer_msg *msg) -{ - return prot_queue_push(&writer->inbox, msg); -} - static inline int ndb_writer_queue_msgs(struct ndb_writer *writer, struct ndb_writer_msg *msgs, int num_msgs) @@ -814,8 +891,8 @@ void ndb_profile_record_builder_free(struct ndb_profile_record_builder *b) b->flatbuf = NULL; } -static int ndb_process_profile_note(struct ndb_note *note, - struct ndb_profile_record_builder *profile) +int ndb_process_profile_note(struct ndb_note *note, + struct ndb_profile_record_builder *profile) { int res; diff --git a/test.c b/test.c @@ -284,7 +284,7 @@ static void test_migrate() { assert(ndb_init(&ndb, v0_dir, mapsize, threads, 0)); ndb_destroy(ndb); assert(ndb_init(&ndb, v0_dir, mapsize, threads, 0)); - assert(ndb_db_version(ndb) == 2); + assert(ndb_db_version(ndb) == 3); test_profile_search(ndb); ndb_destroy(ndb);