nostrdb

commit 201ab6fa16b5ec2445f64e29b76d0e4e94d6a5f1
parent fad218c1bdda74168e4d265b40e176d00590fa39
Author: William Casarin <jb55@jb55.com>
Date:   Thu,  2 Nov 2023 14:12:13 +0900

ndb: migration fix for japanese profile names

Diffstat:
M
nostrdb.c
 | 
119
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++--------------
M
test.c
 | 
2
+-

2 files changed, 99 insertions(+), 22 deletions(-)
diff --git a/nostrdb.c b/nostrdb.c
@@ -64,6 +64,14 @@ struct ndb_ingest_controller
 	struct ndb_lmdb *lmdb;
 };
 
+enum ndb_writer_msgtype {
+	NDB_WRITER_QUIT, // kill thread immediately
+	NDB_WRITER_NOTE, // write a note to the db
+	NDB_WRITER_PROFILE, // write a profile to the db
+	NDB_WRITER_DBMETA, // write ndb metadata
+	NDB_WRITER_PROFILE_LAST_FETCH, // when profiles were last fetched
+};
+
 enum ndb_dbs {
 	NDB_DB_NOTE,
 	NDB_DB_META,
@@ -121,6 +129,7 @@ struct ndb {
 	// lmdb environ handles, etc
 };
 
+
 // A clustered key with an id and a timestamp
 struct ndb_tsid {
 	unsigned char id[32];
@@ -147,7 +156,6 @@ static void lowercase_strncpy(char *dst, const char *src, int n) {
 	}
 }
 
-
 static void ndb_make_search_key(struct ndb_search_key *key, unsigned char *id,
 			        uint64_t timestamp, const char *search)
 {
@@ -319,10 +327,7 @@ static int ndb_migrate_lower_user_search_indices(struct ndb *ndb)
 	return ndb_migrate_user_search_indices(ndb);
 }
 
-static struct ndb_migration MIGRATIONS[] = {
-	{ .fn = ndb_migrate_user_search_indices },
-	{ .fn = ndb_migrate_lower_user_search_indices }
-};
+int ndb_process_profile_note(struct ndb_note *note, struct ndb_profile_record_builder *profile);
 
 
 int ndb_db_version(struct ndb *ndb)
@@ -418,14 +423,6 @@ enum ndb_ingester_msgtype {
 	NDB_INGEST_QUIT,  // kill ingester thread immediately
 };
 
-enum ndb_writer_msgtype {
-	NDB_WRITER_QUIT, // kill thread immediately
-	NDB_WRITER_NOTE, // write a note to the db
-	NDB_WRITER_PROFILE, // write a profile to the db
-	NDB_WRITER_DBMETA, // write ndb metadata
-	NDB_WRITER_PROFILE_LAST_FETCH, // when profiles were last fetched
-};
-
 struct ndb_ingester_event {
 	char *json;
 	unsigned client : 1; // ["EVENT", {...}] messages
@@ -473,6 +470,92 @@ struct ndb_writer_msg {
 	};
 };
 
+static inline int ndb_writer_queue_msg(struct ndb_writer *writer,
+				       struct ndb_writer_msg *msg)
+{
+	return prot_queue_push(&writer->inbox, msg);
+}
+
+static int ndb_migrate_utf8_profile_names(struct ndb *ndb)
+{
+	int rc;
+	MDB_cursor *cur;
+	MDB_val k, v;
+	void *profile_root;
+	NdbProfileRecord_table_t record;
+	struct ndb_txn txn;
+	struct ndb_note *note, *copied_note;
+	uint64_t note_key;
+	size_t len;
+	int count, failed;
+	struct ndb_writer_msg out;
+
+	if (!ndb_begin_rw_query(ndb, &txn)) {
+		fprintf(stderr, "ndb_migrate_utf8_profile_names: ndb_begin_rw_query failed\n");
+		return 0;
+	}
+
+	if ((rc = mdb_cursor_open(txn.mdb_txn, ndb->lmdb.dbs[NDB_DB_PROFILE], &cur))) {
+		fprintf(stderr, "ndb_migrate_utf8_profile_names: mdb_cursor_open failed, error %d\n", rc);
+		return 0;
+	}
+
+	count = 0;
+	failed = 0;
+
+	// loop through all profiles and write search indices
+	while (mdb_cursor_get(cur, &k, &v, MDB_NEXT) == 0) {
+		profile_root = v.mv_data;
+		record = NdbProfileRecord_as_root(profile_root);
+		note_key = NdbProfileRecord_note_key(record);
+		note = ndb_get_note_by_key(&txn, note_key, &len);
+
+		if (note == NULL) {
+			fprintf(stderr, "ndb_migrate_utf8_profile_names: note lookup failed\n");
+			return 0;
+		}
+
+		struct ndb_profile_record_builder *b = &out.profile.record;
+
+		// reprocess profile
+		if (!ndb_process_profile_note(note, b)) {
+			failed++;
+			continue;
+		}
+
+		// the writer needs to own this note, and its expected to free it
+		copied_note = malloc(len);
+		memcpy(copied_note, note, len);
+
+		out.type = NDB_WRITER_PROFILE;
+		out.profile.note.note = copied_note;
+		out.profile.note.note_len = len;
+
+		ndb_writer_queue_msg(&ndb->writer, &out);
+
+		count++;
+	}
+
+	fprintf(stderr, "migrated %d profiles to fix utf8 profile names\n", count);
+
+	if (failed != 0) {
+		fprintf(stderr, "failed to migrate %d profiles to fix utf8 profile names\n", failed);
+	}
+
+	mdb_cursor_close(cur);
+
+	ndb_end_query(&txn);
+
+	return 1;
+}
+
+static struct ndb_migration MIGRATIONS[] = {
+	{ .fn = ndb_migrate_user_search_indices },
+	{ .fn = ndb_migrate_lower_user_search_indices },
+	{ .fn = ndb_migrate_utf8_profile_names }
+};
+
+
 int ndb_end_query(struct ndb_txn *txn)
 {
 	// this works on read or write queries. 
@@ -496,12 +579,6 @@ int ndb_note_verify(void *ctx, unsigned char pubkey[32], unsigned char id[32],
 	return 1;
 }
 
-static inline int ndb_writer_queue_msg(struct ndb_writer *writer,
-				       struct ndb_writer_msg *msg)
-{
-	return prot_queue_push(&writer->inbox, msg);
-}
-
 static inline int ndb_writer_queue_msgs(struct ndb_writer *writer,
 					struct ndb_writer_msg *msgs,
 					int num_msgs)
@@ -814,8 +891,8 @@ void ndb_profile_record_builder_free(struct ndb_profile_record_builder *b)
 	b->flatbuf = NULL;
 }
 
-static int ndb_process_profile_note(struct ndb_note *note,
-				    struct ndb_profile_record_builder *profile)
+int ndb_process_profile_note(struct ndb_note *note,
+			     struct ndb_profile_record_builder *profile)
 {
 	int res;
 
diff --git a/test.c b/test.c
@@ -284,7 +284,7 @@ static void test_migrate() {
 	assert(ndb_init(&ndb, v0_dir, mapsize, threads, 0));
 	ndb_destroy(ndb);
 	assert(ndb_init(&ndb, v0_dir, mapsize, threads, 0));
-	assert(ndb_db_version(ndb) == 2);
+	assert(ndb_db_version(ndb) == 3);
 
 	test_profile_search(ndb);
 	ndb_destroy(ndb);