nostrdb

commit a2bc936c1bf6f788b95024a1e36f916484fef103
parent fe32b87d164cfe2cdf89a7b86af45ad92e442ff1
Author: William Casarin <jb55@jb55.com>
Date:   Tue, 19 Sep 2023 15:20:25 -0400

db: implement profile search index

Diffstat:
M
Makefile
 | 
6
+++---
M
nostrdb.c
 | 
180
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
M
nostrdb.h
 | 
16
++++++++++++++++
M
test.c
 | 
25
+++++++++++++++++++++++++

4 files changed, 224 insertions(+), 3 deletions(-)
diff --git a/Makefile b/Makefile
@@ -131,9 +131,9 @@ testdata/db/ndb-v0.tar: testdata/db/ndb-v0.tar.zst
 	zstd -d < $< > $@
 
 testdata/db/v0/data.mdb: testdata/db/ndb-v0.tar
-	@tar xf $<
-	@rm -rf testdata/db/v0
-	@mv v0 testdata/db
+	tar xf $<
+	rm -rf testdata/db/v0
+	mv v0 testdata/db
 
 testdata/many-events.json.zst:
 	curl https://cdn.jb55.com/s/many-events.json.zst -o $@
diff --git a/nostrdb.c b/nostrdb.c
@@ -69,6 +69,7 @@ enum ndb_dbs {
 	NDB_DB_NOTE_ID,
 	NDB_DB_PROFILE_PK,
 	NDB_DB_NDB_META,
+	NDB_DB_PROFILE_SEARCH,
 	NDB_DBS,
 };
 
@@ -187,6 +188,7 @@ static int ndb_tsid_compare(const MDB_val *a, const MDB_val *b)
 {
 	struct ndb_tsid *tsa, *tsb;
 	MDB_val a2 = *a, b2 = *b;
+
 	a2.mv_size = sizeof(tsa->id);
 	b2.mv_size = sizeof(tsb->id);
 
@@ -674,6 +676,171 @@ static uint64_t ndb_get_last_key(MDB_txn *txn, MDB_dbi db)
         return *((uint64_t*)key.mv_data);
 }
 
+// make a search key meant for user queries without any other note info
+static void ndb_make_search_key_low(struct ndb_search_key *key, const char *search)
+{
+	memset(key->id, 0, sizeof(key->id));
+	key->timestamp = 0;
+	strncpy(key->search, search, sizeof(key->search) - 1);
+	key->search[sizeof(key->search) - 1] = '\0';
+}
+
+int ndb_search_profile(struct ndb_txn *txn, struct ndb_search *search, const char *query)
+{
+	int rc;
+	struct ndb_search_key s;
+	MDB_val k, v;
+	search->cursor = NULL;
+
+	MDB_cursor **cursor = (MDB_cursor **)&search->cursor;
+
+	ndb_make_search_key_low(&s, query);
+
+	k.mv_data = &s;
+	k.mv_size = sizeof(s);
+
+	if ((rc = mdb_cursor_open(txn->mdb_txn,
+			    txn->ndb->lmdb.dbs[NDB_DB_PROFILE_SEARCH],
+			    cursor))) {
+		printf("search_profile: cursor opened failed: %s\n",
+				mdb_strerror(rc));
+		return 0;
+	}
+
+	// Position cursor at the next key greater than or equal to the specified key
+	if (mdb_cursor_get(search->cursor, &k, &v, MDB_SET_RANGE)) {
+		printf("search_profile: cursor get failed\n");
+		goto cleanup;
+	} else {
+		search->key = k.mv_data;
+		assert(v.mv_size == 8);
+		search->profile_key = *((uint64_t*)v.mv_data);
+		return 1;
+	}
+
+cleanup:
+	mdb_cursor_close(search->cursor);
+	search->cursor = NULL;
+	return 0;
+}
+
+void ndb_search_profile_end(struct ndb_search *search)
+{
+	if (search->cursor)
+		mdb_cursor_close(search->cursor);
+}
+
+int ndb_search_profile_next(struct ndb_txn *txn, struct ndb_search *search)
+{
+	MDB_val k, v;
+
+	k.mv_data = search->key;
+	k.mv_size = sizeof(*search->key);
+
+	if (mdb_cursor_get(search->cursor, &k, &v, MDB_NEXT)) {
+		return 0;
+	} else {
+		search->key = k.mv_data;
+		assert(v.mv_size == 8);
+		search->profile_key = *((uint64_t*)v.mv_data);
+	}
+
+	return 1;
+}
+
+static int ndb_search_key_cmp(const MDB_val *a, const MDB_val *b)
+{
+	int cmp;
+	struct ndb_search_key *ska, *skb;
+
+	ska = a->mv_data;
+	skb = b->mv_data;
+
+	MDB_val a2 = *a;
+	MDB_val b2 = *b;
+
+	a2.mv_data = ska->search;
+	a2.mv_size = sizeof(ska->search) + sizeof(ska->id);
+
+	cmp = mdb_cmp_memn(&a2, &b2);
+	if (cmp) return cmp;
+
+	if (ska->timestamp < skb->timestamp)
+		return -1;
+	else if (ska->timestamp > skb->timestamp)
+		return 1;
+
+	return 0;
+}
+
+static void ndb_make_search_key(struct ndb_search_key *key, unsigned char *id,
+			        uint64_t timestamp, const char *search)
+{
+	memcpy(key->id, id, 32);
+	key->timestamp = timestamp;
+	strncpy(key->search, search, sizeof(key->search) - 1);
+	key->search[sizeof(key->search) - 1] = '\0';
+}
+
+static int ndb_write_profile_search_index(struct ndb_lmdb *lmdb,
+					  MDB_txn *txn,
+					  struct ndb_search_key *index_key,
+					  uint64_t profile_key)
+{
+	int rc;
+	MDB_val key, val;
+	
+	key.mv_data = index_key;
+	key.mv_size = sizeof(*index_key);
+	val.mv_data = &profile_key;
+	val.mv_size = sizeof(profile_key);
+
+	if ((rc = mdb_put(txn, lmdb->dbs[NDB_DB_PROFILE_SEARCH], &key, &val, 0))) {
+		ndb_debug("ndb_write_profile_search_index failed: %s\n",
+			  mdb_strerror(rc));
+		return 0;
+	}
+
+	return 1;
+}
+
+// map usernames and display names to profile keys for user searching
+static int ndb_write_profile_search_indices(struct ndb_lmdb *lmdb,
+					    MDB_txn *txn,
+					    struct ndb_note *note,
+					    uint64_t profile_key,
+					    void *profile_root)
+{
+	struct ndb_search_key index;
+	NdbProfileRecord_table_t profile_record;
+	NdbProfile_table_t profile;
+
+	profile_record = NdbProfileRecord_as_root(profile_root);
+	profile = NdbProfileRecord_profile_get(profile_record);
+
+	const char *name = NdbProfile_name_get(profile);
+	const char *display_name = NdbProfile_display_name_get(profile);
+
+	// words + pubkey + created
+	if (name) {
+		ndb_make_search_key(&index, note->pubkey, note->created_at,
+				    name);
+		if (!ndb_write_profile_search_index(lmdb, txn, &index,
+						    profile_key))
+			return 0;
+	}
+
+	if (display_name) {
+		ndb_make_search_key(&index, note->pubkey, note->created_at,
+				    display_name);
+		if (!ndb_write_profile_search_index(lmdb, txn, &index,
+						    profile_key))
+			return 0;
+	}
+
+	return 1;
+}
+
 static int ndb_write_profile(struct ndb_lmdb *lmdb, MDB_txn *txn,
 			     struct ndb_writer_profile *profile,
 			     uint64_t note_key)
@@ -737,6 +904,13 @@ static int ndb_write_profile(struct ndb_lmdb *lmdb, MDB_txn *txn,
 		return 0;
 	}
 
+	// write name, display_name profile search indices
+	if (!ndb_write_profile_search_indices(lmdb, txn, note, profile_key,
+					      flatbuf)) {
+		ndb_debug("failed to write profile search indices\n");
+		return 0;
+	}
+
 	return 1;
 }
 
@@ -1090,6 +1264,12 @@ static int ndb_init_lmdb(const char *filename, struct ndb_lmdb *lmdb, size_t map
 		return 0;
 	}
 
+	// profile search db
+	if ((rc = mdb_dbi_open(txn, "profile_search", MDB_CREATE, &lmdb->dbs[NDB_DB_PROFILE_SEARCH]))) {
+		fprintf(stderr, "mdb_dbi_open profile failed, error %d\n", rc);
+		return 0;
+	}
+
 	// ndb metadata (db version, etc)
 	if ((rc = mdb_dbi_open(txn, "ndb_meta", MDB_CREATE | MDB_INTEGERKEY, &lmdb->dbs[NDB_DB_NDB_META]))) {
 		fprintf(stderr, "mdb_dbi_open ndb_meta failed, error %d\n", rc);
diff --git a/nostrdb.h b/nostrdb.h
@@ -25,6 +25,19 @@ struct ndb_t {
 	struct ndb *ndb;
 };
 
+struct ndb_search_key
+{
+	char search[24];
+	unsigned char id[32];
+	uint64_t timestamp;
+};
+
+struct ndb_search {
+	struct ndb_search_key *key;
+	uint64_t profile_key;
+	void *cursor; // MDB_cursor *
+};
+
 // required to keep a read 
 struct ndb_txn {
 	struct ndb *ndb;
@@ -165,6 +178,9 @@ int ndb_db_version(struct ndb *ndb);
 int ndb_process_event(struct ndb *, const char *json, int len);
 int ndb_process_events(struct ndb *, const char *ldjson, size_t len);
 int ndb_begin_query(struct ndb *, struct ndb_txn *);
+int ndb_search_profile(struct ndb_txn *txn, struct ndb_search *search, const char *query);
+int ndb_search_profile_next(struct ndb_txn *txn, struct ndb_search *search);
+void ndb_search_profile_end(struct ndb_search *search);
 void ndb_end_query(struct ndb_txn *);
 void *ndb_get_profile_by_pubkey(struct ndb_txn *txn, const unsigned char *pubkey, size_t *len, uint64_t *primkey);
 void *ndb_get_profile_by_key(struct ndb_txn *txn, uint64_t key, size_t *len);
diff --git a/test.c b/test.c
@@ -15,6 +15,29 @@
 
 static const char *test_dir = "./testdata/db";
 
+static void test_profile_search(struct ndb *ndb)
+{
+	struct ndb_txn txn;
+	struct ndb_search search;
+	size_t len;
+	void *root;
+
+	assert(ndb_begin_query(ndb, &txn));
+	assert(ndb_search_profile(&txn, &search, "jb"));
+	assert((root = ndb_get_profile_by_key(&txn, search.profile_key, &len)));
+	assert(root);
+
+	ndb_search_profile_end(&search);
+
+	NdbProfileRecord_table_t profile_record = NdbProfileRecord_as_root(root);
+	NdbProfile_table_t profile = NdbProfileRecord_profile_get(profile_record);
+	const char *searched_name = NdbProfile_name_get(profile);
+
+	assert(!strcmp(searched_name, "jb55"));
+
+	ndb_end_query(&txn);
+}
+
 static void test_load_profiles()
 {
 	static const int alloc_size = 1024 * 1024;
@@ -48,6 +71,8 @@ static void test_load_profiles()
 	assert(!strcmp(ndb_note_content(note), expected_content));
 	ndb_end_query(&txn);
 
+	test_profile_search(ndb);
+
 	ndb_destroy(ndb);
 
 	free(json);