nostrdb

commit dbadb428a7cf1fa1daa49f072d60774dd7babe30
parent 05ed6252c0c63503ed7de83cb6296e03a02b1677
Author: William Casarin <jb55@jb55.com>
Date:   Wed, 20 Sep 2023 09:18:35 -0400

search: skip duplicate pubkeys when searching profiles

Sometimes the users name and display_name indices are adjacent. Update
ndb_search_profile_next() to skip these.

Diffstat:
M
nostrdb.c
 | 
20
+++++++++++++++++---
M
nostrdb.h
 | 
3
++-
M
test.c
 | 
42
++++++++++++++++++++++++++++++++++++------

3 files changed, 55 insertions(+), 10 deletions(-)
diff --git a/nostrdb.c b/nostrdb.c
@@ -730,19 +730,29 @@ void ndb_search_profile_end(struct ndb_search *search)
 		mdb_cursor_close(search->cursor);
 }
 
-int ndb_search_profile_next(struct ndb_txn *txn, struct ndb_search *search)
+int ndb_search_profile_next(struct ndb_search *search)
 {
+	int rc;
 	MDB_val k, v;
+	unsigned char *init_id;
 
+	init_id = search->key->id;
 	k.mv_data = search->key;
 	k.mv_size = sizeof(*search->key);
 
-	if (mdb_cursor_get(search->cursor, &k, &v, MDB_NEXT)) {
+retry:
+	if ((rc = mdb_cursor_get(search->cursor, &k, &v, MDB_NEXT))) {
+		ndb_debug("ndb_search_profile_next: %s\n",
+				mdb_strerror(rc));
 		return 0;
 	} else {
 		search->key = k.mv_data;
 		assert(v.mv_size == 8);
 		search->profile_key = *((uint64_t*)v.mv_data);
+
+		// skip duplicate pubkeys
+		if (!memcmp(init_id, search->key->id, 32))
+			goto retry;
 	}
 
 	return 1;
@@ -831,6 +841,10 @@ static int ndb_write_profile_search_indices(struct ndb_lmdb *lmdb,
 	}
 
 	if (display_name) {
+		// don't write the same name/display_name twice
+		if (name && !strcmp(display_name, name)) {
+			return 1;
+		}
 		ndb_make_search_key(&index, note->pubkey, note->created_at,
 				    display_name);
 		if (!ndb_write_profile_search_index(lmdb, txn, &index,
@@ -1266,7 +1280,7 @@ static int ndb_init_lmdb(const char *filename, struct ndb_lmdb *lmdb, size_t map
 
 	// profile search db
 	if ((rc = mdb_dbi_open(txn, "profile_search", MDB_CREATE, &lmdb->dbs[NDB_DB_PROFILE_SEARCH]))) {
-		fprintf(stderr, "mdb_dbi_open profile failed, error %d\n", rc);
+		fprintf(stderr, "mdb_dbi_open profile_search failed, error %d\n", rc);
 		return 0;
 	}
 	mdb_set_compare(txn, lmdb->dbs[NDB_DB_PROFILE_SEARCH], ndb_search_key_cmp);
diff --git a/nostrdb.h b/nostrdb.h
@@ -33,6 +33,7 @@ struct ndb_search_key
 };
 
 struct ndb_search {
+	const char *query;
 	struct ndb_search_key *key;
 	uint64_t profile_key;
 	void *cursor; // MDB_cursor *
@@ -179,7 +180,7 @@ int ndb_process_event(struct ndb *, const char *json, int len);
 int ndb_process_events(struct ndb *, const char *ldjson, size_t len);
 int ndb_begin_query(struct ndb *, struct ndb_txn *);
 int ndb_search_profile(struct ndb_txn *txn, struct ndb_search *search, const char *query);
-int ndb_search_profile_next(struct ndb_txn *txn, struct ndb_search *search);
+int ndb_search_profile_next(struct ndb_search *search);
 void ndb_search_profile_end(struct ndb_search *search);
 void ndb_end_query(struct ndb_txn *);
 void *ndb_get_profile_by_pubkey(struct ndb_txn *txn, const unsigned char *pubkey, size_t *len, uint64_t *primkey);
diff --git a/test.c b/test.c
@@ -15,11 +15,38 @@
 
 static const char *test_dir = "./testdata/db";
 
+static void print_hex(unsigned char* data, size_t size) {
+	size_t i;
+	for (i = 0; i < size; i++) {
+		printf("%02x", data[i]);
+	}
+}
+
+
+static void print_search(struct ndb_txn *txn, struct ndb_search *search)
+{
+	void *root;
+	size_t len;
+	assert((root = ndb_get_profile_by_key(txn, search->profile_key, &len)));
+	assert(root);
+
+	NdbProfileRecord_table_t profile_record = NdbProfileRecord_as_root(root);
+	NdbProfile_table_t profile = NdbProfileRecord_profile_get(profile_record);
+
+	const char *name = NdbProfile_name_get(profile);
+	const char *display_name = NdbProfile_display_name_get(profile);
+	printf("searched_name name:'%s' display_name:'%s' pk:%" PRIu64 " ts:%" PRIu64 " id:", name, display_name, search->profile_key, search->key->timestamp);
+	print_hex(search->key->id, 32);
+	printf("\n");
+}
+
+
 static void test_profile_search(struct ndb *ndb)
 {
 	struct ndb_txn txn;
 	struct ndb_search search;
 	size_t len;
+	int i;
 	void *root;
 
 	assert(ndb_begin_query(ndb, &txn));
@@ -27,14 +54,17 @@ static void test_profile_search(struct ndb *ndb)
 	assert((root = ndb_get_profile_by_key(&txn, search.profile_key, &len)));
 	assert(root);
 
-	ndb_search_profile_end(&search);
+	//assert(!strcmp(searched_name, "jb55"));
+	print_search(&txn, &search);
 
-	NdbProfileRecord_table_t profile_record = NdbProfileRecord_as_root(root);
-	NdbProfile_table_t profile = NdbProfileRecord_profile_get(profile_record);
-	const char *searched_name = NdbProfile_name_get(profile);
+	for (i = 0; i < 20; i++) {
+		assert(ndb_search_profile_next(&search));
+		print_search(&txn, &search);
+	}
 
-	assert(!strcmp(searched_name, "jb55"));
+	//assert(!strcmp(searched_name, "jb55"));
 
+	ndb_search_profile_end(&search);
 	ndb_end_query(&txn);
 }
 
@@ -695,6 +725,7 @@ static void test_fast_strchr()
 
 int main(int argc, const char *argv[]) {
 	test_migrate();
+	test_load_profiles();
 	test_basic_event();
 	test_empty_tags();
 	test_parse_json();
@@ -719,7 +750,6 @@ int main(int argc, const char *argv[]) {
 	test_fast_strchr();
 
 	// profiles
-	test_load_profiles();
 	test_replacement();
 
 	printf("All tests passed!\n");       // Print this if all tests pass.