nostrdb

commit 38028283a4901d925ca402ea0cc47b7e9a47239a
parent c00473a296d69f3ac5db2c814beaf0e75eb0b3a5
Author: William Casarin <jb55@jb55.com>
Date:   Thu,  4 Jan 2024 15:24:40 -0800

query: implement kind queries

Diffstat:
M
src/nostrdb.c
 | 
88
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-----
M
test.c
 | 
31
++++++++++++++++++++++++++-----

2 files changed, 109 insertions(+), 10 deletions(-)
diff --git a/src/nostrdb.c b/src/nostrdb.c
@@ -2337,6 +2337,46 @@ static int ndb_filter_get_since(struct ndb_filter *filter, uint64_t *lim)
 	return ndb_filter_int(filter, NDB_FILTER_SINCE, lim);
 }
 
+static int ndb_query_filter_kind(struct ndb_txn *txn, struct ndb_filter *filter,
+				 MDB_cursor *cur, uint64_t kind, uint64_t since,
+				 int *matched, struct ndb_query_result *res)
+{
+	MDB_val k, v;
+	uint64_t note_id;
+	struct ndb_u64_tsid tsid, *ptsid;
+
+	res->note = NULL;
+
+	ndb_u64_tsid_init(&tsid, kind, since);
+
+	k.mv_data = &tsid;
+	k.mv_size = sizeof(tsid);
+
+	if (!ndb_cursor_start(cur, &k, &v))
+		return 0;
+
+	ptsid = (struct ndb_u64_tsid *)k.mv_data;
+	note_id = *(uint64_t*)v.mv_data;
+
+	if (kind == ptsid->u64)
+		*matched |= 1 << NDB_FILTER_KINDS;
+	else
+		return 1;
+
+	// get the note because we need it to match against the filter
+	if (!(res->note = ndb_get_note_by_key(txn, note_id, NULL)))
+		return 1;
+
+	// Sure this particular lookup matched the index query, but does it
+	// match the entire filter? Check! We also pass in things we've already
+	// matched via the filter so we don't have to check again. This can be
+	// pretty important for filters with a large number of entries.
+	if (!ndb_filter_matches_with(filter, res->note, *matched))
+		return 1;
+
+	return 2;
+}
+
 static int ndb_query_filter_id(struct ndb_txn *txn, struct ndb_filter *filter,
 			       MDB_cursor *cur, const unsigned char *id,
 			       uint64_t since, int *matched,
@@ -2400,6 +2440,14 @@ static int compare_query_results(const void *pa, const void *pb)
 	}
 }
 
+static int query_is_full(struct cursor *results, int limit)
+{
+	if (results->p >= results->end)
+		return 1;
+
+	return cursor_count(results, sizeof(struct ndb_query_result)) >= limit;
+}
+
 static int ndb_query_filter(struct ndb_txn *txn, struct ndb_filter *filter,
 			    struct ndb_query_result *results, int capacity,
 			    int *results_out)
@@ -2407,9 +2455,9 @@ static int ndb_query_filter(struct ndb_txn *txn, struct ndb_filter *filter,
 	struct ndb_filter_elements *els;
 	struct ndb_query_result res;
 	struct cursor results_arr;
-	uint64_t limit, since, until;
+	uint64_t limit, since, until, kind;
 	const unsigned char *id;
-	int i, k, rc;
+	int i, k, rc, matched;
 	MDB_cursor *cur;
 	MDB_dbi db;
 
@@ -2427,20 +2475,20 @@ static int ndb_query_filter(struct ndb_txn *txn, struct ndb_filter *filter,
 		    &results_arr);
 
 	for (i = 0; i < filter->num_elements; i++) {
-		if (results_arr.p >= results_arr.end)
+		matched = 0;
+		if (query_is_full(&results_arr, limit))
 			goto done;
 
 		els = filter->elements[i];
 		switch (els->field.type) {
 		case NDB_FILTER_IDS:
-			int matched = 0;
 			db = txn->lmdb->dbs[NDB_DB_NOTE_ID];
 			if ((rc = mdb_cursor_open(txn->mdb_txn, db, &cur)))
 				return 0;
 
 			// for each id in our ids filter, find in the db
 			for (k = 0; k < els->count; k++) {
-				if (results_arr.p >= results_arr.end) {
+				if (query_is_full(&results_arr, limit)) {
 					mdb_cursor_close(cur);
 					goto done;
 				}
@@ -2477,6 +2525,36 @@ static int ndb_query_filter(struct ndb_txn *txn, struct ndb_filter *filter,
 		case NDB_FILTER_AUTHORS:
 			break;
 		case NDB_FILTER_KINDS:
+			db = txn->lmdb->dbs[NDB_DB_NOTE_KIND];
+			if ((rc = mdb_cursor_open(txn->mdb_txn, db, &cur)))
+				return 0;
+
+			// for each id in our ids filter, find in the db
+			for (k = 0; k < els->count; k++) {
+				if (query_is_full(&results_arr, limit)) {
+					mdb_cursor_close(cur);
+					goto done;
+				}
+
+				kind = els->elements[k].integer;
+				if (!(rc = ndb_query_filter_kind(txn, filter,
+								 cur, kind,
+								 since,
+							         &matched,
+							         &res))) {
+					// there was a fatal error
+					mdb_cursor_close(cur);
+					return 0;
+				}
+
+				// rc > 1, matched!
+				if (!push_query_result(&results_arr, &res)) {
+					mdb_cursor_close(cur);
+					goto done;
+				}
+			}
+
+			mdb_cursor_close(cur);
 			break;
 		case NDB_FILTER_GENERIC:
 			break;
diff --git a/test.c b/test.c
@@ -1147,11 +1147,13 @@ static void test_query()
 	struct ndb_txn txn;
 	struct ndb_filter filter, *f = &filter;
 	struct ndb_config config;
-	struct ndb_query_result results[2];
-	int count;
-	uint64_t subid, note_ids[2];
+	struct ndb_query_result results[4];
+	int count, cap;
+	uint64_t subid, note_ids[4];
 	ndb_default_config(&config);
 
+	cap = sizeof(results) / sizeof(results[0]);
+
 	const unsigned char id[] = {
 	  0x03, 0x36, 0x94, 0x8b, 0xdf, 0xbf, 0x5f, 0x93, 0x98, 0x02, 0xeb, 0xa0,
 	  0x3a, 0xa7, 0x87, 0x35, 0xc8, 0x28, 0x25, 0x21, 0x1e, 0xec, 0xe9, 0x87,
@@ -1169,6 +1171,11 @@ static void test_query()
 
 	const char *ev2 = "[\"EVENT\",\"s\",{\"id\": \"0a350c5851af6f6ce368bab4e2d4fe442a1318642c7fe58de5392103700c10fc\",\"pubkey\": \"dfa3fc062f7430dab3d947417fd3c6fb38a7e60f82ffe3387e2679d4c6919b1d\",\"created_at\": 1704404822,\"kind\": 1,\"tags\": [],\"content\": \"hello2\",\"sig\": \"48a0bb9560b89ee2c6b88edcf1cbeeff04f5e1b10d26da8564cac851065f30fa6961ee51f450cefe5e8f4895e301e8ffb2be06a2ff44259684fbd4ea1c885696\"}]";
 
+
+	const char *ev3 = "[\"EVENT\",\"s\",{\"id\": \"20d2b66e1a3ac4a2afe22866ad742091b6267e6e614303de062adb33e12c9931\",\"pubkey\": \"7987bfb2632d561088fc8e3c30a95836f822e4f53633228ec92ae2f5cd6690aa\",\"created_at\": 1704408561,\"kind\": 2,\"tags\": [],\"content\": \"what\",\"sig\": \"cc8533bf177ac87771a5218a04bed24f7a1706f0b2d92700045cdeb38accc5507c6c8de09525e43190df3652012b554d4efe7b82ab268a87ff6f23da44e16a8f\"}";
+
+	const char *ev4 = "[\"EVENT\",\"s\",{\"id\": \"8a2057c13c1c57b536eab78e6c55428732d33b6b5b234c1f5eab2b5918c37fa1\",\"pubkey\": \"303b5851504da5caa14142e9e2e1b1b60783c48d6f137c205019d46d09244c26\",\"created_at\": 1704408730,\"kind\": 2,\"tags\": [],\"content\": \"hmm\",\"sig\": \"e7cd3029042d41964192411929cade59592840af766da6420077ccc57a61405312db6ca879150db01f53c3b81c477cec5d6bd49f9dc10937267cacf7e5c784b3\"}";
+
 	assert(ndb_init(&ndb, test_dir, &config));
 
 	ndb_filter_init(f);
@@ -1180,12 +1187,26 @@ static void test_query()
 	assert((subid = ndb_subscribe(ndb, f, 1)));
 	assert(ndb_process_event(ndb, ev, strlen(ev)));
 	assert(ndb_process_event(ndb, ev2, strlen(ev2)));
-	assert(ndb_wait_for_notes(ndb, subid, note_ids, 2));
+	assert(ndb_process_event(ndb, ev3, strlen(ev3)));
+	assert(ndb_process_event(ndb, ev4, strlen(ev4)));
+	assert(ndb_wait_for_notes(ndb, subid, note_ids, 4));
 
 	ndb_begin_query(ndb, &txn);
-	assert(ndb_query(&txn, f, 1, results, 2, &count));
+	assert(ndb_query(&txn, f, 1, results, cap, &count));
 	assert(count == 2);
 	assert(0 == memcmp(ndb_note_id(results[0].note), id2, 32));
+
+	ndb_filter_reset(f);
+	ndb_filter_start_field(f, NDB_FILTER_KINDS);
+	ndb_filter_add_int_element(f, 2);
+	ndb_filter_end_field(f);
+	ndb_filter_start_field(f, NDB_FILTER_LIMIT);
+	ndb_filter_add_int_element(f, 1);
+	ndb_filter_end_field(f);
+
+	assert(ndb_query(&txn, f, 1, results, cap, &count));
+	assert(count == 1);
+
 	ndb_end_query(&txn);
 	ndb_destroy(ndb);
 }