nostrdb

commit 1378a29c1ba400dd5a4540717b8af9a29d5a2430
parent 13f662c0c1fd1b8a926c7bc5fc93b92c51b94be5
Author: William Casarin <jb55@jb55.com>
Date:   Thu, 10 Aug 2023 12:10:41 -0700

ndb: add ndb_process_events

A helper for processing many json websocket events all at once

Diffstat:
M
nostrdb.c
 | 
24
+++++++++++++++++++++++-
M
nostrdb.h
 | 
1
+
M
test.c
 | 
29
+++++++++++++++++++++++++++++

3 files changed, 53 insertions(+), 1 deletion(-)
diff --git a/nostrdb.c b/nostrdb.c
@@ -616,7 +616,29 @@ int ndb_process_event(struct ndb *ndb, const char *json, int json_len)
 	if (json_copy == NULL)
 		return 0;
 
-	ndb_ingester_queue_event(&ndb->ingester, json_copy, json_len);
+	return ndb_ingester_queue_event(&ndb->ingester, json_copy, json_len);
+}
+
+int ndb_process_events(struct ndb *ndb, const char *ldjson, int json_len)
+{
+	const char *start, *end, *very_end;
+	start = ldjson;
+	end = start + json_len;
+	very_end = ldjson + json_len;
+	int processed = 0;
+
+	while ((end = fast_strchr(start, '\n', very_end - start))) {
+		//printf("processing '%.*s'\n", (int)(end-start), start);
+		if (!ndb_process_event(ndb, start, end - start)) {
+			ndb_debug("ndb_process_event failed\n");
+			return 0;
+		}
+		start = end + 1;
+		processed++;
+	}
+
+	ndb_debug("ndb_process_events: processed %d events\n", processed);
+
 	return 1;
 }
 
diff --git a/nostrdb.h b/nostrdb.h
@@ -149,6 +149,7 @@ int ndb_note_verify(void *secp_ctx, unsigned char pubkey[32], unsigned char id[3
 // NDB
 int ndb_init(struct ndb **ndb, size_t mapsize, int ingester_threads);
 int ndb_process_event(struct ndb *, const char *json, int len);
+int ndb_process_events(struct ndb *, const char *ldjson, int len);
 void ndb_destroy(struct ndb *);
 
 // BUILDER
diff --git a/test.c b/test.c
@@ -11,6 +11,29 @@
 
 #define ARRAY_SIZE(x) (sizeof(x) / sizeof(x[0]))
 
+static void test_load_profiles()
+{
+	static const int alloc_size = 1024 * 1024;
+	char *json = malloc(alloc_size);
+	unsigned char *buf = malloc(alloc_size);
+	struct ndb *ndb;
+	size_t mapsize;
+	int written, ingester_threads;
+
+	mapsize = 1024 * 1024 * 100;
+	ingester_threads = 1;
+	assert(ndb_init(&ndb, mapsize, ingester_threads));
+
+	read_file("testdata/profiles.json", (unsigned char*)json, alloc_size, &written);
+
+	assert(ndb_process_events(ndb, json, written));
+
+	ndb_destroy(ndb);
+
+	free(json);
+	free(buf);
+}
+
 
 static void test_basic_event() {
 	unsigned char buf[512];
@@ -511,5 +534,11 @@ int main(int argc, const char *argv[]) {
 	// memchr stuff
 	test_fast_strchr();
 
+	// profiles
+	test_load_profiles();
+
 	printf("All tests passed!\n");       // Print this if all tests pass.
 }
+
+
+