nostrdb

commit 050438e62c7fd468fb0b492d6b0bcd62b3aa685a
parent aa1f9ac6bd53fabee4e1b38311de4474ad28dfb9
Author: William Casarin <jb55@jb55.com>
Date:   Fri, 11 Aug 2023 21:28:01 -0700

bench: add 1million note import benchmark

Diffstat:
M
.gitignore
 | 
3
+++
M
Makefile
 | 
8
+++++++-
A
bench-ingest-many.c
 | 
72
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
M
nostrdb.h
 | 
2
+-

4 files changed, 83 insertions(+), 2 deletions(-)
diff --git a/.gitignore b/.gitignore
@@ -4,10 +4,13 @@ test
 .build-result
 shell.nix
 *.dSYM
+*.zst
+testdata/many-events.json
 TODO.bak
 test_contacts_ndb_note
 bench
 bench-ingest
+bench-ingest-many
 configurator
 config.h
 libsecp256k1.a
diff --git a/Makefile b/Makefile
@@ -122,7 +122,13 @@ bench: bench.c $(DEPS)
 bench-ingest: bench-ingest.c $(DEPS)
 	$(CC) $(CFLAGS) bench-ingest.c $(LDS) -o $@
 
-bench-ingest-many: bench-ingest-many.c $(DEPS)
+testdata/many-events.json.zst:
+	curl https://cdn.jb55.com/s/many-events.json.zst -o $@
+
+testdata/many-events.json: testdata/many-events.json.zst
+	zstd -d $<
+
+bench-ingest-many: bench-ingest-many.c $(DEPS) testdata/many-events.json
 	$(CC) $(CFLAGS) $< $(LDS) -o $@
 
 testdata/db/.dir:
diff --git a/bench-ingest-many.c b/bench-ingest-many.c
@@ -0,0 +1,72 @@
+
+
+#include "io.h"
+#include "nostrdb.h"
+#include <sys/mman.h>
+#include <time.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include <sys/stat.h>
+
+int map_file(const char *filename, unsigned char **p, size_t *flen)
+{
+	struct stat st;
+	int des;
+	stat(filename, &st);
+	*flen = st.st_size;
+
+	des = open(filename, O_RDONLY);
+
+	*p = mmap(NULL, *flen, PROT_READ, MAP_PRIVATE, des, 0);
+	close(des);
+
+	return *p != MAP_FAILED;
+}
+
+static int bench_parser()
+{
+	int ingester_threads;
+	long nanos, ms;
+	size_t written, mapsize;
+	struct ndb *ndb;
+	struct timespec t1, t2;
+	char *json;
+	int times = 1;
+
+	mapsize = 1024 * 1024 * 2000;
+	ingester_threads = 10;
+	assert(ndb_init(&ndb, mapsize, ingester_threads));
+	const char *filename = "testdata/many-events.json";
+	if (!map_file(filename, (unsigned char**)&json, &written)) {
+		printf("mapping testdata/many-events.json failed\n");
+		return 2;
+	}
+
+	printf("mapped %ld bytes in %s\n", written, filename);
+
+	clock_gettime(CLOCK_MONOTONIC, &t1);
+
+	ndb_process_events(ndb, json, written);
+
+	ndb_destroy(ndb);
+
+	clock_gettime(CLOCK_MONOTONIC, &t2);
+
+	nanos = (t2.tv_sec - t1.tv_sec) * (long)1e9 + (t2.tv_nsec - t1.tv_nsec);
+	ms = nanos / 1e6;
+	printf("ns/run\t%ld\nms/run\t%f\nns\t%ld\nms\t%ld\n",
+		nanos/times, (double)ms/(double)times, nanos, ms);
+
+	return 1;
+}
+
+int main(int argc, char *argv[], char **env)
+{
+	if (!bench_parser())
+		return 2;
+	
+	return 0;
+}
+
diff --git a/nostrdb.h b/nostrdb.h
@@ -7,7 +7,7 @@
 #define NDB_PACKED_STR     0x1
 #define NDB_PACKED_ID      0x2
 
-#define DEBUG 1
+//#define DEBUG 1
 
 #ifdef DEBUG
 #define ndb_debug(...) printf(__VA_ARGS__)