nostrdb

commit bc4753ca7c039d52ba244e245c5071fca5bdec65
parent 7cd176001388fd977a56bc2c64781a382a69c2ad
Author: William Casarin <jb55@jb55.com>
Date:   Thu, 30 Nov 2023 09:44:10 -0800

flag: add ndb config flag for skipping note verification

makes some large imports a bit faster

Diffstat:
M
bench-ingest-many.c
 | 
3
++-
M
ndb.c
 | 
25
++++++++++++++++++-------
M
nostrdb.c
 | 
38
++++++++++++++++++++++++++------------
M
nostrdb.h
 | 
1
+

4 files changed, 47 insertions(+), 20 deletions(-)
diff --git a/bench-ingest-many.c b/bench-ingest-many.c
@@ -37,7 +37,8 @@ static int bench_parser()
 
 	mapsize = 1024ULL * 1024ULL * 400ULL * 10ULL;
 	ingester_threads = 8;
-	assert(ndb_init(&ndb, "testdata/db", mapsize, ingester_threads, 0));
+	assert(ndb_init(&ndb, "testdata/db", mapsize, ingester_threads,
+			NDB_FLAG_SKIP_NOTE_VERIFY));
 	const char *filename = "testdata/many-events.json";
 	if (!map_file(filename, (unsigned char**)&json, &written)) {
 		printf("mapping testdata/many-events.json failed\n");
diff --git a/ndb.c b/ndb.c
@@ -9,11 +9,14 @@
 
 static int usage()
 {
-	printf("usage: ndb [-d db_dir] <command>\n\n");
+	printf("usage: ndb [--skip-verification] [-d db_dir] <command>\n\n");
 	printf("commands\n\n");
 	printf("	stat\n");
 	printf("	search <fulltext query>\n");
-	printf("	import <line-delimited json file>\n");
+	printf("	import <line-delimited json file>\n\n");
+	printf("settings\n\n");
+	printf("	--skip-verification  skip signature validation\n");
+	printf("	-d <db_dir>          set database directory\n");
 	return 1;
 }
 
@@ -87,7 +90,7 @@ int main(int argc, char *argv[])
 {
 	struct ndb *ndb;
 	int threads = 6;
-	int flags = 0;
+	int i, flags;
 	struct ndb_stat stat;
 	struct ndb_txn txn;
 	struct ndb_text_search_results results;
@@ -101,10 +104,18 @@ int main(int argc, char *argv[])
 	}
 
 	dir = ".";
-	if (!strcmp(argv[1], "-d") && argv[2]) {
-		dir = argv[2];
-		argv += 2;
-		argc -= 2;
+	flags = 0;
+	for (i = 0; i < 2; i++)
+	{
+		if (!strcmp(argv[1], "-d") && argv[2]) {
+			dir = argv[2];
+			argv += 2;
+			argc -= 2;
+		} else if (!strcmp(argv[1], "--skip-verification")) {
+			flags = NDB_FLAG_SKIP_NOTE_VERIFY;
+			argv += 1;
+			argc -= 1;
+		}
 	}
 
 	fprintf(stderr, "using db '%s'\n", dir);
diff --git a/nostrdb.c b/nostrdb.c
@@ -112,6 +112,7 @@ struct ndb_writer {
 };
 
 struct ndb_ingester {
+	uint32_t flags;
 	struct threadpool tp;
 	struct ndb_writer *writer;
 };
@@ -122,6 +123,7 @@ struct ndb {
 	struct ndb_ingester ingester;
 	struct ndb_writer writer;
 	int version;
+	uint32_t flags; // setting flags
 	// lmdb environ handles, etc
 };
 
@@ -1540,17 +1542,22 @@ int ndb_process_profile_note(struct ndb_note *note,
 static int ndb_ingester_process_note(secp256k1_context *ctx,
 				     struct ndb_note *note,
 				     size_t note_size,
-				     struct ndb_writer_msg *out)
+				     struct ndb_writer_msg *out,
+				     uint32_t flags)
 {
 	//printf("ndb_ingester_process_note ");
 	//print_hex(note->id, 32);
 	//printf("\n");
 
-	// Verify! If it's an invalid note we don't need to
-	// bother writing it to the database
-	if (!ndb_note_verify(ctx, note->pubkey, note->id, note->sig)) {
-		ndb_debug("signature verification failed\n");
-		return 0;
+	// some special situations we might want to skip sig validation,
+	// like during large imports
+	if (!(flags & NDB_FLAG_SKIP_NOTE_VERIFY)) {
+		// Verify! If it's an invalid note we don't need to
+		// bother writing it to the database
+		if (!ndb_note_verify(ctx, note->pubkey, note->id, note->sig)) {
+			ndb_debug("signature verification failed\n");
+			return 0;
+		}
 	}
 
 	// we didn't find anything. let's send it
@@ -1636,9 +1643,10 @@ static int ndb_ingester_process_event(secp256k1_context *ctx,
 				goto cleanup;
 			}
 
-			if (!ndb_ingester_process_note(ctx, note, note_size, out))
+			if (!ndb_ingester_process_note(ctx, note, note_size,
+						       out, ingester->flags)) {
 				goto cleanup;
-			else {
+			} else {
 				// we're done with the original json, free it
 				free(ev->json);
 				return 1;
@@ -1656,9 +1664,10 @@ static int ndb_ingester_process_event(secp256k1_context *ctx,
 				goto cleanup;
 			}
 
-			if (!ndb_ingester_process_note(ctx, note, note_size, out))
+			if (!ndb_ingester_process_note(ctx, note, note_size,
+						       out, ingester->flags)) {
 				goto cleanup;
-			else {
+			} else {
 				// we're done with the original json, free it
 				free(ev->json);
 				return 1;
@@ -2902,7 +2911,8 @@ static int ndb_writer_init(struct ndb_writer *writer, struct ndb_lmdb *lmdb)
 
 // initialize the ingester queue and then spawn the thread
 static int ndb_ingester_init(struct ndb_ingester *ingester,
-			     struct ndb_writer *writer, int num_threads)
+			     struct ndb_writer *writer, int num_threads,
+			     int flags)
 {
 	int elem_size, num_elems;
 	static struct ndb_ingester_msg quit_msg = { .type = NDB_INGEST_QUIT };
@@ -2912,6 +2922,7 @@ static int ndb_ingester_init(struct ndb_ingester *ingester,
 	num_elems = DEFAULT_QUEUE_SIZE;
 
 	ingester->writer = writer;
+	ingester->flags = flags;
 
 	if (!threadpool_init(&ingester->tp, num_threads, elem_size, num_elems,
 			     &quit_msg, ingester, ndb_ingester_thread))
@@ -3127,6 +3138,8 @@ int ndb_init(struct ndb **pndb, const char *filename, size_t mapsize, int ingest
 	//MDB_dbi ind_id; // TODO: ind_pk, etc
 
 	ndb = *pndb = calloc(1, sizeof(struct ndb));
+	ndb->flags = flags;
+
 	if (ndb == NULL) {
 		fprintf(stderr, "ndb_init: malloc failed\n");
 		return 0;
@@ -3140,7 +3153,8 @@ int ndb_init(struct ndb **pndb, const char *filename, size_t mapsize, int ingest
 		return 0;
 	}
 
-	if (!ndb_ingester_init(&ndb->ingester, &ndb->writer, ingester_threads)) {
+	if (!ndb_ingester_init(&ndb->ingester, &ndb->writer, ingester_threads,
+			       ndb->flags)) {
 		fprintf(stderr, "failed to initialize %d ingester thread(s)\n",
 				ingester_threads);
 		return 0;
diff --git a/nostrdb.h b/nostrdb.h
@@ -8,6 +8,7 @@
 #define NDB_PACKED_ID      0x2
 
 #define NDB_FLAG_NOMIGRATE (1 << 0)
+#define NDB_FLAG_SKIP_NOTE_VERIFY (1 << 1)
 
 //#define DEBUG 1