nostrdb

an unfairly fast embedded nostr database backed by lmdb
git clone git://jb55.com/nostrdb
Log | Files | Refs | Submodules | README | LICENSE

ndb.c (12153B)


      1 
      2 
      3 #include "nostrdb.h"
      4 #include "lmdb.h"
      5 #include "print_util.h"
      6 #include "hex.h"
      7 #include <time.h>
      8 #include <stdio.h>
      9 #include <stdlib.h>
     10 #include <sys/stat.h>
     11 #include <sys/mman.h>
     12 #include <unistd.h>
     13 #include <fcntl.h>
     14 
     15 static int usage()
     16 {
     17 	printf("usage: ndb [--skip-verification] [-d db_dir] <command>\n\n");
     18 
     19 	printf("commands\n\n");
     20 
     21 	printf("	stat\n");
     22 	printf("	query [--kind 42] [--id abcdef...] [--notekey key] [--search term] [--limit 42] \n");
     23 	printf("	      [-e abcdef...] [--author abcdef... -a bcdef...] [--relay wss://relay.damus.io]\n");
     24 	printf("	profile <pubkey>                            print the raw profile data for a pubkey\n");
     25 	printf("	note-relays <note-id>                       list the relays a given note id has been seen on\n");
     26 	printf("	print-search-keys\n");
     27 	printf("	print-kind-keys\n");
     28 	printf("	print-tag-keys\n");
     29 	printf("	print-relay-kind-index-keys\n");
     30 	printf("	print-author-kind-index-keys\n");
     31 	printf("	import <line-delimited json file>\n\n");
     32 
     33 	printf("settings\n\n");
     34 
     35 	printf("	--skip-verification  skip signature validation\n");
     36 	printf("	-d <db_dir>          set database directory\n");
     37 	return 1;
     38 }
     39 
     40 
     41 static int map_file(const char *filename, unsigned char **p, size_t *flen)
     42 {
     43 	struct stat st;
     44 	int des;
     45 	stat(filename, &st);
     46 	*flen = st.st_size;
     47 
     48 	des = open(filename, O_RDONLY);
     49 
     50 	*p = mmap(NULL, *flen, PROT_READ, MAP_PRIVATE, des, 0);
     51 	close(des);
     52 
     53 	return *p != MAP_FAILED;
     54 }
     55 
     56 static inline void print_stat_counts(struct ndb_stat_counts *counts)
     57 {
     58 	printf("%zu\t%zu\t%zu\t%zu\n",
     59 	       counts->count,
     60 	       counts->key_size,
     61 	       counts->value_size,
     62 	       counts->key_size + counts->value_size);
     63 }
     64 
     65 static void print_stats(struct ndb_stat *stat)
     66 {
     67 	int i;
     68 	const char *name;
     69 	struct ndb_stat_counts *c;
     70 
     71 	struct ndb_stat_counts total;
     72 	ndb_stat_counts_init(&total);
     73 
     74 	printf("name\tcount\tkey_bytes\tvalue_bytes\ttotal_bytes\n");
     75 	printf("---\ndbs\n---\n");
     76 	for (i = 0; i < NDB_DBS; i++) {
     77 		name = ndb_db_name(i);
     78 
     79 		total.count += stat->dbs[i].count;
     80 		total.key_size += stat->dbs[i].key_size;
     81 		total.value_size += stat->dbs[i].value_size;
     82 
     83 		printf("%s\t", name);
     84 		print_stat_counts(&stat->dbs[i]);
     85 	}
     86 
     87 	printf("total\t");
     88 	print_stat_counts(&total);
     89 
     90 	printf("-----\nkinds\n-----\n");
     91 	for (i = 0; i < NDB_CKIND_COUNT; i++) {
     92 		c = &stat->common_kinds[i];
     93 		if (c->count == 0)
     94 			continue;
     95 
     96 		printf("%s\t", ndb_kind_name(i));
     97 		print_stat_counts(c);
     98 	}
     99 
    100 	if (stat->other_kinds.count != 0) {
    101 		printf("other\t");
    102 		print_stat_counts(&stat->other_kinds);
    103 	}
    104 }
    105 
    106 int ndb_print_search_keys(struct ndb_txn *txn);
    107 int ndb_print_kind_keys(struct ndb_txn *txn);
    108 int ndb_print_tag_index(struct ndb_txn *txn);
    109 int ndb_print_relay_kind_index(struct ndb_txn *txn);
    110 int ndb_print_author_kind_index(struct ndb_txn *txn);
    111 
    112 static void print_note(struct ndb_note *note)
    113 {
    114 	static char buf[5000000];
    115 	if (!ndb_note_json(note, buf, sizeof(buf))) {
    116 		print_hex_stream(stderr, ndb_note_id(note), 32);
    117 		fprintf(stderr, " is too big to print! >5mb");
    118 		return;
    119 	}
    120 	puts(buf);
    121 }
    122 
    123 static void ndb_print_text_search_result(struct ndb_txn *txn,
    124 		struct ndb_text_search_result *r)
    125 {
    126 	size_t len;
    127 	struct ndb_note *note;
    128 
    129 	//ndb_print_text_search_key(&r->key);
    130 
    131 	if (!(note = ndb_get_note_by_key(txn, r->key.note_id, &len))) {
    132 		fprintf(stderr,": note not found");
    133 		return;
    134 	}
    135 
    136 	//fprintf(stderr,"\n");
    137 	print_note(note);
    138 }
    139 
    140 
    141 int main(int argc, char *argv[])
    142 {
    143 	struct ndb *ndb;
    144 	int i, flags, limit, count, current_field, len, res;
    145 	long nanos;
    146 	struct ndb_stat stat;
    147 	struct ndb_txn txn;
    148 	const char *dir;
    149 	unsigned char *data;
    150 	size_t data_len;
    151 	struct ndb_config config;
    152 	struct timespec t1, t2;
    153 	unsigned char tmp_id[32];
    154 	char buf[1024];
    155 	buf[0] = 0;
    156 
    157 	// profiles
    158 	const char *arg_str;
    159 	void *ptr;
    160 	size_t profile_len;
    161 	uint64_t key;
    162 
    163 	res = 0;
    164 	ndb_default_config(&config);
    165 	ndb_config_set_mapsize(&config, 1024ULL * 1024ULL * 1024ULL * 1024ULL /* 1 TiB */);
    166 
    167 	if (argc < 2) {
    168 		return usage();
    169 	}
    170 
    171 	dir = ".";
    172 	flags = 0;
    173 	for (i = 0; i < 2; i++)
    174 	{
    175 		if (!strcmp(argv[1], "-d") && argv[2]) {
    176 			dir = argv[2];
    177 			argv += 2;
    178 			argc -= 2;
    179 		} else if (!strcmp(argv[1], "--skip-verification")) {
    180 			flags = NDB_FLAG_SKIP_NOTE_VERIFY;
    181 			argv += 1;
    182 			argc -= 1;
    183 		}
    184 	}
    185 
    186 	ndb_config_set_flags(&config, flags);
    187 
    188 	//fprintf(stderr, "using db '%s'\n", dir);
    189 
    190 	if (!ndb_init(&ndb, dir, &config)) {
    191 		return 2;
    192 	}
    193 
    194 	if (argc == 2 && !strcmp(argv[1], "stat")) {
    195 		if (!ndb_stat(ndb, &stat)) {
    196 			res = 3;
    197 			goto cleanup;
    198 		}
    199 
    200 		print_stats(&stat);
    201 	} else if (argc >= 3 && !strcmp(argv[1], "query")) {
    202 		struct ndb_filter filter, *f = &filter;
    203 		ndb_filter_init(f);
    204 
    205 		argv += 2;
    206 		argc -= 2;
    207 		current_field = 0;
    208 
    209 		for (i = 0; argc && i < 1000; i++) {
    210 			if (!strcmp(argv[0], "-k") || !strcmp(argv[0], "--kind")) {
    211 				if (current_field != NDB_FILTER_KINDS) {
    212 					ndb_filter_end_field(f);
    213 					ndb_filter_start_field(f, NDB_FILTER_KINDS);
    214 				}
    215 				current_field = NDB_FILTER_KINDS;
    216 				ndb_filter_add_int_element(f, atoll(argv[1]));
    217 				argv += 2;
    218 				argc -= 2;
    219 			} else if (!strcmp(argv[0], "--notekey")) {
    220 				key = atol(argv[1]);
    221 				argv += 2;
    222 				argc -= 2;
    223 			} else if (!strcmp(argv[0], "-l") || !strcmp(argv[0], "--limit")) {
    224 				limit = atol(argv[1]);
    225 				if (current_field) {
    226 					ndb_filter_end_field(f);
    227 					current_field = 0;
    228 				}
    229 				ndb_filter_start_field(f, NDB_FILTER_LIMIT);
    230 				current_field = NDB_FILTER_LIMIT;
    231 				ndb_filter_add_int_element(f, limit);
    232 				ndb_filter_end_field(f);
    233 				argv += 2;
    234 				argc -= 2;
    235 			} else if (!strcmp(argv[0], "--since") || !strcmp(argv[0], "-s")) {
    236 				if (current_field) {
    237 					ndb_filter_end_field(f);
    238 					current_field = 0;
    239 				}
    240 				ndb_filter_start_field(f, NDB_FILTER_SINCE);
    241 				ndb_filter_add_int_element(f, atoll(argv[1]));
    242 				ndb_filter_end_field(f);
    243 				argv += 2;
    244 				argc -= 2;
    245 			} else if (!strcmp(argv[0], "--until") || !strcmp(argv[0], "-u")) {
    246 				if (current_field) {
    247 					ndb_filter_end_field(f);
    248 					current_field = 0;
    249 				}
    250 				ndb_filter_start_field(f, NDB_FILTER_UNTIL);
    251 				ndb_filter_add_int_element(f, atoll(argv[1]));
    252 				ndb_filter_end_field(f);
    253 				argv += 2;
    254 				argc -= 2;
    255 			} else if (!strcmp(argv[0], "-t")) {
    256 				if (current_field) {
    257 					ndb_filter_end_field(f);
    258 					current_field = 0;
    259 				}
    260 				ndb_filter_start_tag_field(f, 't');
    261 				ndb_filter_add_str_element(f, argv[1]);
    262 				ndb_filter_end_field(f);
    263 				argv += 2;
    264 				argc -= 2;
    265 			} else if (!strcmp(argv[0], "--search") || !strcmp(argv[0], "-S")) {
    266 				if (current_field) {
    267 					ndb_filter_end_field(f);
    268 					current_field = 0;
    269 				}
    270 				ndb_filter_start_field(f, NDB_FILTER_SEARCH);
    271 				ndb_filter_add_str_element(f, argv[1]);
    272 				ndb_filter_end_field(f);
    273 				argv += 2;
    274 				argc -= 2;
    275 			} else if (!strcmp(argv[0], "--relay") || !strcmp(argv[0], "-r")) {
    276 				if (current_field) {
    277 					ndb_filter_end_field(f);
    278 					current_field = 0;
    279 				}
    280 				ndb_filter_start_field(f, NDB_FILTER_RELAYS);
    281 				ndb_filter_add_str_element(f, argv[1]);
    282 				ndb_filter_end_field(f);
    283 				argv += 2;
    284 				argc -= 2;
    285 			} else if (!strcmp(argv[0], "-i") || !strcmp(argv[0], "--id")) {
    286 				if (current_field != NDB_FILTER_IDS) {
    287 					ndb_filter_end_field(f);
    288 					ndb_filter_start_field(f, NDB_FILTER_IDS);
    289 					current_field = NDB_FILTER_IDS;
    290 				}
    291 
    292 				len = strlen(argv[1]);
    293 				if (len != 64 || !hex_decode(argv[1], 64, tmp_id, sizeof(tmp_id))) {
    294 					fprintf(stderr, "invalid hex id\n");
    295 					res = 42;
    296 					goto cleanup;
    297 				}
    298 
    299 				ndb_filter_add_id_element(f, tmp_id);
    300 				argv += 2;
    301 				argc -= 2;
    302 			} else if (!strcmp(argv[0], "-e")) {
    303 				if (current_field != 'e') {
    304 					if (!ndb_filter_start_tag_field(f, 'e')) {
    305 						fprintf(stderr, "field already started\n");
    306 						res = 44;
    307 						goto cleanup;
    308 					}
    309 				}
    310 				current_field = 'e';
    311 
    312 				if (len != 64 || !hex_decode(argv[1], 64, tmp_id, sizeof(tmp_id))) {
    313 					fprintf(stderr, "invalid hex id\n");
    314 					res = 42;
    315 					goto cleanup;
    316 				}
    317 
    318 				if (!ndb_filter_add_id_element(f, tmp_id)) {
    319 					fprintf(stderr, "too many event ids\n");
    320 					res = 43;
    321 					goto cleanup;
    322 				}
    323 
    324 				argv += 2;
    325 				argc -= 2;
    326 			} else if (!strcmp(argv[0], "-a") || !strcmp(argv[0], "--author")) {
    327 				if (current_field != NDB_FILTER_AUTHORS) {
    328 					ndb_filter_end_field(f);
    329 					ndb_filter_start_field(f, NDB_FILTER_AUTHORS);
    330 					current_field = NDB_FILTER_AUTHORS;
    331 				}
    332 
    333 				len = strlen(argv[1]);
    334 				if (len != 64 || !hex_decode(argv[1], 64, tmp_id, sizeof(tmp_id))) {
    335 					fprintf(stderr, "invalid hex pubkey\n");
    336 					res = 42;
    337 					goto cleanup;
    338 				}
    339 
    340 				if (!ndb_filter_add_id_element(f, tmp_id)) {
    341 					fprintf(stderr, "too many author pubkeys\n");
    342 					res = 43;
    343 					goto cleanup;
    344 				}
    345 
    346 				argv += 2;
    347 				argc -= 2;
    348 			} else {
    349 				fprintf(stderr, "unrecognized option: %s\n", argv[0]);
    350 				res = 100;
    351 				goto cleanup;
    352 			}
    353 		}
    354 
    355 		if (current_field) {
    356 			ndb_filter_end_field(f);
    357 			current_field = 0;
    358 		}
    359 
    360 		ndb_filter_end(f);
    361 
    362 		ndb_filter_json(f, buf, sizeof(buf));
    363 		fprintf(stderr, "using filter '%s'\n", buf);
    364 
    365 		int rsize = 30000;
    366 		struct ndb_query_result *results = malloc(sizeof(struct ndb_query_result) * rsize);
    367 		assert(results);
    368 		ndb_begin_query(ndb, &txn);
    369 
    370 		clock_gettime(CLOCK_MONOTONIC, &t1);
    371 		if (key) {
    372 			results[0].note = ndb_get_note_by_key(&txn, key, NULL);
    373 			if (results[0].note != NULL)
    374 				count = 1;
    375 			else
    376 				count = 0;
    377 		} else if (!ndb_query(&txn, f, 1, results, rsize, &count)) {
    378 			fprintf(stderr, "query error\n");
    379 		}
    380 		clock_gettime(CLOCK_MONOTONIC, &t2);
    381 
    382 		nanos = (t2.tv_sec - t1.tv_sec) * (long)1e9 + (t2.tv_nsec - t1.tv_nsec);
    383 
    384 		fprintf(stderr, "%d results in %f ms\n", count, nanos/1000000.0);
    385 		for (i = 0; i < count; i++) {
    386 			print_note(results[i].note);
    387 		}
    388 
    389 		ndb_end_query(&txn);
    390 		ndb_filter_destroy(f);
    391 
    392 		free(results);
    393 	} else if (argc == 3 && !strcmp(argv[1], "import")) {
    394 		if (!strcmp(argv[2], "-")) {
    395 			ndb_process_events_stream(ndb, stdin);
    396 		} else {
    397 			map_file(argv[2], &data, &data_len);
    398 			ndb_process_events(ndb, (const char *)data, data_len);
    399 			//ndb_process_client_events(ndb, (const char *)data, data_len);
    400 		}
    401 	} else if (argc == 2 && !strcmp(argv[1], "print-search-keys")) {
    402 		ndb_begin_query(ndb, &txn);
    403 		ndb_print_search_keys(&txn);
    404 		ndb_end_query(&txn);
    405 	} else if (argc == 2 && !strcmp(argv[1], "print-kind-keys")) {
    406 		ndb_begin_query(ndb, &txn);
    407 		ndb_print_kind_keys(&txn);
    408 		ndb_end_query(&txn);
    409 	} else if (argc == 2 && !strcmp(argv[1], "print-tag-keys")) {
    410 		ndb_begin_query(ndb, &txn);
    411 		ndb_print_tag_index(&txn);
    412 		ndb_end_query(&txn);
    413 	} else if (argc == 2 && !strcmp(argv[1], "print-relay-kind-index-keys")) {
    414 		ndb_begin_query(ndb, &txn);
    415 		ndb_print_relay_kind_index(&txn);
    416 		ndb_end_query(&txn);
    417 	} else if (argc == 2 && !strcmp(argv[1], "print-author-kind-index-keys")) {
    418 		ndb_begin_query(ndb, &txn);
    419 		ndb_print_author_kind_index(&txn);
    420 		ndb_end_query(&txn);
    421 	} else if (argc == 3 && !strcmp(argv[1], "note-relays")) {
    422 		struct ndb_note_relay_iterator iter;
    423 		const char *relay;
    424 
    425 		ndb_begin_query(ndb, &txn);
    426 		arg_str = argv[2];
    427 
    428 		if (!hex_decode(arg_str, strlen(arg_str), tmp_id, sizeof(tmp_id))) {
    429 			fprintf(stderr, "failed to decode hex pubkey '%s'\n", arg_str);
    430 			res = 88;
    431 			goto cleanup;
    432 		}
    433 
    434 		if (!(key = ndb_get_notekey_by_id(&txn, tmp_id))) {
    435 			fprintf(stderr, "noteid '%s' not found\n", arg_str);
    436 			res = 89;
    437 			goto cleanup;
    438 		}
    439 
    440 		ndb_note_relay_iterate_start(&txn, &iter, key);
    441 
    442 		for (i = 0; (relay = ndb_note_relay_iterate_next(&iter)); i++) {
    443 			printf("%s\n", relay);
    444 		}
    445 
    446 		fprintf(stderr, "seen on %d relays\n", i);
    447 
    448 		ndb_end_query(&txn);
    449 	} else if (argc == 3 && !strcmp(argv[1], "profile")) {
    450 		arg_str = argv[2];
    451 		if (!hex_decode(arg_str, strlen(arg_str), tmp_id, sizeof(tmp_id))) {
    452 			fprintf(stderr, "failed to decode hex pubkey '%s'\n", arg_str);
    453 			res = 88;
    454 			goto cleanup;
    455 		}
    456 		ndb_begin_query(ndb, &txn);
    457 		if (!(ptr = ndb_get_profile_by_pubkey(&txn, tmp_id, &profile_len, &key))) {
    458 			ndb_end_query(&txn);
    459 			fprintf(stderr, "profile not found\n");
    460 			res = 89;
    461 			goto cleanup;
    462 		}
    463 		ndb_end_query(&txn);
    464 		print_hex(ptr, profile_len);
    465 		printf("\n");
    466 	} else {
    467 		ndb_destroy(ndb);
    468 		return usage();
    469 	}
    470 
    471 cleanup:
    472 	ndb_destroy(ndb);
    473 	return res;
    474 }