ndb.c (12153B)
1 2 3 #include "nostrdb.h" 4 #include "lmdb.h" 5 #include "print_util.h" 6 #include "hex.h" 7 #include <time.h> 8 #include <stdio.h> 9 #include <stdlib.h> 10 #include <sys/stat.h> 11 #include <sys/mman.h> 12 #include <unistd.h> 13 #include <fcntl.h> 14 15 static int usage() 16 { 17 printf("usage: ndb [--skip-verification] [-d db_dir] <command>\n\n"); 18 19 printf("commands\n\n"); 20 21 printf(" stat\n"); 22 printf(" query [--kind 42] [--id abcdef...] [--notekey key] [--search term] [--limit 42] \n"); 23 printf(" [-e abcdef...] [--author abcdef... -a bcdef...] [--relay wss://relay.damus.io]\n"); 24 printf(" profile <pubkey> print the raw profile data for a pubkey\n"); 25 printf(" note-relays <note-id> list the relays a given note id has been seen on\n"); 26 printf(" print-search-keys\n"); 27 printf(" print-kind-keys\n"); 28 printf(" print-tag-keys\n"); 29 printf(" print-relay-kind-index-keys\n"); 30 printf(" print-author-kind-index-keys\n"); 31 printf(" import <line-delimited json file>\n\n"); 32 33 printf("settings\n\n"); 34 35 printf(" --skip-verification skip signature validation\n"); 36 printf(" -d <db_dir> set database directory\n"); 37 return 1; 38 } 39 40 41 static int map_file(const char *filename, unsigned char **p, size_t *flen) 42 { 43 struct stat st; 44 int des; 45 stat(filename, &st); 46 *flen = st.st_size; 47 48 des = open(filename, O_RDONLY); 49 50 *p = mmap(NULL, *flen, PROT_READ, MAP_PRIVATE, des, 0); 51 close(des); 52 53 return *p != MAP_FAILED; 54 } 55 56 static inline void print_stat_counts(struct ndb_stat_counts *counts) 57 { 58 printf("%zu\t%zu\t%zu\t%zu\n", 59 counts->count, 60 counts->key_size, 61 counts->value_size, 62 counts->key_size + counts->value_size); 63 } 64 65 static void print_stats(struct ndb_stat *stat) 66 { 67 int i; 68 const char *name; 69 struct ndb_stat_counts *c; 70 71 struct ndb_stat_counts total; 72 ndb_stat_counts_init(&total); 73 74 printf("name\tcount\tkey_bytes\tvalue_bytes\ttotal_bytes\n"); 75 printf("---\ndbs\n---\n"); 76 for (i = 0; i < NDB_DBS; i++) { 77 name = ndb_db_name(i); 78 79 total.count += stat->dbs[i].count; 80 total.key_size += stat->dbs[i].key_size; 81 total.value_size += stat->dbs[i].value_size; 82 83 printf("%s\t", name); 84 print_stat_counts(&stat->dbs[i]); 85 } 86 87 printf("total\t"); 88 print_stat_counts(&total); 89 90 printf("-----\nkinds\n-----\n"); 91 for (i = 0; i < NDB_CKIND_COUNT; i++) { 92 c = &stat->common_kinds[i]; 93 if (c->count == 0) 94 continue; 95 96 printf("%s\t", ndb_kind_name(i)); 97 print_stat_counts(c); 98 } 99 100 if (stat->other_kinds.count != 0) { 101 printf("other\t"); 102 print_stat_counts(&stat->other_kinds); 103 } 104 } 105 106 int ndb_print_search_keys(struct ndb_txn *txn); 107 int ndb_print_kind_keys(struct ndb_txn *txn); 108 int ndb_print_tag_index(struct ndb_txn *txn); 109 int ndb_print_relay_kind_index(struct ndb_txn *txn); 110 int ndb_print_author_kind_index(struct ndb_txn *txn); 111 112 static void print_note(struct ndb_note *note) 113 { 114 static char buf[5000000]; 115 if (!ndb_note_json(note, buf, sizeof(buf))) { 116 print_hex_stream(stderr, ndb_note_id(note), 32); 117 fprintf(stderr, " is too big to print! >5mb"); 118 return; 119 } 120 puts(buf); 121 } 122 123 static void ndb_print_text_search_result(struct ndb_txn *txn, 124 struct ndb_text_search_result *r) 125 { 126 size_t len; 127 struct ndb_note *note; 128 129 //ndb_print_text_search_key(&r->key); 130 131 if (!(note = ndb_get_note_by_key(txn, r->key.note_id, &len))) { 132 fprintf(stderr,": note not found"); 133 return; 134 } 135 136 //fprintf(stderr,"\n"); 137 print_note(note); 138 } 139 140 141 int main(int argc, char *argv[]) 142 { 143 struct ndb *ndb; 144 int i, flags, limit, count, current_field, len, res; 145 long nanos; 146 struct ndb_stat stat; 147 struct ndb_txn txn; 148 const char *dir; 149 unsigned char *data; 150 size_t data_len; 151 struct ndb_config config; 152 struct timespec t1, t2; 153 unsigned char tmp_id[32]; 154 char buf[1024]; 155 buf[0] = 0; 156 157 // profiles 158 const char *arg_str; 159 void *ptr; 160 size_t profile_len; 161 uint64_t key; 162 163 res = 0; 164 ndb_default_config(&config); 165 ndb_config_set_mapsize(&config, 1024ULL * 1024ULL * 1024ULL * 1024ULL /* 1 TiB */); 166 167 if (argc < 2) { 168 return usage(); 169 } 170 171 dir = "."; 172 flags = 0; 173 for (i = 0; i < 2; i++) 174 { 175 if (!strcmp(argv[1], "-d") && argv[2]) { 176 dir = argv[2]; 177 argv += 2; 178 argc -= 2; 179 } else if (!strcmp(argv[1], "--skip-verification")) { 180 flags = NDB_FLAG_SKIP_NOTE_VERIFY; 181 argv += 1; 182 argc -= 1; 183 } 184 } 185 186 ndb_config_set_flags(&config, flags); 187 188 //fprintf(stderr, "using db '%s'\n", dir); 189 190 if (!ndb_init(&ndb, dir, &config)) { 191 return 2; 192 } 193 194 if (argc == 2 && !strcmp(argv[1], "stat")) { 195 if (!ndb_stat(ndb, &stat)) { 196 res = 3; 197 goto cleanup; 198 } 199 200 print_stats(&stat); 201 } else if (argc >= 3 && !strcmp(argv[1], "query")) { 202 struct ndb_filter filter, *f = &filter; 203 ndb_filter_init(f); 204 205 argv += 2; 206 argc -= 2; 207 current_field = 0; 208 209 for (i = 0; argc && i < 1000; i++) { 210 if (!strcmp(argv[0], "-k") || !strcmp(argv[0], "--kind")) { 211 if (current_field != NDB_FILTER_KINDS) { 212 ndb_filter_end_field(f); 213 ndb_filter_start_field(f, NDB_FILTER_KINDS); 214 } 215 current_field = NDB_FILTER_KINDS; 216 ndb_filter_add_int_element(f, atoll(argv[1])); 217 argv += 2; 218 argc -= 2; 219 } else if (!strcmp(argv[0], "--notekey")) { 220 key = atol(argv[1]); 221 argv += 2; 222 argc -= 2; 223 } else if (!strcmp(argv[0], "-l") || !strcmp(argv[0], "--limit")) { 224 limit = atol(argv[1]); 225 if (current_field) { 226 ndb_filter_end_field(f); 227 current_field = 0; 228 } 229 ndb_filter_start_field(f, NDB_FILTER_LIMIT); 230 current_field = NDB_FILTER_LIMIT; 231 ndb_filter_add_int_element(f, limit); 232 ndb_filter_end_field(f); 233 argv += 2; 234 argc -= 2; 235 } else if (!strcmp(argv[0], "--since") || !strcmp(argv[0], "-s")) { 236 if (current_field) { 237 ndb_filter_end_field(f); 238 current_field = 0; 239 } 240 ndb_filter_start_field(f, NDB_FILTER_SINCE); 241 ndb_filter_add_int_element(f, atoll(argv[1])); 242 ndb_filter_end_field(f); 243 argv += 2; 244 argc -= 2; 245 } else if (!strcmp(argv[0], "--until") || !strcmp(argv[0], "-u")) { 246 if (current_field) { 247 ndb_filter_end_field(f); 248 current_field = 0; 249 } 250 ndb_filter_start_field(f, NDB_FILTER_UNTIL); 251 ndb_filter_add_int_element(f, atoll(argv[1])); 252 ndb_filter_end_field(f); 253 argv += 2; 254 argc -= 2; 255 } else if (!strcmp(argv[0], "-t")) { 256 if (current_field) { 257 ndb_filter_end_field(f); 258 current_field = 0; 259 } 260 ndb_filter_start_tag_field(f, 't'); 261 ndb_filter_add_str_element(f, argv[1]); 262 ndb_filter_end_field(f); 263 argv += 2; 264 argc -= 2; 265 } else if (!strcmp(argv[0], "--search") || !strcmp(argv[0], "-S")) { 266 if (current_field) { 267 ndb_filter_end_field(f); 268 current_field = 0; 269 } 270 ndb_filter_start_field(f, NDB_FILTER_SEARCH); 271 ndb_filter_add_str_element(f, argv[1]); 272 ndb_filter_end_field(f); 273 argv += 2; 274 argc -= 2; 275 } else if (!strcmp(argv[0], "--relay") || !strcmp(argv[0], "-r")) { 276 if (current_field) { 277 ndb_filter_end_field(f); 278 current_field = 0; 279 } 280 ndb_filter_start_field(f, NDB_FILTER_RELAYS); 281 ndb_filter_add_str_element(f, argv[1]); 282 ndb_filter_end_field(f); 283 argv += 2; 284 argc -= 2; 285 } else if (!strcmp(argv[0], "-i") || !strcmp(argv[0], "--id")) { 286 if (current_field != NDB_FILTER_IDS) { 287 ndb_filter_end_field(f); 288 ndb_filter_start_field(f, NDB_FILTER_IDS); 289 current_field = NDB_FILTER_IDS; 290 } 291 292 len = strlen(argv[1]); 293 if (len != 64 || !hex_decode(argv[1], 64, tmp_id, sizeof(tmp_id))) { 294 fprintf(stderr, "invalid hex id\n"); 295 res = 42; 296 goto cleanup; 297 } 298 299 ndb_filter_add_id_element(f, tmp_id); 300 argv += 2; 301 argc -= 2; 302 } else if (!strcmp(argv[0], "-e")) { 303 if (current_field != 'e') { 304 if (!ndb_filter_start_tag_field(f, 'e')) { 305 fprintf(stderr, "field already started\n"); 306 res = 44; 307 goto cleanup; 308 } 309 } 310 current_field = 'e'; 311 312 if (len != 64 || !hex_decode(argv[1], 64, tmp_id, sizeof(tmp_id))) { 313 fprintf(stderr, "invalid hex id\n"); 314 res = 42; 315 goto cleanup; 316 } 317 318 if (!ndb_filter_add_id_element(f, tmp_id)) { 319 fprintf(stderr, "too many event ids\n"); 320 res = 43; 321 goto cleanup; 322 } 323 324 argv += 2; 325 argc -= 2; 326 } else if (!strcmp(argv[0], "-a") || !strcmp(argv[0], "--author")) { 327 if (current_field != NDB_FILTER_AUTHORS) { 328 ndb_filter_end_field(f); 329 ndb_filter_start_field(f, NDB_FILTER_AUTHORS); 330 current_field = NDB_FILTER_AUTHORS; 331 } 332 333 len = strlen(argv[1]); 334 if (len != 64 || !hex_decode(argv[1], 64, tmp_id, sizeof(tmp_id))) { 335 fprintf(stderr, "invalid hex pubkey\n"); 336 res = 42; 337 goto cleanup; 338 } 339 340 if (!ndb_filter_add_id_element(f, tmp_id)) { 341 fprintf(stderr, "too many author pubkeys\n"); 342 res = 43; 343 goto cleanup; 344 } 345 346 argv += 2; 347 argc -= 2; 348 } else { 349 fprintf(stderr, "unrecognized option: %s\n", argv[0]); 350 res = 100; 351 goto cleanup; 352 } 353 } 354 355 if (current_field) { 356 ndb_filter_end_field(f); 357 current_field = 0; 358 } 359 360 ndb_filter_end(f); 361 362 ndb_filter_json(f, buf, sizeof(buf)); 363 fprintf(stderr, "using filter '%s'\n", buf); 364 365 int rsize = 30000; 366 struct ndb_query_result *results = malloc(sizeof(struct ndb_query_result) * rsize); 367 assert(results); 368 ndb_begin_query(ndb, &txn); 369 370 clock_gettime(CLOCK_MONOTONIC, &t1); 371 if (key) { 372 results[0].note = ndb_get_note_by_key(&txn, key, NULL); 373 if (results[0].note != NULL) 374 count = 1; 375 else 376 count = 0; 377 } else if (!ndb_query(&txn, f, 1, results, rsize, &count)) { 378 fprintf(stderr, "query error\n"); 379 } 380 clock_gettime(CLOCK_MONOTONIC, &t2); 381 382 nanos = (t2.tv_sec - t1.tv_sec) * (long)1e9 + (t2.tv_nsec - t1.tv_nsec); 383 384 fprintf(stderr, "%d results in %f ms\n", count, nanos/1000000.0); 385 for (i = 0; i < count; i++) { 386 print_note(results[i].note); 387 } 388 389 ndb_end_query(&txn); 390 ndb_filter_destroy(f); 391 392 free(results); 393 } else if (argc == 3 && !strcmp(argv[1], "import")) { 394 if (!strcmp(argv[2], "-")) { 395 ndb_process_events_stream(ndb, stdin); 396 } else { 397 map_file(argv[2], &data, &data_len); 398 ndb_process_events(ndb, (const char *)data, data_len); 399 //ndb_process_client_events(ndb, (const char *)data, data_len); 400 } 401 } else if (argc == 2 && !strcmp(argv[1], "print-search-keys")) { 402 ndb_begin_query(ndb, &txn); 403 ndb_print_search_keys(&txn); 404 ndb_end_query(&txn); 405 } else if (argc == 2 && !strcmp(argv[1], "print-kind-keys")) { 406 ndb_begin_query(ndb, &txn); 407 ndb_print_kind_keys(&txn); 408 ndb_end_query(&txn); 409 } else if (argc == 2 && !strcmp(argv[1], "print-tag-keys")) { 410 ndb_begin_query(ndb, &txn); 411 ndb_print_tag_index(&txn); 412 ndb_end_query(&txn); 413 } else if (argc == 2 && !strcmp(argv[1], "print-relay-kind-index-keys")) { 414 ndb_begin_query(ndb, &txn); 415 ndb_print_relay_kind_index(&txn); 416 ndb_end_query(&txn); 417 } else if (argc == 2 && !strcmp(argv[1], "print-author-kind-index-keys")) { 418 ndb_begin_query(ndb, &txn); 419 ndb_print_author_kind_index(&txn); 420 ndb_end_query(&txn); 421 } else if (argc == 3 && !strcmp(argv[1], "note-relays")) { 422 struct ndb_note_relay_iterator iter; 423 const char *relay; 424 425 ndb_begin_query(ndb, &txn); 426 arg_str = argv[2]; 427 428 if (!hex_decode(arg_str, strlen(arg_str), tmp_id, sizeof(tmp_id))) { 429 fprintf(stderr, "failed to decode hex pubkey '%s'\n", arg_str); 430 res = 88; 431 goto cleanup; 432 } 433 434 if (!(key = ndb_get_notekey_by_id(&txn, tmp_id))) { 435 fprintf(stderr, "noteid '%s' not found\n", arg_str); 436 res = 89; 437 goto cleanup; 438 } 439 440 ndb_note_relay_iterate_start(&txn, &iter, key); 441 442 for (i = 0; (relay = ndb_note_relay_iterate_next(&iter)); i++) { 443 printf("%s\n", relay); 444 } 445 446 fprintf(stderr, "seen on %d relays\n", i); 447 448 ndb_end_query(&txn); 449 } else if (argc == 3 && !strcmp(argv[1], "profile")) { 450 arg_str = argv[2]; 451 if (!hex_decode(arg_str, strlen(arg_str), tmp_id, sizeof(tmp_id))) { 452 fprintf(stderr, "failed to decode hex pubkey '%s'\n", arg_str); 453 res = 88; 454 goto cleanup; 455 } 456 ndb_begin_query(ndb, &txn); 457 if (!(ptr = ndb_get_profile_by_pubkey(&txn, tmp_id, &profile_len, &key))) { 458 ndb_end_query(&txn); 459 fprintf(stderr, "profile not found\n"); 460 res = 89; 461 goto cleanup; 462 } 463 ndb_end_query(&txn); 464 print_hex(ptr, profile_len); 465 printf("\n"); 466 } else { 467 ndb_destroy(ndb); 468 return usage(); 469 } 470 471 cleanup: 472 ndb_destroy(ndb); 473 return res; 474 }