ndb.c (9544B)
1 2 3 #include "nostrdb.h" 4 #include "lmdb.h" 5 #include "print_util.h" 6 #include "hex.h" 7 #include <time.h> 8 #include <stdio.h> 9 #include <stdlib.h> 10 #include <sys/stat.h> 11 #include <sys/mman.h> 12 #include <unistd.h> 13 #include <fcntl.h> 14 15 static int usage() 16 { 17 printf("usage: ndb [--skip-verification] [-d db_dir] <command>\n\n"); 18 printf("commands\n\n"); 19 printf(" stat\n"); 20 printf(" query [--kind 42] [--search term] [--limit 42] [-e abcdef...] [--author abcdef... -a bcdef...]\n"); 21 printf(" profile <pubkey>\n"); 22 printf(" print-search-keys\n"); 23 printf(" print-kind-keys\n"); 24 printf(" print-tag-keys\n"); 25 printf(" print-relay-kind-index-keys\n"); 26 printf(" import <line-delimited json file>\n\n"); 27 printf("settings\n\n"); 28 printf(" --skip-verification skip signature validation\n"); 29 printf(" -d <db_dir> set database directory\n"); 30 return 1; 31 } 32 33 34 static int map_file(const char *filename, unsigned char **p, size_t *flen) 35 { 36 struct stat st; 37 int des; 38 stat(filename, &st); 39 *flen = st.st_size; 40 41 des = open(filename, O_RDONLY); 42 43 *p = mmap(NULL, *flen, PROT_READ, MAP_PRIVATE, des, 0); 44 close(des); 45 46 return *p != MAP_FAILED; 47 } 48 49 static inline void print_stat_counts(struct ndb_stat_counts *counts) 50 { 51 printf("%zu\t%zu\t%zu\t%zu\n", 52 counts->count, 53 counts->key_size, 54 counts->value_size, 55 counts->key_size + counts->value_size); 56 } 57 58 static void print_stats(struct ndb_stat *stat) 59 { 60 int i; 61 const char *name; 62 struct ndb_stat_counts *c; 63 64 struct ndb_stat_counts total; 65 ndb_stat_counts_init(&total); 66 67 printf("name\tcount\tkey_bytes\tvalue_bytes\ttotal_bytes\n"); 68 printf("---\ndbs\n---\n"); 69 for (i = 0; i < NDB_DBS; i++) { 70 name = ndb_db_name(i); 71 72 total.count += stat->dbs[i].count; 73 total.key_size += stat->dbs[i].key_size; 74 total.value_size += stat->dbs[i].value_size; 75 76 printf("%s\t", name); 77 print_stat_counts(&stat->dbs[i]); 78 } 79 80 printf("total\t"); 81 print_stat_counts(&total); 82 83 printf("-----\nkinds\n-----\n"); 84 for (i = 0; i < NDB_CKIND_COUNT; i++) { 85 c = &stat->common_kinds[i]; 86 if (c->count == 0) 87 continue; 88 89 printf("%s\t", ndb_kind_name(i)); 90 print_stat_counts(c); 91 } 92 93 if (stat->other_kinds.count != 0) { 94 printf("other\t"); 95 print_stat_counts(&stat->other_kinds); 96 } 97 } 98 99 int ndb_print_search_keys(struct ndb_txn *txn); 100 int ndb_print_kind_keys(struct ndb_txn *txn); 101 int ndb_print_tag_index(struct ndb_txn *txn); 102 int ndb_print_relay_kind_index(struct ndb_txn *txn); 103 104 static void print_note(struct ndb_note *note) 105 { 106 static char buf[5000000]; 107 if (!ndb_note_json(note, buf, sizeof(buf))) { 108 print_hex_stream(stderr, ndb_note_id(note), 32); 109 fprintf(stderr, " is too big to print! >5mb"); 110 return; 111 } 112 puts(buf); 113 } 114 115 static void ndb_print_text_search_result(struct ndb_txn *txn, 116 struct ndb_text_search_result *r) 117 { 118 size_t len; 119 struct ndb_note *note; 120 121 //ndb_print_text_search_key(&r->key); 122 123 if (!(note = ndb_get_note_by_key(txn, r->key.note_id, &len))) { 124 fprintf(stderr,": note not found"); 125 return; 126 } 127 128 //fprintf(stderr,"\n"); 129 print_note(note); 130 } 131 132 133 int main(int argc, char *argv[]) 134 { 135 struct ndb *ndb; 136 int i, flags, limit, count, current_field, len, res; 137 long nanos; 138 struct ndb_stat stat; 139 struct ndb_txn txn; 140 const char *dir; 141 unsigned char *data; 142 size_t data_len; 143 struct ndb_config config; 144 struct timespec t1, t2; 145 unsigned char tmp_id[32]; 146 147 // profiles 148 const char *pk_str; 149 void *ptr; 150 size_t profile_len; 151 uint64_t key; 152 153 res = 0; 154 ndb_default_config(&config); 155 ndb_config_set_mapsize(&config, 1024ULL * 1024ULL * 1024ULL * 1024ULL /* 1 TiB */); 156 157 if (argc < 2) { 158 return usage(); 159 } 160 161 dir = "."; 162 flags = 0; 163 for (i = 0; i < 2; i++) 164 { 165 if (!strcmp(argv[1], "-d") && argv[2]) { 166 dir = argv[2]; 167 argv += 2; 168 argc -= 2; 169 } else if (!strcmp(argv[1], "--skip-verification")) { 170 flags = NDB_FLAG_SKIP_NOTE_VERIFY; 171 argv += 1; 172 argc -= 1; 173 } 174 } 175 176 ndb_config_set_flags(&config, flags); 177 178 fprintf(stderr, "using db '%s'\n", dir); 179 180 if (!ndb_init(&ndb, dir, &config)) { 181 return 2; 182 } 183 184 if (argc == 2 && !strcmp(argv[1], "stat")) { 185 if (!ndb_stat(ndb, &stat)) { 186 res = 3; 187 goto cleanup; 188 } 189 190 print_stats(&stat); 191 } else if (argc >= 3 && !strcmp(argv[1], "query")) { 192 struct ndb_filter filter, *f = &filter; 193 ndb_filter_init(f); 194 195 argv += 2; 196 argc -= 2; 197 current_field = 0; 198 199 for (i = 0; argc && i < 1000; i++) { 200 if (!strcmp(argv[0], "-k") || !strcmp(argv[0], "--kind")) { 201 if (current_field != NDB_FILTER_KINDS) { 202 ndb_filter_end_field(f); 203 ndb_filter_start_field(f, NDB_FILTER_KINDS); 204 } 205 current_field = NDB_FILTER_KINDS; 206 ndb_filter_add_int_element(f, atoll(argv[1])); 207 argv += 2; 208 argc -= 2; 209 } else if (!strcmp(argv[0], "-l") || !strcmp(argv[0], "--limit")) { 210 limit = atol(argv[1]); 211 if (current_field) { 212 ndb_filter_end_field(f); 213 current_field = 0; 214 } 215 ndb_filter_start_field(f, NDB_FILTER_LIMIT); 216 current_field = NDB_FILTER_LIMIT; 217 ndb_filter_add_int_element(f, limit); 218 ndb_filter_end_field(f); 219 argv += 2; 220 argc -= 2; 221 } else if (!strcmp(argv[0], "--since") || !strcmp(argv[0], "-s")) { 222 if (current_field) { 223 ndb_filter_end_field(f); 224 current_field = 0; 225 } 226 ndb_filter_start_field(f, NDB_FILTER_SINCE); 227 ndb_filter_add_int_element(f, atoll(argv[1])); 228 ndb_filter_end_field(f); 229 argv += 2; 230 argc -= 2; 231 } else if (!strcmp(argv[0], "--until") || !strcmp(argv[0], "-u")) { 232 if (current_field) { 233 ndb_filter_end_field(f); 234 current_field = 0; 235 } 236 ndb_filter_start_field(f, NDB_FILTER_UNTIL); 237 ndb_filter_add_int_element(f, atoll(argv[1])); 238 ndb_filter_end_field(f); 239 argv += 2; 240 argc -= 2; 241 } else if (!strcmp(argv[0], "-t")) { 242 if (current_field) { 243 ndb_filter_end_field(f); 244 current_field = 0; 245 } 246 ndb_filter_start_tag_field(f, 't'); 247 ndb_filter_add_str_element(f, argv[1]); 248 ndb_filter_end_field(f); 249 argv += 2; 250 argc -= 2; 251 } else if (!strcmp(argv[0], "--search") || !strcmp(argv[0], "-S")) { 252 if (current_field) { 253 ndb_filter_end_field(f); 254 current_field = 0; 255 } 256 ndb_filter_start_field(f, NDB_FILTER_SEARCH); 257 ndb_filter_add_str_element(f, argv[1]); 258 ndb_filter_end_field(f); 259 argv += 2; 260 argc -= 2; 261 } else if (!strcmp(argv[0], "-e")) { 262 if (current_field != 'e') { 263 if (!ndb_filter_start_tag_field(f, 'e')) { 264 fprintf(stderr, "field already started\n"); 265 res = 44; 266 goto cleanup; 267 } 268 } 269 current_field = 'e'; 270 271 if (len != 64 || !hex_decode(argv[1], 64, tmp_id, sizeof(tmp_id))) { 272 fprintf(stderr, "invalid hex id\n"); 273 res = 42; 274 goto cleanup; 275 } 276 277 if (!ndb_filter_add_id_element(f, tmp_id)) { 278 fprintf(stderr, "too many event ids\n"); 279 res = 43; 280 goto cleanup; 281 } 282 283 argv += 2; 284 argc -= 2; 285 } else if (!strcmp(argv[0], "-a") || !strcmp(argv[0], "--author")) { 286 if (current_field) { 287 ndb_filter_end_field(f); 288 current_field = 0; 289 } 290 if (current_field != NDB_FILTER_AUTHORS) { 291 ndb_filter_start_field(f, NDB_FILTER_AUTHORS); 292 } 293 current_field = NDB_FILTER_AUTHORS; 294 295 len = strlen(argv[1]); 296 if (len != 64 || !hex_decode(argv[1], 64, tmp_id, sizeof(tmp_id))) { 297 fprintf(stderr, "invalid hex pubkey\n"); 298 res = 42; 299 goto cleanup; 300 } 301 302 if (!ndb_filter_add_id_element(f, tmp_id)) { 303 fprintf(stderr, "too many author pubkeys\n"); 304 res = 43; 305 goto cleanup; 306 } 307 308 argv += 2; 309 argc -= 2; 310 } 311 } 312 313 if (current_field) { 314 ndb_filter_end_field(f); 315 current_field = 0; 316 } 317 318 struct ndb_query_result results[10000]; 319 ndb_begin_query(ndb, &txn); 320 321 clock_gettime(CLOCK_MONOTONIC, &t1); 322 if (!ndb_query(&txn, f, 1, results, 10000, &count)) { 323 fprintf(stderr, "query error\n"); 324 } 325 clock_gettime(CLOCK_MONOTONIC, &t2); 326 327 nanos = (t2.tv_sec - t1.tv_sec) * (long)1e9 + (t2.tv_nsec - t1.tv_nsec); 328 329 fprintf(stderr, "%d results in %f ms\n", count, nanos/1000000.0); 330 for (i = 0; i < count; i++) { 331 print_note(results[i].note); 332 } 333 334 ndb_end_query(&txn); 335 336 } else if (argc == 3 && !strcmp(argv[1], "import")) { 337 if (!strcmp(argv[2], "-")) { 338 ndb_process_events_stream(ndb, stdin); 339 } else { 340 map_file(argv[2], &data, &data_len); 341 ndb_process_events(ndb, (const char *)data, data_len); 342 //ndb_process_client_events(ndb, (const char *)data, data_len); 343 } 344 } else if (argc == 2 && !strcmp(argv[1], "print-search-keys")) { 345 ndb_begin_query(ndb, &txn); 346 ndb_print_search_keys(&txn); 347 ndb_end_query(&txn); 348 } else if (argc == 2 && !strcmp(argv[1], "print-kind-keys")) { 349 ndb_begin_query(ndb, &txn); 350 ndb_print_kind_keys(&txn); 351 ndb_end_query(&txn); 352 } else if (argc == 2 && !strcmp(argv[1], "print-tag-keys")) { 353 ndb_begin_query(ndb, &txn); 354 ndb_print_tag_index(&txn); 355 ndb_end_query(&txn); 356 } else if (argc == 2 && !strcmp(argv[1], "print-relay-kind-index-keys")) { 357 ndb_begin_query(ndb, &txn); 358 ndb_print_relay_kind_index(&txn); 359 ndb_end_query(&txn); 360 } else if (argc == 3 && !strcmp(argv[1], "profile")) { 361 pk_str = argv[2]; 362 if (!hex_decode(pk_str, strlen(pk_str), tmp_id, sizeof(tmp_id))) { 363 fprintf(stderr, "failed to decode hex pubkey '%s'\n", pk_str); 364 res = 88; 365 goto cleanup; 366 } 367 ndb_begin_query(ndb, &txn); 368 if (!(ptr = ndb_get_profile_by_pubkey(&txn, tmp_id, &profile_len, &key))) { 369 ndb_end_query(&txn); 370 fprintf(stderr, "profile not found\n"); 371 res = 89; 372 goto cleanup; 373 } 374 ndb_end_query(&txn); 375 print_hex(ptr, profile_len); 376 printf("\n"); 377 } else { 378 ndb_destroy(ndb); 379 return usage(); 380 } 381 382 cleanup: 383 ndb_destroy(ndb); 384 return res; 385 }