nostrdb.c (211299B)
1 2 #include "nostrdb.h" 3 #include "jsmn.h" 4 #include "hex.h" 5 #include "cursor.h" 6 #include "random.h" 7 #include "ccan/crypto/sha256/sha256.h" 8 #include "bolt11/bolt11.h" 9 #include "bolt11/amount.h" 10 #include "lmdb.h" 11 #include "util.h" 12 #include "cpu.h" 13 #include "block.h" 14 #include "threadpool.h" 15 #include "thread.h" 16 #include "protected_queue.h" 17 #include "memchr.h" 18 #include "print_util.h" 19 #include <stdlib.h> 20 #include <limits.h> 21 #include <assert.h> 22 23 #include "bindings/c/profile_json_parser.h" 24 #include "bindings/c/profile_builder.h" 25 #include "bindings/c/meta_builder.h" 26 #include "bindings/c/meta_reader.h" 27 #include "bindings/c/profile_verifier.h" 28 #include "secp256k1.h" 29 #include "secp256k1_ecdh.h" 30 #include "secp256k1_schnorrsig.h" 31 32 #define max(a,b) ((a) > (b) ? (a) : (b)) 33 #define min(a,b) ((a) < (b) ? (a) : (b)) 34 35 // the maximum number of things threads pop and push in bulk 36 #define THREAD_QUEUE_BATCH 4096 37 38 // maximum number of active subscriptions 39 #define MAX_SUBSCRIPTIONS 256 40 #define MAX_SCAN_CURSORS 12 41 #define MAX_FILTERS 16 42 43 // the maximum size of inbox queues 44 static const int DEFAULT_QUEUE_SIZE = 32768; 45 46 // 2mb scratch size for the writer thread 47 static const int DEFAULT_WRITER_SCRATCH_SIZE = 2097152; 48 49 // increase if we need bigger filters 50 #define NDB_FILTER_PAGES 64 51 52 #define ndb_flag_set(flags, f) ((flags & f) == f) 53 54 #define NDB_PARSED_ID (1 << 0) 55 #define NDB_PARSED_PUBKEY (1 << 1) 56 #define NDB_PARSED_SIG (1 << 2) 57 #define NDB_PARSED_CREATED_AT (1 << 3) 58 #define NDB_PARSED_KIND (1 << 4) 59 #define NDB_PARSED_CONTENT (1 << 5) 60 #define NDB_PARSED_TAGS (1 << 6) 61 #define NDB_PARSED_ALL (NDB_PARSED_ID|NDB_PARSED_PUBKEY|NDB_PARSED_SIG|NDB_PARSED_CREATED_AT|NDB_PARSED_KIND|NDB_PARSED_CONTENT|NDB_PARSED_TAGS) 62 63 typedef int (*ndb_migrate_fn)(struct ndb_txn *); 64 typedef int (*ndb_word_parser_fn)(void *, const char *word, int word_len, 65 int word_index); 66 67 // these must be byte-aligned, they are directly accessing the serialized data 68 // representation 69 #pragma pack(push, 1) 70 71 union ndb_packed_str { 72 struct { 73 char str[3]; 74 // we assume little endian everywhere. sorry not sorry. 75 unsigned char flag; // NDB_PACKED_STR, etc 76 } packed; 77 78 uint32_t offset; 79 unsigned char bytes[4]; 80 }; 81 82 struct ndb_tag { 83 uint16_t count; 84 union ndb_packed_str strs[0]; 85 }; 86 87 struct ndb_tags { 88 uint16_t padding; 89 uint16_t count; 90 }; 91 92 // v1 93 struct ndb_note { 94 unsigned char version; // v=1 95 unsigned char padding[3]; // keep things aligned 96 unsigned char id[32]; 97 unsigned char pubkey[32]; 98 unsigned char sig[64]; 99 100 uint64_t created_at; 101 uint32_t kind; 102 uint32_t content_length; 103 union ndb_packed_str content; 104 uint32_t strings; 105 // nothing can come after tags since it contains variadic data 106 struct ndb_tags tags; 107 }; 108 109 #pragma pack(pop) 110 111 112 struct ndb_migration { 113 ndb_migrate_fn fn; 114 }; 115 116 struct ndb_profile_record_builder { 117 flatcc_builder_t *builder; 118 void *flatbuf; 119 }; 120 121 // controls whether to continue or stop the json parser 122 enum ndb_idres { 123 NDB_IDRES_CONT, 124 NDB_IDRES_STOP, 125 }; 126 127 // closure data for the id-detecting ingest controller 128 struct ndb_ingest_controller 129 { 130 MDB_txn *read_txn; 131 struct ndb_lmdb *lmdb; 132 struct ndb_note *note; 133 uint64_t note_key; 134 }; 135 136 enum ndb_writer_msgtype { 137 NDB_WRITER_QUIT, // kill thread immediately 138 NDB_WRITER_NOTE, // write a note to the db 139 NDB_WRITER_PROFILE, // write a profile to the db 140 NDB_WRITER_DBMETA, // write ndb metadata 141 NDB_WRITER_PROFILE_LAST_FETCH, // when profiles were last fetched 142 NDB_WRITER_BLOCKS, // write parsed note blocks 143 NDB_WRITER_MIGRATE, // migrate the database 144 NDB_WRITER_NOTE_RELAY, // we already have the note, but we have more relays to write 145 }; 146 147 // keys used for storing data in the NDB metadata database (NDB_DB_NDB_META) 148 enum ndb_meta_key { 149 NDB_META_KEY_VERSION = 1 150 }; 151 152 struct ndb_json_parser { 153 const char *json; 154 int json_len; 155 struct ndb_builder builder; 156 jsmn_parser json_parser; 157 jsmntok_t *toks, *toks_end; 158 int i; 159 int num_tokens; 160 }; 161 162 // useful to pass to threads on its own 163 struct ndb_lmdb { 164 MDB_env *env; 165 MDB_dbi dbs[NDB_DBS]; 166 }; 167 168 struct ndb_writer { 169 struct ndb_lmdb *lmdb; 170 struct ndb_monitor *monitor; 171 172 int scratch_size; 173 uint32_t ndb_flags; 174 void *queue_buf; 175 int queue_buflen; 176 pthread_t thread_id; 177 178 struct prot_queue inbox; 179 }; 180 181 struct ndb_ingester { 182 struct ndb_lmdb *lmdb; 183 uint32_t flags; 184 struct threadpool tp; 185 struct prot_queue *writer_inbox; 186 void *filter_context; 187 ndb_ingest_filter_fn filter; 188 189 int scratch_size; 190 }; 191 192 struct ndb_filter_group { 193 struct ndb_filter filters[MAX_FILTERS]; 194 int num_filters; 195 }; 196 197 struct ndb_subscription { 198 uint64_t subid; 199 struct ndb_filter_group group; 200 struct prot_queue inbox; 201 }; 202 203 struct ndb_monitor { 204 struct ndb_subscription subscriptions[MAX_SUBSCRIPTIONS]; 205 ndb_sub_fn sub_cb; 206 void *sub_cb_ctx; 207 int num_subscriptions; 208 209 // monitor isn't a full inbox. We want pollers to be able to poll 210 // subscriptions efficiently without going through a message queue, so 211 // we use a simple mutex here. 212 pthread_mutex_t mutex; 213 }; 214 215 struct ndb { 216 struct ndb_lmdb lmdb; 217 struct ndb_ingester ingester; 218 struct ndb_monitor monitor; 219 struct ndb_writer writer; 220 int version; 221 uint32_t flags; // setting flags 222 // lmdb environ handles, etc 223 }; 224 225 /// 226 /// Query Plans 227 /// 228 /// There are general strategies for performing certain types of query 229 /// depending on the filter. For example, for large contact list queries 230 /// with many authors, we simply do a descending scan on created_at 231 /// instead of doing 1000s of pubkey scans. 232 /// 233 /// Query plans are calculated from filters via `ndb_filter_plan` 234 /// 235 enum ndb_query_plan { 236 NDB_PLAN_KINDS, 237 NDB_PLAN_IDS, 238 NDB_PLAN_AUTHORS, 239 NDB_PLAN_AUTHOR_KINDS, 240 NDB_PLAN_CREATED, 241 NDB_PLAN_TAGS, 242 NDB_PLAN_SEARCH, 243 NDB_PLAN_RELAY_KINDS, 244 NDB_PLAN_PROFILE_SEARCH, 245 }; 246 247 // A id + u64 + timestamp 248 struct ndb_id_u64_ts { 249 unsigned char id[32]; // pubkey, id, etc 250 uint64_t u64; // kind, etc 251 uint64_t timestamp; 252 }; 253 254 // A clustered key with an id and a timestamp 255 struct ndb_tsid { 256 unsigned char id[32]; 257 uint64_t timestamp; 258 }; 259 260 // A u64 + timestamp id. Just using this for kinds at the moment. 261 struct ndb_u64_ts { 262 uint64_t u64; // kind, etc 263 uint64_t timestamp; 264 }; 265 266 struct ndb_word 267 { 268 const char *word; 269 int word_len; 270 }; 271 272 struct ndb_search_words 273 { 274 struct ndb_word words[MAX_TEXT_SEARCH_WORDS]; 275 int num_words; 276 }; 277 278 279 static inline int is_replaceable_kind(uint64_t kind) 280 { 281 return kind == 0 || kind == 3 282 || (10000 <= kind && kind < 20000) 283 || (30000 <= kind && kind < 40000); 284 } 285 286 287 // ndb_text_search_key 288 // 289 // This is compressed when in lmdb: 290 // 291 // note_id: varint 292 // strlen: varint 293 // str: cstr 294 // timestamp: varint 295 // word_index: varint 296 // 297 static int ndb_make_text_search_key(unsigned char *buf, int bufsize, 298 int word_index, int word_len, const char *str, 299 uint64_t timestamp, uint64_t note_id, 300 int *keysize) 301 { 302 struct cursor cur; 303 make_cursor(buf, buf + bufsize, &cur); 304 305 // TODO: need update this to uint64_t 306 // we push this first because our query function can pull this off 307 // quickly to check matches 308 if (!cursor_push_varint(&cur, (int32_t)note_id)) 309 return 0; 310 311 // string length 312 if (!cursor_push_varint(&cur, word_len)) 313 return 0; 314 315 // non-null terminated, lowercase string 316 if (!cursor_push_lowercase(&cur, str, word_len)) 317 return 0; 318 319 // TODO: need update this to uint64_t 320 if (!cursor_push_varint(&cur, (int)timestamp)) 321 return 0; 322 323 // the index of the word in the content so that we can do more accurate 324 // phrase searches 325 if (!cursor_push_varint(&cur, word_index)) 326 return 0; 327 328 // pad to 8-byte alignment 329 if (!cursor_align(&cur, 8)) 330 return 0; 331 332 *keysize = cur.p - cur.start; 333 assert((*keysize % 8) == 0); 334 335 return 1; 336 } 337 338 static int ndb_make_noted_text_search_key(unsigned char *buf, int bufsize, 339 int wordlen, const char *word, 340 uint64_t timestamp, uint64_t note_id, 341 int *keysize) 342 { 343 return ndb_make_text_search_key(buf, bufsize, 0, wordlen, word, 344 timestamp, note_id, keysize); 345 } 346 347 static int ndb_make_text_search_key_low(unsigned char *buf, int bufsize, 348 int wordlen, const char *word, 349 uint64_t since, 350 int *keysize) 351 { 352 uint64_t note_id; 353 note_id = 0; 354 return ndb_make_text_search_key(buf, bufsize, 0, wordlen, word, 355 since, note_id, keysize); 356 } 357 358 static int ndb_make_text_search_key_high(unsigned char *buf, int bufsize, 359 int wordlen, const char *word, 360 uint64_t until, 361 int *keysize) 362 { 363 uint64_t note_id; 364 note_id = INT32_MAX; 365 return ndb_make_text_search_key(buf, bufsize, 0, wordlen, word, 366 until, note_id, keysize); 367 } 368 369 typedef int (*ndb_text_search_key_order_fn)(unsigned char *buf, int bufsize, int wordlen, const char *word, uint64_t timestamp, int *keysize); 370 371 /** From LMDB: Compare two items lexically */ 372 static int mdb_cmp_memn(const MDB_val *a, const MDB_val *b) { 373 int diff; 374 ssize_t len_diff; 375 unsigned int len; 376 377 len = a->mv_size; 378 len_diff = (ssize_t) a->mv_size - (ssize_t) b->mv_size; 379 if (len_diff > 0) { 380 len = b->mv_size; 381 len_diff = 1; 382 } 383 384 diff = memcmp(a->mv_data, b->mv_data, len); 385 return diff ? diff : len_diff<0 ? -1 : len_diff; 386 } 387 388 static int ndb_tag_key_compare(const MDB_val *a, const MDB_val *b) 389 { 390 MDB_val va, vb; 391 uint64_t ts_a, ts_b; 392 int cmp; 393 394 va.mv_data = a->mv_data; 395 va.mv_size = a->mv_size - 8; 396 397 vb.mv_data = b->mv_data; 398 vb.mv_size = b->mv_size - 8; 399 400 if ((cmp = mdb_cmp_memn(&va, &vb))) 401 return cmp; 402 403 ts_a = *(uint64_t*)((unsigned char *)va.mv_data + va.mv_size); 404 ts_b = *(uint64_t*)((unsigned char *)vb.mv_data + vb.mv_size); 405 406 if (ts_a < ts_b) 407 return -1; 408 else if (ts_a > ts_b) 409 return 1; 410 411 return 0; 412 } 413 414 static int ndb_text_search_key_compare(const MDB_val *a, const MDB_val *b) 415 { 416 struct cursor ca, cb; 417 uint64_t sa, sb, nid_a, nid_b; 418 MDB_val a2, b2; 419 420 make_cursor(a->mv_data, (unsigned char *)a->mv_data + a->mv_size, &ca); 421 make_cursor(b->mv_data, (unsigned char *)b->mv_data + b->mv_size, &cb); 422 423 // note_id 424 if (unlikely(!cursor_pull_varint(&ca, &nid_a) || !cursor_pull_varint(&cb, &nid_b))) 425 return 0; 426 427 // string size 428 if (unlikely(!cursor_pull_varint(&ca, &sa) || !cursor_pull_varint(&cb, &sb))) 429 return 0; 430 431 a2.mv_data = ca.p; 432 a2.mv_size = sa; 433 434 b2.mv_data = cb.p; 435 b2.mv_size = sb; 436 437 int cmp = mdb_cmp_memn(&a2, &b2); 438 if (cmp) return cmp; 439 440 // skip over string 441 ca.p += sa; 442 cb.p += sb; 443 444 // timestamp 445 if (unlikely(!cursor_pull_varint(&ca, &sa) || !cursor_pull_varint(&cb, &sb))) 446 return 0; 447 448 if (sa < sb) return -1; 449 else if (sa > sb) return 1; 450 451 // note_id 452 if (nid_a < nid_b) return -1; 453 else if (nid_a > nid_b) return 1; 454 455 // word index 456 if (unlikely(!cursor_pull_varint(&ca, &sa) || !cursor_pull_varint(&cb, &sb))) 457 return 0; 458 459 if (sa < sb) return -1; 460 else if (sa > sb) return 1; 461 462 return 0; 463 } 464 465 static inline int ndb_unpack_text_search_key_noteid( 466 struct cursor *cur, uint64_t *note_id) 467 { 468 if (!cursor_pull_varint(cur, note_id)) 469 return 0; 470 471 return 1; 472 } 473 474 // faster peek of just the string instead of unpacking everything 475 // this is used to quickly discard range query matches if there is no 476 // common prefix 477 static inline int ndb_unpack_text_search_key_string(struct cursor *cur, 478 const char **str, 479 int *str_len) 480 { 481 uint64_t len; 482 483 if (!cursor_pull_varint(cur, &len)) 484 return 0; 485 486 *str_len = len; 487 488 *str = (const char *)cur->p; 489 490 if (!cursor_skip(cur, *str_len)) 491 return 0; 492 493 return 1; 494 } 495 496 // should be called after ndb_unpack_text_search_key_string. It continues 497 // the unpacking of a text search key if we've already started it. 498 static inline int 499 ndb_unpack_remaining_text_search_key(struct cursor *cur, 500 struct ndb_text_search_key *key) 501 { 502 if (!cursor_pull_varint(cur, &key->timestamp)) 503 return 0; 504 505 if (!cursor_pull_varint(cur, &key->word_index)) 506 return 0; 507 508 return 1; 509 } 510 511 // unpack a fulltext search key 512 // 513 // full version of string + unpack remaining. This is split up because text 514 // searching only requires to pull the string for prefix searching, and the 515 // remaining is optional 516 static inline int ndb_unpack_text_search_key(unsigned char *p, int len, 517 struct ndb_text_search_key *key) 518 { 519 struct cursor c; 520 make_cursor(p, p + len, &c); 521 522 if (!ndb_unpack_text_search_key_noteid(&c, &key->note_id)) 523 return 0; 524 525 if (!ndb_unpack_text_search_key_string(&c, &key->str, &key->str_len)) 526 return 0; 527 528 return ndb_unpack_remaining_text_search_key(&c, key); 529 } 530 531 // Copies only lowercase characters to the destination string and fills the rest with null bytes. 532 // `dst` and `src` are pointers to the destination and source strings, respectively. 533 // `n` is the maximum number of characters to copy. 534 static void lowercase_strncpy(char *dst, const char *src, int n) { 535 int j = 0, i = 0; 536 537 if (!dst || !src || n == 0) { 538 return; 539 } 540 541 while (src[i] != '\0' && j < n) { 542 dst[j++] = tolower(src[i++]); 543 } 544 545 // Null-terminate and fill the destination string 546 while (j < n) { 547 dst[j++] = '\0'; 548 } 549 } 550 551 static inline int ndb_filter_elem_is_ptr(struct ndb_filter_field *field) { 552 return field->elem_type == NDB_ELEMENT_STRING || field->elem_type == NDB_ELEMENT_ID; 553 } 554 555 // Copy the filter 556 int ndb_filter_clone(struct ndb_filter *dst, struct ndb_filter *src) 557 { 558 size_t src_size, elem_size, data_size; 559 560 memcpy(dst, src, sizeof(*src)); 561 562 elem_size = src->elem_buf.end - src->elem_buf.start; 563 data_size = src->data_buf.end - src->data_buf.start; 564 src_size = data_size + elem_size; 565 566 // let's only allow finalized filters to be cloned 567 if (!src || !src->finalized) 568 return 0; 569 570 dst->elem_buf.start = malloc(src_size); 571 dst->elem_buf.end = dst->elem_buf.start + elem_size; 572 dst->elem_buf.p = dst->elem_buf.end; 573 574 dst->data_buf.start = dst->elem_buf.start + elem_size; 575 dst->data_buf.end = dst->data_buf.start + data_size; 576 dst->data_buf.p = dst->data_buf.end; 577 578 if (dst->elem_buf.start == NULL) 579 return 0; 580 581 memcpy(dst->elem_buf.start, src->elem_buf.start, src_size); 582 583 return 1; 584 } 585 586 // "Finalize" the filter. This resizes the allocated heap buffers so that they 587 // are as small as possible. This also prevents new fields from being added 588 int ndb_filter_end(struct ndb_filter *filter) 589 { 590 #ifdef NDB_LOG 591 size_t orig_size; 592 #endif 593 size_t data_len, elem_len; 594 unsigned char *rel; 595 596 assert(filter); 597 assert(filter->elem_buf.start); 598 599 if (filter->finalized == 1) 600 return 0; 601 602 // move the data buffer to the end of the element buffer and update 603 // all of the element pointers accordingly 604 data_len = filter->data_buf.p - filter->data_buf.start; 605 elem_len = filter->elem_buf.p - filter->elem_buf.start; 606 #ifdef NDB_LOG 607 orig_size = filter->data_buf.end - filter->elem_buf.start; 608 #endif 609 610 // cap the elem buff 611 filter->elem_buf.end = filter->elem_buf.p; 612 613 // move the data buffer to the end of the element buffer 614 memmove(filter->elem_buf.p, filter->data_buf.start, data_len); 615 616 // realloc the whole thing 617 rel = realloc(filter->elem_buf.start, elem_len + data_len); 618 if (rel) 619 filter->elem_buf.start = rel; 620 assert(filter->elem_buf.start); 621 filter->elem_buf.end = filter->elem_buf.start + elem_len; 622 filter->elem_buf.p = filter->elem_buf.end; 623 624 filter->data_buf.start = filter->elem_buf.end; 625 filter->data_buf.end = filter->data_buf.start + data_len; 626 filter->data_buf.p = filter->data_buf.end; 627 628 filter->finalized = 1; 629 630 ndb_debug("ndb_filter_end: %ld -> %ld\n", orig_size, elem_len + data_len); 631 632 return 1; 633 } 634 635 static inline struct ndb_filter_elements * 636 ndb_filter_get_elements_by_offset(const struct ndb_filter *filter, int offset) 637 { 638 struct ndb_filter_elements *els; 639 640 if (offset < 0) 641 return NULL; 642 643 els = (struct ndb_filter_elements *)(filter->elem_buf.start + offset); 644 645 if ((unsigned char *)els > filter->elem_buf.p) 646 return NULL; 647 648 return els; 649 } 650 651 struct ndb_filter_elements * 652 ndb_filter_current_element(const struct ndb_filter *filter) 653 { 654 return ndb_filter_get_elements_by_offset(filter, filter->current); 655 } 656 657 struct ndb_filter_elements * 658 ndb_filter_get_elements(const struct ndb_filter *filter, int index) 659 { 660 if (filter->num_elements <= 0) 661 return NULL; 662 663 if (index > filter->num_elements-1) 664 return NULL; 665 666 return ndb_filter_get_elements_by_offset(filter, filter->elements[index]); 667 } 668 669 static inline unsigned char * 670 ndb_filter_elements_data(const struct ndb_filter *filter, int offset) 671 { 672 unsigned char *data; 673 674 if (offset < 0) 675 return NULL; 676 677 data = filter->data_buf.start + offset; 678 if (data > filter->data_buf.p) 679 return NULL; 680 681 return data; 682 } 683 684 struct ndb_filter_custom * 685 ndb_filter_get_custom_element(const struct ndb_filter *filter, const struct ndb_filter_elements *els) 686 { 687 return (struct ndb_filter_custom *)ndb_filter_elements_data(filter, els->elements[0]); 688 } 689 690 unsigned char * 691 ndb_filter_get_id_element(const struct ndb_filter *filter, const struct ndb_filter_elements *els, int index) 692 { 693 return ndb_filter_elements_data(filter, els->elements[index]); 694 } 695 696 const char * 697 ndb_filter_get_string_element(const struct ndb_filter *filter, const struct ndb_filter_elements *els, int index) 698 { 699 return (const char *)ndb_filter_elements_data(filter, els->elements[index]); 700 } 701 702 uint64_t * 703 ndb_filter_get_int_element_ptr(struct ndb_filter_elements *els, int index) 704 { 705 return &els->elements[index]; 706 } 707 708 uint64_t 709 ndb_filter_get_int_element(const struct ndb_filter_elements *els, int index) 710 { 711 return els->elements[index]; 712 } 713 714 int ndb_filter_init_with(struct ndb_filter *filter, int pages) 715 { 716 struct cursor cur; 717 int page_size, elem_size, data_size, buf_size; 718 719 page_size = 4096; // assuming this, not a big deal if we're wrong 720 721 buf_size = page_size * pages; 722 elem_size = buf_size / 4; 723 data_size = buf_size - elem_size; 724 725 unsigned char *buf = malloc(buf_size); 726 if (!buf) 727 return 0; 728 729 // init memory arena for the cursor 730 make_cursor(buf, buf + buf_size, &cur); 731 732 cursor_slice(&cur, &filter->elem_buf, elem_size); 733 cursor_slice(&cur, &filter->data_buf, data_size); 734 735 // make sure we are fully allocated 736 assert(cur.p == cur.end); 737 738 // make sure elem_buf is the start of the buffer 739 assert(filter->elem_buf.start == cur.start); 740 741 filter->num_elements = 0; 742 filter->elements[0] = 0; 743 filter->current = -1; 744 filter->finalized = 0; 745 746 return 1; 747 } 748 749 int ndb_filter_init(struct ndb_filter *filter) { 750 return ndb_filter_init_with(filter, NDB_FILTER_PAGES); 751 } 752 753 void ndb_filter_destroy(struct ndb_filter *filter) 754 { 755 if (filter->elem_buf.start) 756 free(filter->elem_buf.start); 757 758 memset(filter, 0, sizeof(*filter)); 759 } 760 761 static const char *ndb_filter_field_name(enum ndb_filter_fieldtype field) 762 { 763 switch (field) { 764 case NDB_FILTER_IDS: return "ids"; 765 case NDB_FILTER_AUTHORS: return "authors"; 766 case NDB_FILTER_KINDS: return "kinds"; 767 case NDB_FILTER_TAGS: return "tags"; 768 case NDB_FILTER_SINCE: return "since"; 769 case NDB_FILTER_UNTIL: return "until"; 770 case NDB_FILTER_LIMIT: return "limit"; 771 case NDB_FILTER_SEARCH: return "search"; 772 case NDB_FILTER_RELAYS: return "relays"; 773 case NDB_FILTER_CUSTOM: return "custom"; 774 } 775 776 return "unknown"; 777 } 778 779 static int ndb_filter_start_field_impl(struct ndb_filter *filter, enum ndb_filter_fieldtype field, char tag) 780 { 781 int i; 782 struct ndb_filter_elements *els, *el; 783 784 if (ndb_filter_current_element(filter)) { 785 fprintf(stderr, "ndb_filter_start_field: filter field already in progress, did you forget to call ndb_filter_end_field?\n"); 786 return 0; 787 } 788 789 // you can only start and end fields once 790 for (i = 0; i < filter->num_elements; i++) { 791 el = ndb_filter_get_elements(filter, i); 792 assert(el); 793 // TODO: fix this tags check to try to find matching tags 794 if (el->field.type == field && field != NDB_FILTER_TAGS) { 795 fprintf(stderr, "ndb_filter_start_field: field '%s' already exists\n", 796 ndb_filter_field_name(field)); 797 return 0; 798 } 799 } 800 801 filter->current = filter->elem_buf.p - filter->elem_buf.start; 802 els = ndb_filter_current_element(filter); 803 assert(els); 804 805 // advance elem buffer to the variable data section 806 if (!cursor_skip(&filter->elem_buf, sizeof(struct ndb_filter_elements))) { 807 fprintf(stderr, "ndb_filter_start_field: '%s' oom (todo: realloc?)\n", 808 ndb_filter_field_name(field)); 809 return 0; 810 } 811 812 els->field.type = field; 813 els->field.tag = tag; 814 els->field.elem_type = 0; 815 els->count = 0; 816 817 return 1; 818 } 819 820 int ndb_filter_start_field(struct ndb_filter *filter, enum ndb_filter_fieldtype field) 821 { 822 return ndb_filter_start_field_impl(filter, field, 0); 823 } 824 825 int ndb_filter_start_tag_field(struct ndb_filter *filter, char tag) 826 { 827 return ndb_filter_start_field_impl(filter, NDB_FILTER_TAGS, tag); 828 } 829 830 static int ndb_filter_add_element(struct ndb_filter *filter, union ndb_filter_element el) 831 { 832 struct ndb_filter_elements *current; 833 uint64_t offset; 834 835 if (!(current = ndb_filter_current_element(filter))) 836 return 0; 837 838 offset = filter->data_buf.p - filter->data_buf.start; 839 840 switch (current->field.type) { 841 case NDB_FILTER_CUSTOM: 842 if (!cursor_push(&filter->data_buf, (unsigned char *)&el, sizeof(el))) 843 return 0; 844 break; 845 case NDB_FILTER_IDS: 846 case NDB_FILTER_AUTHORS: 847 if (!cursor_push(&filter->data_buf, (unsigned char *)el.id, 32)) 848 return 0; 849 break; 850 case NDB_FILTER_KINDS: 851 offset = el.integer; 852 break; 853 case NDB_FILTER_SINCE: 854 case NDB_FILTER_UNTIL: 855 case NDB_FILTER_LIMIT: 856 // only one allowed for since/until 857 if (current->count != 0) 858 return 0; 859 offset = el.integer; 860 break; 861 case NDB_FILTER_SEARCH: 862 if (current->field.elem_type != NDB_ELEMENT_STRING) { 863 return 0; 864 } 865 if (!cursor_push(&filter->data_buf, (unsigned char *)el.string.string, el.string.len)) 866 return 0; 867 if (!cursor_push_byte(&filter->data_buf, 0)) 868 return 0; 869 break; 870 case NDB_FILTER_TAGS: 871 switch (current->field.elem_type) { 872 case NDB_ELEMENT_ID: 873 if (!cursor_push(&filter->data_buf, (unsigned char *)el.id, 32)) 874 return 0; 875 break; 876 case NDB_ELEMENT_STRING: 877 if (!cursor_push(&filter->data_buf, (unsigned char *)el.string.string, el.string.len)) 878 return 0; 879 if (!cursor_push_byte(&filter->data_buf, 0)) 880 return 0; 881 break; 882 case NDB_ELEMENT_INT: 883 // ints are not allowed in tag filters 884 case NDB_ELEMENT_UNKNOWN: 885 case NDB_ELEMENT_CUSTOM: 886 return 0; 887 } 888 // push a pointer of the string in the databuf as an element 889 break; 890 case NDB_FILTER_RELAYS: 891 if (current->field.elem_type != NDB_ELEMENT_STRING) { 892 return 0; 893 } 894 if (!cursor_push(&filter->data_buf, (unsigned char *)el.string.string, el.string.len)) 895 return 0; 896 if (!cursor_push_byte(&filter->data_buf, 0)) 897 return 0; 898 break; 899 } 900 901 if (!cursor_push(&filter->elem_buf, (unsigned char *)&offset, 902 sizeof(offset))) { 903 return 0; 904 } 905 906 current->count++; 907 908 return 1; 909 } 910 911 static int ndb_filter_set_elem_type(struct ndb_filter *filter, 912 enum ndb_generic_element_type elem_type) 913 { 914 enum ndb_generic_element_type current_elem_type; 915 struct ndb_filter_elements *current; 916 917 if (!(current = ndb_filter_current_element(filter))) 918 return 0; 919 920 current_elem_type = current->field.elem_type; 921 922 // element types must be uniform 923 if (current_elem_type != elem_type && current_elem_type != NDB_ELEMENT_UNKNOWN) { 924 fprintf(stderr, "ndb_filter_set_elem_type: element types must be uniform\n"); 925 return 0; 926 } 927 928 current->field.elem_type = elem_type; 929 930 return 1; 931 } 932 933 934 int ndb_filter_add_str_element_len(struct ndb_filter *filter, const char *str, int len) 935 { 936 union ndb_filter_element el; 937 struct ndb_filter_elements *current; 938 939 if (!(current = ndb_filter_current_element(filter))) 940 return 0; 941 942 // only generic tags and search queries are allowed to have strings 943 switch (current->field.type) { 944 case NDB_FILTER_SINCE: 945 case NDB_FILTER_UNTIL: 946 case NDB_FILTER_LIMIT: 947 case NDB_FILTER_IDS: 948 case NDB_FILTER_AUTHORS: 949 case NDB_FILTER_KINDS: 950 case NDB_FILTER_CUSTOM: 951 return 0; 952 case NDB_FILTER_SEARCH: 953 if (current->count == 1) { 954 // you can't add more than one string to a search 955 return 0; 956 } 957 break; 958 case NDB_FILTER_RELAYS: 959 case NDB_FILTER_TAGS: 960 break; 961 } 962 963 if (!ndb_filter_set_elem_type(filter, NDB_ELEMENT_STRING)) 964 return 0; 965 966 el.string.string = str; 967 el.string.len = len; 968 969 return ndb_filter_add_element(filter, el); 970 } 971 972 int ndb_filter_add_str_element(struct ndb_filter *filter, const char *str) 973 { 974 return ndb_filter_add_str_element_len(filter, str, strlen(str)); 975 } 976 977 int ndb_filter_add_custom_filter_element(struct ndb_filter *filter, ndb_filter_callback_fn *cb, void *ctx) 978 { 979 union ndb_filter_element el; 980 struct ndb_filter_elements *current; 981 struct ndb_filter_custom custom; 982 983 custom.cb = cb; 984 custom.ctx = ctx; 985 986 if (!(current = ndb_filter_current_element(filter))) 987 return 0; 988 989 switch (current->field.type) { 990 case NDB_FILTER_CUSTOM: 991 break; 992 case NDB_FILTER_IDS: 993 case NDB_FILTER_AUTHORS: 994 case NDB_FILTER_TAGS: 995 case NDB_FILTER_SEARCH: 996 case NDB_FILTER_RELAYS: 997 case NDB_FILTER_KINDS: 998 case NDB_FILTER_SINCE: 999 case NDB_FILTER_UNTIL: 1000 case NDB_FILTER_LIMIT: 1001 return 0; 1002 } 1003 1004 if (!ndb_filter_set_elem_type(filter, NDB_ELEMENT_CUSTOM)) 1005 return 0; 1006 1007 el.custom_filter = custom; 1008 1009 return ndb_filter_add_element(filter, el); 1010 } 1011 1012 int ndb_filter_add_int_element(struct ndb_filter *filter, uint64_t integer) 1013 { 1014 union ndb_filter_element el; 1015 struct ndb_filter_elements *current; 1016 if (!(current = ndb_filter_current_element(filter))) 1017 return 0; 1018 1019 switch (current->field.type) { 1020 case NDB_FILTER_IDS: 1021 case NDB_FILTER_AUTHORS: 1022 case NDB_FILTER_TAGS: 1023 case NDB_FILTER_SEARCH: 1024 case NDB_FILTER_RELAYS: 1025 case NDB_FILTER_CUSTOM: 1026 return 0; 1027 case NDB_FILTER_KINDS: 1028 case NDB_FILTER_SINCE: 1029 case NDB_FILTER_UNTIL: 1030 case NDB_FILTER_LIMIT: 1031 break; 1032 } 1033 1034 if (!ndb_filter_set_elem_type(filter, NDB_ELEMENT_INT)) 1035 return 0; 1036 1037 el.integer = integer; 1038 1039 return ndb_filter_add_element(filter, el); 1040 } 1041 1042 int ndb_filter_add_id_element(struct ndb_filter *filter, const unsigned char *id) 1043 { 1044 union ndb_filter_element el; 1045 struct ndb_filter_elements *current; 1046 1047 if (!(current = ndb_filter_current_element(filter))) 1048 return 0; 1049 1050 // only certain filter types allow pushing id elements 1051 switch (current->field.type) { 1052 case NDB_FILTER_SINCE: 1053 case NDB_FILTER_UNTIL: 1054 case NDB_FILTER_LIMIT: 1055 case NDB_FILTER_KINDS: 1056 case NDB_FILTER_SEARCH: 1057 case NDB_FILTER_RELAYS: 1058 case NDB_FILTER_CUSTOM: 1059 return 0; 1060 case NDB_FILTER_IDS: 1061 case NDB_FILTER_AUTHORS: 1062 case NDB_FILTER_TAGS: 1063 break; 1064 } 1065 1066 if (!ndb_filter_set_elem_type(filter, NDB_ELEMENT_ID)) 1067 return 0; 1068 1069 // this is needed so that generic filters know its an id 1070 el.id = id; 1071 1072 return ndb_filter_add_element(filter, el); 1073 } 1074 1075 static int ndb_tag_filter_matches(struct ndb_filter *filter, 1076 struct ndb_filter_elements *els, 1077 struct ndb_note *note) 1078 { 1079 int i; 1080 const unsigned char *id; 1081 const char *el_str; 1082 struct ndb_iterator iter, *it = &iter; 1083 struct ndb_str str; 1084 1085 ndb_tags_iterate_start(note, it); 1086 1087 while (ndb_tags_iterate_next(it)) { 1088 // we're looking for tags with 2 or more entries: ["p", id], etc 1089 if (it->tag->count < 2) 1090 continue; 1091 1092 str = ndb_tag_str(note, it->tag, 0); 1093 1094 // we only care about packed strings (single char, etc) 1095 if (str.flag != NDB_PACKED_STR) 1096 continue; 1097 1098 // do we have #e matching e (or p, etc) 1099 if (str.str[0] != els->field.tag || str.str[1] != 0) 1100 continue; 1101 1102 str = ndb_tag_str(note, it->tag, 1); 1103 1104 switch (els->field.elem_type) { 1105 case NDB_ELEMENT_ID: 1106 // if our filter element type is an id, then we 1107 // expect a packed id in the tag, otherwise skip 1108 if (str.flag != NDB_PACKED_ID) 1109 continue; 1110 break; 1111 case NDB_ELEMENT_STRING: 1112 // if our filter element type is a string, then 1113 // we should not expect an id 1114 if (str.flag == NDB_PACKED_ID) 1115 continue; 1116 1117 break; 1118 case NDB_ELEMENT_UNKNOWN: 1119 default: 1120 // For some reason the element type is not set. It's 1121 // possible nothing was added to the generic filter? 1122 // Let's just fail here and log a note for debugging 1123 fprintf(stderr, "UNUSUAL ndb_tag_filter_matches: have unknown element type %d\n", els->field.elem_type); 1124 return 0; 1125 } 1126 1127 for (i = 0; i < els->count; i++) { 1128 switch (els->field.elem_type) { 1129 case NDB_ELEMENT_ID: 1130 id = ndb_filter_get_id_element(filter, els, i); 1131 if (!memcmp(id, str.id, 32)) 1132 return 1; 1133 break; 1134 case NDB_ELEMENT_STRING: 1135 el_str = ndb_filter_get_string_element(filter, els, i); 1136 if (!strcmp(el_str, str.str)) 1137 return 1; 1138 break; 1139 case NDB_ELEMENT_INT: 1140 // int elements int tag queries are not supported 1141 case NDB_ELEMENT_UNKNOWN: 1142 case NDB_ELEMENT_CUSTOM: 1143 return 0; 1144 } 1145 } 1146 } 1147 1148 return 0; 1149 } 1150 1151 struct search_id_state { 1152 struct ndb_filter *filter; 1153 struct ndb_filter_elements *els; 1154 unsigned char *key; 1155 }; 1156 1157 static int compare_ids(const void *pa, const void *pb) 1158 { 1159 unsigned char *a = *(unsigned char **)pa; 1160 unsigned char *b = *(unsigned char **)pb; 1161 1162 return memcmp(a, b, 32); 1163 } 1164 1165 static int compare_strs(const void *pa, const void *pb) 1166 { 1167 const char *a = *(const char **)pa; 1168 const char *b = *(const char **)pb; 1169 1170 return strcmp(a, b); 1171 } 1172 1173 static int search_strs(const void *ctx, const void *mstr_ptr) 1174 { 1175 // we reuse search_id_state here and just cast to (const char *) when 1176 // needed 1177 struct search_id_state *state; 1178 const char *mstr_str; 1179 uint32_t mstr; 1180 1181 state = (struct search_id_state *)ctx; 1182 mstr = *(uint32_t *)mstr_ptr; 1183 1184 mstr_str = (const char *)ndb_filter_elements_data(state->filter, mstr); 1185 assert(mstr_str); 1186 1187 return strcmp((const char *)state->key, mstr_str); 1188 } 1189 1190 static int search_ids(const void *ctx, const void *mid_ptr) 1191 { 1192 struct search_id_state *state; 1193 unsigned char *mid_id; 1194 uint32_t mid; 1195 1196 state = (struct search_id_state *)ctx; 1197 mid = *(uint32_t *)mid_ptr; 1198 1199 mid_id = ndb_filter_elements_data(state->filter, mid); 1200 assert(mid_id); 1201 1202 return memcmp(state->key, mid_id, 32); 1203 } 1204 1205 static int compare_kinds(const void *pa, const void *pb) 1206 { 1207 1208 // NOTE: this should match type in `union ndb_filter_element` 1209 uint64_t a = *(uint64_t *)pa; 1210 uint64_t b = *(uint64_t *)pb; 1211 1212 if (a < b) { 1213 return -1; 1214 } else if (a > b) { 1215 return 1; 1216 } else { 1217 return 0; 1218 } 1219 } 1220 1221 // 1222 // returns 1 if a filter matches a note 1223 static int ndb_filter_matches_with(struct ndb_filter *filter, 1224 struct ndb_note *note, int already_matched, 1225 struct ndb_note_relay_iterator *relay_iter) 1226 { 1227 int i, j; 1228 struct ndb_filter_elements *els; 1229 struct search_id_state state; 1230 struct ndb_filter_custom *custom; 1231 1232 state.filter = filter; 1233 1234 for (i = 0; i < filter->num_elements; i++) { 1235 els = ndb_filter_get_elements(filter, i); 1236 state.els = els; 1237 assert(els); 1238 1239 // if we know we already match from a query scan result, 1240 // we can skip this check 1241 if ((1 << els->field.type) & already_matched) 1242 continue; 1243 1244 switch (els->field.type) { 1245 case NDB_FILTER_KINDS: 1246 for (j = 0; j < els->count; j++) { 1247 if ((unsigned int)els->elements[j] == note->kind) 1248 goto cont; 1249 } 1250 break; 1251 case NDB_FILTER_RELAYS: 1252 // for each relay the note was seen on, see if any match 1253 if (!relay_iter) { 1254 assert(!"expected relay iterator..."); 1255 break; 1256 } 1257 while ((state.key = (unsigned char *)ndb_note_relay_iterate_next(relay_iter))) { 1258 // relays in filters are always sorted 1259 if (bsearch(&state, &els->elements[0], els->count, 1260 sizeof(els->elements[0]), search_strs)) { 1261 ndb_note_relay_iterate_close(relay_iter); 1262 goto cont; 1263 } 1264 } 1265 ndb_note_relay_iterate_close(relay_iter); 1266 break; 1267 case NDB_FILTER_IDS: 1268 state.key = ndb_note_id(note); 1269 if (bsearch(&state, &els->elements[0], els->count, 1270 sizeof(els->elements[0]), search_ids)) { 1271 continue; 1272 } 1273 break; 1274 case NDB_FILTER_AUTHORS: 1275 state.key = ndb_note_pubkey(note); 1276 if (bsearch(&state, &els->elements[0], els->count, 1277 sizeof(els->elements[0]), search_ids)) { 1278 continue; 1279 } 1280 break; 1281 case NDB_FILTER_TAGS: 1282 if (ndb_tag_filter_matches(filter, els, note)) 1283 continue; 1284 break; 1285 case NDB_FILTER_SINCE: 1286 assert(els->count == 1); 1287 if (note->created_at >= els->elements[0]) 1288 continue; 1289 break; 1290 case NDB_FILTER_UNTIL: 1291 assert(els->count == 1); 1292 if (note->created_at < els->elements[0]) 1293 continue; 1294 break; 1295 case NDB_FILTER_SEARCH: 1296 // TODO: matching search filters will need an accelerated 1297 // data structure, like our minimal perfect hashmap 1298 // idea for mutewords. 1299 // 1300 // We'll also want to store tokenized words in the filter 1301 // itself, so that we can walk over each word and check 1302 // the hashmap to see if the note contains at least 1303 // one word. 1304 // 1305 // For now we always return true, since we assume 1306 // the search index will be walked for these kinds 1307 // of queries. 1308 continue; 1309 case NDB_FILTER_CUSTOM: 1310 custom = ndb_filter_get_custom_element(filter, els); 1311 if (custom->cb(custom->ctx, note)) 1312 continue; 1313 break; 1314 1315 case NDB_FILTER_LIMIT: 1316 cont: 1317 continue; 1318 } 1319 1320 // all need to match 1321 return 0; 1322 } 1323 1324 return 1; 1325 } 1326 1327 int ndb_filter_matches(struct ndb_filter *filter, struct ndb_note *note) 1328 { 1329 return ndb_filter_matches_with(filter, note, 0, NULL); 1330 } 1331 1332 int ndb_filter_matches_with_relay(struct ndb_filter *filter, 1333 struct ndb_note *note, 1334 struct ndb_note_relay_iterator *note_relay_iter) 1335 { 1336 return ndb_filter_matches_with(filter, note, 0, note_relay_iter); 1337 } 1338 1339 // because elements are stored as offsets and qsort doesn't support context, 1340 // we do a dumb thing where we convert elements to pointers and back if we 1341 // are doing an id or string sort 1342 static void sort_filter_elements(struct ndb_filter *filter, 1343 struct ndb_filter_elements *els, 1344 int (*cmp)(const void *, const void *)) 1345 { 1346 int i; 1347 1348 assert(ndb_filter_elem_is_ptr(&els->field)); 1349 1350 for (i = 0; i < els->count; i++) 1351 els->elements[i] += (uint64_t)filter->data_buf.start; 1352 1353 qsort(&els->elements[0], els->count, sizeof(els->elements[0]), cmp); 1354 1355 for (i = 0; i < els->count; i++) 1356 els->elements[i] -= (uint64_t)filter->data_buf.start; 1357 } 1358 1359 static int ndb_filter_field_eq(struct ndb_filter *a_filt, 1360 struct ndb_filter_elements *a_field, 1361 struct ndb_filter *b_filt, 1362 struct ndb_filter_elements *b_field) 1363 { 1364 int i; 1365 const char *a_str, *b_str; 1366 unsigned char *a_id, *b_id; 1367 uint64_t a_int, b_int; 1368 struct ndb_filter_custom *a_custom, *b_custom; 1369 1370 if (a_field->count != b_field->count) 1371 return 0; 1372 1373 if (a_field->field.type != b_field->field.type) { 1374 ndb_debug("UNUSUAL: field types do not match in ndb_filter_field_eq\n"); 1375 return 0; 1376 } 1377 1378 if (a_field->field.elem_type != b_field->field.elem_type) { 1379 ndb_debug("UNUSUAL: field element types do not match in ndb_filter_field_eq\n"); 1380 return 0; 1381 } 1382 1383 if (a_field->field.elem_type == NDB_ELEMENT_UNKNOWN) { 1384 ndb_debug("UNUSUAL: field element types are unknown\n"); 1385 return 0; 1386 } 1387 1388 for (i = 0; i < a_field->count; i++) { 1389 switch (a_field->field.elem_type) { 1390 case NDB_ELEMENT_CUSTOM: 1391 a_custom = ndb_filter_get_custom_element(a_filt, a_field); 1392 b_custom = ndb_filter_get_custom_element(b_filt, b_field); 1393 if (memcmp(a_custom, b_custom, sizeof(*a_custom))) 1394 return 0; 1395 break; 1396 case NDB_ELEMENT_UNKNOWN: 1397 return 0; 1398 case NDB_ELEMENT_STRING: 1399 a_str = ndb_filter_get_string_element(a_filt, a_field, i); 1400 b_str = ndb_filter_get_string_element(b_filt, b_field, i); 1401 if (strcmp(a_str, b_str)) 1402 return 0; 1403 break; 1404 case NDB_ELEMENT_ID: 1405 a_id = ndb_filter_get_id_element(a_filt, a_field, i); 1406 b_id = ndb_filter_get_id_element(b_filt, b_field, i); 1407 if (memcmp(a_id, b_id, 32)) 1408 return 0; 1409 break; 1410 case NDB_ELEMENT_INT: 1411 a_int = ndb_filter_get_int_element(a_field, i); 1412 b_int = ndb_filter_get_int_element(b_field, i); 1413 if (a_int != b_int) 1414 return 0; 1415 break; 1416 } 1417 } 1418 1419 return 1; 1420 } 1421 1422 void ndb_filter_end_field(struct ndb_filter *filter) 1423 { 1424 int cur_offset; 1425 struct ndb_filter_elements *cur; 1426 1427 cur_offset = filter->current; 1428 1429 if (!(cur = ndb_filter_current_element(filter))) 1430 return; 1431 1432 filter->elements[filter->num_elements++] = cur_offset; 1433 1434 // sort elements for binary search 1435 switch (cur->field.type) { 1436 case NDB_FILTER_IDS: 1437 case NDB_FILTER_AUTHORS: 1438 sort_filter_elements(filter, cur, compare_ids); 1439 break; 1440 case NDB_FILTER_RELAYS: 1441 sort_filter_elements(filter, cur, compare_strs); 1442 break; 1443 case NDB_FILTER_KINDS: 1444 qsort(&cur->elements[0], cur->count, 1445 sizeof(cur->elements[0]), compare_kinds); 1446 break; 1447 case NDB_FILTER_TAGS: 1448 // TODO: generic tag search sorting 1449 break; 1450 case NDB_FILTER_SINCE: 1451 case NDB_FILTER_CUSTOM: 1452 case NDB_FILTER_UNTIL: 1453 case NDB_FILTER_LIMIT: 1454 case NDB_FILTER_SEARCH: 1455 // don't need to sort these 1456 break; 1457 } 1458 1459 filter->current = -1; 1460 1461 } 1462 1463 static void ndb_filter_group_init(struct ndb_filter_group *group) 1464 { 1465 group->num_filters = 0; 1466 } 1467 1468 static int ndb_filter_group_add(struct ndb_filter_group *group, 1469 struct ndb_filter *filter) 1470 { 1471 if (group->num_filters + 1 > MAX_FILTERS) 1472 return 0; 1473 1474 return ndb_filter_clone(&group->filters[group->num_filters++], filter); 1475 } 1476 1477 static int ndb_filter_group_matches(struct ndb_filter_group *group, 1478 struct ndb_note *note) 1479 { 1480 int i; 1481 struct ndb_filter *filter; 1482 1483 if (group->num_filters == 0) 1484 return 1; 1485 1486 for (i = 0; i < group->num_filters; i++) { 1487 filter = &group->filters[i]; 1488 1489 if (ndb_filter_matches(filter, note)) 1490 return 1; 1491 } 1492 1493 return 0; 1494 } 1495 1496 static void ndb_make_search_key(struct ndb_search_key *key, unsigned char *id, 1497 uint64_t timestamp, const char *search) 1498 { 1499 memcpy(key->id, id, 32); 1500 key->timestamp = timestamp; 1501 lowercase_strncpy(key->search, search, sizeof(key->search) - 1); 1502 key->search[sizeof(key->search) - 1] = '\0'; 1503 } 1504 1505 static int ndb_write_profile_search_index(struct ndb_txn *txn, 1506 struct ndb_search_key *index_key, 1507 uint64_t profile_key) 1508 { 1509 int rc; 1510 MDB_val key, val; 1511 1512 key.mv_data = index_key; 1513 key.mv_size = sizeof(*index_key); 1514 val.mv_data = &profile_key; 1515 val.mv_size = sizeof(profile_key); 1516 1517 if ((rc = mdb_put(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_PROFILE_SEARCH], 1518 &key, &val, 0))) 1519 { 1520 ndb_debug("ndb_write_profile_search_index failed: %s\n", 1521 mdb_strerror(rc)); 1522 return 0; 1523 } 1524 1525 return 1; 1526 } 1527 1528 1529 // map usernames and display names to profile keys for user searching 1530 static int ndb_write_profile_search_indices(struct ndb_txn *txn, 1531 struct ndb_note *note, 1532 uint64_t profile_key, 1533 void *profile_root) 1534 { 1535 struct ndb_search_key index; 1536 NdbProfileRecord_table_t profile_record; 1537 NdbProfile_table_t profile; 1538 1539 profile_record = NdbProfileRecord_as_root(profile_root); 1540 profile = NdbProfileRecord_profile_get(profile_record); 1541 1542 const char *name = NdbProfile_name_get(profile); 1543 const char *display_name = NdbProfile_display_name_get(profile); 1544 1545 // words + pubkey + created 1546 if (name) { 1547 ndb_make_search_key(&index, note->pubkey, note->created_at, 1548 name); 1549 if (!ndb_write_profile_search_index(txn, &index, profile_key)) 1550 return 0; 1551 } 1552 1553 if (display_name) { 1554 // don't write the same name/display_name twice 1555 if (name && !strcmp(display_name, name)) { 1556 return 1; 1557 } 1558 ndb_make_search_key(&index, note->pubkey, note->created_at, 1559 display_name); 1560 if (!ndb_write_profile_search_index(txn, &index, profile_key)) 1561 return 0; 1562 } 1563 1564 return 1; 1565 } 1566 1567 static inline void ndb_tsid_init(struct ndb_tsid *key, unsigned char *id, 1568 uint64_t timestamp) 1569 { 1570 memcpy(key->id, id, 32); 1571 key->timestamp = timestamp; 1572 } 1573 1574 static inline void ndb_tsid_low(struct ndb_tsid *key, unsigned char *id) 1575 { 1576 memcpy(key->id, id, 32); 1577 key->timestamp = 0; 1578 } 1579 1580 static inline void ndb_u64_ts_init(struct ndb_u64_ts *key, uint64_t integer, 1581 uint64_t timestamp) 1582 { 1583 key->u64 = integer; 1584 key->timestamp = timestamp; 1585 } 1586 1587 // useful for range-searching for the latest key with a clustered created_at timen 1588 static inline void ndb_tsid_high(struct ndb_tsid *key, const unsigned char *id) 1589 { 1590 memcpy(key->id, id, 32); 1591 key->timestamp = UINT64_MAX; 1592 } 1593 1594 static int _ndb_begin_query(struct ndb *ndb, struct ndb_txn *txn, int flags) 1595 { 1596 txn->lmdb = &ndb->lmdb; 1597 MDB_txn **mdb_txn = (MDB_txn **)&txn->mdb_txn; 1598 if (!txn->lmdb->env) 1599 return 0; 1600 return mdb_txn_begin(txn->lmdb->env, NULL, flags, mdb_txn) == 0; 1601 } 1602 1603 int ndb_begin_query(struct ndb *ndb, struct ndb_txn *txn) 1604 { 1605 return _ndb_begin_query(ndb, txn, MDB_RDONLY); 1606 } 1607 1608 // this should only be used in migrations, etc 1609 static int ndb_begin_rw_query(struct ndb *ndb, struct ndb_txn *txn) 1610 { 1611 return _ndb_begin_query(ndb, txn, 0); 1612 } 1613 1614 static int ndb_db_is_index(enum ndb_dbs index) 1615 { 1616 switch (index) { 1617 case NDB_DB_NOTE: 1618 case NDB_DB_META: 1619 case NDB_DB_PROFILE: 1620 case NDB_DB_NOTE_ID: 1621 case NDB_DB_NDB_META: 1622 case NDB_DB_PROFILE_SEARCH: 1623 case NDB_DB_PROFILE_LAST_FETCH: 1624 case NDB_DB_NOTE_RELAYS: 1625 case NDB_DBS: 1626 return 0; 1627 case NDB_DB_PROFILE_PK: 1628 case NDB_DB_NOTE_KIND: 1629 case NDB_DB_NOTE_TEXT: 1630 case NDB_DB_NOTE_BLOCKS: 1631 case NDB_DB_NOTE_TAGS: 1632 case NDB_DB_NOTE_PUBKEY: 1633 case NDB_DB_NOTE_PUBKEY_KIND: 1634 case NDB_DB_NOTE_RELAY_KIND: 1635 return 1; 1636 } 1637 1638 return 0; 1639 } 1640 1641 static inline void ndb_id_u64_ts_init(struct ndb_id_u64_ts *key, 1642 unsigned char *id, uint64_t iu64, 1643 uint64_t timestamp) 1644 { 1645 memcpy(key->id, id, 32); 1646 key->u64 = iu64; 1647 key->timestamp = timestamp; 1648 } 1649 1650 // formats the relay url buffer for the NDB_DB_NOTE_RELAYS value. It's a 1651 // null terminated string padded to 8 bytes (we must keep the entire database 1652 // aligned to 8 bytes at all times) 1653 static int prepare_relay_buf(char *relay_buf, int bufsize, const char *relay, 1654 int relay_len) 1655 { 1656 struct cursor cur; 1657 1658 // make sure the size of the buffer is aligned 1659 assert((bufsize % 8) == 0); 1660 1661 make_cursor((unsigned char *)relay_buf, (unsigned char *)relay_buf + bufsize, &cur); 1662 1663 // push the relay string 1664 if (!cursor_push(&cur, (unsigned char *)relay, relay_len)) 1665 return 0; 1666 1667 // relay urls are null terminated for convenience 1668 if (!cursor_push_byte(&cur, 0)) 1669 return 0; 1670 1671 // align the buffer 1672 if (!cursor_align(&cur, 8)) 1673 return 0; 1674 1675 return cur.p - cur.start; 1676 } 1677 1678 // Write to the note_id -> relay_url database. This records where notes 1679 // have been seen 1680 static int ndb_write_note_relay(struct ndb_txn *txn, uint64_t note_key, 1681 const char *relay, uint8_t relay_len) 1682 { 1683 char relay_buf[256]; 1684 int rc, len; 1685 MDB_val k, v; 1686 1687 if (relay == NULL || relay_len == 0) { 1688 ndb_debug("relay is NULL in ndb_write_note_relay? '%s' %d\n", relay, relay_len); 1689 return 0; 1690 } 1691 1692 ndb_debug("writing note_relay '%s' for notekey:%" PRIu64 "\n", relay, note_key); 1693 1694 if (!(len = prepare_relay_buf(relay_buf, sizeof(relay_buf), relay, relay_len))) { 1695 fprintf(stderr, "relay url '%s' too large when writing note relay index\n", relay); 1696 return 0; 1697 } 1698 1699 assert((len % 8) == 0); 1700 1701 k.mv_data = ¬e_key; 1702 k.mv_size = sizeof(note_key); 1703 1704 v.mv_data = relay_buf; 1705 v.mv_size = len; 1706 1707 // NODUPDATA is specified so that we don't accidently add duplicate 1708 // key/value pairs 1709 if ((rc = mdb_put(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_NOTE_RELAYS], 1710 &k, &v, MDB_NODUPDATA))) 1711 { 1712 ndb_debug("ndb_write_note_relay failed: %s\n", mdb_strerror(rc)); 1713 return 0; 1714 } 1715 1716 ndb_debug("wrote %d bytes to note relay: '%s'\n", len, relay_buf); 1717 1718 return 1; 1719 } 1720 1721 struct ndb_relay_kind_key { 1722 uint64_t note_key; 1723 uint64_t kind; 1724 uint64_t created_at; 1725 uint8_t relay_len; 1726 const char *relay; 1727 }; 1728 1729 static int ndb_relay_kind_key_init( 1730 struct ndb_relay_kind_key *key, 1731 uint64_t note_key, 1732 uint64_t kind, 1733 uint64_t created_at, 1734 const char *relay) 1735 { 1736 if (relay == NULL) 1737 return 0; 1738 1739 key->relay = relay; 1740 key->relay_len = strlen(relay); 1741 if (key->relay_len > 248) 1742 return 0; 1743 1744 key->note_key = note_key; 1745 key->kind = kind; 1746 key->created_at = created_at; 1747 return 1; 1748 } 1749 1750 1751 // create a range key for a relay kind query 1752 static int ndb_relay_kind_key_init_high( 1753 struct ndb_relay_kind_key *key, 1754 const char *relay, 1755 uint64_t kind, 1756 uint64_t until) 1757 { 1758 return ndb_relay_kind_key_init(key, UINT64_MAX, kind, UINT64_MAX, relay); 1759 } 1760 1761 static void ndb_parse_relay_kind_key(struct ndb_relay_kind_key *key, unsigned char *buf) 1762 { 1763 // WE ARE ASSUMING WE ARE PARSING FROM AN ALIGNED BUFFER HERE 1764 assert((uint64_t)buf % 8 == 0); 1765 // - note_id: 00 + 8 bytes 1766 // - kind: 08 + 8 bytes 1767 // - created_at: 16 + 8 bytes 1768 // - relay_url_size: 24 + 1 byte 1769 // - relay_url: 25 + n byte null-terminated string 1770 // - pad to 8 byte alignment 1771 key->note_key = *(uint64_t*) (buf + 0); 1772 key->kind = *(uint64_t*) (buf + 8); 1773 key->created_at = *(uint64_t*) (buf + 16); 1774 key->relay_len = *(uint8_t*) (buf + 24); 1775 key->relay = (const char*) (buf + 25); 1776 } 1777 1778 static void ndb_debug_relay_kind_key(struct ndb_relay_kind_key *key) 1779 { 1780 ndb_debug("note_key:%" PRIu64 " kind:%" PRIu64 " created_at:%" PRIu64 " '%s'\n", 1781 key->note_key, key->kind, key->created_at, key->relay); 1782 } 1783 1784 static int ndb_build_relay_kind_key(unsigned char *buf, int bufsize, struct ndb_relay_kind_key *key) 1785 { 1786 struct cursor cur; 1787 make_cursor(buf, buf + bufsize, &cur); 1788 1789 if (!cursor_push(&cur, (unsigned char *)&key->note_key, 8)) return 0; 1790 if (!cursor_push(&cur, (unsigned char *)&key->kind, 8)) return 0; 1791 if (!cursor_push(&cur, (unsigned char *)&key->created_at, 8)) return 0; 1792 if (!cursor_push_byte(&cur, key->relay_len)) return 0; 1793 if (!cursor_push(&cur, (unsigned char *)key->relay, key->relay_len)) return 0; 1794 if (!cursor_push_byte(&cur, 0)) return 0; 1795 if (!cursor_align(&cur, 8)) return 0; 1796 1797 assert(((cur.p-cur.start)%8) == 0); 1798 1799 return cur.p - cur.start; 1800 } 1801 1802 static int ndb_write_note_relay_kind_index( 1803 struct ndb_txn *txn, 1804 struct ndb_relay_kind_key *key) 1805 { 1806 // The relay kind key has a layout like so 1807 // 1808 // - note_key: 00 + 8 bytes 1809 // - kind: 08 + 8 bytes 1810 // - created_at: 16 + 8 bytes 1811 // - relay_url_size: 24 + 1 byte 1812 // - relay_url: 25 + n byte null-terminated string 1813 // - pad to 8 byte alignment 1814 1815 unsigned char buf[256]; 1816 int rc, len; 1817 MDB_val k, v; 1818 1819 // come on bro 1820 if (key->relay_len > 248 || key->relay == NULL || key->relay_len == 0) 1821 return 0; 1822 1823 ndb_debug("writing note_relay_kind_index '%s' for notekey:%" PRIu64 "\n", key->relay, key->note_key); 1824 1825 if (!(len = ndb_build_relay_kind_key(buf, sizeof(buf), key))) 1826 return 0; 1827 1828 k.mv_data = buf; 1829 k.mv_size = len; 1830 1831 v.mv_data = NULL; 1832 v.mv_size = 0; 1833 1834 if ((rc = mdb_put(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_NOTE_RELAY_KIND], &k, &v, 0))) { 1835 fprintf(stderr, "write note relay kind index failed: %s\n", 1836 mdb_strerror(rc)); 1837 return 0; 1838 } 1839 1840 return 1; 1841 } 1842 1843 // writes the relay note kind index and the note_id -> relay db 1844 static int ndb_write_note_relay_indexes(struct ndb_txn *txn, struct ndb_relay_kind_key *key) 1845 { 1846 ndb_write_note_relay_kind_index(txn, key); 1847 ndb_write_note_relay(txn, key->note_key, key->relay, key->relay_len); 1848 return 1; 1849 } 1850 1851 static int ndb_write_note_pubkey_index(struct ndb_txn *txn, struct ndb_note *note, 1852 uint64_t note_key) 1853 { 1854 int rc; 1855 struct ndb_tsid key; 1856 MDB_val k, v; 1857 1858 ndb_tsid_init(&key, ndb_note_pubkey(note), ndb_note_created_at(note)); 1859 1860 k.mv_data = &key; 1861 k.mv_size = sizeof(key); 1862 1863 v.mv_data = ¬e_key; 1864 v.mv_size = sizeof(note_key); 1865 1866 if ((rc = mdb_put(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_NOTE_PUBKEY], &k, &v, 0))) { 1867 fprintf(stderr, "write note pubkey index failed: %s\n", 1868 mdb_strerror(rc)); 1869 return 0; 1870 } 1871 1872 return 1; 1873 } 1874 1875 static int ndb_write_note_pubkey_kind_index(struct ndb_txn *txn, 1876 struct ndb_note *note, 1877 uint64_t note_key) 1878 { 1879 int rc; 1880 struct ndb_id_u64_ts key; 1881 MDB_val k, v; 1882 1883 ndb_id_u64_ts_init(&key, ndb_note_pubkey(note), ndb_note_kind(note), 1884 ndb_note_created_at(note)); 1885 1886 k.mv_data = &key; 1887 k.mv_size = sizeof(key); 1888 1889 v.mv_data = ¬e_key; 1890 v.mv_size = sizeof(note_key); 1891 1892 if ((rc = mdb_put(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_NOTE_PUBKEY_KIND], &k, &v, 0))) { 1893 fprintf(stderr, "write note pubkey_kind index failed: %s\n", 1894 mdb_strerror(rc)); 1895 return 0; 1896 } 1897 1898 return 1; 1899 } 1900 1901 1902 static int ndb_rebuild_note_indices(struct ndb_txn *txn, enum ndb_dbs *indices, int num_indices) 1903 { 1904 MDB_val k, v; 1905 MDB_cursor *cur; 1906 int i, drop_dbi, count, rc; 1907 uint64_t note_key; 1908 struct ndb_note *note; 1909 enum ndb_dbs index; 1910 1911 // 0 means empty, not delete the dbi 1912 drop_dbi = 0; 1913 1914 // ensure they are all index dbs 1915 for (i = 0; i < num_indices; i++) { 1916 if (!ndb_db_is_index(indices[i])) { 1917 fprintf(stderr, "ndb_rebuild_note_index: %s is not an index db\n", ndb_db_name(indices[i])); 1918 return -1; 1919 } 1920 } 1921 1922 // empty the index dbs before we rebuild 1923 for (i = 0; i < num_indices; i++) { 1924 index = indices[i]; 1925 if (mdb_drop(txn->mdb_txn, index, drop_dbi)) { 1926 fprintf(stderr, "ndb_rebuild_pubkey_index: mdb_drop failed for %s\n", ndb_db_name(index)); 1927 return -1; 1928 } 1929 } 1930 1931 if ((rc = mdb_cursor_open(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_NOTE], &cur))) { 1932 fprintf(stderr, "ndb_migrate_user_search_indices: mdb_cursor_open failed, error %d\n", rc); 1933 return -1; 1934 } 1935 1936 count = 0; 1937 1938 // loop through all notes and write search indices 1939 while (mdb_cursor_get(cur, &k, &v, MDB_NEXT) == 0) { 1940 note = v.mv_data; 1941 note_key = *((uint64_t*)k.mv_data); 1942 1943 for (i = 0; i < num_indices; i++) { 1944 index = indices[i]; 1945 switch (index) { 1946 case NDB_DB_NOTE: 1947 case NDB_DB_META: 1948 case NDB_DB_PROFILE: 1949 case NDB_DB_NOTE_ID: 1950 case NDB_DB_NDB_META: 1951 case NDB_DB_PROFILE_SEARCH: 1952 case NDB_DB_PROFILE_LAST_FETCH: 1953 case NDB_DB_NOTE_RELAYS: 1954 case NDB_DBS: 1955 // this should never happen since we check at 1956 // the start 1957 count = -1; 1958 goto cleanup; 1959 case NDB_DB_PROFILE_PK: 1960 case NDB_DB_NOTE_KIND: 1961 case NDB_DB_NOTE_TEXT: 1962 case NDB_DB_NOTE_BLOCKS: 1963 case NDB_DB_NOTE_TAGS: 1964 fprintf(stderr, "%s index rebuild not supported yet. sorry.\n", ndb_db_name(index)); 1965 count = -1; 1966 goto cleanup; 1967 case NDB_DB_NOTE_PUBKEY: 1968 if (!ndb_write_note_pubkey_index(txn, note, note_key)) { 1969 count = -1; 1970 goto cleanup; 1971 } 1972 break; 1973 case NDB_DB_NOTE_RELAY_KIND: 1974 fprintf(stderr, "it doesn't make sense to rebuild note relay kind index\n"); 1975 return 0; 1976 case NDB_DB_NOTE_PUBKEY_KIND: 1977 if (!ndb_write_note_pubkey_kind_index(txn, note, note_key)) { 1978 count = -1; 1979 goto cleanup; 1980 } 1981 break; 1982 } 1983 } 1984 1985 count++; 1986 } 1987 1988 cleanup: 1989 mdb_cursor_close(cur); 1990 1991 return count; 1992 } 1993 1994 1995 // Migrations 1996 // 1997 1998 // This was before we had note_profile_pubkey{,_kind} indices. Let's create them. 1999 static int ndb_migrate_profile_indices(struct ndb_txn *txn) 2000 { 2001 int count; 2002 2003 enum ndb_dbs indices[] = {NDB_DB_NOTE_PUBKEY, NDB_DB_NOTE_PUBKEY_KIND}; 2004 if ((count = ndb_rebuild_note_indices(txn, indices, 2)) != -1) { 2005 fprintf(stderr, "migrated %d notes to have pubkey and pubkey_kind indices\n", count); 2006 return 1; 2007 } else { 2008 fprintf(stderr, "error migrating notes to have pubkey and pubkey_kind indices, aborting.\n"); 2009 return 0; 2010 } 2011 } 2012 2013 static int ndb_migrate_user_search_indices(struct ndb_txn *txn) 2014 { 2015 int rc; 2016 MDB_cursor *cur; 2017 MDB_val k, v; 2018 void *profile_root; 2019 NdbProfileRecord_table_t record; 2020 struct ndb_note *note; 2021 uint64_t note_key, profile_key; 2022 size_t len; 2023 int count; 2024 2025 if ((rc = mdb_cursor_open(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_PROFILE], &cur))) { 2026 fprintf(stderr, "ndb_migrate_user_search_indices: mdb_cursor_open failed, error %d\n", rc); 2027 return 0; 2028 } 2029 2030 count = 0; 2031 2032 // loop through all profiles and write search indices 2033 while (mdb_cursor_get(cur, &k, &v, MDB_NEXT) == 0) { 2034 profile_root = v.mv_data; 2035 profile_key = *((uint64_t*)k.mv_data); 2036 record = NdbProfileRecord_as_root(profile_root); 2037 note_key = NdbProfileRecord_note_key(record); 2038 note = ndb_get_note_by_key(txn, note_key, &len); 2039 2040 if (note == NULL) { 2041 continue; 2042 } 2043 2044 if (!ndb_write_profile_search_indices(txn, note, profile_key, 2045 profile_root)) { 2046 fprintf(stderr, "ndb_migrate_user_search_indices: ndb_write_profile_search_indices failed\n"); 2047 continue; 2048 } 2049 2050 count++; 2051 } 2052 2053 fprintf(stderr, "migrated %d profiles to include search indices\n", count); 2054 2055 mdb_cursor_close(cur); 2056 2057 return 1; 2058 } 2059 2060 static int ndb_migrate_lower_user_search_indices(struct ndb_txn *txn) 2061 { 2062 // just drop the search db so we can rebuild it 2063 if (mdb_drop(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_PROFILE_SEARCH], 0)) { 2064 fprintf(stderr, "ndb_migrate_lower_user_search_indices: mdb_drop failed\n"); 2065 return 0; 2066 } 2067 2068 return ndb_migrate_user_search_indices(txn); 2069 } 2070 2071 int ndb_process_profile_note(struct ndb_note *note, struct ndb_profile_record_builder *profile); 2072 2073 2074 int ndb_db_version(struct ndb_txn *txn) 2075 { 2076 uint64_t version, version_key; 2077 MDB_val k, v; 2078 2079 version_key = NDB_META_KEY_VERSION; 2080 k.mv_data = &version_key; 2081 k.mv_size = sizeof(version_key); 2082 2083 if (mdb_get(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_NDB_META], &k, &v)) { 2084 version = -1; 2085 } else { 2086 if (v.mv_size != 8) { 2087 fprintf(stderr, "run_migrations: invalid version size?"); 2088 return 0; 2089 } 2090 version = *((uint64_t*)v.mv_data); 2091 } 2092 2093 return version; 2094 } 2095 2096 // custom pubkey+kind+timestamp comparison function. This is used by lmdb to 2097 // perform b+ tree searches over the pubkey+kind+timestamp index 2098 static int ndb_id_u64_ts_compare(const MDB_val *a, const MDB_val *b) 2099 { 2100 struct ndb_id_u64_ts *tsa, *tsb; 2101 MDB_val a2 = *a, b2 = *b; 2102 2103 a2.mv_size = sizeof(tsa->id); 2104 b2.mv_size = sizeof(tsb->id); 2105 2106 int cmp = mdb_cmp_memn(&a2, &b2); 2107 if (cmp) return cmp; 2108 2109 tsa = a->mv_data; 2110 tsb = b->mv_data; 2111 2112 if (tsa->u64 < tsb->u64) 2113 return -1; 2114 else if (tsa->u64 > tsb->u64) 2115 return 1; 2116 2117 if (tsa->timestamp < tsb->timestamp) 2118 return -1; 2119 else if (tsa->timestamp > tsb->timestamp) 2120 return 1; 2121 2122 return 0; 2123 } 2124 2125 // custom kind+timestamp comparison function. This is used by lmdb to perform 2126 // b+ tree searches over the kind+timestamp index 2127 static int ndb_u64_ts_compare(const MDB_val *a, const MDB_val *b) 2128 { 2129 struct ndb_u64_ts *tsa, *tsb; 2130 tsa = a->mv_data; 2131 tsb = b->mv_data; 2132 2133 if (tsa->u64 < tsb->u64) 2134 return -1; 2135 else if (tsa->u64 > tsb->u64) 2136 return 1; 2137 2138 if (tsa->timestamp < tsb->timestamp) 2139 return -1; 2140 else if (tsa->timestamp > tsb->timestamp) 2141 return 1; 2142 2143 return 0; 2144 } 2145 2146 static int ndb_tsid_compare(const MDB_val *a, const MDB_val *b) 2147 { 2148 struct ndb_tsid *tsa, *tsb; 2149 MDB_val a2 = *a, b2 = *b; 2150 2151 a2.mv_size = sizeof(tsa->id); 2152 b2.mv_size = sizeof(tsb->id); 2153 2154 int cmp = mdb_cmp_memn(&a2, &b2); 2155 if (cmp) return cmp; 2156 2157 tsa = a->mv_data; 2158 tsb = b->mv_data; 2159 2160 if (tsa->timestamp < tsb->timestamp) 2161 return -1; 2162 else if (tsa->timestamp > tsb->timestamp) 2163 return 1; 2164 return 0; 2165 } 2166 2167 enum ndb_ingester_msgtype { 2168 NDB_INGEST_EVENT, // write json to the ingester queue for processing 2169 NDB_INGEST_QUIT, // kill ingester thread immediately 2170 }; 2171 2172 struct ndb_ingester_event { 2173 const char *relay; 2174 char *json; 2175 unsigned client : 1; // ["EVENT", {...}] messages 2176 unsigned len : 31; 2177 }; 2178 2179 struct ndb_writer_note_relay { 2180 const char *relay; 2181 uint64_t note_key; 2182 uint64_t kind; 2183 uint64_t created_at; 2184 }; 2185 2186 struct ndb_writer_note { 2187 struct ndb_note *note; 2188 size_t note_len; 2189 const char *relay; 2190 }; 2191 2192 static void ndb_writer_note_init(struct ndb_writer_note *writer_note, struct ndb_note *note, size_t note_len, const char *relay) 2193 { 2194 writer_note->note = note; 2195 writer_note->note_len = note_len; 2196 writer_note->relay = relay; 2197 } 2198 2199 struct ndb_writer_profile { 2200 struct ndb_writer_note note; 2201 struct ndb_profile_record_builder record; 2202 }; 2203 2204 struct ndb_ingester_msg { 2205 enum ndb_ingester_msgtype type; 2206 union { 2207 struct ndb_ingester_event event; 2208 }; 2209 }; 2210 2211 struct ndb_writer_ndb_meta { 2212 // these are 64 bit because I'm paranoid of db-wide alignment issues 2213 uint64_t version; 2214 }; 2215 2216 // Used in the writer thread when writing ndb_profile_fetch_record's 2217 // kv = pubkey: recor 2218 struct ndb_writer_last_fetch { 2219 unsigned char pubkey[32]; 2220 uint64_t fetched_at; 2221 }; 2222 2223 // write note blocks 2224 struct ndb_writer_blocks { 2225 struct ndb_blocks *blocks; 2226 uint64_t note_key; 2227 }; 2228 2229 // The different types of messages that the writer thread can write to the 2230 // database 2231 struct ndb_writer_msg { 2232 enum ndb_writer_msgtype type; 2233 union { 2234 struct ndb_writer_note_relay note_relay; 2235 struct ndb_writer_note note; 2236 struct ndb_writer_profile profile; 2237 struct ndb_writer_ndb_meta ndb_meta; 2238 struct ndb_writer_last_fetch last_fetch; 2239 struct ndb_writer_blocks blocks; 2240 }; 2241 }; 2242 2243 static inline int ndb_writer_queue_msg(struct ndb_writer *writer, 2244 struct ndb_writer_msg *msg) 2245 { 2246 return prot_queue_push(&writer->inbox, msg); 2247 } 2248 2249 static uint64_t ndb_write_note_and_profile(struct ndb_txn *txn, struct ndb_writer_profile *profile, unsigned char *scratch, size_t scratch_size, uint32_t ndb_flags); 2250 static int ndb_migrate_utf8_profile_names(struct ndb_txn *txn) 2251 { 2252 int rc; 2253 MDB_cursor *cur; 2254 MDB_val k, v; 2255 void *profile_root; 2256 NdbProfileRecord_table_t record; 2257 struct ndb_note *note, *copied_note; 2258 uint64_t note_key; 2259 size_t len; 2260 int count, failed, ret; 2261 struct ndb_writer_profile profile; 2262 2263 if ((rc = mdb_cursor_open(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_PROFILE], &cur))) { 2264 fprintf(stderr, "ndb_migrate_utf8_profile_names: mdb_cursor_open failed, error %d\n", rc); 2265 return 0; 2266 } 2267 2268 size_t scratch_size = 8 * 1024 * 1024; 2269 unsigned char *scratch = malloc(scratch_size); 2270 2271 ret = 1; 2272 count = 0; 2273 failed = 0; 2274 2275 // loop through all profiles and write search indices 2276 while (mdb_cursor_get(cur, &k, &v, MDB_NEXT) == 0) { 2277 profile_root = v.mv_data; 2278 record = NdbProfileRecord_as_root(profile_root); 2279 note_key = NdbProfileRecord_note_key(record); 2280 note = ndb_get_note_by_key(txn, note_key, &len); 2281 2282 if (note == NULL) { 2283 failed++; 2284 continue; 2285 } 2286 2287 struct ndb_profile_record_builder *b = &profile.record; 2288 2289 // reprocess profile 2290 if (!ndb_process_profile_note(note, b)) { 2291 failed++; 2292 continue; 2293 } 2294 2295 // the writer needs to own this note, and its expected to free it 2296 copied_note = malloc(len); 2297 memcpy(copied_note, note, len); 2298 2299 ndb_writer_note_init(&profile.note, copied_note, len, NULL); 2300 2301 // we don't pass in flags when migrating... a bit sketchy but 2302 // whatever. noone is using this to customize nostrdb atm 2303 if (ndb_write_note_and_profile(txn, &profile, scratch, scratch_size, 0)) { 2304 count++; 2305 } 2306 } 2307 2308 fprintf(stderr, "migrated %d profiles to fix utf8 profile names\n", count); 2309 2310 if (failed != 0) { 2311 fprintf(stderr, "failed to migrate %d profiles to fix utf8 profile names\n", failed); 2312 } 2313 2314 free(scratch); 2315 mdb_cursor_close(cur); 2316 2317 return ret; 2318 } 2319 2320 static struct ndb_migration MIGRATIONS[] = { 2321 { .fn = ndb_migrate_user_search_indices }, 2322 { .fn = ndb_migrate_lower_user_search_indices }, 2323 { .fn = ndb_migrate_utf8_profile_names }, 2324 { .fn = ndb_migrate_profile_indices }, 2325 }; 2326 2327 2328 int ndb_end_query(struct ndb_txn *txn) 2329 { 2330 // this works on read or write queries. 2331 return mdb_txn_commit(txn->mdb_txn) == 0; 2332 } 2333 2334 int ndb_note_verify(void *ctx, unsigned char *scratch, size_t scratch_size, 2335 struct ndb_note *note) 2336 { 2337 unsigned char id[32]; 2338 secp256k1_xonly_pubkey xonly_pubkey; 2339 int ok; 2340 2341 // first, we ensure the id is valid by calculating the id independently 2342 // from what is given to us 2343 if (!ndb_calculate_id(note, scratch, scratch_size, id)) { 2344 ndb_debug("ndb_note_verify: scratch buffer size too small"); 2345 return 0; 2346 } 2347 2348 if (memcmp(id, note->id, 32)) { 2349 ndb_debug("ndb_note_verify: note id does not match!"); 2350 return 0; 2351 } 2352 2353 // id is ok, let's check signature 2354 2355 ok = secp256k1_xonly_pubkey_parse((secp256k1_context*)ctx, 2356 &xonly_pubkey, 2357 ndb_note_pubkey(note)) != 0; 2358 if (!ok) return 0; 2359 2360 ok = secp256k1_schnorrsig_verify((secp256k1_context*)ctx, 2361 ndb_note_sig(note), id, 32, 2362 &xonly_pubkey) > 0; 2363 if (!ok) return 0; 2364 2365 return 1; 2366 } 2367 2368 static void ndb_writer_last_profile_fetch(struct ndb_txn *txn, 2369 const unsigned char *pubkey, 2370 uint64_t fetched_at) 2371 { 2372 int rc; 2373 MDB_val key, val; 2374 2375 key.mv_data = (unsigned char*)pubkey; 2376 key.mv_size = 32; 2377 val.mv_data = &fetched_at; 2378 val.mv_size = sizeof(fetched_at); 2379 2380 if ((rc = mdb_put(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_PROFILE_LAST_FETCH], 2381 &key, &val, 0))) 2382 { 2383 ndb_debug("write version to ndb_meta failed: %s\n", 2384 mdb_strerror(rc)); 2385 return; 2386 } 2387 2388 //fprintf(stderr, "writing version %" PRIu64 "\n", version); 2389 } 2390 2391 2392 // We just received a profile that we haven't processed yet, but it could 2393 // be an older one! Make sure we only write last fetched profile if it's a new 2394 // one 2395 // 2396 // To do this, we first check the latest profile in the database. If the 2397 // created_date for this profile note is newer, then we write a 2398 // last_profile_fetch record, otherwise we do not. 2399 // 2400 // WARNING: This function is only valid when called from the writer thread 2401 static int ndb_maybe_write_last_profile_fetch(struct ndb_txn *txn, 2402 struct ndb_note *note) 2403 { 2404 size_t len; 2405 uint64_t profile_key, note_key; 2406 void *root; 2407 struct ndb_note *last_profile; 2408 NdbProfileRecord_table_t record; 2409 2410 if ((root = ndb_get_profile_by_pubkey(txn, note->pubkey, &len, &profile_key))) { 2411 record = NdbProfileRecord_as_root(root); 2412 note_key = NdbProfileRecord_note_key(record); 2413 last_profile = ndb_get_note_by_key(txn, note_key, &len); 2414 if (last_profile == NULL) { 2415 return 0; 2416 } 2417 2418 // found profile, let's see if it's newer than ours 2419 if (note->created_at > last_profile->created_at) { 2420 // this is a new profile note, record last fetched time 2421 ndb_writer_last_profile_fetch(txn, note->pubkey, time(NULL)); 2422 } 2423 } else { 2424 // couldn't fetch profile. record last fetched time 2425 ndb_writer_last_profile_fetch(txn, note->pubkey, time(NULL)); 2426 } 2427 2428 return 1; 2429 } 2430 2431 int ndb_write_last_profile_fetch(struct ndb *ndb, const unsigned char *pubkey, 2432 uint64_t fetched_at) 2433 { 2434 struct ndb_writer_msg msg; 2435 msg.type = NDB_WRITER_PROFILE_LAST_FETCH; 2436 memcpy(&msg.last_fetch.pubkey[0], pubkey, 32); 2437 msg.last_fetch.fetched_at = fetched_at; 2438 2439 return ndb_writer_queue_msg(&ndb->writer, &msg); 2440 } 2441 2442 2443 // When doing cursor scans from greatest to lowest, this function positions the 2444 // cursor at the first element before descending. MDB_SET_RANGE puts us right 2445 // after the first element, so we have to go back one. 2446 static int ndb_cursor_start(MDB_cursor *cur, MDB_val *k, MDB_val *v) 2447 { 2448 int rc; 2449 // Position cursor at the next key greater than or equal to the 2450 // specified key 2451 2452 if ((rc = mdb_cursor_get(cur, k, v, MDB_SET_RANGE))) { 2453 ndb_debug("MDB_SET_RANGE failed: '%s'\n", mdb_strerror(rc)); 2454 // Failed :(. It could be the last element? 2455 if ((rc = mdb_cursor_get(cur, k, v, MDB_LAST))) { 2456 ndb_debug("MDB_LAST failed: '%s'\n", mdb_strerror(rc)); 2457 return 0; 2458 } 2459 } else { 2460 // if set range worked and our key exists, it should be 2461 // the one right before this one 2462 if ((rc = mdb_cursor_get(cur, k, v, MDB_PREV))) { 2463 ndb_debug("moving back failed: '%s'\n", mdb_strerror(rc)); 2464 return 0; 2465 } 2466 } 2467 2468 return 1; 2469 } 2470 2471 2472 // get some value based on a clustered id key 2473 int ndb_get_tsid(struct ndb_txn *txn, enum ndb_dbs db, const unsigned char *id, 2474 MDB_val *val) 2475 { 2476 MDB_val k, v; 2477 MDB_cursor *cur; 2478 int success = 0, rc; 2479 struct ndb_tsid tsid; 2480 2481 // position at the most recent 2482 ndb_tsid_high(&tsid, id); 2483 2484 k.mv_data = &tsid; 2485 k.mv_size = sizeof(tsid); 2486 2487 if ((rc = mdb_cursor_open(txn->mdb_txn, txn->lmdb->dbs[db], &cur))) { 2488 ndb_debug("ndb_get_tsid: failed to open cursor: '%s'\n", mdb_strerror(rc)); 2489 return 0; 2490 } 2491 2492 if (!ndb_cursor_start(cur, &k, &v)) 2493 goto cleanup; 2494 2495 if (memcmp(k.mv_data, id, 32) == 0) { 2496 *val = v; 2497 success = 1; 2498 } 2499 2500 cleanup: 2501 mdb_cursor_close(cur); 2502 return success; 2503 } 2504 2505 static void *ndb_lookup_by_key(struct ndb_txn *txn, uint64_t key, 2506 enum ndb_dbs store, size_t *len) 2507 { 2508 MDB_val k, v; 2509 2510 k.mv_data = &key; 2511 k.mv_size = sizeof(key); 2512 2513 if (mdb_get(txn->mdb_txn, txn->lmdb->dbs[store], &k, &v)) { 2514 ndb_debug("ndb_lookup_by_key: mdb_get note failed\n"); 2515 return NULL; 2516 } 2517 2518 if (len) 2519 *len = v.mv_size; 2520 2521 return v.mv_data; 2522 } 2523 2524 static void *ndb_lookup_tsid(struct ndb_txn *txn, enum ndb_dbs ind, 2525 enum ndb_dbs store, const unsigned char *pk, 2526 size_t *len, uint64_t *primkey) 2527 { 2528 MDB_val k, v; 2529 void *res = NULL; 2530 if (len) 2531 *len = 0; 2532 2533 if (!ndb_get_tsid(txn, ind, pk, &k)) { 2534 //ndb_debug("ndb_get_profile_by_pubkey: ndb_get_tsid failed\n"); 2535 return 0; 2536 } 2537 2538 if (primkey) 2539 *primkey = *(uint64_t*)k.mv_data; 2540 2541 if (mdb_get(txn->mdb_txn, txn->lmdb->dbs[store], &k, &v)) { 2542 ndb_debug("ndb_get_profile_by_pubkey: mdb_get note failed\n"); 2543 return 0; 2544 } 2545 2546 res = v.mv_data; 2547 assert(((uint64_t)res % 4) == 0); 2548 if (len) 2549 *len = v.mv_size; 2550 return res; 2551 } 2552 2553 void *ndb_get_profile_by_pubkey(struct ndb_txn *txn, const unsigned char *pk, size_t *len, uint64_t *key) 2554 { 2555 return ndb_lookup_tsid(txn, NDB_DB_PROFILE_PK, NDB_DB_PROFILE, pk, len, key); 2556 } 2557 2558 struct ndb_note *ndb_get_note_by_id(struct ndb_txn *txn, const unsigned char *id, size_t *len, uint64_t *key) 2559 { 2560 return ndb_lookup_tsid(txn, NDB_DB_NOTE_ID, NDB_DB_NOTE, id, len, key); 2561 } 2562 2563 static inline uint64_t ndb_get_indexkey_by_id(struct ndb_txn *txn, 2564 enum ndb_dbs db, 2565 const unsigned char *id) 2566 { 2567 MDB_val k; 2568 2569 if (!ndb_get_tsid(txn, db, id, &k)) 2570 return 0; 2571 2572 return *(uint32_t*)k.mv_data; 2573 } 2574 2575 uint64_t ndb_get_notekey_by_id(struct ndb_txn *txn, const unsigned char *id) 2576 { 2577 return ndb_get_indexkey_by_id(txn, NDB_DB_NOTE_ID, id); 2578 } 2579 2580 uint64_t ndb_get_profilekey_by_pubkey(struct ndb_txn *txn, const unsigned char *id) 2581 { 2582 return ndb_get_indexkey_by_id(txn, NDB_DB_PROFILE_PK, id); 2583 } 2584 2585 struct ndb_note *ndb_get_note_by_key(struct ndb_txn *txn, uint64_t key, size_t *len) 2586 { 2587 return ndb_lookup_by_key(txn, key, NDB_DB_NOTE, len); 2588 } 2589 2590 void *ndb_get_profile_by_key(struct ndb_txn *txn, uint64_t key, size_t *len) 2591 { 2592 return ndb_lookup_by_key(txn, key, NDB_DB_PROFILE, len); 2593 } 2594 2595 uint64_t 2596 ndb_read_last_profile_fetch(struct ndb_txn *txn, const unsigned char *pubkey) 2597 { 2598 MDB_val k, v; 2599 2600 k.mv_data = (unsigned char*)pubkey; 2601 k.mv_size = 32; 2602 2603 if (mdb_get(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_PROFILE_LAST_FETCH], &k, &v)) { 2604 //ndb_debug("ndb_read_last_profile_fetch: mdb_get note failed\n"); 2605 return 0; 2606 } 2607 2608 return *((uint64_t*)v.mv_data); 2609 } 2610 2611 2612 static int ndb_has_note(struct ndb_txn *txn, const unsigned char *id) 2613 { 2614 MDB_val val; 2615 2616 if (!ndb_get_tsid(txn, NDB_DB_NOTE_ID, id, &val)) 2617 return 0; 2618 2619 return 1; 2620 } 2621 2622 static void ndb_txn_from_mdb(struct ndb_txn *txn, struct ndb_lmdb *lmdb, 2623 MDB_txn *mdb_txn) 2624 { 2625 txn->lmdb = lmdb; 2626 txn->mdb_txn = mdb_txn; 2627 } 2628 2629 static enum ndb_idres ndb_ingester_json_controller(void *data, const char *hexid) 2630 { 2631 unsigned char id[32]; 2632 struct ndb_ingest_controller *c = data; 2633 struct ndb_txn txn; 2634 2635 hex_decode(hexid, 64, id, sizeof(id)); 2636 2637 // let's see if we already have it 2638 ndb_txn_from_mdb(&txn, c->lmdb, c->read_txn); 2639 c->note = ndb_get_note_by_id(&txn, id, NULL, &c->note_key); 2640 2641 if (c->note == NULL) 2642 return NDB_IDRES_CONT; 2643 2644 return NDB_IDRES_STOP; 2645 } 2646 2647 static int ndbprofile_parse_json(flatcc_builder_t *B, 2648 const char *buf, size_t bufsiz, int flags, NdbProfile_ref_t *profile) 2649 { 2650 flatcc_json_parser_t parser, *ctx = &parser; 2651 flatcc_json_parser_init(ctx, B, buf, buf + bufsiz, flags); 2652 2653 if (flatcc_builder_start_buffer(B, 0, 0, 0)) 2654 return 0; 2655 2656 NdbProfile_parse_json_table(ctx, buf, buf + bufsiz, profile); 2657 if (ctx->error) 2658 return 0; 2659 2660 if (!flatcc_builder_end_buffer(B, *profile)) 2661 return 0; 2662 2663 ctx->end_loc = buf; 2664 2665 2666 return 1; 2667 } 2668 2669 void ndb_profile_record_builder_init(struct ndb_profile_record_builder *b) 2670 { 2671 b->builder = malloc(sizeof(*b->builder)); 2672 b->flatbuf = NULL; 2673 } 2674 2675 void ndb_profile_record_builder_free(struct ndb_profile_record_builder *b) 2676 { 2677 if (b->flatbuf) 2678 flatcc_builder_aligned_free(b->flatbuf); 2679 if (b->builder) { 2680 flatcc_builder_clear(b->builder); 2681 free(b->builder); 2682 } 2683 2684 b->builder = NULL; 2685 b->flatbuf = NULL; 2686 2687 } 2688 2689 int ndb_process_profile_note(struct ndb_note *note, 2690 struct ndb_profile_record_builder *profile) 2691 { 2692 int res; 2693 2694 NdbProfile_ref_t profile_table; 2695 flatcc_builder_t *builder; 2696 2697 ndb_profile_record_builder_init(profile); 2698 builder = profile->builder; 2699 flatcc_builder_init(builder); 2700 2701 NdbProfileRecord_start_as_root(builder); 2702 2703 //printf("parsing profile '%.*s'\n", note->content_length, ndb_note_content(note)); 2704 if (!(res = ndbprofile_parse_json(builder, ndb_note_content(note), 2705 note->content_length, 2706 flatcc_json_parser_f_skip_unknown, 2707 &profile_table))) 2708 { 2709 ndb_debug("profile_parse_json failed %d '%.*s'\n", res, 2710 note->content_length, ndb_note_content(note)); 2711 ndb_profile_record_builder_free(profile); 2712 return 0; 2713 } 2714 2715 uint64_t received_at = time(NULL); 2716 const char *lnurl = "fixme"; 2717 2718 NdbProfileRecord_profile_add(builder, profile_table); 2719 NdbProfileRecord_received_at_add(builder, received_at); 2720 2721 flatcc_builder_ref_t lnurl_off; 2722 lnurl_off = flatcc_builder_create_string_str(builder, lnurl); 2723 2724 NdbProfileRecord_lnurl_add(builder, lnurl_off); 2725 2726 //*profile = flatcc_builder_finalize_aligned_buffer(builder, profile_len); 2727 return 1; 2728 } 2729 2730 static int ndb_ingester_queue_event(struct ndb_ingester *ingester, 2731 char *json, unsigned len, 2732 unsigned client, const char *relay) 2733 { 2734 struct ndb_ingester_msg msg; 2735 msg.type = NDB_INGEST_EVENT; 2736 2737 msg.event.json = json; 2738 msg.event.len = len; 2739 msg.event.client = client; 2740 msg.event.relay = relay; 2741 2742 return threadpool_dispatch(&ingester->tp, &msg); 2743 } 2744 2745 void ndb_ingest_meta_init(struct ndb_ingest_meta *meta, unsigned client, const char *relay) 2746 { 2747 meta->client = client; 2748 meta->relay = relay; 2749 } 2750 2751 static int ndb_ingest_event(struct ndb_ingester *ingester, const char *json, 2752 int len, struct ndb_ingest_meta *meta) 2753 { 2754 const char *relay = meta->relay; 2755 2756 // Without this, we get bus errors in the json parser inside when 2757 // trying to ingest empty kind 6 reposts... we should probably do fuzz 2758 // testing on inputs to the json parser 2759 if (len == 0) 2760 return 0; 2761 2762 // Since we need to return as soon as possible, and we're not 2763 // making any assumptions about the lifetime of the string, we 2764 // definitely need to copy the json here. In the future once we 2765 // have our thread that manages a websocket connection, we can 2766 // avoid the copy and just use the buffer we get from that 2767 // thread. 2768 char *json_copy = strdupn(json, len); 2769 if (json_copy == NULL) 2770 return 0; 2771 2772 if (relay != NULL) { 2773 relay = strdup(meta->relay); 2774 if (relay == NULL) 2775 return 0; 2776 } 2777 2778 return ndb_ingester_queue_event(ingester, json_copy, len, meta->client, relay); 2779 } 2780 2781 2782 static int ndb_ingester_process_note(secp256k1_context *ctx, 2783 struct ndb_note *note, 2784 size_t note_size, 2785 struct ndb_writer_msg *out, 2786 struct ndb_ingester *ingester, 2787 unsigned char *scratch, 2788 const char *relay) 2789 { 2790 enum ndb_ingest_filter_action action; 2791 struct ndb_ingest_meta meta; 2792 2793 action = NDB_INGEST_ACCEPT; 2794 2795 if (ingester->filter) 2796 action = ingester->filter(ingester->filter_context, note); 2797 2798 if (action == NDB_INGEST_REJECT) 2799 return 0; 2800 2801 // some special situations we might want to skip sig validation, 2802 // like during large imports 2803 if (action == NDB_INGEST_SKIP_VALIDATION || (ingester->flags & NDB_FLAG_SKIP_NOTE_VERIFY)) { 2804 // if we're skipping validation we don't need to verify 2805 } else { 2806 // verify! If it's an invalid note we don't need to 2807 // bother writing it to the database 2808 if (!ndb_note_verify(ctx, scratch, ingester->scratch_size, note)) { 2809 ndb_debug("note verification failed\n"); 2810 return 0; 2811 } 2812 } 2813 2814 // we didn't find anything. let's send it 2815 // to the writer thread 2816 note = realloc(note, note_size); 2817 assert(((uint64_t)note % 4) == 0); 2818 2819 if (note->kind == 0) { 2820 struct ndb_profile_record_builder *b = 2821 &out->profile.record; 2822 2823 ndb_process_profile_note(note, b); 2824 2825 out->type = NDB_WRITER_PROFILE; 2826 ndb_writer_note_init(&out->profile.note, note, note_size, relay); 2827 return 1; 2828 } else if (note->kind == 6) { 2829 // process the repost if we have a repost event 2830 ndb_debug("processing kind 6 repost\n"); 2831 // dup the relay string 2832 ndb_ingest_meta_init(&meta, 0, relay); 2833 ndb_ingest_event(ingester, ndb_note_content(note), 2834 ndb_note_content_length(note), 2835 &meta); 2836 } 2837 2838 out->type = NDB_WRITER_NOTE; 2839 ndb_writer_note_init(&out->note, note, note_size, relay); 2840 2841 return 1; 2842 } 2843 2844 int ndb_note_seen_on_relay(struct ndb_txn *txn, uint64_t note_key, const char *relay) 2845 { 2846 MDB_val k, v; 2847 MDB_cursor *cur; 2848 int rc, len; 2849 char relay_buf[256]; 2850 2851 if (relay == NULL) 2852 return 0; 2853 2854 len = strlen(relay); 2855 2856 if (!(len = prepare_relay_buf(relay_buf, sizeof(relay_buf), relay, len))) 2857 return 0; 2858 2859 assert((len % 8) == 0); 2860 2861 k.mv_data = ¬e_key; 2862 k.mv_size = sizeof(note_key); 2863 2864 v.mv_data = relay_buf; 2865 v.mv_size = len; 2866 2867 if ((rc = mdb_cursor_open(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_NOTE_RELAYS], &cur)) != MDB_SUCCESS) 2868 return 0; 2869 2870 rc = mdb_cursor_get(cur, &k, &v, MDB_GET_BOTH); 2871 ndb_debug("seen_on_relay result: %s\n", mdb_strerror(rc)); 2872 mdb_cursor_close(cur); 2873 2874 return rc == MDB_SUCCESS; 2875 } 2876 2877 // process the relay for the note. this is called when we already have the 2878 // note in the database but still need to check if the relay needs to be 2879 // written to the relay indexes for corresponding note 2880 static int ndb_process_note_relay(struct ndb_txn *txn, struct ndb_writer_msg *out, 2881 uint64_t note_key, struct ndb_note *note, 2882 const char *relay) 2883 { 2884 // query to see if we already have the relay on this note 2885 if (ndb_note_seen_on_relay(txn, note_key, relay)) { 2886 return 0; 2887 } 2888 2889 // if not, tell the writer thread to emit a NOTE_RELAY event 2890 out->type = NDB_WRITER_NOTE_RELAY; 2891 2892 ndb_debug("pushing NDB_WRITER_NOTE_RELAY with note_key %" PRIu64 "\n", note_key); 2893 out->note_relay.relay = relay; 2894 out->note_relay.note_key = note_key; 2895 out->note_relay.kind = ndb_note_kind(note); 2896 out->note_relay.created_at = ndb_note_created_at(note); 2897 2898 return 1; 2899 } 2900 2901 static int ndb_ingester_process_event(secp256k1_context *ctx, 2902 struct ndb_ingester *ingester, 2903 struct ndb_ingester_event *ev, 2904 struct ndb_writer_msg *out, 2905 unsigned char *scratch, 2906 MDB_txn *read_txn) 2907 { 2908 struct ndb_tce tce; 2909 struct ndb_fce fce; 2910 struct ndb_note *note; 2911 struct ndb_ingest_controller controller; 2912 struct ndb_id_cb cb; 2913 void *buf; 2914 int ok; 2915 size_t bufsize, note_size; 2916 2917 ok = 0; 2918 2919 // we will use this to check if we already have it in the DB during 2920 // ID parsing 2921 controller.read_txn = read_txn; 2922 controller.lmdb = ingester->lmdb; 2923 cb.fn = ndb_ingester_json_controller; 2924 cb.data = &controller; 2925 2926 // since we're going to be passing this allocated note to a different 2927 // thread, we can't use thread-local buffers. just allocate a block 2928 bufsize = max(ev->len * 8.0, 4096); 2929 buf = malloc(bufsize); 2930 if (!buf) { 2931 ndb_debug("couldn't malloc buf\n"); 2932 return 0; 2933 } 2934 2935 note_size = 2936 ev->client ? 2937 ndb_client_event_from_json(ev->json, ev->len, &fce, buf, bufsize, &cb) : 2938 ndb_ws_event_from_json(ev->json, ev->len, &tce, buf, bufsize, &cb); 2939 2940 // This is a result from our special json parser. It parsed the id 2941 // and found that we already have it in the database 2942 if ((int)note_size == -42) { 2943 assert(controller.note != NULL); 2944 assert(controller.note_key != 0); 2945 struct ndb_txn txn; 2946 ndb_txn_from_mdb(&txn, ingester->lmdb, read_txn); 2947 2948 // we still need to process the relays on the note even 2949 // if we already have it 2950 if (ev->relay && ndb_process_note_relay(&txn, out, 2951 controller.note_key, 2952 controller.note, 2953 ev->relay)) 2954 { 2955 // free note buf here since we don't pass the note to the writer thread 2956 free(buf); 2957 goto success; 2958 } else { 2959 // we already have the note and there are no new 2960 // relays to process. nothing to write. 2961 goto cleanup; 2962 } 2963 } else if (note_size == 0) { 2964 ndb_debug("failed to parse '%.*s'\n", ev->len, ev->json); 2965 goto cleanup; 2966 } 2967 2968 //ndb_debug("parsed evtype:%d '%.*s'\n", tce.evtype, ev->len, ev->json); 2969 2970 if (ev->client) { 2971 switch (fce.evtype) { 2972 case NDB_FCE_EVENT: 2973 note = fce.event.note; 2974 if (note != buf) { 2975 ndb_debug("note buffer not equal to malloc'd buffer\n"); 2976 goto cleanup; 2977 } 2978 2979 if (!ndb_ingester_process_note(ctx, note, note_size, 2980 out, ingester, scratch, 2981 ev->relay)) { 2982 ndb_debug("failed to process note\n"); 2983 goto cleanup; 2984 } else { 2985 goto success; 2986 } 2987 } 2988 } else { 2989 switch (tce.evtype) { 2990 case NDB_TCE_AUTH: goto cleanup; 2991 case NDB_TCE_NOTICE: goto cleanup; 2992 case NDB_TCE_EOSE: goto cleanup; 2993 case NDB_TCE_OK: goto cleanup; 2994 case NDB_TCE_EVENT: 2995 note = tce.event.note; 2996 if (note != buf) { 2997 ndb_debug("note buffer not equal to malloc'd buffer\n"); 2998 goto cleanup; 2999 } 3000 3001 if (!ndb_ingester_process_note(ctx, note, note_size, 3002 out, ingester, scratch, 3003 ev->relay)) { 3004 ndb_debug("failed to process note\n"); 3005 goto cleanup; 3006 } else { 3007 goto success; 3008 } 3009 } 3010 } 3011 3012 3013 success: 3014 free(ev->json); 3015 // we don't free relay or buf since those are passed to the writer thread 3016 return 1; 3017 3018 cleanup: 3019 free(ev->json); 3020 if (ev->relay) 3021 free((void*)ev->relay); 3022 free(buf); 3023 3024 return ok; 3025 } 3026 3027 static uint64_t ndb_get_last_key(MDB_txn *txn, MDB_dbi db) 3028 { 3029 MDB_cursor *mc; 3030 MDB_val key, val; 3031 3032 if (mdb_cursor_open(txn, db, &mc)) 3033 return 0; 3034 3035 if (mdb_cursor_get(mc, &key, &val, MDB_LAST)) { 3036 mdb_cursor_close(mc); 3037 return 0; 3038 } 3039 3040 mdb_cursor_close(mc); 3041 3042 assert(key.mv_size == 8); 3043 return *((uint64_t*)key.mv_data); 3044 } 3045 3046 // 3047 // make a search key meant for user queries without any other note info 3048 static void ndb_make_search_key_low(struct ndb_search_key *key, const char *search) 3049 { 3050 memset(key->id, 0, sizeof(key->id)); 3051 key->timestamp = 0; 3052 lowercase_strncpy(key->search, search, sizeof(key->search) - 1); 3053 key->search[sizeof(key->search) - 1] = '\0'; 3054 } 3055 3056 int ndb_search_profile(struct ndb_txn *txn, struct ndb_search *search, const char *query) 3057 { 3058 int rc; 3059 struct ndb_search_key s; 3060 MDB_val k, v; 3061 search->cursor = NULL; 3062 3063 MDB_cursor **cursor = (MDB_cursor **)&search->cursor; 3064 3065 ndb_make_search_key_low(&s, query); 3066 3067 k.mv_data = &s; 3068 k.mv_size = sizeof(s); 3069 3070 if ((rc = mdb_cursor_open(txn->mdb_txn, 3071 txn->lmdb->dbs[NDB_DB_PROFILE_SEARCH], 3072 cursor))) { 3073 printf("search_profile: cursor opened failed: %s\n", 3074 mdb_strerror(rc)); 3075 return 0; 3076 } 3077 3078 // Position cursor at the next key greater than or equal to the specified key 3079 if (mdb_cursor_get(search->cursor, &k, &v, MDB_SET_RANGE)) { 3080 printf("search_profile: cursor get failed\n"); 3081 goto cleanup; 3082 } else { 3083 search->key = k.mv_data; 3084 assert(v.mv_size == 8); 3085 search->profile_key = *((uint64_t*)v.mv_data); 3086 return 1; 3087 } 3088 3089 cleanup: 3090 mdb_cursor_close(search->cursor); 3091 search->cursor = NULL; 3092 return 0; 3093 } 3094 3095 void ndb_search_profile_end(struct ndb_search *search) 3096 { 3097 if (search->cursor) 3098 mdb_cursor_close(search->cursor); 3099 } 3100 3101 int ndb_search_profile_next(struct ndb_search *search) 3102 { 3103 int rc; 3104 MDB_val k, v; 3105 unsigned char *init_id; 3106 3107 init_id = search->key->id; 3108 k.mv_data = search->key; 3109 k.mv_size = sizeof(*search->key); 3110 3111 retry: 3112 if ((rc = mdb_cursor_get(search->cursor, &k, &v, MDB_NEXT))) { 3113 ndb_debug("ndb_search_profile_next: %s\n", 3114 mdb_strerror(rc)); 3115 return 0; 3116 } else { 3117 search->key = k.mv_data; 3118 assert(v.mv_size == 8); 3119 search->profile_key = *((uint64_t*)v.mv_data); 3120 3121 // skip duplicate pubkeys 3122 if (!memcmp(init_id, search->key->id, 32)) 3123 goto retry; 3124 } 3125 3126 return 1; 3127 } 3128 3129 // 3130 // The relay kind index has a layout like so (so we don't need dupsort) 3131 // 3132 // - note_id: 00 + 8 bytes 3133 // - kind: 08 + 8 bytes 3134 // - created_at: 16 + 8 bytes 3135 // - relay_url_size: 24 + 1 byte 3136 // - relay_url: 25 + n byte null-terminated string 3137 // - pad to 8 byte alignment 3138 // 3139 // The key sort order is: 3140 // 3141 // relay_url, kind, created_at 3142 // 3143 static int ndb_relay_kind_cmp(const MDB_val *a, const MDB_val *b) 3144 { 3145 int cmp; 3146 MDB_val va, vb; 3147 uint64_t iva, ivb; 3148 unsigned char *ad = (unsigned char *)a->mv_data; 3149 unsigned char *bd = (unsigned char *)b->mv_data; 3150 assert(((uint64_t)a->mv_data % 8) == 0); 3151 3152 va.mv_size = *(ad + 24); 3153 va.mv_data = ad + 25; 3154 3155 vb.mv_size = *(bd + 24); 3156 vb.mv_data = bd + 25; 3157 3158 cmp = mdb_cmp_memn(&va, &vb); 3159 if (cmp) return cmp; 3160 3161 // kind 3162 iva = *(uint64_t*)(ad + 8); 3163 ivb = *(uint64_t*)(bd + 8); 3164 3165 if (iva < ivb) 3166 return -1; 3167 else if (iva > ivb) 3168 return 1; 3169 3170 // created_at 3171 iva = *(uint64_t*)(ad + 16); 3172 ivb = *(uint64_t*)(bd + 16); 3173 3174 if (iva < ivb) 3175 return -1; 3176 else if (iva > ivb) 3177 return 1; 3178 3179 // note_id (so we don't need dupsort logic) 3180 iva = *(uint64_t*)ad; 3181 ivb = *(uint64_t*)bd; 3182 3183 if (iva < ivb) 3184 return -1; 3185 else if (iva > ivb) 3186 return 1; 3187 3188 return 0; 3189 } 3190 3191 static int ndb_search_key_cmp(const MDB_val *a, const MDB_val *b) 3192 { 3193 int cmp; 3194 struct ndb_search_key *ska, *skb; 3195 3196 ska = a->mv_data; 3197 skb = b->mv_data; 3198 3199 MDB_val a2 = *a; 3200 MDB_val b2 = *b; 3201 3202 a2.mv_data = ska->search; 3203 a2.mv_size = sizeof(ska->search) + sizeof(ska->id); 3204 3205 cmp = mdb_cmp_memn(&a2, &b2); 3206 if (cmp) return cmp; 3207 3208 if (ska->timestamp < skb->timestamp) 3209 return -1; 3210 else if (ska->timestamp > skb->timestamp) 3211 return 1; 3212 3213 return 0; 3214 } 3215 3216 static int ndb_write_profile_pk_index(struct ndb_txn *txn, struct ndb_note *note, uint64_t profile_key) 3217 3218 { 3219 MDB_val key, val; 3220 int rc; 3221 struct ndb_tsid tsid; 3222 MDB_dbi pk_db; 3223 3224 pk_db = txn->lmdb->dbs[NDB_DB_PROFILE_PK]; 3225 3226 // write profile_pk + created_at index 3227 ndb_tsid_init(&tsid, note->pubkey, note->created_at); 3228 3229 key.mv_data = &tsid; 3230 key.mv_size = sizeof(tsid); 3231 val.mv_data = &profile_key; 3232 val.mv_size = sizeof(profile_key); 3233 3234 if ((rc = mdb_put(txn->mdb_txn, pk_db, &key, &val, 0))) { 3235 ndb_debug("write profile_pk(%" PRIu64 ") to db failed: %s\n", 3236 profile_key, mdb_strerror(rc)); 3237 return 0; 3238 } 3239 3240 return 1; 3241 } 3242 3243 static int ndb_write_profile(struct ndb_txn *txn, 3244 struct ndb_writer_profile *profile, 3245 uint64_t note_key) 3246 { 3247 uint64_t profile_key; 3248 struct ndb_note *note; 3249 void *flatbuf; 3250 size_t flatbuf_len; 3251 int rc; 3252 3253 MDB_val key, val; 3254 MDB_dbi profile_db; 3255 3256 note = profile->note.note; 3257 3258 // add note_key to profile record 3259 NdbProfileRecord_note_key_add(profile->record.builder, note_key); 3260 NdbProfileRecord_end_as_root(profile->record.builder); 3261 3262 flatbuf = profile->record.flatbuf = 3263 flatcc_builder_finalize_aligned_buffer(profile->record.builder, &flatbuf_len); 3264 3265 assert(((uint64_t)flatbuf % 8) == 0); 3266 3267 // TODO: this may not be safe!? 3268 flatbuf_len = (flatbuf_len + 7) & ~7; 3269 3270 //assert(NdbProfileRecord_verify_as_root(flatbuf, flatbuf_len) == 0); 3271 3272 // get dbs 3273 profile_db = txn->lmdb->dbs[NDB_DB_PROFILE]; 3274 3275 // get new key 3276 profile_key = ndb_get_last_key(txn->mdb_txn, profile_db) + 1; 3277 3278 // write profile to profile store 3279 key.mv_data = &profile_key; 3280 key.mv_size = sizeof(profile_key); 3281 val.mv_data = flatbuf; 3282 val.mv_size = flatbuf_len; 3283 3284 if ((rc = mdb_put(txn->mdb_txn, profile_db, &key, &val, 0))) { 3285 ndb_debug("write profile to db failed: %s\n", mdb_strerror(rc)); 3286 return 0; 3287 } 3288 3289 // write last fetched record 3290 if (!ndb_maybe_write_last_profile_fetch(txn, note)) { 3291 ndb_debug("failed to write last profile fetched record\n"); 3292 } 3293 3294 // write profile pubkey index 3295 if (!ndb_write_profile_pk_index(txn, note, profile_key)) { 3296 ndb_debug("failed to write profile pubkey index\n"); 3297 return 0; 3298 } 3299 3300 // write name, display_name profile search indices 3301 if (!ndb_write_profile_search_indices(txn, note, profile_key, 3302 flatbuf)) { 3303 ndb_debug("failed to write profile search indices\n"); 3304 return 0; 3305 } 3306 3307 return 1; 3308 } 3309 3310 // find the last id tag in a note (e, p, etc) 3311 static unsigned char *ndb_note_last_id_tag(struct ndb_note *note, char type) 3312 { 3313 unsigned char *last = NULL; 3314 struct ndb_iterator iter; 3315 struct ndb_str str; 3316 3317 // get the liked event id (last id) 3318 ndb_tags_iterate_start(note, &iter); 3319 3320 while (ndb_tags_iterate_next(&iter)) { 3321 if (iter.tag->count < 2) 3322 continue; 3323 3324 str = ndb_tag_str(note, iter.tag, 0); 3325 3326 // assign liked to the last e tag 3327 if (str.flag == NDB_PACKED_STR && str.str[0] == type) { 3328 str = ndb_tag_str(note, iter.tag, 1); 3329 if (str.flag == NDB_PACKED_ID) 3330 last = str.id; 3331 } 3332 } 3333 3334 return last; 3335 } 3336 3337 void *ndb_get_note_meta(struct ndb_txn *txn, const unsigned char *id, size_t *len) 3338 { 3339 MDB_val k, v; 3340 3341 k.mv_data = (unsigned char*)id; 3342 k.mv_size = 32; 3343 3344 if (mdb_get(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_META], &k, &v)) { 3345 //ndb_debug("ndb_get_note_meta: mdb_get note failed\n"); 3346 return NULL; 3347 } 3348 3349 if (len) 3350 *len = v.mv_size; 3351 3352 return v.mv_data; 3353 } 3354 3355 // When receiving a reaction note, look for the liked id and increase the 3356 // reaction counter in the note metadata database 3357 static int ndb_write_reaction_stats(struct ndb_txn *txn, struct ndb_note *note) 3358 { 3359 size_t len; 3360 void *root; 3361 int reactions, rc; 3362 MDB_val key, val; 3363 NdbEventMeta_table_t meta; 3364 unsigned char *liked = ndb_note_last_id_tag(note, 'e'); 3365 3366 if (liked == NULL) 3367 return 0; 3368 3369 root = ndb_get_note_meta(txn, liked, &len); 3370 3371 flatcc_builder_t builder; 3372 flatcc_builder_init(&builder); 3373 NdbEventMeta_start_as_root(&builder); 3374 3375 // no meta record, let's make one 3376 if (root == NULL) { 3377 NdbEventMeta_reactions_add(&builder, 1); 3378 } else { 3379 // clone existing and add to it 3380 meta = NdbEventMeta_as_root(root); 3381 3382 reactions = NdbEventMeta_reactions_get(meta); 3383 NdbEventMeta_clone(&builder, meta); 3384 NdbEventMeta_reactions_add(&builder, reactions + 1); 3385 } 3386 3387 NdbProfileRecord_end_as_root(&builder); 3388 root = flatcc_builder_finalize_aligned_buffer(&builder, &len); 3389 assert(((uint64_t)root % 8) == 0); 3390 3391 if (root == NULL) { 3392 ndb_debug("failed to create note metadata record\n"); 3393 goto fail; 3394 } 3395 3396 // metadata is keyed on id because we want to collect stats regardless 3397 // if we have the note yet or not 3398 key.mv_data = liked; 3399 key.mv_size = 32; 3400 3401 val.mv_data = root; 3402 val.mv_size = len; 3403 3404 // write the new meta record 3405 //ndb_debug("writing stats record for "); 3406 //print_hex(liked, 32); 3407 //ndb_debug("\n"); 3408 3409 if ((rc = mdb_put(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_META], &key, &val, 0))) { 3410 ndb_debug("write reaction stats to db failed: %s\n", mdb_strerror(rc)); 3411 goto fail; 3412 } 3413 3414 free(root); 3415 flatcc_builder_clear(&builder); 3416 3417 return 1; 3418 3419 fail: 3420 free(root); 3421 flatcc_builder_clear(&builder); 3422 return 0; 3423 } 3424 3425 3426 static int ndb_write_note_id_index(struct ndb_txn *txn, struct ndb_note *note, 3427 uint64_t note_key) 3428 3429 { 3430 struct ndb_tsid tsid; 3431 int rc; 3432 MDB_val key, val; 3433 MDB_dbi id_db; 3434 3435 ndb_tsid_init(&tsid, note->id, note->created_at); 3436 3437 key.mv_data = &tsid; 3438 key.mv_size = sizeof(tsid); 3439 val.mv_data = ¬e_key; 3440 val.mv_size = sizeof(note_key); 3441 3442 id_db = txn->lmdb->dbs[NDB_DB_NOTE_ID]; 3443 3444 if ((rc = mdb_put(txn->mdb_txn, id_db, &key, &val, 0))) { 3445 ndb_debug("write note id index to db failed: %s\n", 3446 mdb_strerror(rc)); 3447 return 0; 3448 } 3449 3450 return 1; 3451 } 3452 3453 static int ndb_filter_group_add_filters(struct ndb_filter_group *group, 3454 struct ndb_filter *filters, 3455 int num_filters) 3456 { 3457 int i; 3458 3459 for (i = 0; i < num_filters; i++) { 3460 if (!ndb_filter_group_add(group, &filters[i])) 3461 return 0; 3462 } 3463 3464 return 1; 3465 } 3466 3467 3468 static struct ndb_filter_elements * 3469 ndb_filter_find_elements(struct ndb_filter *filter, enum ndb_filter_fieldtype typ) 3470 { 3471 int i; 3472 struct ndb_filter_elements *els; 3473 3474 for (i = 0; i < filter->num_elements; i++) { 3475 els = ndb_filter_get_elements(filter, i); 3476 assert(els); 3477 if (els->field.type == typ) { 3478 return els; 3479 } 3480 } 3481 3482 return NULL; 3483 } 3484 3485 static const char *ndb_filter_find_search(struct ndb_filter *filter) 3486 { 3487 struct ndb_filter_elements *els; 3488 3489 if (!(els = ndb_filter_find_elements(filter, NDB_FILTER_SEARCH))) 3490 return NULL; 3491 3492 return ndb_filter_get_string_element(filter, els, 0); 3493 } 3494 3495 int ndb_filter_is_subset_of(const struct ndb_filter *a, const struct ndb_filter *b) 3496 { 3497 int i; 3498 struct ndb_filter_elements *b_field, *a_field; 3499 3500 // Everything is always a subset of {} 3501 if (b->num_elements == 0) 3502 return 1; 3503 3504 // We can't be a subset if the number of elements in the other 3505 // filter is larger then the number of elements we have. 3506 if (b->num_elements > a->num_elements) 3507 return 0; 3508 3509 // If our filter count matches, we can only be a subset if we are 3510 // equal 3511 if (b->num_elements == a->num_elements) 3512 return ndb_filter_eq(a, b); 3513 3514 // If our element count is larger than the other filter, then we 3515 // must see if every element in the other filter exists in ours. If 3516 // so, then we are a subset of the other. 3517 // 3518 // eg: B={k:1, a:b} <- A={t:x, k:1, a:b} 3519 // 3520 // A is a subset of B because `k:1` and `a:b` both exist in A 3521 3522 for (i = 0; i < b->num_elements; i++) { 3523 b_field = ndb_filter_get_elements((struct ndb_filter*)b, i); 3524 a_field = ndb_filter_find_elements((struct ndb_filter*)a, 3525 b_field->field.type); 3526 3527 if (a_field == NULL) 3528 return 0; 3529 3530 if (!ndb_filter_field_eq((struct ndb_filter*)a, a_field, 3531 (struct ndb_filter*)b, b_field)) 3532 return 0; 3533 } 3534 3535 return 1; 3536 } 3537 3538 int ndb_filter_eq(const struct ndb_filter *a, const struct ndb_filter *b) 3539 { 3540 int i; 3541 struct ndb_filter_elements *a_els, *b_els; 3542 3543 if (a->num_elements != b->num_elements) 3544 return 0; 3545 3546 for (i = 0; i < a->num_elements; i++) { 3547 a_els = ndb_filter_get_elements((struct ndb_filter*)a, i); 3548 b_els = ndb_filter_find_elements((struct ndb_filter *)b, 3549 a_els->field.type); 3550 3551 if (b_els == NULL) 3552 return 0; 3553 3554 if (!ndb_filter_field_eq((struct ndb_filter*)a, a_els, 3555 (struct ndb_filter*)b, b_els)) 3556 return 0; 3557 } 3558 3559 return 1; 3560 } 3561 3562 3563 static uint64_t * 3564 ndb_filter_get_elem(struct ndb_filter *filter, enum ndb_filter_fieldtype typ) 3565 { 3566 struct ndb_filter_elements *els; 3567 if ((els = ndb_filter_find_elements(filter, typ))) 3568 return &els->elements[0]; 3569 return NULL; 3570 } 3571 3572 static uint64_t *ndb_filter_get_int(struct ndb_filter *filter, 3573 enum ndb_filter_fieldtype typ) 3574 { 3575 uint64_t *el; 3576 if (NULL == (el = ndb_filter_get_elem(filter, typ))) 3577 return NULL; 3578 return el; 3579 } 3580 3581 static inline int push_query_result(struct ndb_query_results *results, 3582 struct ndb_query_result *result) 3583 { 3584 return cursor_push(&results->cur, (unsigned char*)result, sizeof(*result)); 3585 } 3586 3587 static int compare_query_results(const void *pa, const void *pb) 3588 { 3589 struct ndb_query_result *a, *b; 3590 3591 a = (struct ndb_query_result *)pa; 3592 b = (struct ndb_query_result *)pb; 3593 3594 if (a->note->created_at == b->note->created_at) { 3595 return 0; 3596 } else if (a->note->created_at > b->note->created_at) { 3597 return -1; 3598 } else { 3599 return 1; 3600 } 3601 } 3602 3603 static void ndb_query_result_init(struct ndb_query_result *res, 3604 struct ndb_note *note, 3605 uint64_t note_size, 3606 uint64_t note_id) 3607 { 3608 *res = (struct ndb_query_result){ 3609 .note_id = note_id, 3610 .note_size = note_size, 3611 .note = note, 3612 }; 3613 } 3614 3615 static int query_is_full(struct ndb_query_results *results, int limit) 3616 { 3617 if (results->cur.p >= results->cur.end) 3618 return 1; 3619 3620 return cursor_count(&results->cur, sizeof(struct ndb_query_result)) >= limit; 3621 } 3622 3623 static int ndb_query_plan_execute_search(struct ndb_txn *txn, 3624 struct ndb_filter *filter, 3625 struct ndb_query_results *results, 3626 int limit) 3627 { 3628 const char *search; 3629 int i; 3630 struct ndb_text_search_results text_results; 3631 struct ndb_text_search_result *text_result; 3632 struct ndb_text_search_config config; 3633 struct ndb_query_result result; 3634 3635 ndb_default_text_search_config(&config); 3636 3637 if (!(search = ndb_filter_find_search(filter))) 3638 return 0; 3639 3640 if (!ndb_text_search_with(txn, search, &text_results, &config, filter)) 3641 return 0; 3642 3643 for (i = 0; i < text_results.num_results; i++) { 3644 if (query_is_full(results, limit)) 3645 break; 3646 3647 text_result = &text_results.results[i]; 3648 3649 result.note = text_result->note; 3650 result.note_size = text_result->note_size; 3651 result.note_id = text_result->key.note_id; 3652 3653 if (!push_query_result(results, &result)) 3654 break; 3655 } 3656 3657 return 1; 3658 } 3659 3660 static int ndb_query_plan_execute_ids(struct ndb_txn *txn, 3661 struct ndb_filter *filter, 3662 struct ndb_query_results *results, 3663 int limit) 3664 { 3665 MDB_cursor *cur; 3666 MDB_dbi db; 3667 MDB_val k, v; 3668 int rc, i, need_relays = 0; 3669 struct ndb_filter_elements *ids; 3670 struct ndb_note *note; 3671 struct ndb_query_result res; 3672 struct ndb_tsid tsid, *ptsid; 3673 uint64_t note_id, until, *pint; 3674 size_t note_size; 3675 unsigned char *id; 3676 struct ndb_note_relay_iterator note_relay_iter = {0}; 3677 struct ndb_note_relay_iterator *relay_iter = NULL; 3678 3679 until = UINT64_MAX; 3680 3681 if (!(ids = ndb_filter_find_elements(filter, NDB_FILTER_IDS))) 3682 return 0; 3683 3684 if ((pint = ndb_filter_get_int(filter, NDB_FILTER_UNTIL))) 3685 until = *pint; 3686 3687 if (ndb_filter_find_elements(filter, NDB_FILTER_RELAYS)) 3688 need_relays = 1; 3689 3690 db = txn->lmdb->dbs[NDB_DB_NOTE_ID]; 3691 if ((rc = mdb_cursor_open(txn->mdb_txn, db, &cur))) 3692 return 0; 3693 3694 // for each id in our ids filter, find in the db 3695 for (i = 0; i < ids->count; i++) { 3696 if (query_is_full(results, limit)) 3697 break; 3698 3699 id = ndb_filter_get_id_element(filter, ids, i); 3700 ndb_tsid_init(&tsid, (unsigned char *)id, until); 3701 3702 k.mv_data = &tsid; 3703 k.mv_size = sizeof(tsid); 3704 3705 if (!ndb_cursor_start(cur, &k, &v)) 3706 continue; 3707 3708 ptsid = (struct ndb_tsid *)k.mv_data; 3709 note_id = *(uint64_t*)v.mv_data; 3710 3711 if (memcmp(id, ptsid->id, 32)) 3712 continue; 3713 3714 // get the note because we need it to match against the filter 3715 if (!(note = ndb_get_note_by_key(txn, note_id, ¬e_size))) 3716 continue; 3717 3718 relay_iter = need_relays ? ¬e_relay_iter : NULL; 3719 if (relay_iter) 3720 ndb_note_relay_iterate_start(txn, relay_iter, note_id); 3721 3722 // Sure this particular lookup matched the index query, but 3723 // does it match the entire filter? Check! We also pass in 3724 // things we've already matched via the filter so we don't have 3725 // to check again. This can be pretty important for filters 3726 // with a large number of entries. 3727 if (!ndb_filter_matches_with(filter, note, 1 << NDB_FILTER_IDS, relay_iter)) { 3728 ndb_note_relay_iterate_close(relay_iter); 3729 continue; 3730 } 3731 ndb_note_relay_iterate_close(relay_iter); 3732 3733 ndb_query_result_init(&res, note, note_size, note_id); 3734 if (!push_query_result(results, &res)) 3735 break; 3736 } 3737 3738 mdb_cursor_close(cur); 3739 return 1; 3740 } 3741 3742 // 3743 // encode a tag index key 3744 // 3745 // consists of: 3746 // 3747 // u8 tag 3748 // u8 tag_val_len 3749 // [u8] tag_val_bytes 3750 // u64 created_at 3751 // 3752 static int ndb_encode_tag_key(unsigned char *buf, int buf_size, 3753 char tag, const unsigned char *val, 3754 unsigned char val_len, 3755 uint64_t timestamp) 3756 { 3757 struct cursor writer; 3758 int ok; 3759 3760 // quick exit for obvious case where it will be too big. There can be 3761 // values of val_len that still fail, but we just let the writer handle 3762 // those failure cases 3763 if (val_len >= buf_size) 3764 return 0; 3765 3766 make_cursor(buf, buf + buf_size, &writer); 3767 3768 ok = 3769 cursor_push_byte(&writer, tag) && 3770 cursor_push(&writer, (unsigned char*)val, val_len) && 3771 cursor_push(&writer, (unsigned char*)×tamp, sizeof(timestamp)); 3772 3773 if (!ok) 3774 return 0; 3775 3776 return writer.p - writer.start; 3777 } 3778 3779 static int ndb_query_plan_execute_authors(struct ndb_txn *txn, 3780 struct ndb_filter *filter, 3781 struct ndb_query_results *results, 3782 int limit) 3783 { 3784 MDB_val k, v; 3785 MDB_cursor *cur; 3786 int rc, i, need_relays = 0; 3787 uint64_t *pint, until, since, note_key; 3788 unsigned char *author; 3789 struct ndb_note *note; 3790 size_t note_size; 3791 struct ndb_filter_elements *authors; 3792 struct ndb_query_result res; 3793 struct ndb_tsid tsid, *ptsid; 3794 struct ndb_note_relay_iterator note_relay_iter; 3795 enum ndb_dbs db; 3796 3797 db = txn->lmdb->dbs[NDB_DB_NOTE_PUBKEY]; 3798 3799 if (!(authors = ndb_filter_find_elements(filter, NDB_FILTER_AUTHORS))) 3800 return 0; 3801 3802 until = UINT64_MAX; 3803 if ((pint = ndb_filter_get_int(filter, NDB_FILTER_UNTIL))) 3804 until = *pint; 3805 3806 since = 0; 3807 if ((pint = ndb_filter_get_int(filter, NDB_FILTER_SINCE))) 3808 since = *pint; 3809 3810 if (ndb_filter_find_elements(filter, NDB_FILTER_RELAYS)) 3811 need_relays = 1; 3812 3813 if ((rc = mdb_cursor_open(txn->mdb_txn, db, &cur))) 3814 return 0; 3815 3816 for (i = 0; i < authors->count; i++) { 3817 author = ndb_filter_get_id_element(filter, authors, i); 3818 3819 ndb_tsid_init(&tsid, author, until); 3820 3821 k.mv_data = &tsid; 3822 k.mv_size = sizeof(tsid); 3823 3824 if (!ndb_cursor_start(cur, &k, &v)) 3825 continue; 3826 3827 // for each id in our ids filter, find in the db 3828 while (!query_is_full(results, limit)) { 3829 ptsid = (struct ndb_tsid *)k.mv_data; 3830 note_key = *(uint64_t*)v.mv_data; 3831 3832 // don't continue the scan if we're below `since` 3833 if (ptsid->timestamp < since) 3834 break; 3835 3836 // our author should match, if not bail 3837 if (memcmp(author, ptsid->id, 32)) 3838 break; 3839 3840 // fetch the note, we need it for our query results 3841 // and to match further against the filter 3842 if (!(note = ndb_get_note_by_key(txn, note_key, ¬e_size))) 3843 goto next; 3844 3845 if (need_relays) 3846 ndb_note_relay_iterate_start(txn, ¬e_relay_iter, note_key); 3847 3848 if (!ndb_filter_matches_with(filter, note, 3849 1 << NDB_FILTER_AUTHORS, 3850 need_relays ? ¬e_relay_iter : NULL)) 3851 { 3852 goto next; 3853 } 3854 3855 ndb_query_result_init(&res, note, note_size, note_key); 3856 if (!push_query_result(results, &res)) 3857 break; 3858 3859 next: 3860 if (mdb_cursor_get(cur, &k, &v, MDB_PREV)) 3861 break; 3862 } 3863 } 3864 3865 mdb_cursor_close(cur); 3866 return 1; 3867 } 3868 3869 static int ndb_query_plan_execute_created_at(struct ndb_txn *txn, 3870 struct ndb_filter *filter, 3871 struct ndb_query_results *results, 3872 int limit) 3873 { 3874 MDB_dbi db; 3875 MDB_val k, v; 3876 MDB_cursor *cur; 3877 int rc, need_relays = 0; 3878 struct ndb_note *note; 3879 struct ndb_tsid key, *pkey; 3880 uint64_t *pint, until, since, note_id; 3881 size_t note_size; 3882 struct ndb_query_result res; 3883 struct ndb_note_relay_iterator note_relay_iter; 3884 unsigned char high_key[32] = {0xFF}; 3885 3886 db = txn->lmdb->dbs[NDB_DB_NOTE_ID]; 3887 3888 until = UINT64_MAX; 3889 if ((pint = ndb_filter_get_int(filter, NDB_FILTER_UNTIL))) 3890 until = *pint; 3891 3892 since = 0; 3893 if ((pint = ndb_filter_get_int(filter, NDB_FILTER_SINCE))) 3894 since = *pint; 3895 3896 if (ndb_filter_find_elements(filter, NDB_FILTER_RELAYS)) 3897 need_relays = 1; 3898 3899 if ((rc = mdb_cursor_open(txn->mdb_txn, db, &cur))) 3900 return 0; 3901 3902 // if we have until, start there, otherwise just use max 3903 ndb_tsid_init(&key, high_key, until); 3904 k.mv_data = &key; 3905 k.mv_size = sizeof(key); 3906 3907 if (!ndb_cursor_start(cur, &k, &v)) 3908 return 1; 3909 3910 while (!query_is_full(results, limit)) { 3911 pkey = (struct ndb_tsid *)k.mv_data; 3912 note_id = *(uint64_t*)v.mv_data; 3913 assert(v.mv_size == 8); 3914 3915 // don't continue the scan if we're below `since` 3916 if (pkey->timestamp < since) 3917 break; 3918 3919 if (!(note = ndb_get_note_by_key(txn, note_id, ¬e_size))) 3920 goto next; 3921 3922 if (need_relays) 3923 ndb_note_relay_iterate_start(txn, ¬e_relay_iter, note_id); 3924 3925 // does this entry match our filter? 3926 if (!ndb_filter_matches_with(filter, note, 0, need_relays ? ¬e_relay_iter : NULL)) 3927 goto next; 3928 3929 ndb_query_result_init(&res, note, (uint64_t)note_size, note_id); 3930 if (!push_query_result(results, &res)) 3931 break; 3932 next: 3933 if (mdb_cursor_get(cur, &k, &v, MDB_PREV)) 3934 break; 3935 } 3936 3937 mdb_cursor_close(cur); 3938 return 1; 3939 } 3940 3941 static int ndb_query_plan_execute_tags(struct ndb_txn *txn, 3942 struct ndb_filter *filter, 3943 struct ndb_query_results *results, 3944 int limit) 3945 { 3946 MDB_cursor *cur; 3947 MDB_dbi db; 3948 MDB_val k, v; 3949 int len, taglen, rc, i, need_relays = 0; 3950 uint64_t *pint, until, note_id; 3951 size_t note_size; 3952 unsigned char key_buffer[255]; 3953 struct ndb_note *note; 3954 struct ndb_filter_elements *tags; 3955 unsigned char *tag; 3956 struct ndb_query_result res; 3957 struct ndb_note_relay_iterator note_relay_iter; 3958 3959 db = txn->lmdb->dbs[NDB_DB_NOTE_TAGS]; 3960 3961 if (!(tags = ndb_filter_find_elements(filter, NDB_FILTER_TAGS))) 3962 return 0; 3963 3964 if (ndb_filter_find_elements(filter, NDB_FILTER_RELAYS)) 3965 need_relays = 1; 3966 3967 until = UINT64_MAX; 3968 if ((pint = ndb_filter_get_int(filter, NDB_FILTER_UNTIL))) 3969 until = *pint; 3970 3971 if ((rc = mdb_cursor_open(txn->mdb_txn, db, &cur))) 3972 return 0; 3973 3974 for (i = 0; i < tags->count; i++) { 3975 tag = ndb_filter_get_id_element(filter, tags, i); 3976 3977 taglen = tags->field.elem_type == NDB_ELEMENT_ID 3978 ? 32 : strlen((const char*)tag); 3979 3980 if (!(len = ndb_encode_tag_key(key_buffer, sizeof(key_buffer), 3981 tags->field.tag, tag, taglen, 3982 until))) { 3983 goto fail; 3984 } 3985 3986 k.mv_data = key_buffer; 3987 k.mv_size = len; 3988 3989 if (!ndb_cursor_start(cur, &k, &v)) 3990 continue; 3991 3992 // for each id in our ids filter, find in the db 3993 while (!query_is_full(results, limit)) { 3994 // check if tag value matches, bail if not 3995 if (((unsigned char *)k.mv_data)[0] != tags->field.tag) 3996 break; 3997 3998 // check if tag value matches, bail if not 3999 if (taglen != k.mv_size - 9) 4000 break; 4001 4002 if (memcmp((unsigned char *)k.mv_data+1, tag, k.mv_size-9)) 4003 break; 4004 4005 note_id = *(uint64_t*)v.mv_data; 4006 4007 if (!(note = ndb_get_note_by_key(txn, note_id, ¬e_size))) 4008 goto next; 4009 4010 if (need_relays) 4011 ndb_note_relay_iterate_start(txn, ¬e_relay_iter, note_id); 4012 4013 if (!ndb_filter_matches_with(filter, note, 4014 1 << NDB_FILTER_TAGS, 4015 need_relays ? ¬e_relay_iter : NULL)) 4016 goto next; 4017 4018 ndb_query_result_init(&res, note, note_size, note_id); 4019 if (!push_query_result(results, &res)) 4020 break; 4021 4022 next: 4023 if (mdb_cursor_get(cur, &k, &v, MDB_PREV)) 4024 break; 4025 } 4026 } 4027 4028 mdb_cursor_close(cur); 4029 return 1; 4030 fail: 4031 mdb_cursor_close(cur); 4032 return 0; 4033 } 4034 4035 static int ndb_query_plan_execute_author_kinds( 4036 struct ndb_txn *txn, 4037 struct ndb_filter *filter, 4038 struct ndb_query_results *results, 4039 int limit) 4040 { 4041 MDB_cursor *cur; 4042 MDB_dbi db; 4043 MDB_val k, v; 4044 struct ndb_note *note; 4045 struct ndb_filter_elements *kinds, *relays, *authors; 4046 struct ndb_query_result res; 4047 uint64_t kind, note_id, until, since, *pint; 4048 size_t note_size; 4049 unsigned char *author; 4050 int i, j, rc; 4051 struct ndb_id_u64_ts key, *pkey; 4052 struct ndb_note_relay_iterator note_relay_iter; 4053 4054 // we should have kinds in a kinds filter! 4055 if (!(kinds = ndb_filter_find_elements(filter, NDB_FILTER_KINDS))) 4056 return 0; 4057 // 4058 // we should have kinds in a kinds filter! 4059 if (!(authors = ndb_filter_find_elements(filter, NDB_FILTER_AUTHORS))) 4060 return 0; 4061 4062 relays = ndb_filter_find_elements(filter, NDB_FILTER_RELAYS); 4063 4064 until = UINT64_MAX; 4065 if ((pint = ndb_filter_get_int(filter, NDB_FILTER_UNTIL))) 4066 until = *pint; 4067 4068 since = 0; 4069 if ((pint = ndb_filter_get_int(filter, NDB_FILTER_SINCE))) 4070 since = *pint; 4071 4072 db = txn->lmdb->dbs[NDB_DB_NOTE_PUBKEY_KIND]; 4073 4074 if ((rc = mdb_cursor_open(txn->mdb_txn, db, &cur))) 4075 return 0; 4076 4077 for (j = 0; j < authors->count; j++) { 4078 if (query_is_full(results, limit)) 4079 break; 4080 4081 if (!(author = ndb_filter_get_id_element(filter, authors, j))) 4082 continue; 4083 4084 for (i = 0; i < kinds->count; i++) { 4085 if (query_is_full(results, limit)) 4086 break; 4087 4088 kind = kinds->elements[i]; 4089 4090 ndb_debug("finding kind %"PRIu64"\n", kind); 4091 4092 ndb_id_u64_ts_init(&key, author, kind, until); 4093 4094 k.mv_data = &key; 4095 k.mv_size = sizeof(key); 4096 4097 if (!ndb_cursor_start(cur, &k, &v)) 4098 continue; 4099 4100 // scan the kind subindex 4101 while (!query_is_full(results, limit)) { 4102 pkey = (struct ndb_id_u64_ts*)k.mv_data; 4103 4104 ndb_debug("scanning subindex kind:%"PRIu64" created_at:%"PRIu64" pubkey:", 4105 pkey->u64, 4106 pkey->timestamp); 4107 4108 if (pkey->u64 != kind) 4109 break; 4110 4111 // don't continue the scan if we're below `since` 4112 if (pkey->timestamp < since) 4113 break; 4114 4115 if (memcmp(pkey->id, author, 32)) 4116 break; 4117 4118 note_id = *(uint64_t*)v.mv_data; 4119 if (!(note = ndb_get_note_by_key(txn, note_id, ¬e_size))) 4120 goto next; 4121 4122 if (relays) 4123 ndb_note_relay_iterate_start(txn, ¬e_relay_iter, note_id); 4124 4125 if (!ndb_filter_matches_with(filter, note, 4126 (1 << NDB_FILTER_KINDS) | (1 << NDB_FILTER_AUTHORS), 4127 relays? ¬e_relay_iter : NULL)) 4128 goto next; 4129 4130 ndb_query_result_init(&res, note, note_size, note_id); 4131 if (!push_query_result(results, &res)) 4132 break; 4133 4134 next: 4135 if (mdb_cursor_get(cur, &k, &v, MDB_PREV)) 4136 break; 4137 } 4138 } 4139 } 4140 4141 mdb_cursor_close(cur); 4142 return 1; 4143 } 4144 4145 static int ndb_query_plan_execute_profile_search( 4146 struct ndb_txn *txn, 4147 struct ndb_filter *filter, 4148 struct ndb_query_results *results, 4149 int limit) 4150 { 4151 const char *search; 4152 int i; 4153 4154 // The filter pubkey is updated inplace for each note search 4155 unsigned char *filter_pubkey; 4156 unsigned char pubkey[32] = {0}; 4157 struct ndb_filter_elements *els; 4158 struct ndb_search profile_search; 4159 struct ndb_filter note_filter, *f = ¬e_filter; 4160 4161 if (!(search = ndb_filter_find_search(filter))) 4162 return 0; 4163 4164 if (!ndb_filter_init_with(f, 1)) 4165 return 0; 4166 4167 ndb_filter_start_field(f, NDB_FILTER_KINDS); 4168 ndb_filter_add_int_element(f, 0); 4169 ndb_filter_end_field(f); 4170 4171 ndb_filter_start_field(f, NDB_FILTER_AUTHORS); 4172 ndb_filter_add_id_element(f, pubkey); 4173 ndb_filter_end_field(f); 4174 ndb_filter_end(f); 4175 4176 // get the authors element after we finalize the filter, since 4177 // the data could have moved 4178 if (!(els = ndb_filter_find_elements(f, NDB_FILTER_AUTHORS))) 4179 goto fail; 4180 4181 // grab pointer to pubkey in the filter so that we can 4182 // update the filter as we go 4183 if (!(filter_pubkey = ndb_filter_get_id_element(f, els, 0))) 4184 goto fail; 4185 4186 for (i = 0; !query_is_full(results, limit); i++) { 4187 if (i == 0) { 4188 if (!ndb_search_profile(txn, &profile_search, search)) 4189 break; 4190 } else { 4191 if (!ndb_search_profile_next(&profile_search)) 4192 break; 4193 } 4194 4195 // Copy pubkey into filter 4196 memcpy(filter_pubkey, profile_search.key->id, 32); 4197 4198 // Look up the corresponding note associated with that pubkey 4199 if (!ndb_query_plan_execute_author_kinds(txn, f, results, limit)) 4200 goto fail; 4201 } 4202 4203 ndb_search_profile_end(&profile_search); 4204 ndb_filter_destroy(f); 4205 return 1; 4206 4207 fail: 4208 ndb_filter_destroy(f); 4209 return 0; 4210 } 4211 4212 static int ndb_query_plan_execute_relay_kinds( 4213 struct ndb_txn *txn, 4214 struct ndb_filter *filter, 4215 struct ndb_query_results *results, 4216 int limit) 4217 { 4218 MDB_cursor *cur; 4219 MDB_dbi db; 4220 MDB_val k, v; 4221 struct ndb_note *note; 4222 struct ndb_filter_elements *kinds, *relays; 4223 struct ndb_query_result res; 4224 uint64_t kind, note_id, until, since, *pint; 4225 size_t note_size; 4226 const char *relay; 4227 int i, j, rc, len; 4228 struct ndb_relay_kind_key relay_key; 4229 unsigned char keybuf[256]; 4230 4231 // we should have kinds in a kinds filter! 4232 if (!(kinds = ndb_filter_find_elements(filter, NDB_FILTER_KINDS))) 4233 return 0; 4234 4235 if (!(relays = ndb_filter_find_elements(filter, NDB_FILTER_RELAYS))) 4236 return 0; 4237 4238 until = UINT64_MAX; 4239 if ((pint = ndb_filter_get_int(filter, NDB_FILTER_UNTIL))) 4240 until = *pint; 4241 4242 since = 0; 4243 if ((pint = ndb_filter_get_int(filter, NDB_FILTER_SINCE))) 4244 since = *pint; 4245 4246 db = txn->lmdb->dbs[NDB_DB_NOTE_RELAY_KIND]; 4247 4248 if ((rc = mdb_cursor_open(txn->mdb_txn, db, &cur))) 4249 return 0; 4250 4251 for (j = 0; j < relays->count; j++) { 4252 if (query_is_full(results, limit)) 4253 break; 4254 4255 if (!(relay = ndb_filter_get_string_element(filter, relays, j))) 4256 continue; 4257 4258 for (i = 0; i < kinds->count; i++) { 4259 if (query_is_full(results, limit)) 4260 break; 4261 4262 kind = kinds->elements[i]; 4263 ndb_debug("kind %" PRIu64 "\n", kind); 4264 4265 if (!ndb_relay_kind_key_init_high(&relay_key, relay, kind, until)) { 4266 ndb_debug("ndb_relay_kind_key_init_high failed in relay query\n"); 4267 continue; 4268 } 4269 4270 if (!(len = ndb_build_relay_kind_key(keybuf, sizeof(keybuf), &relay_key))) { 4271 ndb_debug("ndb_build_relay_kind_key failed in relay query\n"); 4272 ndb_debug_relay_kind_key(&relay_key); 4273 continue; 4274 } 4275 4276 k.mv_data = keybuf; 4277 k.mv_size = len; 4278 4279 ndb_debug("starting with key "); 4280 ndb_debug_relay_kind_key(&relay_key); 4281 4282 if (!ndb_cursor_start(cur, &k, &v)) 4283 continue; 4284 4285 // scan the kind subindex 4286 while (!query_is_full(results, limit)) { 4287 ndb_parse_relay_kind_key(&relay_key, k.mv_data); 4288 4289 ndb_debug("inside kind subindex "); 4290 ndb_debug_relay_kind_key(&relay_key); 4291 4292 if (relay_key.kind != kind) 4293 break; 4294 4295 if (strcmp(relay_key.relay, relay)) 4296 break; 4297 4298 // don't continue the scan if we're below `since` 4299 if (relay_key.created_at < since) 4300 break; 4301 4302 note_id = relay_key.note_key; 4303 if (!(note = ndb_get_note_by_key(txn, note_id, ¬e_size))) 4304 goto next; 4305 4306 if (!ndb_filter_matches_with(filter, note, 4307 (1 << NDB_FILTER_KINDS) | (1 << NDB_FILTER_RELAYS), 4308 NULL)) 4309 goto next; 4310 4311 ndb_query_result_init(&res, note, note_size, note_id); 4312 if (!push_query_result(results, &res)) 4313 break; 4314 4315 next: 4316 if (mdb_cursor_get(cur, &k, &v, MDB_PREV)) 4317 break; 4318 } 4319 } 4320 } 4321 4322 mdb_cursor_close(cur); 4323 return 1; 4324 } 4325 4326 static int ndb_query_plan_execute_kinds(struct ndb_txn *txn, 4327 struct ndb_filter *filter, 4328 struct ndb_query_results *results, 4329 int limit) 4330 { 4331 MDB_cursor *cur; 4332 MDB_dbi db; 4333 MDB_val k, v; 4334 struct ndb_note *note; 4335 struct ndb_u64_ts tsid, *ptsid; 4336 struct ndb_filter_elements *kinds; 4337 struct ndb_query_result res; 4338 uint64_t kind, note_id, until, since, *pint; 4339 size_t note_size; 4340 int i, rc, need_relays = 0; 4341 struct ndb_note_relay_iterator note_relay_iter; 4342 4343 // we should have kinds in a kinds filter! 4344 if (!(kinds = ndb_filter_find_elements(filter, NDB_FILTER_KINDS))) 4345 return 0; 4346 4347 if (ndb_filter_find_elements(filter, NDB_FILTER_RELAYS)) 4348 need_relays = 1; 4349 4350 until = UINT64_MAX; 4351 if ((pint = ndb_filter_get_int(filter, NDB_FILTER_UNTIL))) 4352 until = *pint; 4353 4354 since = 0; 4355 if ((pint = ndb_filter_get_int(filter, NDB_FILTER_SINCE))) 4356 since = *pint; 4357 4358 db = txn->lmdb->dbs[NDB_DB_NOTE_KIND]; 4359 4360 if ((rc = mdb_cursor_open(txn->mdb_txn, db, &cur))) 4361 return 0; 4362 4363 for (i = 0; i < kinds->count; i++) { 4364 if (query_is_full(results, limit)) 4365 break; 4366 4367 kind = kinds->elements[i]; 4368 ndb_debug("kind %" PRIu64 "\n", kind); 4369 ndb_u64_ts_init(&tsid, kind, until); 4370 4371 k.mv_data = &tsid; 4372 k.mv_size = sizeof(tsid); 4373 4374 if (!ndb_cursor_start(cur, &k, &v)) 4375 continue; 4376 4377 // for each id in our ids filter, find in the db 4378 while (!query_is_full(results, limit)) { 4379 ptsid = (struct ndb_u64_ts *)k.mv_data; 4380 if (ptsid->u64 != kind) 4381 break; 4382 4383 // don't continue the scan if we're below `since` 4384 if (ptsid->timestamp < since) 4385 break; 4386 4387 note_id = *(uint64_t*)v.mv_data; 4388 if (!(note = ndb_get_note_by_key(txn, note_id, ¬e_size))) 4389 goto next; 4390 4391 if (need_relays) 4392 ndb_note_relay_iterate_start(txn, ¬e_relay_iter, note_id); 4393 4394 if (!ndb_filter_matches_with(filter, note, 4395 1 << NDB_FILTER_KINDS, 4396 need_relays ? ¬e_relay_iter : NULL)) 4397 goto next; 4398 4399 ndb_query_result_init(&res, note, note_size, note_id); 4400 if (!push_query_result(results, &res)) 4401 break; 4402 4403 next: 4404 if (mdb_cursor_get(cur, &k, &v, MDB_PREV)) 4405 break; 4406 } 4407 } 4408 4409 mdb_cursor_close(cur); 4410 return 1; 4411 } 4412 4413 static enum ndb_query_plan ndb_filter_plan(struct ndb_filter *filter) 4414 { 4415 struct ndb_filter_elements *ids, *kinds, *authors, *tags, *search, *relays; 4416 4417 ids = ndb_filter_find_elements(filter, NDB_FILTER_IDS); 4418 search = ndb_filter_find_elements(filter, NDB_FILTER_SEARCH); 4419 kinds = ndb_filter_find_elements(filter, NDB_FILTER_KINDS); 4420 authors = ndb_filter_find_elements(filter, NDB_FILTER_AUTHORS); 4421 tags = ndb_filter_find_elements(filter, NDB_FILTER_TAGS); 4422 relays = ndb_filter_find_elements(filter, NDB_FILTER_RELAYS); 4423 4424 // profile search 4425 if (kinds && kinds->count == 1 && kinds->elements[0] == 0 && search) { 4426 return NDB_PLAN_PROFILE_SEARCH; 4427 } 4428 4429 // TODO: fix multi-author queries 4430 if (search) { 4431 return NDB_PLAN_SEARCH; 4432 } else if (ids) { 4433 return NDB_PLAN_IDS; 4434 } else if (relays && kinds && !authors) { 4435 return NDB_PLAN_RELAY_KINDS; 4436 } else if (kinds && authors && authors->count == 1) { 4437 return NDB_PLAN_AUTHOR_KINDS; 4438 } else if (authors && authors->count == 1) { 4439 return NDB_PLAN_AUTHORS; 4440 } else if (tags && tags->count == 1) { 4441 return NDB_PLAN_TAGS; 4442 } else if (kinds) { 4443 return NDB_PLAN_KINDS; 4444 } 4445 4446 return NDB_PLAN_CREATED; 4447 } 4448 4449 static const char *ndb_query_plan_name(enum ndb_query_plan plan_id) 4450 { 4451 switch (plan_id) { 4452 case NDB_PLAN_IDS: return "ids"; 4453 case NDB_PLAN_SEARCH: return "search"; 4454 case NDB_PLAN_KINDS: return "kinds"; 4455 case NDB_PLAN_TAGS: return "tags"; 4456 case NDB_PLAN_CREATED: return "created"; 4457 case NDB_PLAN_AUTHORS: return "authors"; 4458 case NDB_PLAN_RELAY_KINDS: return "relay_kinds"; 4459 case NDB_PLAN_AUTHOR_KINDS: return "author_kinds"; 4460 case NDB_PLAN_PROFILE_SEARCH: return "profile_search"; 4461 } 4462 4463 return "unknown"; 4464 } 4465 4466 static int ndb_query_filter(struct ndb_txn *txn, struct ndb_filter *filter, 4467 struct ndb_query_result *res, int capacity, 4468 int *results_out) 4469 { 4470 struct ndb_query_results results; 4471 uint64_t limit, *pint; 4472 enum ndb_query_plan plan; 4473 limit = capacity; 4474 4475 if ((pint = ndb_filter_get_int(filter, NDB_FILTER_LIMIT))) 4476 limit = *pint; 4477 4478 limit = min(capacity, limit); 4479 make_cursor((unsigned char *)res, 4480 ((unsigned char *)res) + limit * sizeof(*res), 4481 &results.cur); 4482 4483 plan = ndb_filter_plan(filter); 4484 ndb_debug("using query plan '%s'\n", ndb_query_plan_name(plan)); 4485 switch (plan) { 4486 // We have a list of ids, just open a cursor and jump to each once 4487 case NDB_PLAN_IDS: 4488 if (!ndb_query_plan_execute_ids(txn, filter, &results, limit)) 4489 return 0; 4490 break; 4491 case NDB_PLAN_RELAY_KINDS: 4492 if (!ndb_query_plan_execute_relay_kinds(txn, filter, &results, limit)) 4493 return 0; 4494 break; 4495 case NDB_PLAN_SEARCH: 4496 if (!ndb_query_plan_execute_search(txn, filter, &results, limit)) 4497 return 0; 4498 break; 4499 4500 case NDB_PLAN_PROFILE_SEARCH: 4501 if (!ndb_query_plan_execute_profile_search(txn, filter, &results, limit)) 4502 return 0; 4503 break; 4504 4505 // We have just kinds, just scan the kind index 4506 case NDB_PLAN_KINDS: 4507 if (!ndb_query_plan_execute_kinds(txn, filter, &results, limit)) 4508 return 0; 4509 break; 4510 case NDB_PLAN_TAGS: 4511 if (!ndb_query_plan_execute_tags(txn, filter, &results, limit)) 4512 return 0; 4513 break; 4514 case NDB_PLAN_CREATED: 4515 if (!ndb_query_plan_execute_created_at(txn, filter, &results, limit)) 4516 return 0; 4517 break; 4518 case NDB_PLAN_AUTHORS: 4519 if (!ndb_query_plan_execute_authors(txn, filter, &results, limit)) 4520 return 0; 4521 break; 4522 case NDB_PLAN_AUTHOR_KINDS: 4523 if (!ndb_query_plan_execute_author_kinds(txn, filter, &results, limit)) 4524 return 0; 4525 break; 4526 } 4527 4528 *results_out = cursor_count(&results.cur, sizeof(*res)); 4529 return 1; 4530 } 4531 4532 int ndb_query(struct ndb_txn *txn, struct ndb_filter *filters, int num_filters, 4533 struct ndb_query_result *results, int result_capacity, int *count) 4534 { 4535 int i, out; 4536 struct ndb_query_result *p = results; 4537 4538 out = 0; 4539 *count = 0; 4540 4541 for (i = 0; i < num_filters; i++) { 4542 if (!ndb_query_filter(txn, &filters[i], p, 4543 result_capacity, &out)) { 4544 return 0; 4545 } 4546 4547 *count += out; 4548 p += out; 4549 result_capacity -= out; 4550 if (result_capacity <= 0) 4551 break; 4552 } 4553 4554 // sort results 4555 qsort(results, *count, sizeof(*results), compare_query_results); 4556 return 1; 4557 } 4558 4559 static int ndb_write_note_tag_index(struct ndb_txn *txn, struct ndb_note *note, 4560 uint64_t note_key) 4561 { 4562 unsigned char key_buffer[255]; 4563 struct ndb_iterator iter; 4564 struct ndb_str tkey, tval; 4565 char tchar; 4566 int len, rc; 4567 MDB_val key, val; 4568 MDB_dbi tags_db; 4569 4570 tags_db = txn->lmdb->dbs[NDB_DB_NOTE_TAGS]; 4571 4572 ndb_tags_iterate_start(note, &iter); 4573 4574 while (ndb_tags_iterate_next(&iter)) { 4575 if (iter.tag->count < 2) 4576 continue; 4577 4578 tkey = ndb_tag_str(note, iter.tag, 0); 4579 4580 // we only write indices for 1-char tags. 4581 tchar = tkey.str[0]; 4582 if (tchar == 0 || tkey.str[1] != 0) 4583 continue; 4584 4585 tval = ndb_tag_str(note, iter.tag, 1); 4586 len = ndb_str_len(&tval); 4587 4588 if (!(len = ndb_encode_tag_key(key_buffer, sizeof(key_buffer), 4589 tchar, tval.id, (unsigned char)len, 4590 ndb_note_created_at(note)))) { 4591 // this will fail when we try to encode a key that is 4592 // too big 4593 continue; 4594 } 4595 4596 //ndb_debug("writing tag '%c':'data:%d' to index\n", tchar, len); 4597 4598 key.mv_data = key_buffer; 4599 key.mv_size = len; 4600 4601 val.mv_data = ¬e_key; 4602 val.mv_size = sizeof(note_key); 4603 4604 if ((rc = mdb_put(txn->mdb_txn, tags_db, &key, &val, 0))) { 4605 ndb_debug("write note tag index to db failed: %s\n", 4606 mdb_strerror(rc)); 4607 return 0; 4608 } 4609 } 4610 4611 return 1; 4612 } 4613 4614 static int ndb_write_note_kind_index(struct ndb_txn *txn, struct ndb_note *note, 4615 uint64_t note_key) 4616 { 4617 struct ndb_u64_ts tsid; 4618 int rc; 4619 MDB_val key, val; 4620 MDB_dbi kind_db; 4621 4622 ndb_u64_ts_init(&tsid, note->kind, note->created_at); 4623 4624 key.mv_data = &tsid; 4625 key.mv_size = sizeof(tsid); 4626 val.mv_data = ¬e_key; 4627 val.mv_size = sizeof(note_key); 4628 4629 kind_db = txn->lmdb->dbs[NDB_DB_NOTE_KIND]; 4630 4631 if ((rc = mdb_put(txn->mdb_txn, kind_db, &key, &val, 0))) { 4632 ndb_debug("write note kind index to db failed: %s\n", 4633 mdb_strerror(rc)); 4634 return 0; 4635 } 4636 4637 return 1; 4638 } 4639 4640 static int ndb_write_word_to_index(struct ndb_txn *txn, const char *word, 4641 int word_len, int word_index, 4642 uint64_t timestamp, uint64_t note_id) 4643 { 4644 // cap to some reasonable key size 4645 unsigned char buffer[1024]; 4646 int keysize, rc; 4647 MDB_val k, v; 4648 MDB_dbi text_db; 4649 4650 // build our compressed text index key 4651 if (!ndb_make_text_search_key(buffer, sizeof(buffer), word_index, 4652 word_len, word, timestamp, note_id, 4653 &keysize)) { 4654 // probably too big 4655 4656 return 0; 4657 } 4658 4659 k.mv_data = buffer; 4660 k.mv_size = keysize; 4661 4662 v.mv_data = NULL; 4663 v.mv_size = 0; 4664 4665 text_db = txn->lmdb->dbs[NDB_DB_NOTE_TEXT]; 4666 4667 if ((rc = mdb_put(txn->mdb_txn, text_db, &k, &v, 0))) { 4668 ndb_debug("write note text index to db failed: %s\n", 4669 mdb_strerror(rc)); 4670 return 0; 4671 } 4672 4673 return 1; 4674 } 4675 4676 4677 4678 // break a string into individual words for querying or for building the 4679 // fulltext search index. This is callback based so we don't need to 4680 // build up an intermediate structure 4681 static int ndb_parse_words(struct cursor *cur, void *ctx, ndb_word_parser_fn fn) 4682 { 4683 int word_len, words; 4684 const char *word; 4685 4686 words = 0; 4687 4688 while (cur->p < cur->end) { 4689 consume_whitespace_or_punctuation(cur); 4690 if (cur->p >= cur->end) 4691 break; 4692 word = (const char *)cur->p; 4693 4694 if (!consume_until_boundary(cur)) 4695 break; 4696 4697 // start of word or end 4698 word_len = cur->p - (unsigned char *)word; 4699 if (word_len == 0 && cur->p >= cur->end) 4700 break; 4701 4702 if (word_len == 0) { 4703 if (!cursor_skip(cur, 1)) 4704 break; 4705 continue; 4706 } 4707 4708 //ndb_debug("writing word index '%.*s'\n", word_len, word); 4709 4710 if (!fn(ctx, word, word_len, words)) 4711 continue; 4712 4713 words++; 4714 } 4715 4716 return 1; 4717 } 4718 4719 struct ndb_word_writer_ctx 4720 { 4721 struct ndb_txn *txn; 4722 struct ndb_note *note; 4723 uint64_t note_id; 4724 }; 4725 4726 static int ndb_fulltext_word_writer(void *ctx, 4727 const char *word, int word_len, int words) 4728 { 4729 struct ndb_word_writer_ctx *wctx = ctx; 4730 4731 if (!ndb_write_word_to_index(wctx->txn, word, word_len, words, 4732 wctx->note->created_at, wctx->note_id)) { 4733 // too big to write this one, just skip it 4734 ndb_debug("failed to write word '%.*s' to index\n", word_len, word); 4735 4736 return 0; 4737 } 4738 4739 //fprintf(stderr, "wrote '%.*s' to note text index\n", word_len, word); 4740 return 1; 4741 } 4742 4743 static int ndb_write_note_fulltext_index(struct ndb_txn *txn, 4744 struct ndb_note *note, 4745 uint64_t note_id) 4746 { 4747 struct cursor cur; 4748 unsigned char *content; 4749 struct ndb_str str; 4750 struct ndb_word_writer_ctx ctx; 4751 4752 str = ndb_note_str(note, ¬e->content); 4753 // I don't think this should happen? 4754 if (unlikely(str.flag == NDB_PACKED_ID)) 4755 return 0; 4756 4757 content = (unsigned char *)str.str; 4758 4759 make_cursor(content, content + note->content_length, &cur); 4760 4761 ctx.txn = txn; 4762 ctx.note = note; 4763 ctx.note_id = note_id; 4764 4765 ndb_parse_words(&cur, &ctx, ndb_fulltext_word_writer); 4766 4767 return 1; 4768 } 4769 4770 static int ndb_parse_search_words(void *ctx, const char *word_str, int word_len, int word_index) 4771 { 4772 (void)word_index; 4773 struct ndb_search_words *words = ctx; 4774 struct ndb_word *word; 4775 4776 if (words->num_words + 1 > MAX_TEXT_SEARCH_WORDS) 4777 return 0; 4778 4779 word = &words->words[words->num_words++]; 4780 word->word = word_str; 4781 word->word_len = word_len; 4782 4783 return 1; 4784 } 4785 4786 static void ndb_search_words_init(struct ndb_search_words *words) 4787 { 4788 words->num_words = 0; 4789 } 4790 4791 static int prefix_count(const char *str1, int len1, const char *str2, int len2) { 4792 int i, count = 0; 4793 int min_len = len1 < len2 ? len1 : len2; 4794 4795 for (i = 0; i < min_len; i++) { 4796 // case insensitive 4797 if (tolower(str1[i]) == tolower(str2[i])) 4798 count++; 4799 else 4800 break; 4801 } 4802 4803 return count; 4804 } 4805 4806 static int ndb_prefix_matches(struct ndb_text_search_result *result, 4807 struct ndb_word *search_word) 4808 { 4809 // Empty strings shouldn't happen but let's 4810 if (result->key.str_len < 2 || search_word->word_len < 2) 4811 return 0; 4812 4813 // make sure we at least have two matching prefix characters. exact 4814 // matches are nice but range searches allow us to match prefixes as 4815 // well. A double-char prefix is suffient, but maybe we could up this 4816 // in the future. 4817 // 4818 // TODO: How are we handling utf-8 prefix matches like 4819 // japanese? 4820 // 4821 if ( result->key.str[0] != tolower(search_word->word[0]) 4822 && result->key.str[1] != tolower(search_word->word[1]) 4823 ) 4824 return 0; 4825 4826 // count the number of prefix-matched characters. This will be used 4827 // for ranking search results 4828 result->prefix_chars = prefix_count(result->key.str, 4829 result->key.str_len, 4830 search_word->word, 4831 search_word->word_len); 4832 4833 if (result->prefix_chars <= (int)((double)search_word->word_len / 1.5)) 4834 return 0; 4835 4836 return 1; 4837 } 4838 4839 // This is called when scanning the full text search index. Scanning stops 4840 // when we no longer have a prefix match for the word 4841 static int ndb_text_search_next_word(MDB_cursor *cursor, MDB_cursor_op op, 4842 MDB_val *k, struct ndb_word *search_word, 4843 struct ndb_text_search_result *last_result, 4844 struct ndb_text_search_result *result, 4845 MDB_cursor_op order_op) 4846 { 4847 struct cursor key_cursor; 4848 //struct ndb_text_search_key search_key; 4849 MDB_val v; 4850 int retries; 4851 retries = -1; 4852 4853 make_cursor(k->mv_data, (unsigned char *)k->mv_data + k->mv_size, &key_cursor); 4854 4855 // When op is MDB_SET_RANGE, this initializes the search. Position 4856 // the cursor at the next key greater than or equal to the specified 4857 // key. 4858 // 4859 // Subsequent searches should use MDB_NEXT 4860 if (mdb_cursor_get(cursor, k, &v, op)) { 4861 // we should only do this if we're going in reverse 4862 if (op == MDB_SET_RANGE && order_op == MDB_PREV) { 4863 // if set range worked and our key exists, it should be 4864 // the one right before this one 4865 if (mdb_cursor_get(cursor, k, &v, MDB_PREV)) 4866 return 0; 4867 } else { 4868 return 0; 4869 } 4870 } 4871 4872 retry: 4873 retries++; 4874 /* 4875 printf("continuing from "); 4876 if (ndb_unpack_text_search_key(k->mv_data, k->mv_size, &search_key)) { 4877 ndb_print_text_search_key(&search_key); 4878 } else { printf("??"); } 4879 printf("\n"); 4880 */ 4881 4882 make_cursor(k->mv_data, (unsigned char *)k->mv_data + k->mv_size, &key_cursor); 4883 4884 if (unlikely(!ndb_unpack_text_search_key_noteid(&key_cursor, &result->key.note_id))) { 4885 fprintf(stderr, "UNUSUAL: failed to unpack text search key note_id\n"); 4886 return 0; 4887 } 4888 4889 // if the note id doesn't match the last result, then we stop trying 4890 // each search word 4891 if (last_result && last_result->key.note_id != result->key.note_id) { 4892 return 0; 4893 } 4894 4895 // On success, this could still be not related at all. 4896 // It could just be adjacent to the word. Let's check 4897 // if we have a matching prefix at least. 4898 4899 // Before we unpack the entire key, let's quickly 4900 // unpack just the string to check the prefix. We don't 4901 // need to unpack the entire key if the prefix doesn't 4902 // match 4903 if (!ndb_unpack_text_search_key_string(&key_cursor, 4904 &result->key.str, 4905 &result->key.str_len)) { 4906 // this should never happen 4907 fprintf(stderr, "UNUSUAL: failed to unpack text search key string\n"); 4908 return 0; 4909 } 4910 4911 if (!ndb_prefix_matches(result, search_word)) { 4912 /* 4913 printf("result prefix '%.*s' didn't match search word '%.*s'\n", 4914 result->key.str_len, result->key.str, 4915 search_word->word_len, search_word->word); 4916 */ 4917 // we should only do this if we're going in reverse 4918 if (retries == 0 && op == MDB_SET_RANGE && order_op == MDB_PREV) { 4919 // if set range worked and our key exists, it should be 4920 // the one right before this one 4921 mdb_cursor_get(cursor, k, &v, MDB_PREV); 4922 goto retry; 4923 } else { 4924 return 0; 4925 } 4926 } 4927 4928 // Unpack the remaining text search key, we will need this information 4929 // when building up our search results. 4930 if (!ndb_unpack_remaining_text_search_key(&key_cursor, &result->key)) { 4931 // This should never happen 4932 fprintf(stderr, "UNUSUAL: failed to unpack text search key\n"); 4933 return 0; 4934 } 4935 4936 /* 4937 if (last_result) { 4938 if (result->key.word_index < last_result->key.word_index) { 4939 fprintf(stderr, "skipping '%.*s' because it is before last result '%.*s'\n", 4940 result->key.str_len, result->key.str, 4941 last_result->key.str_len, last_result->key.str); 4942 return 0; 4943 } 4944 } 4945 */ 4946 4947 return 1; 4948 } 4949 4950 static void ndb_text_search_results_init( 4951 struct ndb_text_search_results *results) { 4952 results->num_results = 0; 4953 } 4954 4955 void ndb_default_text_search_config(struct ndb_text_search_config *cfg) 4956 { 4957 cfg->order = NDB_ORDER_DESCENDING; 4958 cfg->limit = MAX_TEXT_SEARCH_RESULTS; 4959 } 4960 4961 void ndb_text_search_config_set_order(struct ndb_text_search_config *cfg, 4962 enum ndb_search_order order) 4963 { 4964 cfg->order = order; 4965 } 4966 4967 void ndb_text_search_config_set_limit(struct ndb_text_search_config *cfg, int limit) 4968 { 4969 cfg->limit = limit; 4970 } 4971 4972 static int compare_search_words(const void *pa, const void *pb) 4973 { 4974 struct ndb_word *a, *b; 4975 4976 a = (struct ndb_word *)pa; 4977 b = (struct ndb_word *)pb; 4978 4979 if (a->word_len == b->word_len) { 4980 return 0; 4981 } else if (a->word_len > b->word_len) { 4982 // biggest words should be at the front of the list, 4983 // so we say it's "smaller" here 4984 return -1; 4985 } else { 4986 return 1; 4987 } 4988 } 4989 4990 // Sort search words from largest to smallest. Larger words are less likely 4991 // in the index, allowing our scan to walk fewer words at the root when 4992 // recursively matching. 4993 void sort_largest_to_smallest(struct ndb_search_words *words) 4994 { 4995 qsort(words->words, words->num_words, sizeof(words->words[0]), compare_search_words); 4996 } 4997 4998 4999 int ndb_text_search_with(struct ndb_txn *txn, const char *query, 5000 struct ndb_text_search_results *results, 5001 struct ndb_text_search_config *config, 5002 struct ndb_filter *filter) 5003 { 5004 unsigned char buffer[1024], *buf; 5005 unsigned char saved_buf[1024], *saved; 5006 struct ndb_text_search_result *result, *last_result; 5007 struct ndb_text_search_result candidate, last_candidate; 5008 struct ndb_search_words search_words; 5009 //struct ndb_text_search_key search_key; 5010 struct ndb_word *search_word; 5011 struct ndb_note *note; 5012 struct cursor cur; 5013 uint64_t since, until, timestamp_op, *pint; 5014 size_t note_size; 5015 ndb_text_search_key_order_fn key_order_fn; 5016 MDB_dbi text_db; 5017 MDB_cursor *cursor; 5018 MDB_val k, v; 5019 int i, j, keysize, saved_size, limit; 5020 MDB_cursor_op op, order_op; 5021 5022 note_size = 0; 5023 note = 0; 5024 saved = NULL; 5025 ndb_text_search_results_init(results); 5026 ndb_search_words_init(&search_words); 5027 5028 until = UINT64_MAX; 5029 since = 0; 5030 limit = MAX_TEXT_SEARCH_RESULTS; 5031 5032 // until, since from filter 5033 if (filter != NULL) { 5034 if ((pint = ndb_filter_get_int(filter, NDB_FILTER_UNTIL))) 5035 until = *pint; 5036 5037 if ((pint = ndb_filter_get_int(filter, NDB_FILTER_SINCE))) 5038 since = *pint; 5039 5040 if ((pint = ndb_filter_get_int(filter, NDB_FILTER_LIMIT))) 5041 limit = *pint; 5042 } 5043 5044 order_op = MDB_PREV; 5045 key_order_fn = ndb_make_text_search_key_high; 5046 timestamp_op = until; 5047 if (config) { 5048 if (config->order == NDB_ORDER_ASCENDING) { 5049 order_op = MDB_NEXT; 5050 // set the min timestamp value to since when ascending 5051 timestamp_op = since; 5052 key_order_fn = ndb_make_text_search_key_low; 5053 } 5054 limit = min(limit, config->limit); 5055 } 5056 // end search config 5057 5058 text_db = txn->lmdb->dbs[NDB_DB_NOTE_TEXT]; 5059 make_cursor((unsigned char *)query, (unsigned char *)query + strlen(query), &cur); 5060 5061 ndb_parse_words(&cur, &search_words, ndb_parse_search_words); 5062 if (search_words.num_words == 0) 5063 return 0; 5064 5065 if ((i = mdb_cursor_open(txn->mdb_txn, text_db, &cursor))) { 5066 fprintf(stderr, "nd_text_search: mdb_cursor_open failed, error %d\n", i); 5067 return 0; 5068 } 5069 5070 // This should complete the query quicker because the larger words are 5071 // likely to have fewer entries in the search index. This is not always 5072 // true. Words with higher frequency (like bitcoin on nostr in 2024) 5073 // may be slower. TODO: Skip word recursion by leveraging a minimal 5074 // perfect hashmap of parsed words on a note 5075 sort_largest_to_smallest(&search_words); 5076 5077 // for each word, we recursively find all of the submatches 5078 while (results->num_results < limit) { 5079 last_result = NULL; 5080 result = &results->results[results->num_results]; 5081 5082 // if we have saved, then we continue from the last root search 5083 // sequence 5084 if (saved) { 5085 buf = saved_buf; 5086 saved = NULL; 5087 keysize = saved_size; 5088 5089 k.mv_data = buf; 5090 k.mv_size = saved_size; 5091 5092 // reposition the cursor so we can continue 5093 if (mdb_cursor_get(cursor, &k, &v, MDB_SET_RANGE)) 5094 break; 5095 5096 op = order_op; 5097 } else { 5098 // construct a packed fulltext search key using this 5099 // word. This key doesn't contain any timestamp or index 5100 // info, so it should range match instead of exact 5101 // match 5102 if (!key_order_fn(buffer, sizeof(buffer), 5103 search_words.words[0].word_len, 5104 search_words.words[0].word, 5105 timestamp_op, 5106 &keysize)) 5107 { 5108 // word is too big to fit in 1024-sized key 5109 continue; 5110 } 5111 5112 buf = buffer; 5113 op = MDB_SET_RANGE; 5114 } 5115 5116 for (j = 0; j < search_words.num_words; j++) { 5117 search_word = &search_words.words[j]; 5118 5119 // shouldn't happen but let's be defensive a bit 5120 if (search_word->word_len == 0) 5121 continue; 5122 5123 // if we already matched a note in this phrase, make 5124 // sure we're including the note id in the query 5125 if (last_result) { 5126 // we are narrowing down a search. 5127 // if we already have this note id, just continue 5128 for (i = 0; i < results->num_results; i++) { 5129 if (results->results[i].key.note_id == last_result->key.note_id) 5130 // we can't break here to 5131 // leave the word loop so 5132 // have to use a goto 5133 goto cont; 5134 } 5135 5136 if (!ndb_make_noted_text_search_key( 5137 buffer, sizeof(buffer), 5138 search_word->word_len, 5139 search_word->word, 5140 last_result->key.timestamp, 5141 last_result->key.note_id, 5142 &keysize)) 5143 { 5144 continue; 5145 } 5146 5147 buf = buffer; 5148 } 5149 5150 k.mv_data = buf; 5151 k.mv_size = keysize; 5152 5153 // TODO: we can speed this up with the minimal perfect 5154 // hashmap by quickly rejecting the remaining words 5155 // by looking in the word hashmap on the note. This 5156 // would allow us to skip the recursive word lookup 5157 // thing 5158 if (!ndb_text_search_next_word(cursor, op, &k, 5159 search_word, 5160 last_result, 5161 &candidate, 5162 order_op)) { 5163 // we didn't find a match for this note_id 5164 if (j == 0) 5165 // if we're at one of the root words, 5166 // this means that there are no further 5167 // root word matches for any note, so 5168 // we know we're done 5169 goto done; 5170 else 5171 break; 5172 } 5173 5174 *result = candidate; 5175 op = MDB_SET_RANGE; 5176 5177 // save the first key match, since we will continue from 5178 // this on the next root word result 5179 if (j == 0) { 5180 if (!saved) { 5181 memcpy(saved_buf, k.mv_data, k.mv_size); 5182 saved = saved_buf; 5183 saved_size = k.mv_size; 5184 } 5185 5186 // since we will be trying to match the same 5187 // note_id on all subsequent word matches, 5188 // let's lookup this note and make sure it 5189 // matches the filter if we have one. If it 5190 // doesn't match, we can quickly skip the 5191 // remaining word queries 5192 if (filter) { 5193 if ((note = ndb_get_note_by_key(txn, 5194 result->key.note_id, 5195 ¬e_size))) 5196 { 5197 if (!ndb_filter_matches(filter, note)) { 5198 break; 5199 } 5200 result->note = note; 5201 result->note_size = note_size; 5202 } 5203 } 5204 } 5205 5206 result->note = note; 5207 result->note_size = note_size; 5208 last_candidate = *result; 5209 last_result = &last_candidate; 5210 } 5211 5212 // we matched all of the queries! 5213 if (j == search_words.num_words) { 5214 results->num_results++; 5215 } 5216 5217 cont: 5218 ; 5219 } 5220 5221 done: 5222 mdb_cursor_close(cursor); 5223 5224 return 1; 5225 } 5226 5227 int ndb_text_search(struct ndb_txn *txn, const char *query, 5228 struct ndb_text_search_results *results, 5229 struct ndb_text_search_config *config) 5230 { 5231 return ndb_text_search_with(txn, query, results, config, NULL); 5232 } 5233 5234 static void ndb_write_blocks(struct ndb_txn *txn, uint64_t note_key, 5235 struct ndb_blocks *blocks) 5236 { 5237 int rc; 5238 MDB_val key, val; 5239 5240 // make sure we're not writing the owned flag to the db 5241 blocks->flags &= ~NDB_BLOCK_FLAG_OWNED; 5242 5243 key.mv_data = ¬e_key; 5244 key.mv_size = sizeof(note_key); 5245 val.mv_data = blocks; 5246 val.mv_size = ndb_blocks_total_size(blocks); 5247 assert((val.mv_size % 8) == 0); 5248 5249 if ((rc = mdb_put(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_NOTE_BLOCKS], &key, &val, 0))) { 5250 ndb_debug("write version to note_blocks failed: %s\n", 5251 mdb_strerror(rc)); 5252 return; 5253 } 5254 } 5255 5256 static int ndb_write_new_blocks(struct ndb_txn *txn, struct ndb_note *note, 5257 uint64_t note_key, unsigned char *scratch, 5258 size_t scratch_size) 5259 { 5260 size_t content_len; 5261 const char *content; 5262 struct ndb_blocks *blocks; 5263 5264 content_len = ndb_note_content_length(note); 5265 content = ndb_note_content(note); 5266 5267 if (!ndb_parse_content(scratch, scratch_size, content, content_len, &blocks)) { 5268 //ndb_debug("failed to parse content '%.*s'\n", content_len, content); 5269 return 0; 5270 } 5271 5272 ndb_write_blocks(txn, note_key, blocks); 5273 return 1; 5274 } 5275 5276 static uint64_t ndb_write_note(struct ndb_txn *txn, 5277 struct ndb_writer_note *note, 5278 unsigned char *scratch, size_t scratch_size, 5279 uint32_t ndb_flags) 5280 { 5281 int rc; 5282 uint64_t note_key, kind; 5283 struct ndb_relay_kind_key relay_key; 5284 MDB_dbi note_db; 5285 MDB_val key, val; 5286 5287 kind = note->note->kind; 5288 5289 // let's quickly sanity check if we already have this note 5290 if ((note_key = ndb_get_notekey_by_id(txn, note->note->id))) { 5291 if (ndb_relay_kind_key_init(&relay_key, note_key, kind, ndb_note_created_at(note->note), note->relay)) 5292 ndb_write_note_relay_indexes(txn, &relay_key); 5293 return 0; 5294 } 5295 5296 // get dbs 5297 note_db = txn->lmdb->dbs[NDB_DB_NOTE]; 5298 5299 // get new key 5300 note_key = ndb_get_last_key(txn->mdb_txn, note_db) + 1; 5301 5302 // write note to event store 5303 key.mv_data = ¬e_key; 5304 key.mv_size = sizeof(note_key); 5305 val.mv_data = note->note; 5306 val.mv_size = note->note_len; 5307 5308 if ((rc = mdb_put(txn->mdb_txn, note_db, &key, &val, 0))) { 5309 ndb_debug("write note to db failed: %s\n", mdb_strerror(rc)); 5310 return 0; 5311 } 5312 5313 ndb_write_note_id_index(txn, note->note, note_key); 5314 ndb_write_note_kind_index(txn, note->note, note_key); 5315 ndb_write_note_tag_index(txn, note->note, note_key); 5316 ndb_write_note_pubkey_index(txn, note->note, note_key); 5317 ndb_write_note_pubkey_kind_index(txn, note->note, note_key); 5318 5319 if (ndb_relay_kind_key_init(&relay_key, note_key, kind, ndb_note_created_at(note->note), note->relay)) 5320 ndb_write_note_relay_indexes(txn, &relay_key); 5321 5322 // only parse content and do fulltext index on text and longform notes 5323 if (kind == 1 || kind == 30023) { 5324 if (!ndb_flag_set(ndb_flags, NDB_FLAG_NO_FULLTEXT)) { 5325 if (!ndb_write_note_fulltext_index(txn, note->note, note_key)) 5326 return 0; 5327 } 5328 5329 // write note blocks 5330 if (!ndb_flag_set(ndb_flags, NDB_FLAG_NO_NOTE_BLOCKS)) { 5331 ndb_write_new_blocks(txn, note->note, note_key, scratch, scratch_size); 5332 } 5333 } else if (kind == 7 && !ndb_flag_set(ndb_flags, NDB_FLAG_NO_STATS)) { 5334 ndb_write_reaction_stats(txn, note->note); 5335 } 5336 5337 return note_key; 5338 } 5339 5340 static void ndb_monitor_lock(struct ndb_monitor *mon) { 5341 pthread_mutex_lock(&mon->mutex); 5342 } 5343 5344 static void ndb_monitor_unlock(struct ndb_monitor *mon) { 5345 pthread_mutex_unlock(&mon->mutex); 5346 } 5347 5348 struct written_note { 5349 uint64_t note_id; 5350 struct ndb_writer_note *note; 5351 }; 5352 5353 // When the data has been committed to the database, take all of the written 5354 // notes, check them against subscriptions, and then write to the subscription 5355 // inbox for all matching notes 5356 static void ndb_notify_subscriptions(struct ndb_monitor *monitor, 5357 struct written_note *wrote, int num_notes) 5358 { 5359 int i, k; 5360 int pushed; 5361 struct written_note *written; 5362 struct ndb_note *note; 5363 struct ndb_subscription *sub; 5364 5365 ndb_monitor_lock(monitor); 5366 5367 for (i = 0; i < monitor->num_subscriptions; i++) { 5368 sub = &monitor->subscriptions[i]; 5369 ndb_debug("checking subscription %d, %d notes\n", i, num_notes); 5370 5371 pushed = 0; 5372 for (k = 0; k < num_notes; k++) { 5373 written = &wrote[k]; 5374 note = written->note->note; 5375 5376 if (ndb_filter_group_matches(&sub->group, note)) { 5377 ndb_debug("pushing note\n"); 5378 5379 if (!prot_queue_push(&sub->inbox, &written->note_id)) { 5380 ndb_debug("couldn't push note to subscriber"); 5381 } else { 5382 pushed++; 5383 } 5384 } else { 5385 ndb_debug("not pushing note\n"); 5386 } 5387 } 5388 5389 // After pushing all of the matching notes, check to see if we 5390 // have a registered subscription callback. If so, we call it. 5391 // The callback needs to call ndb_poll_for_notes to pull data 5392 // that was just pushed to the queue in the for loop above. 5393 if (monitor->sub_cb != NULL && pushed > 0) { 5394 monitor->sub_cb(monitor->sub_cb_ctx, sub->subid); 5395 } 5396 } 5397 5398 ndb_monitor_unlock(monitor); 5399 } 5400 5401 uint64_t ndb_write_note_and_profile( 5402 struct ndb_txn *txn, 5403 struct ndb_writer_profile *profile, 5404 unsigned char *scratch, 5405 size_t scratch_size, 5406 uint32_t ndb_flags) 5407 { 5408 uint64_t note_nkey; 5409 5410 note_nkey = ndb_write_note(txn, &profile->note, scratch, scratch_size, ndb_flags); 5411 5412 if (profile->record.builder) { 5413 // only write if parsing didn't fail 5414 ndb_write_profile(txn, profile, note_nkey); 5415 } 5416 5417 return note_nkey; 5418 } 5419 5420 // only to be called from the writer thread 5421 static int ndb_write_version(struct ndb_txn *txn, uint64_t version) 5422 { 5423 int rc; 5424 MDB_val key, val; 5425 uint64_t version_key; 5426 5427 version_key = NDB_META_KEY_VERSION; 5428 5429 key.mv_data = &version_key; 5430 key.mv_size = sizeof(version_key); 5431 val.mv_data = &version; 5432 val.mv_size = sizeof(version); 5433 5434 if ((rc = mdb_put(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_NDB_META], &key, &val, 0))) { 5435 ndb_debug("write version to ndb_meta failed: %s\n", 5436 mdb_strerror(rc)); 5437 return 0; 5438 } 5439 5440 //fprintf(stderr, "writing version %" PRIu64 "\n", version); 5441 return 1; 5442 } 5443 5444 5445 static int ndb_run_migrations(struct ndb_txn *txn) 5446 { 5447 int64_t version, latest_version, i; 5448 5449 latest_version = sizeof(MIGRATIONS) / sizeof(MIGRATIONS[0]); 5450 5451 if ((version = ndb_db_version(txn)) == -1) { 5452 ndb_debug("run_migrations: no version found, assuming new db\n"); 5453 version = latest_version; 5454 5455 // no version found. fresh db? 5456 if (!ndb_write_version(txn, version)) { 5457 fprintf(stderr, "run_migrations: failed writing db version"); 5458 return 0; 5459 } 5460 5461 return 1; 5462 } else { 5463 ndb_debug("ndb: version %" PRIu64 " found\n", version); 5464 } 5465 5466 if (version < latest_version) 5467 fprintf(stderr, "nostrdb: migrating v%d -> v%d\n", 5468 (int)version, (int)latest_version); 5469 5470 for (i = version; i < latest_version; i++) { 5471 if (!MIGRATIONS[i].fn(txn)) { 5472 fprintf(stderr, "run_migrations: migration v%d -> v%d failed\n", (int)i, (int)(i+1)); 5473 return 0; 5474 } 5475 5476 if (!ndb_write_version(txn, i+1)) { 5477 fprintf(stderr, "run_migrations: failed writing db version"); 5478 return 0; 5479 } 5480 5481 version = i+1; 5482 } 5483 5484 return 1; 5485 } 5486 5487 5488 static void *ndb_writer_thread(void *data) 5489 { 5490 ndb_debug("started writer thread\n"); 5491 struct ndb_writer *writer = data; 5492 struct ndb_writer_msg msgs[THREAD_QUEUE_BATCH], *msg; 5493 struct written_note written_notes[THREAD_QUEUE_BATCH]; 5494 int i, popped, done, needs_commit, num_notes; 5495 uint64_t note_nkey; 5496 struct ndb_txn txn; 5497 unsigned char *scratch; 5498 struct ndb_relay_kind_key relay_key; 5499 5500 // 2MB scratch buffer for parsing note content 5501 scratch = malloc(writer->scratch_size); 5502 MDB_txn *mdb_txn = NULL; 5503 ndb_txn_from_mdb(&txn, writer->lmdb, mdb_txn); 5504 5505 done = 0; 5506 while (!done) { 5507 txn.mdb_txn = NULL; 5508 num_notes = 0; 5509 ndb_debug("writer waiting for items\n"); 5510 popped = prot_queue_pop_all(&writer->inbox, msgs, THREAD_QUEUE_BATCH); 5511 ndb_debug("writer popped %d items\n", popped); 5512 5513 needs_commit = 0; 5514 for (i = 0 ; i < popped; i++) { 5515 msg = &msgs[i]; 5516 switch (msg->type) { 5517 case NDB_WRITER_NOTE: needs_commit = 1; break; 5518 case NDB_WRITER_PROFILE: needs_commit = 1; break; 5519 case NDB_WRITER_DBMETA: needs_commit = 1; break; 5520 case NDB_WRITER_PROFILE_LAST_FETCH: needs_commit = 1; break; 5521 case NDB_WRITER_BLOCKS: needs_commit = 1; break; 5522 case NDB_WRITER_MIGRATE: needs_commit = 1; break; 5523 case NDB_WRITER_NOTE_RELAY: needs_commit = 1; break; 5524 case NDB_WRITER_QUIT: break; 5525 } 5526 } 5527 5528 if (needs_commit && mdb_txn_begin(txn.lmdb->env, NULL, 0, (MDB_txn **)&txn.mdb_txn)) 5529 { 5530 fprintf(stderr, "writer thread txn_begin failed"); 5531 // should definitely not happen unless DB is full 5532 // or something ? 5533 continue; 5534 } 5535 5536 for (i = 0; i < popped; i++) { 5537 msg = &msgs[i]; 5538 5539 switch (msg->type) { 5540 case NDB_WRITER_QUIT: 5541 // quits are handled before this 5542 ndb_debug("writer thread got quit message\n"); 5543 done = 1; 5544 continue; 5545 case NDB_WRITER_PROFILE: 5546 note_nkey = 5547 ndb_write_note_and_profile( 5548 &txn, 5549 &msg->profile, 5550 scratch, 5551 writer->scratch_size, 5552 writer->ndb_flags); 5553 5554 if (note_nkey > 0) { 5555 written_notes[num_notes++] = 5556 (struct written_note){ 5557 .note_id = note_nkey, 5558 .note = &msg->profile.note, 5559 }; 5560 } else { 5561 ndb_debug("failed to write note\n"); 5562 } 5563 break; 5564 case NDB_WRITER_NOTE: 5565 note_nkey = ndb_write_note(&txn, &msg->note, 5566 scratch, 5567 writer->scratch_size, 5568 writer->ndb_flags); 5569 5570 if (note_nkey > 0) { 5571 written_notes[num_notes++] = (struct written_note){ 5572 .note_id = note_nkey, 5573 .note = &msg->note, 5574 }; 5575 } 5576 break; 5577 case NDB_WRITER_NOTE_RELAY: 5578 if (ndb_relay_kind_key_init(&relay_key, 5579 msg->note_relay.note_key, 5580 msg->note_relay.kind, 5581 msg->note_relay.created_at, 5582 msg->note_relay.relay)) 5583 { 5584 ndb_write_note_relay_indexes(&txn, &relay_key); 5585 } 5586 break; 5587 case NDB_WRITER_DBMETA: 5588 ndb_write_version(&txn, msg->ndb_meta.version); 5589 break; 5590 case NDB_WRITER_BLOCKS: 5591 ndb_write_blocks(&txn, msg->blocks.note_key, 5592 msg->blocks.blocks); 5593 break; 5594 case NDB_WRITER_MIGRATE: 5595 if (!ndb_run_migrations(&txn)) { 5596 mdb_txn_abort(txn.mdb_txn); 5597 goto bail; 5598 } 5599 break; 5600 case NDB_WRITER_PROFILE_LAST_FETCH: 5601 ndb_writer_last_profile_fetch(&txn, 5602 msg->last_fetch.pubkey, 5603 msg->last_fetch.fetched_at 5604 ); 5605 break; 5606 } 5607 } 5608 5609 // commit writes 5610 if (needs_commit) { 5611 if (!ndb_end_query(&txn)) { 5612 ndb_debug("writer thread txn commit failed\n"); 5613 } else { 5614 ndb_debug("notifying subscriptions, %d notes\n", num_notes); 5615 ndb_notify_subscriptions(writer->monitor, 5616 written_notes, 5617 num_notes); 5618 // update subscriptions 5619 } 5620 } 5621 5622 // free notes 5623 for (i = 0; i < popped; i++) { 5624 msg = &msgs[i]; 5625 if (msg->type == NDB_WRITER_NOTE) { 5626 free(msg->note.note); 5627 if (msg->note.relay) 5628 free((void*)msg->note.relay); 5629 } else if (msg->type == NDB_WRITER_PROFILE) { 5630 free(msg->profile.note.note); 5631 ndb_profile_record_builder_free(&msg->profile.record); 5632 } else if (msg->type == NDB_WRITER_BLOCKS) { 5633 ndb_blocks_free(msg->blocks.blocks); 5634 } else if (msg->type == NDB_WRITER_NOTE_RELAY) { 5635 free((void*)msg->note_relay.relay); 5636 } 5637 } 5638 } 5639 5640 bail: 5641 free(scratch); 5642 ndb_debug("quitting writer thread\n"); 5643 return NULL; 5644 } 5645 5646 static void *ndb_ingester_thread(void *data) 5647 { 5648 secp256k1_context *ctx; 5649 struct thread *thread = data; 5650 struct ndb_ingester *ingester = (struct ndb_ingester *)thread->ctx; 5651 struct ndb_lmdb *lmdb = ingester->lmdb; 5652 struct ndb_ingester_msg msgs[THREAD_QUEUE_BATCH], *msg; 5653 struct ndb_writer_msg outs[THREAD_QUEUE_BATCH], *out; 5654 int i, to_write, popped, done, any_event; 5655 MDB_txn *read_txn = NULL; 5656 unsigned char *scratch; 5657 int rc; 5658 5659 // this is used in note verification and anything else that 5660 // needs a temporary buffer 5661 scratch = malloc(ingester->scratch_size); 5662 5663 ctx = secp256k1_context_create(SECP256K1_CONTEXT_VERIFY); 5664 //ndb_debug("started ingester thread\n"); 5665 5666 done = 0; 5667 while (!done) { 5668 to_write = 0; 5669 any_event = 0; 5670 5671 popped = prot_queue_pop_all(&thread->inbox, msgs, THREAD_QUEUE_BATCH); 5672 ndb_debug("ingester popped %d items\n", popped); 5673 5674 for (i = 0; i < popped; i++) { 5675 msg = &msgs[i]; 5676 if (msg->type == NDB_INGEST_EVENT) { 5677 any_event = 1; 5678 break; 5679 } 5680 } 5681 5682 if (any_event && (rc = mdb_txn_begin(lmdb->env, NULL, MDB_RDONLY, &read_txn))) { 5683 // this is bad 5684 fprintf(stderr, "UNUSUAL ndb_ingester: mdb_txn_begin failed: '%s'\n", 5685 mdb_strerror(rc)); 5686 continue; 5687 } 5688 5689 for (i = 0; i < popped; i++) { 5690 msg = &msgs[i]; 5691 switch (msg->type) { 5692 case NDB_INGEST_QUIT: 5693 done = 1; 5694 break; 5695 5696 case NDB_INGEST_EVENT: 5697 out = &outs[to_write]; 5698 if (ndb_ingester_process_event(ctx, ingester, 5699 &msg->event, out, 5700 scratch, 5701 read_txn)) { 5702 to_write++; 5703 } 5704 } 5705 } 5706 5707 if (any_event) 5708 mdb_txn_abort(read_txn); 5709 5710 if (to_write > 0) { 5711 ndb_debug("pushing %d events to write queue\n", to_write); 5712 if (!prot_queue_push_all(ingester->writer_inbox, outs, to_write)) { 5713 ndb_debug("failed pushing %d events to write queue\n", to_write); 5714 } 5715 } 5716 } 5717 5718 ndb_debug("quitting ingester thread\n"); 5719 secp256k1_context_destroy(ctx); 5720 free(scratch); 5721 return NULL; 5722 } 5723 5724 5725 static int ndb_writer_init(struct ndb_writer *writer, struct ndb_lmdb *lmdb, 5726 struct ndb_monitor *monitor, uint32_t ndb_flags, 5727 int scratch_size) 5728 { 5729 writer->lmdb = lmdb; 5730 writer->monitor = monitor; 5731 writer->ndb_flags = ndb_flags; 5732 writer->scratch_size = scratch_size; 5733 writer->queue_buflen = sizeof(struct ndb_writer_msg) * DEFAULT_QUEUE_SIZE; 5734 writer->queue_buf = malloc(writer->queue_buflen); 5735 if (writer->queue_buf == NULL) { 5736 fprintf(stderr, "ndb: failed to allocate space for writer queue"); 5737 return 0; 5738 } 5739 5740 // init the writer queue. 5741 prot_queue_init(&writer->inbox, writer->queue_buf, 5742 writer->queue_buflen, sizeof(struct ndb_writer_msg)); 5743 5744 // spin up the writer thread 5745 if (THREAD_CREATE(writer->thread_id, ndb_writer_thread, writer)) 5746 { 5747 fprintf(stderr, "ndb writer thread failed to create\n"); 5748 return 0; 5749 } 5750 5751 return 1; 5752 } 5753 5754 // initialize the ingester queue and then spawn the thread 5755 static int ndb_ingester_init(struct ndb_ingester *ingester, 5756 struct ndb_lmdb *lmdb, 5757 struct prot_queue *writer_inbox, 5758 int scratch_size, 5759 const struct ndb_config *config) 5760 { 5761 int elem_size, num_elems; 5762 static struct ndb_ingester_msg quit_msg = { .type = NDB_INGEST_QUIT }; 5763 5764 // TODO: configurable queue sizes 5765 elem_size = sizeof(struct ndb_ingester_msg); 5766 num_elems = DEFAULT_QUEUE_SIZE; 5767 5768 ingester->scratch_size = scratch_size; 5769 ingester->writer_inbox = writer_inbox; 5770 ingester->lmdb = lmdb; 5771 ingester->flags = config->flags; 5772 ingester->filter = config->ingest_filter; 5773 ingester->filter_context = config->filter_context; 5774 5775 if (!threadpool_init(&ingester->tp, config->ingester_threads, 5776 elem_size, num_elems, &quit_msg, ingester, 5777 ndb_ingester_thread)) 5778 { 5779 fprintf(stderr, "ndb ingester threadpool failed to init\n"); 5780 return 0; 5781 } 5782 5783 return 1; 5784 } 5785 5786 static int ndb_writer_destroy(struct ndb_writer *writer) 5787 { 5788 struct ndb_writer_msg msg; 5789 5790 // kill thread 5791 msg.type = NDB_WRITER_QUIT; 5792 ndb_debug("writer: pushing quit message\n"); 5793 if (!prot_queue_push(&writer->inbox, &msg)) { 5794 // queue is too full to push quit message. just kill it. 5795 ndb_debug("writer: terminating thread\n"); 5796 THREAD_TERMINATE(writer->thread_id); 5797 } else { 5798 ndb_debug("writer: joining thread\n"); 5799 THREAD_FINISH(writer->thread_id); 5800 } 5801 5802 // cleanup 5803 ndb_debug("writer: cleaning up protected queue\n"); 5804 prot_queue_destroy(&writer->inbox); 5805 5806 free(writer->queue_buf); 5807 5808 return 1; 5809 } 5810 5811 static int ndb_ingester_destroy(struct ndb_ingester *ingester) 5812 { 5813 threadpool_destroy(&ingester->tp); 5814 return 1; 5815 } 5816 5817 static int ndb_init_lmdb(const char *filename, struct ndb_lmdb *lmdb, size_t mapsize) 5818 { 5819 int rc; 5820 MDB_txn *txn; 5821 5822 if ((rc = mdb_env_create(&lmdb->env))) { 5823 fprintf(stderr, "mdb_env_create failed, error %d\n", rc); 5824 return 0; 5825 } 5826 5827 if ((rc = mdb_env_set_mapsize(lmdb->env, mapsize))) { 5828 fprintf(stderr, "mdb_env_set_mapsize failed, error %d\n", rc); 5829 return 0; 5830 } 5831 5832 if ((rc = mdb_env_set_maxdbs(lmdb->env, NDB_DBS))) { 5833 fprintf(stderr, "mdb_env_set_maxdbs failed, error %d\n", rc); 5834 return 0; 5835 } 5836 5837 if ((rc = mdb_env_open(lmdb->env, filename, 0, 0664))) { 5838 fprintf(stderr, "mdb_env_open failed, error %d\n", rc); 5839 return 0; 5840 } 5841 5842 // Initialize DBs 5843 if ((rc = mdb_txn_begin(lmdb->env, NULL, 0, &txn))) { 5844 fprintf(stderr, "mdb_txn_begin failed, error %d\n", rc); 5845 return 0; 5846 } 5847 5848 // note flatbuffer db 5849 if ((rc = mdb_dbi_open(txn, "note", MDB_CREATE | MDB_INTEGERKEY, &lmdb->dbs[NDB_DB_NOTE]))) { 5850 fprintf(stderr, "mdb_dbi_open event failed, error %d\n", rc); 5851 return 0; 5852 } 5853 5854 // note metadata db 5855 if ((rc = mdb_dbi_open(txn, "meta", MDB_CREATE, &lmdb->dbs[NDB_DB_META]))) { 5856 fprintf(stderr, "mdb_dbi_open meta failed, error %d\n", rc); 5857 return 0; 5858 } 5859 5860 // profile flatbuffer db 5861 if ((rc = mdb_dbi_open(txn, "profile", MDB_CREATE | MDB_INTEGERKEY, &lmdb->dbs[NDB_DB_PROFILE]))) { 5862 fprintf(stderr, "mdb_dbi_open profile failed, error %d\n", rc); 5863 return 0; 5864 } 5865 5866 // profile search db 5867 if ((rc = mdb_dbi_open(txn, "profile_search", MDB_CREATE, &lmdb->dbs[NDB_DB_PROFILE_SEARCH]))) { 5868 fprintf(stderr, "mdb_dbi_open profile_search failed, error %d\n", rc); 5869 return 0; 5870 } 5871 mdb_set_compare(txn, lmdb->dbs[NDB_DB_PROFILE_SEARCH], ndb_search_key_cmp); 5872 5873 // ndb metadata (db version, etc) 5874 if ((rc = mdb_dbi_open(txn, "ndb_meta", MDB_CREATE | MDB_INTEGERKEY, &lmdb->dbs[NDB_DB_NDB_META]))) { 5875 fprintf(stderr, "mdb_dbi_open ndb_meta failed, error %d\n", rc); 5876 return 0; 5877 } 5878 5879 // profile last fetches 5880 if ((rc = mdb_dbi_open(txn, "profile_last_fetch", MDB_CREATE, &lmdb->dbs[NDB_DB_PROFILE_LAST_FETCH]))) { 5881 fprintf(stderr, "mdb_dbi_open profile last fetch, error %d\n", rc); 5882 return 0; 5883 } 5884 5885 // relay kind index. maps <relay_url><kind><created><note_id> primary keys to relay records 5886 // see ndb_relay_kind_cmp function for more details on the key format 5887 if ((rc = mdb_dbi_open(txn, "relay_kind", MDB_CREATE, &lmdb->dbs[NDB_DB_NOTE_RELAY_KIND]))) { 5888 fprintf(stderr, "mdb_dbi_open profile last fetch, error %d\n", rc); 5889 return 0; 5890 } 5891 mdb_set_compare(txn, lmdb->dbs[NDB_DB_NOTE_RELAY_KIND], ndb_relay_kind_cmp); 5892 5893 // note_id -> relay index 5894 if ((rc = mdb_dbi_open(txn, "note_relays", MDB_CREATE | MDB_DUPSORT, &lmdb->dbs[NDB_DB_NOTE_RELAYS]))) { 5895 fprintf(stderr, "mdb_dbi_open profile last fetch, error %d\n", rc); 5896 return 0; 5897 } 5898 5899 // id+ts index flags 5900 unsigned int tsid_flags = MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED; 5901 5902 // index dbs 5903 if ((rc = mdb_dbi_open(txn, "note_id", tsid_flags, &lmdb->dbs[NDB_DB_NOTE_ID]))) { 5904 fprintf(stderr, "mdb_dbi_open id failed: %s\n", mdb_strerror(rc)); 5905 return 0; 5906 } 5907 mdb_set_compare(txn, lmdb->dbs[NDB_DB_NOTE_ID], ndb_tsid_compare); 5908 5909 if ((rc = mdb_dbi_open(txn, "profile_pk", tsid_flags, &lmdb->dbs[NDB_DB_PROFILE_PK]))) { 5910 fprintf(stderr, "mdb_dbi_open profile_pk failed: %s\n", mdb_strerror(rc)); 5911 return 0; 5912 } 5913 mdb_set_compare(txn, lmdb->dbs[NDB_DB_PROFILE_PK], ndb_tsid_compare); 5914 5915 if ((rc = mdb_dbi_open(txn, "note_kind", 5916 MDB_CREATE | MDB_DUPSORT | MDB_INTEGERDUP | MDB_DUPFIXED, 5917 &lmdb->dbs[NDB_DB_NOTE_KIND]))) { 5918 fprintf(stderr, "mdb_dbi_open note_kind failed: %s\n", mdb_strerror(rc)); 5919 return 0; 5920 } 5921 mdb_set_compare(txn, lmdb->dbs[NDB_DB_NOTE_KIND], ndb_u64_ts_compare); 5922 5923 if ((rc = mdb_dbi_open(txn, "note_pubkey", 5924 MDB_CREATE | MDB_DUPSORT | MDB_INTEGERDUP | MDB_DUPFIXED, 5925 &lmdb->dbs[NDB_DB_NOTE_PUBKEY]))) { 5926 fprintf(stderr, "mdb_dbi_open note_pubkey failed: %s\n", mdb_strerror(rc)); 5927 return 0; 5928 } 5929 mdb_set_compare(txn, lmdb->dbs[NDB_DB_NOTE_PUBKEY], ndb_tsid_compare); 5930 5931 if ((rc = mdb_dbi_open(txn, "note_pubkey_kind", 5932 MDB_CREATE | MDB_DUPSORT | MDB_INTEGERDUP | MDB_DUPFIXED, 5933 &lmdb->dbs[NDB_DB_NOTE_PUBKEY_KIND]))) { 5934 fprintf(stderr, "mdb_dbi_open note_pubkey_kind failed: %s\n", mdb_strerror(rc)); 5935 return 0; 5936 } 5937 mdb_set_compare(txn, lmdb->dbs[NDB_DB_NOTE_PUBKEY_KIND], ndb_id_u64_ts_compare); 5938 5939 if ((rc = mdb_dbi_open(txn, "note_text", MDB_CREATE | MDB_DUPSORT, 5940 &lmdb->dbs[NDB_DB_NOTE_TEXT]))) { 5941 fprintf(stderr, "mdb_dbi_open note_text failed: %s\n", mdb_strerror(rc)); 5942 return 0; 5943 } 5944 mdb_set_compare(txn, lmdb->dbs[NDB_DB_NOTE_TEXT], ndb_text_search_key_compare); 5945 5946 if ((rc = mdb_dbi_open(txn, "note_blocks", MDB_CREATE | MDB_INTEGERKEY, 5947 &lmdb->dbs[NDB_DB_NOTE_BLOCKS]))) { 5948 fprintf(stderr, "mdb_dbi_open note_blocks failed: %s\n", mdb_strerror(rc)); 5949 return 0; 5950 } 5951 5952 if ((rc = mdb_dbi_open(txn, "note_tags", MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, 5953 &lmdb->dbs[NDB_DB_NOTE_TAGS]))) { 5954 fprintf(stderr, "mdb_dbi_open note_tags failed: %s\n", mdb_strerror(rc)); 5955 return 0; 5956 } 5957 mdb_set_compare(txn, lmdb->dbs[NDB_DB_NOTE_TAGS], ndb_tag_key_compare); 5958 5959 // Commit the transaction 5960 if ((rc = mdb_txn_commit(txn))) { 5961 fprintf(stderr, "mdb_txn_commit failed, error %d\n", rc); 5962 return 0; 5963 } 5964 5965 return 1; 5966 } 5967 5968 static int ndb_queue_write_version(struct ndb *ndb, uint64_t version) 5969 { 5970 struct ndb_writer_msg msg; 5971 msg.type = NDB_WRITER_DBMETA; 5972 msg.ndb_meta.version = version; 5973 return ndb_writer_queue_msg(&ndb->writer, &msg); 5974 } 5975 5976 static void ndb_monitor_init(struct ndb_monitor *monitor, ndb_sub_fn cb, 5977 void *sub_cb_ctx) 5978 { 5979 monitor->num_subscriptions = 0; 5980 monitor->sub_cb = cb; 5981 monitor->sub_cb_ctx = sub_cb_ctx; 5982 pthread_mutex_init(&monitor->mutex, NULL); 5983 } 5984 5985 void ndb_filter_group_destroy(struct ndb_filter_group *group) 5986 { 5987 struct ndb_filter *filter; 5988 int i; 5989 for (i = 0; i < group->num_filters; i++) { 5990 filter = &group->filters[i]; 5991 ndb_filter_destroy(filter); 5992 } 5993 } 5994 5995 static void ndb_subscription_destroy(struct ndb_subscription *sub) 5996 { 5997 ndb_filter_group_destroy(&sub->group); 5998 // this was malloc'd inside ndb_subscribe 5999 free(sub->inbox.buf); 6000 prot_queue_destroy(&sub->inbox); 6001 sub->subid = 0; 6002 } 6003 6004 static void ndb_monitor_destroy(struct ndb_monitor *monitor) 6005 { 6006 int i; 6007 6008 ndb_monitor_lock(monitor); 6009 6010 for (i = 0; i < monitor->num_subscriptions; i++) { 6011 ndb_subscription_destroy(&monitor->subscriptions[i]); 6012 } 6013 6014 monitor->num_subscriptions = 0; 6015 6016 ndb_monitor_unlock(monitor); 6017 6018 pthread_mutex_destroy(&monitor->mutex); 6019 } 6020 6021 int ndb_init(struct ndb **pndb, const char *filename, const struct ndb_config *config) 6022 { 6023 struct ndb *ndb; 6024 //MDB_dbi ind_id; // TODO: ind_pk, etc 6025 6026 ndb = *pndb = calloc(1, sizeof(struct ndb)); 6027 ndb->flags = config->flags; 6028 6029 if (ndb == NULL) { 6030 fprintf(stderr, "ndb_init: malloc failed\n"); 6031 return 0; 6032 } 6033 6034 if (!ndb_init_lmdb(filename, &ndb->lmdb, config->mapsize)) 6035 return 0; 6036 6037 ndb_monitor_init(&ndb->monitor, config->sub_cb, config->sub_cb_ctx); 6038 6039 if (!ndb_writer_init(&ndb->writer, &ndb->lmdb, &ndb->monitor, ndb->flags, 6040 config->writer_scratch_buffer_size)) { 6041 fprintf(stderr, "ndb_writer_init failed\n"); 6042 return 0; 6043 } 6044 6045 if (!ndb_ingester_init(&ndb->ingester, &ndb->lmdb, &ndb->writer.inbox, 6046 config->writer_scratch_buffer_size, config)) { 6047 fprintf(stderr, "failed to initialize %d ingester thread(s)\n", 6048 config->ingester_threads); 6049 return 0; 6050 } 6051 6052 if (!ndb_flag_set(config->flags, NDB_FLAG_NOMIGRATE)) { 6053 struct ndb_writer_msg msg = { .type = NDB_WRITER_MIGRATE }; 6054 ndb_writer_queue_msg(&ndb->writer, &msg); 6055 } 6056 6057 // Initialize LMDB environment and spin up threads 6058 return 1; 6059 } 6060 6061 void ndb_destroy(struct ndb *ndb) 6062 { 6063 if (ndb == NULL) 6064 return; 6065 6066 // ingester depends on writer and must be destroyed first 6067 ndb_debug("destroying ingester\n"); 6068 ndb_ingester_destroy(&ndb->ingester); 6069 ndb_debug("destroying writer\n"); 6070 ndb_writer_destroy(&ndb->writer); 6071 ndb_debug("destroying monitor\n"); 6072 ndb_monitor_destroy(&ndb->monitor); 6073 6074 ndb_debug("closing env\n"); 6075 mdb_env_close(ndb->lmdb.env); 6076 6077 ndb_debug("ndb destroyed\n"); 6078 free(ndb); 6079 } 6080 6081 6082 // Process a nostr event from a client 6083 // 6084 // ie: ["EVENT", {"content":"..."} ...] 6085 // 6086 // The client-sent variation of ndb_process_event 6087 int ndb_process_client_event(struct ndb *ndb, const char *json, int len) 6088 { 6089 struct ndb_ingest_meta meta; 6090 ndb_ingest_meta_init(&meta, 1, NULL); 6091 6092 return ndb_ingest_event(&ndb->ingester, json, len, &meta); 6093 } 6094 6095 // Process anostr event from a relay, 6096 // 6097 // ie: ["EVENT", "subid", {"content":"..."}...] 6098 // 6099 // This function returns as soon as possible, first copying the passed 6100 // json and then queueing it up for processing. Worker threads then take 6101 // the json and process it. 6102 // 6103 // Processing: 6104 // 6105 // 1. The event is parsed into ndb_notes and the signature is validated 6106 // 2. A quick lookup is made on the database to see if we already have 6107 // the note id, if we do we don't need to waste time on json parsing 6108 // or note validation. 6109 // 3. Once validation is done we pass it to the writer queue for writing 6110 // to LMDB. 6111 // 6112 int ndb_process_event(struct ndb *ndb, const char *json, int json_len) 6113 { 6114 struct ndb_ingest_meta meta; 6115 ndb_ingest_meta_init(&meta, 0, NULL); 6116 6117 return ndb_ingest_event(&ndb->ingester, json, json_len, &meta); 6118 } 6119 6120 int ndb_process_event_with(struct ndb *ndb, const char *json, int json_len, 6121 struct ndb_ingest_meta *meta) 6122 { 6123 return ndb_ingest_event(&ndb->ingester, json, json_len, meta); 6124 } 6125 6126 int _ndb_process_events(struct ndb *ndb, const char *ldjson, size_t json_len, 6127 struct ndb_ingest_meta *meta) 6128 { 6129 const char *start, *end, *very_end; 6130 start = ldjson; 6131 end = start + json_len; 6132 very_end = ldjson + json_len; 6133 #if DEBUG 6134 int processed = 0; 6135 #endif 6136 6137 while ((end = fast_strchr(start, '\n', very_end - start))) { 6138 //printf("processing '%.*s'\n", (int)(end-start), start); 6139 if (!ndb_process_event_with(ndb, start, end - start, meta)) { 6140 ndb_debug("ndb_process_client_event failed\n"); 6141 return 0; 6142 } 6143 start = end + 1; 6144 #if DEBUG 6145 processed++; 6146 #endif 6147 } 6148 6149 #if DEBUG 6150 ndb_debug("ndb_process_events: processed %d events\n", processed); 6151 #endif 6152 6153 return 1; 6154 } 6155 6156 #ifndef _WIN32 6157 // TODO: windows 6158 int ndb_process_events_stream(struct ndb *ndb, FILE* fp) 6159 { 6160 char *line = NULL; 6161 size_t len = 0; 6162 ssize_t nread; 6163 6164 while ((nread = getline(&line, &len, fp)) != -1) { 6165 if (line == NULL) 6166 break; 6167 ndb_process_event(ndb, line, len); 6168 } 6169 6170 if (line) 6171 free(line); 6172 6173 return 1; 6174 } 6175 #endif 6176 6177 int ndb_process_events_with(struct ndb *ndb, const char *ldjson, size_t json_len, 6178 struct ndb_ingest_meta *meta) 6179 { 6180 return _ndb_process_events(ndb, ldjson, json_len, meta); 6181 } 6182 6183 int ndb_process_client_events(struct ndb *ndb, const char *ldjson, size_t json_len) 6184 { 6185 struct ndb_ingest_meta meta; 6186 ndb_ingest_meta_init(&meta, 1, NULL); 6187 6188 return _ndb_process_events(ndb, ldjson, json_len, &meta); 6189 } 6190 6191 int ndb_process_events(struct ndb *ndb, const char *ldjson, size_t json_len) 6192 { 6193 struct ndb_ingest_meta meta; 6194 ndb_ingest_meta_init(&meta, 0, NULL); 6195 6196 return _ndb_process_events(ndb, ldjson, json_len, &meta); 6197 } 6198 6199 static inline int cursor_push_tag(struct cursor *cur, struct ndb_tag *tag) 6200 { 6201 return cursor_push_u16(cur, tag->count); 6202 } 6203 6204 int ndb_builder_init(struct ndb_builder *builder, unsigned char *buf, 6205 size_t bufsize) 6206 { 6207 struct ndb_note *note; 6208 int half, size, str_indices_size; 6209 6210 // come on bruh 6211 if (bufsize < sizeof(struct ndb_note) * 2) 6212 return 0; 6213 6214 str_indices_size = bufsize / 32; 6215 size = bufsize - str_indices_size; 6216 half = size / 2; 6217 6218 //debug("size %d half %d str_indices %d\n", size, half, str_indices_size); 6219 6220 // make a safe cursor of our available memory 6221 make_cursor(buf, buf + bufsize, &builder->mem); 6222 6223 note = builder->note = (struct ndb_note *)buf; 6224 6225 // take slices of the memory into subcursors 6226 if (!(cursor_slice(&builder->mem, &builder->note_cur, half) && 6227 cursor_slice(&builder->mem, &builder->strings, half) && 6228 cursor_slice(&builder->mem, &builder->str_indices, str_indices_size))) { 6229 return 0; 6230 } 6231 6232 memset(note, 0, sizeof(*note)); 6233 builder->note_cur.p += sizeof(*note); 6234 6235 note->strings = builder->strings.start - buf; 6236 note->version = 1; 6237 6238 return 1; 6239 } 6240 6241 6242 6243 static inline int ndb_json_parser_init(struct ndb_json_parser *p, 6244 const char *json, int json_len, 6245 unsigned char *buf, int bufsize) 6246 { 6247 int half = bufsize / 2; 6248 6249 unsigned char *tok_start = buf + half; 6250 unsigned char *tok_end = buf + bufsize; 6251 6252 p->toks = (jsmntok_t*)tok_start; 6253 p->toks_end = (jsmntok_t*)tok_end; 6254 p->num_tokens = 0; 6255 p->json = json; 6256 p->json_len = json_len; 6257 6258 // ndb_builder gets the first half of the buffer, and jsmn gets the 6259 // second half. I like this way of alloating memory (without actually 6260 // dynamically allocating memory). You get one big chunk upfront and 6261 // then submodules can recursively subdivide it. Maybe you could do 6262 // something even more clever like golden-ratio style subdivision where 6263 // the more important stuff gets a larger chunk and then it spirals 6264 // downward into smaller chunks. Thanks for coming to my TED talk. 6265 6266 if (!ndb_builder_init(&p->builder, buf, half)) 6267 return 0; 6268 6269 jsmn_init(&p->json_parser); 6270 6271 return 1; 6272 } 6273 6274 static inline int ndb_json_parser_parse(struct ndb_json_parser *p, 6275 struct ndb_id_cb *cb) 6276 { 6277 jsmntok_t *tok; 6278 int cap = ((unsigned char *)p->toks_end - (unsigned char*)p->toks)/sizeof(*p->toks); 6279 int res = 6280 jsmn_parse(&p->json_parser, p->json, p->json_len, p->toks, cap, cb != NULL); 6281 6282 // got an ID! 6283 if (res == -42) { 6284 tok = &p->toks[p->json_parser.toknext-1]; 6285 6286 switch (cb->fn(cb->data, p->json + tok->start)) { 6287 case NDB_IDRES_CONT: 6288 res = jsmn_parse(&p->json_parser, p->json, p->json_len, 6289 p->toks, cap, 0); 6290 break; 6291 case NDB_IDRES_STOP: 6292 return -42; 6293 } 6294 } else if (res == 0) { 6295 return 0; 6296 } 6297 6298 p->num_tokens = res; 6299 p->i = 0; 6300 6301 return 1; 6302 } 6303 6304 static inline int toksize(jsmntok_t *tok) 6305 { 6306 return tok->end - tok->start; 6307 } 6308 6309 6310 6311 static int cursor_push_unescaped_char(struct cursor *cur, char c1, char c2) 6312 { 6313 switch (c2) { 6314 case 't': return cursor_push_byte(cur, '\t'); 6315 case 'n': return cursor_push_byte(cur, '\n'); 6316 case 'r': return cursor_push_byte(cur, '\r'); 6317 case 'b': return cursor_push_byte(cur, '\b'); 6318 case 'f': return cursor_push_byte(cur, '\f'); 6319 case '\\': return cursor_push_byte(cur, '\\'); 6320 case '/': return cursor_push_byte(cur, '/'); 6321 case '"': return cursor_push_byte(cur, '"'); 6322 case 'u': 6323 // these aren't handled yet 6324 return 0; 6325 default: 6326 return cursor_push_byte(cur, c1) && cursor_push_byte(cur, c2); 6327 } 6328 } 6329 6330 static int cursor_push_escaped_char(struct cursor *cur, char c) 6331 { 6332 switch (c) { 6333 case '"': return cursor_push_str(cur, "\\\""); 6334 case '\\': return cursor_push_str(cur, "\\\\"); 6335 case '\b': return cursor_push_str(cur, "\\b"); 6336 case '\f': return cursor_push_str(cur, "\\f"); 6337 case '\n': return cursor_push_str(cur, "\\n"); 6338 case '\r': return cursor_push_str(cur, "\\r"); 6339 case '\t': return cursor_push_str(cur, "\\t"); 6340 // TODO: \u hex hex hex hex 6341 } 6342 return cursor_push_byte(cur, c); 6343 } 6344 6345 static int cursor_push_hex_str(struct cursor *cur, unsigned char *buf, int len) 6346 { 6347 int i; 6348 6349 if (len % 2 != 0) 6350 return 0; 6351 6352 if (!cursor_push_byte(cur, '"')) 6353 return 0; 6354 6355 for (i = 0; i < len; i++) { 6356 unsigned int c = ((const unsigned char *)buf)[i]; 6357 if (!cursor_push_byte(cur, hexchar(c >> 4))) 6358 return 0; 6359 if (!cursor_push_byte(cur, hexchar(c & 0xF))) 6360 return 0; 6361 } 6362 6363 if (!cursor_push_byte(cur, '"')) 6364 return 0; 6365 6366 return 1; 6367 } 6368 6369 static int cursor_push_jsonstr(struct cursor *cur, const char *str) 6370 { 6371 int i; 6372 int len; 6373 6374 len = strlen(str); 6375 6376 if (!cursor_push_byte(cur, '"')) 6377 return 0; 6378 6379 for (i = 0; i < len; i++) { 6380 if (!cursor_push_escaped_char(cur, str[i])) 6381 return 0; 6382 } 6383 6384 if (!cursor_push_byte(cur, '"')) 6385 return 0; 6386 6387 return 1; 6388 } 6389 6390 6391 static inline int cursor_push_json_tag_str(struct cursor *cur, struct ndb_str str) 6392 { 6393 if (str.flag == NDB_PACKED_ID) 6394 return cursor_push_hex_str(cur, str.id, 32); 6395 6396 return cursor_push_jsonstr(cur, str.str); 6397 } 6398 6399 static int cursor_push_json_tag(struct cursor *cur, struct ndb_note *note, 6400 struct ndb_tag *tag) 6401 { 6402 int i; 6403 6404 if (!cursor_push_byte(cur, '[')) 6405 return 0; 6406 6407 for (i = 0; i < tag->count; i++) { 6408 if (!cursor_push_json_tag_str(cur, ndb_tag_str(note, tag, i))) 6409 return 0; 6410 if (i != tag->count-1 && !cursor_push_byte(cur, ',')) 6411 return 0; 6412 } 6413 6414 return cursor_push_byte(cur, ']'); 6415 } 6416 6417 static int cursor_push_json_tags(struct cursor *cur, struct ndb_note *note) 6418 { 6419 int i; 6420 struct ndb_iterator iter, *it = &iter; 6421 ndb_tags_iterate_start(note, it); 6422 6423 if (!cursor_push_byte(cur, '[')) 6424 return 0; 6425 6426 i = 0; 6427 while (ndb_tags_iterate_next(it)) { 6428 if (!cursor_push_json_tag(cur, note, it->tag)) 6429 return 0; 6430 if (i != note->tags.count-1 && !cursor_push_str(cur, ",")) 6431 return 0; 6432 i++; 6433 } 6434 6435 if (!cursor_push_byte(cur, ']')) 6436 return 0; 6437 6438 return 1; 6439 } 6440 6441 static int ndb_event_commitment(struct ndb_note *ev, unsigned char *buf, int buflen) 6442 { 6443 char timebuf[16] = {0}; 6444 char kindbuf[16] = {0}; 6445 char pubkey[65]; 6446 struct cursor cur; 6447 int ok; 6448 6449 if (!hex_encode(ev->pubkey, sizeof(ev->pubkey), pubkey)) 6450 return 0; 6451 6452 make_cursor(buf, buf + buflen, &cur); 6453 6454 // TODO: update in 2106 ... 6455 snprintf(timebuf, sizeof(timebuf), "%d", (uint32_t)ev->created_at); 6456 snprintf(kindbuf, sizeof(kindbuf), "%d", ev->kind); 6457 6458 ok = 6459 cursor_push_str(&cur, "[0,\"") && 6460 cursor_push_str(&cur, pubkey) && 6461 cursor_push_str(&cur, "\",") && 6462 cursor_push_str(&cur, timebuf) && 6463 cursor_push_str(&cur, ",") && 6464 cursor_push_str(&cur, kindbuf) && 6465 cursor_push_str(&cur, ",") && 6466 cursor_push_json_tags(&cur, ev) && 6467 cursor_push_str(&cur, ",") && 6468 cursor_push_jsonstr(&cur, ndb_note_str(ev, &ev->content).str) && 6469 cursor_push_str(&cur, "]"); 6470 6471 if (!ok) 6472 return 0; 6473 6474 return cur.p - cur.start; 6475 } 6476 6477 static int cursor_push_hex(struct cursor *c, unsigned char *bytes, int len) 6478 { 6479 int i; 6480 unsigned char chr; 6481 if (c->p + (len * 2) >= c->end) 6482 return 0; 6483 6484 for (i = 0; i < len; i++) { 6485 chr = bytes[i]; 6486 6487 *(c->p++) = hexchar(chr >> 4); 6488 *(c->p++) = hexchar(chr & 0xF); 6489 } 6490 6491 return 1; 6492 } 6493 6494 static int cursor_push_int_str(struct cursor *c, uint64_t num) 6495 { 6496 char timebuf[16] = {0}; 6497 snprintf(timebuf, sizeof(timebuf), "%" PRIu64, num); 6498 return cursor_push_str(c, timebuf); 6499 } 6500 6501 int ndb_note_json(struct ndb_note *note, char *buf, int buflen) 6502 { 6503 struct cursor cur, *c = &cur; 6504 6505 make_cursor((unsigned char *)buf, (unsigned char*)buf + buflen, &cur); 6506 6507 int ok = cursor_push_str(c, "{\"id\":\"") && 6508 cursor_push_hex(c, ndb_note_id(note), 32) && 6509 cursor_push_str(c, "\",\"pubkey\":\"") && 6510 cursor_push_hex(c, ndb_note_pubkey(note), 32) && 6511 cursor_push_str(c, "\",\"created_at\":") && 6512 cursor_push_int_str(c, ndb_note_created_at(note)) && 6513 cursor_push_str(c, ",\"kind\":") && 6514 cursor_push_int_str(c, ndb_note_kind(note)) && 6515 cursor_push_str(c, ",\"tags\":") && 6516 cursor_push_json_tags(c, note) && 6517 cursor_push_str(c, ",\"content\":") && 6518 cursor_push_jsonstr(c, ndb_note_content(note)) && 6519 cursor_push_str(c, ",\"sig\":\"") && 6520 cursor_push_hex(c, ndb_note_sig(note), 64) && 6521 cursor_push_c_str(c, "\"}"); 6522 6523 if (!ok) { 6524 return 0; 6525 } 6526 6527 return cur.p - cur.start; 6528 } 6529 6530 static int cursor_push_json_elem_array(struct cursor *cur, 6531 const struct ndb_filter *filter, 6532 struct ndb_filter_elements *elems) 6533 { 6534 int i; 6535 unsigned char *id; 6536 const char *str; 6537 uint64_t val; 6538 6539 if (!cursor_push_byte(cur, '[')) 6540 return 0; 6541 6542 for (i = 0; i < elems->count; i++) { 6543 6544 switch (elems->field.elem_type) { 6545 case NDB_ELEMENT_CUSTOM: 6546 // can't serialize custom functions 6547 break; 6548 case NDB_ELEMENT_STRING: 6549 str = ndb_filter_get_string_element(filter, elems, i); 6550 if (!cursor_push_jsonstr(cur, str)) 6551 return 0; 6552 break; 6553 case NDB_ELEMENT_ID: 6554 id = ndb_filter_get_id_element(filter, elems, i); 6555 if (!cursor_push_hex_str(cur, id, 32)) 6556 return 0; 6557 break; 6558 case NDB_ELEMENT_INT: 6559 val = ndb_filter_get_int_element(elems, i); 6560 if (!cursor_push_int_str(cur, val)) 6561 return 0; 6562 break; 6563 case NDB_ELEMENT_UNKNOWN: 6564 ndb_debug("unknown element in cursor_push_json_elem_array"); 6565 return 0; 6566 } 6567 6568 if (i != elems->count-1) { 6569 if (!cursor_push_byte(cur, ',')) 6570 return 0; 6571 } 6572 } 6573 6574 if (!cursor_push_str(cur, "]")) 6575 return 0; 6576 6577 return 1; 6578 } 6579 6580 int ndb_filter_json(const struct ndb_filter *filter, char *buf, int buflen) 6581 { 6582 const char *str; 6583 struct cursor cur, *c = &cur; 6584 struct ndb_filter_elements *elems; 6585 int i; 6586 6587 if (!filter->finalized) { 6588 ndb_debug("filter not finalized in ndb_filter_json\n"); 6589 return 0; 6590 } 6591 6592 make_cursor((unsigned char *)buf, (unsigned char*)buf + buflen, c); 6593 6594 if (!cursor_push_str(c, "{")) 6595 return 0; 6596 6597 for (i = 0; i < filter->num_elements; i++) { 6598 elems = ndb_filter_get_elements(filter, i); 6599 switch (elems->field.type) { 6600 case NDB_FILTER_CUSTOM: 6601 // nothing to encode these as 6602 break; 6603 case NDB_FILTER_IDS: 6604 if (!cursor_push_str(c, "\"ids\":")) 6605 return 0; 6606 if (!cursor_push_json_elem_array(c, filter, elems)) 6607 return 0; 6608 break; 6609 case NDB_FILTER_SEARCH: 6610 if (!cursor_push_str(c, "\"search\":")) 6611 return 0; 6612 if (!(str = ndb_filter_get_string_element(filter, elems, 0))) 6613 return 0; 6614 if (!cursor_push_jsonstr(c, str)) 6615 return 0; 6616 break; 6617 case NDB_FILTER_AUTHORS: 6618 if (!cursor_push_str(c, "\"authors\":")) 6619 return 0; 6620 if (!cursor_push_json_elem_array(c, filter, elems)) 6621 return 0; 6622 break; 6623 case NDB_FILTER_KINDS: 6624 if (!cursor_push_str(c, "\"kinds\":")) 6625 return 0; 6626 if (!cursor_push_json_elem_array(c, filter, elems)) 6627 return 0; 6628 break; 6629 case NDB_FILTER_TAGS: 6630 if (!cursor_push_str(c, "\"#")) 6631 return 0; 6632 if (!cursor_push_byte(c, elems->field.tag)) 6633 return 0; 6634 if (!cursor_push_str(c, "\":")) 6635 return 0; 6636 if (!cursor_push_json_elem_array(c, filter, elems)) 6637 return 0; 6638 break; 6639 case NDB_FILTER_SINCE: 6640 if (!cursor_push_str(c, "\"since\":")) 6641 return 0; 6642 if (!cursor_push_int_str(c, ndb_filter_get_int_element(elems, 0))) 6643 return 0; 6644 break; 6645 case NDB_FILTER_UNTIL: 6646 if (!cursor_push_str(c, "\"until\":")) 6647 return 0; 6648 if (!cursor_push_int_str(c, ndb_filter_get_int_element(elems, 0))) 6649 return 0; 6650 break; 6651 case NDB_FILTER_LIMIT: 6652 if (!cursor_push_str(c, "\"limit\":")) 6653 return 0; 6654 if (!cursor_push_int_str(c, ndb_filter_get_int_element(elems, 0))) 6655 return 0; 6656 break; 6657 case NDB_FILTER_RELAYS: 6658 if (!cursor_push_str(c, "\"relays\":")) 6659 return 0; 6660 if (!cursor_push_json_elem_array(c, filter, elems)) 6661 return 0; 6662 break; 6663 } 6664 6665 if (i != filter->num_elements-1) { 6666 if (!cursor_push_byte(c, ',')) { 6667 return 0; 6668 } 6669 } 6670 6671 } 6672 6673 if (!cursor_push_c_str(c, "}")) 6674 return 0; 6675 6676 return cur.p - cur.start; 6677 } 6678 6679 int ndb_calculate_id(struct ndb_note *note, unsigned char *buf, int buflen, unsigned char *id) { 6680 int len; 6681 6682 if (!(len = ndb_event_commitment(note, buf, buflen))) 6683 return 0; 6684 6685 //fprintf(stderr, "%.*s\n", len, buf); 6686 6687 sha256((struct sha256*)id, buf, len); 6688 6689 return 1; 6690 } 6691 6692 int ndb_sign_id(struct ndb_keypair *keypair, unsigned char id[32], 6693 unsigned char sig[64]) 6694 { 6695 unsigned char aux[32]; 6696 secp256k1_keypair *pair = (secp256k1_keypair*) keypair->pair; 6697 6698 if (!fill_random(aux, sizeof(aux))) 6699 return 0; 6700 6701 secp256k1_context *ctx = 6702 secp256k1_context_create(SECP256K1_CONTEXT_NONE); 6703 6704 return secp256k1_schnorrsig_sign32(ctx, sig, id, pair, aux); 6705 } 6706 6707 int ndb_create_keypair(struct ndb_keypair *kp) 6708 { 6709 secp256k1_keypair *keypair = (secp256k1_keypair*)kp->pair; 6710 secp256k1_xonly_pubkey pubkey; 6711 6712 secp256k1_context *ctx = 6713 secp256k1_context_create(SECP256K1_CONTEXT_NONE);; 6714 6715 /* Try to create a keypair with a valid context, it should only 6716 * fail if the secret key is zero or out of range. */ 6717 if (!secp256k1_keypair_create(ctx, keypair, kp->secret)) 6718 return 0; 6719 6720 if (!secp256k1_keypair_xonly_pub(ctx, &pubkey, NULL, keypair)) 6721 return 0; 6722 6723 /* Serialize the public key. Should always return 1 for a valid public key. */ 6724 return secp256k1_xonly_pubkey_serialize(ctx, kp->pubkey, &pubkey); 6725 } 6726 6727 int ndb_decode_key(const char *secstr, struct ndb_keypair *keypair) 6728 { 6729 if (!hex_decode(secstr, strlen(secstr), keypair->secret, 32)) { 6730 fprintf(stderr, "could not hex decode secret key\n"); 6731 return 0; 6732 } 6733 6734 return ndb_create_keypair(keypair); 6735 } 6736 6737 int ndb_builder_finalize(struct ndb_builder *builder, struct ndb_note **note, 6738 struct ndb_keypair *keypair) 6739 { 6740 int strings_len = builder->strings.p - builder->strings.start; 6741 unsigned char *note_end = builder->note_cur.p + strings_len; 6742 int total_size = note_end - builder->note_cur.start; 6743 6744 // move the strings buffer next to the end of our ndb_note 6745 memmove(builder->note_cur.p, builder->strings.start, strings_len); 6746 6747 // set the strings location 6748 builder->note->strings = builder->note_cur.p - builder->note_cur.start; 6749 6750 // record the total size 6751 //builder->note->size = total_size; 6752 6753 *note = builder->note; 6754 6755 // generate id and sign if we're building this manually 6756 if (keypair) { 6757 // use the remaining memory for building our id buffer 6758 unsigned char *end = builder->mem.end; 6759 unsigned char *start = (unsigned char*)(*note) + total_size; 6760 6761 ndb_builder_set_pubkey(builder, keypair->pubkey); 6762 6763 if (!ndb_calculate_id(builder->note, start, end - start, 6764 builder->note->id)) 6765 return 0; 6766 6767 if (!ndb_sign_id(keypair, (*note)->id, (*note)->sig)) 6768 return 0; 6769 } 6770 6771 // make sure we're aligned as a whole 6772 total_size = (total_size + 7) & ~7; 6773 assert((total_size % 8) == 0); 6774 return total_size; 6775 } 6776 6777 struct ndb_note * ndb_builder_note(struct ndb_builder *builder) 6778 { 6779 return builder->note; 6780 } 6781 6782 static union ndb_packed_str ndb_offset_str(uint32_t offset) 6783 { 6784 // ensure accidents like -1 don't corrupt our packed_str 6785 union ndb_packed_str str; 6786 // most significant byte is reserved for ndb_packtype 6787 str.offset = offset & 0xFFFFFF; 6788 return str; 6789 } 6790 6791 6792 /// find an existing string via str_indices. these indices only exist in the 6793 /// builder phase just for this purpose. 6794 static inline int ndb_builder_find_str(struct ndb_builder *builder, 6795 const char *str, int len, 6796 union ndb_packed_str *pstr) 6797 { 6798 // find existing matching string to avoid duplicate strings 6799 int indices = cursor_count(&builder->str_indices, sizeof(uint32_t)); 6800 for (int i = 0; i < indices; i++) { 6801 uint32_t index = ((uint32_t*)builder->str_indices.start)[i]; 6802 const char *some_str = (const char*)builder->strings.start + index; 6803 6804 if (!memcmp(some_str, str, len) && some_str[len] == '\0') { 6805 // found an existing matching str, use that index 6806 *pstr = ndb_offset_str(index); 6807 return 1; 6808 } 6809 } 6810 6811 return 0; 6812 } 6813 6814 static int ndb_builder_push_str(struct ndb_builder *builder, const char *str, 6815 int len, union ndb_packed_str *pstr) 6816 { 6817 uint32_t loc; 6818 6819 // no string found, push a new one 6820 loc = builder->strings.p - builder->strings.start; 6821 if (!(cursor_push(&builder->strings, (unsigned char*)str, len) && 6822 cursor_push_byte(&builder->strings, '\0'))) { 6823 return 0; 6824 } 6825 6826 *pstr = ndb_offset_str(loc); 6827 6828 // record in builder indices. ignore return value, if we can't cache it 6829 // then whatever 6830 cursor_push_u32(&builder->str_indices, loc); 6831 6832 return 1; 6833 } 6834 6835 static int ndb_builder_push_packed_id(struct ndb_builder *builder, 6836 unsigned char *id, 6837 union ndb_packed_str *pstr) 6838 { 6839 // Don't both find id duplicates. very rarely are they duplicated 6840 // and it slows things down quite a bit. If we really care about this 6841 // We can switch to a hash table. 6842 //if (ndb_builder_find_str(builder, (const char*)id, 32, pstr)) { 6843 // pstr->packed.flag = NDB_PACKED_ID; 6844 // return 1; 6845 //} 6846 6847 if (ndb_builder_push_str(builder, (const char*)id, 32, pstr)) { 6848 pstr->packed.flag = NDB_PACKED_ID; 6849 return 1; 6850 } 6851 6852 return 0; 6853 } 6854 6855 union ndb_packed_str ndb_chars_to_packed_str(char c1, char c2) 6856 { 6857 union ndb_packed_str str; 6858 str.packed.flag = NDB_PACKED_STR; 6859 str.packed.str[0] = c1; 6860 str.packed.str[1] = c2; 6861 str.packed.str[2] = '\0'; 6862 return str; 6863 } 6864 6865 static union ndb_packed_str ndb_char_to_packed_str(char c) 6866 { 6867 union ndb_packed_str str; 6868 str.packed.flag = NDB_PACKED_STR; 6869 str.packed.str[0] = c; 6870 str.packed.str[1] = '\0'; 6871 return str; 6872 } 6873 6874 6875 /// Check for small strings to pack 6876 static inline int ndb_builder_try_compact_str(struct ndb_builder *builder, 6877 const char *str, int len, 6878 union ndb_packed_str *pstr, 6879 int pack_ids) 6880 { 6881 unsigned char id_buf[32]; 6882 6883 if (len == 0) { 6884 *pstr = ndb_char_to_packed_str(0); 6885 return 1; 6886 } else if (len == 1) { 6887 *pstr = ndb_char_to_packed_str(str[0]); 6888 return 1; 6889 } else if (len == 2) { 6890 *pstr = ndb_chars_to_packed_str(str[0], str[1]); 6891 return 1; 6892 } else if (pack_ids && len == 64 && hex_decode(str, 64, id_buf, 32)) { 6893 return ndb_builder_push_packed_id(builder, id_buf, pstr); 6894 } 6895 6896 return 0; 6897 } 6898 6899 6900 static int ndb_builder_push_unpacked_str(struct ndb_builder *builder, 6901 const char *str, int len, 6902 union ndb_packed_str *pstr) 6903 { 6904 if (ndb_builder_find_str(builder, str, len, pstr)) 6905 return 1; 6906 6907 return ndb_builder_push_str(builder, str, len, pstr); 6908 } 6909 6910 int ndb_builder_make_str(struct ndb_builder *builder, const char *str, int len, 6911 union ndb_packed_str *pstr, int pack_ids) 6912 { 6913 if (ndb_builder_try_compact_str(builder, str, len, pstr, pack_ids)) 6914 return 1; 6915 6916 return ndb_builder_push_unpacked_str(builder, str, len, pstr); 6917 } 6918 6919 int ndb_builder_set_content(struct ndb_builder *builder, const char *content, 6920 int len) 6921 { 6922 int pack_ids = 0; 6923 builder->note->content_length = len; 6924 return ndb_builder_make_str(builder, content, len, 6925 &builder->note->content, pack_ids); 6926 } 6927 6928 6929 static inline int jsoneq(const char *json, jsmntok_t *tok, int tok_len, 6930 const char *s) 6931 { 6932 if (tok->type == JSMN_STRING && (int)strlen(s) == tok_len && 6933 memcmp(json + tok->start, s, tok_len) == 0) { 6934 return 1; 6935 } 6936 return 0; 6937 } 6938 6939 static int ndb_builder_finalize_tag(struct ndb_builder *builder, 6940 union ndb_packed_str offset) 6941 { 6942 if (!cursor_push_u32(&builder->note_cur, offset.offset)) 6943 return 0; 6944 builder->current_tag->count++; 6945 return 1; 6946 } 6947 6948 /// Unescape and push json strings 6949 static int ndb_builder_make_json_str(struct ndb_builder *builder, 6950 const char *str, int len, 6951 union ndb_packed_str *pstr, 6952 int *written, int pack_ids) 6953 { 6954 // let's not care about de-duping these. we should just unescape 6955 // in-place directly into the strings table. 6956 if (written) 6957 *written = len; 6958 6959 const char *p, *end, *start; 6960 unsigned char *builder_start; 6961 6962 // always try compact strings first 6963 if (ndb_builder_try_compact_str(builder, str, len, pstr, pack_ids)) 6964 return 1; 6965 6966 end = str + len; 6967 start = str; // Initialize start to the beginning of the string 6968 6969 *pstr = ndb_offset_str(builder->strings.p - builder->strings.start); 6970 builder_start = builder->strings.p; 6971 6972 for (p = str; p < end; p++) { 6973 if (*p == '\\' && p+1 < end) { 6974 // Push the chunk of unescaped characters before this escape sequence 6975 if (start < p && !cursor_push(&builder->strings, 6976 (unsigned char *)start, 6977 p - start)) { 6978 return 0; 6979 } 6980 6981 if (!cursor_push_unescaped_char(&builder->strings, *p, *(p+1))) 6982 return 0; 6983 6984 p++; // Skip the character following the backslash 6985 start = p + 1; // Update the start pointer to the next character 6986 } 6987 } 6988 6989 // Handle the last chunk after the last escape sequence (or if there are no escape sequences at all) 6990 if (start < p && !cursor_push(&builder->strings, (unsigned char *)start, 6991 p - start)) { 6992 return 0; 6993 } 6994 6995 if (written) 6996 *written = builder->strings.p - builder_start; 6997 6998 // TODO: dedupe these!? 6999 return cursor_push_byte(&builder->strings, '\0'); 7000 } 7001 7002 static int ndb_builder_push_json_tag(struct ndb_builder *builder, 7003 const char *str, int len) 7004 { 7005 union ndb_packed_str pstr; 7006 int pack_ids = 1; 7007 if (!ndb_builder_make_json_str(builder, str, len, &pstr, NULL, pack_ids)) 7008 return 0; 7009 return ndb_builder_finalize_tag(builder, pstr); 7010 } 7011 7012 // Push a json array into an ndb tag ["p", "abcd..."] -> struct ndb_tag 7013 static int ndb_builder_tag_from_json_array(struct ndb_json_parser *p, 7014 jsmntok_t *array) 7015 { 7016 jsmntok_t *str_tok; 7017 const char *str; 7018 7019 if (array->size == 0) 7020 return 0; 7021 7022 if (!ndb_builder_new_tag(&p->builder)) 7023 return 0; 7024 7025 for (int i = 0; i < array->size; i++) { 7026 str_tok = &array[i+1]; 7027 str = p->json + str_tok->start; 7028 7029 if (!ndb_builder_push_json_tag(&p->builder, str, 7030 toksize(str_tok))) { 7031 return 0; 7032 } 7033 } 7034 7035 return 1; 7036 } 7037 7038 // Push json tags into ndb data 7039 // [["t", "hashtag"], ["p", "abcde..."]] -> struct ndb_tags 7040 static inline int ndb_builder_process_json_tags(struct ndb_json_parser *p, 7041 jsmntok_t *array) 7042 { 7043 jsmntok_t *tag = array; 7044 7045 if (array->size == 0) 7046 return 1; 7047 7048 for (int i = 0; i < array->size; i++) { 7049 if (!ndb_builder_tag_from_json_array(p, &tag[i+1])) 7050 return 0; 7051 tag += tag[i+1].size; 7052 } 7053 7054 return 1; 7055 } 7056 7057 static int parse_unsigned_int(const char *start, int len, unsigned int *num) 7058 { 7059 unsigned int number = 0; 7060 const char *p = start, *end = start + len; 7061 int digits = 0; 7062 7063 while (p < end) { 7064 char c = *p; 7065 7066 if (c < '0' || c > '9') 7067 break; 7068 7069 // Check for overflow 7070 char digit = c - '0'; 7071 if (number > (UINT_MAX - digit) / 10) 7072 return 0; // Overflow detected 7073 7074 number = number * 10 + digit; 7075 7076 p++; 7077 digits++; 7078 } 7079 7080 if (digits == 0) 7081 return 0; 7082 7083 *num = number; 7084 return 1; 7085 } 7086 7087 int ndb_client_event_from_json(const char *json, int len, struct ndb_fce *fce, 7088 unsigned char *buf, int bufsize, struct ndb_id_cb *cb) 7089 { 7090 jsmntok_t *tok = NULL; 7091 int tok_len, res; 7092 struct ndb_json_parser parser; 7093 struct ndb_event *ev = &fce->event; 7094 7095 ndb_json_parser_init(&parser, json, len, buf, bufsize); 7096 7097 if ((res = ndb_json_parser_parse(&parser, cb)) < 0) 7098 return res; 7099 7100 if (parser.toks[0].type == JSMN_OBJECT) { 7101 ndb_debug("got raw json in client_event_from_json\n"); 7102 fce->evtype = NDB_FCE_EVENT; 7103 return ndb_parse_json_note(&parser, &ev->note); 7104 } 7105 7106 if (parser.num_tokens <= 3 || parser.toks[0].type != JSMN_ARRAY) 7107 return 0; 7108 7109 parser.i = 1; 7110 tok = &parser.toks[parser.i++]; 7111 tok_len = toksize(tok); 7112 if (tok->type != JSMN_STRING) 7113 return 0; 7114 7115 if (tok_len == 5 && !memcmp("EVENT", json + tok->start, 5)) { 7116 fce->evtype = NDB_FCE_EVENT; 7117 return ndb_parse_json_note(&parser, &ev->note); 7118 } 7119 7120 return 0; 7121 } 7122 7123 7124 int ndb_ws_event_from_json(const char *json, int len, struct ndb_tce *tce, 7125 unsigned char *buf, int bufsize, 7126 struct ndb_id_cb *cb) 7127 { 7128 jsmntok_t *tok = NULL; 7129 int tok_len, res; 7130 struct ndb_json_parser parser; 7131 struct ndb_event *ev = &tce->event; 7132 7133 tce->subid_len = 0; 7134 tce->subid = ""; 7135 7136 ndb_json_parser_init(&parser, json, len, buf, bufsize); 7137 7138 if ((res = ndb_json_parser_parse(&parser, cb)) < 0) 7139 return res; 7140 7141 if (parser.toks[0].type == JSMN_OBJECT) { 7142 ndb_debug("got raw json in ws_event_from_json\n"); 7143 tce->evtype = NDB_TCE_EVENT; 7144 return ndb_parse_json_note(&parser, &ev->note); 7145 } 7146 7147 if (parser.num_tokens < 3 || parser.toks[0].type != JSMN_ARRAY) 7148 return 0; 7149 7150 parser.i = 1; 7151 tok = &parser.toks[parser.i++]; 7152 tok_len = toksize(tok); 7153 if (tok->type != JSMN_STRING) 7154 return 0; 7155 7156 if (tok_len == 5 && !memcmp("EVENT", json + tok->start, 5)) { 7157 tce->evtype = NDB_TCE_EVENT; 7158 7159 tok = &parser.toks[parser.i++]; 7160 if (tok->type != JSMN_STRING) 7161 return 0; 7162 7163 tce->subid = json + tok->start; 7164 tce->subid_len = toksize(tok); 7165 7166 return ndb_parse_json_note(&parser, &ev->note); 7167 } else if (tok_len == 4 && !memcmp("EOSE", json + tok->start, 4)) { 7168 tce->evtype = NDB_TCE_EOSE; 7169 7170 tok = &parser.toks[parser.i++]; 7171 if (tok->type != JSMN_STRING) 7172 return 0; 7173 7174 tce->subid = json + tok->start; 7175 tce->subid_len = toksize(tok); 7176 return 1; 7177 } else if (tok_len == 2 && !memcmp("OK", json + tok->start, 2)) { 7178 if (parser.num_tokens != 5) 7179 return 0; 7180 7181 struct ndb_command_result *cr = &tce->command_result; 7182 7183 tce->evtype = NDB_TCE_OK; 7184 7185 tok = &parser.toks[parser.i++]; 7186 if (tok->type != JSMN_STRING) 7187 return 0; 7188 7189 tce->subid = json + tok->start; 7190 tce->subid_len = toksize(tok); 7191 7192 tok = &parser.toks[parser.i++]; 7193 if (tok->type != JSMN_PRIMITIVE || toksize(tok) == 0) 7194 return 0; 7195 7196 cr->ok = (json + tok->start)[0] == 't'; 7197 7198 tok = &parser.toks[parser.i++]; 7199 if (tok->type != JSMN_STRING) 7200 return 0; 7201 7202 tce->command_result.msg = json + tok->start; 7203 tce->command_result.msglen = toksize(tok); 7204 7205 return 1; 7206 } else if (tok_len == 4 && !memcmp("AUTH", json + tok->start, 4)) { 7207 tce->evtype = NDB_TCE_AUTH; 7208 7209 tok = &parser.toks[parser.i++]; 7210 if (tok->type != JSMN_STRING) 7211 return 0; 7212 7213 tce->subid = json + tok->start; 7214 tce->subid_len = toksize(tok); 7215 7216 return 1; 7217 } 7218 7219 return 0; 7220 } 7221 7222 static enum ndb_filter_fieldtype 7223 ndb_filter_parse_field(const char *tok, int len, char *tagchar) 7224 { 7225 *tagchar = 0; 7226 7227 if (len == 0) 7228 return 0; 7229 7230 if (len == 7 && !strncmp(tok, "authors", 7)) { 7231 return NDB_FILTER_AUTHORS; 7232 } else if (len == 3 && !strncmp(tok, "ids", 3)) { 7233 return NDB_FILTER_IDS; 7234 } else if (len == 5 && !strncmp(tok, "kinds", 5)) { 7235 return NDB_FILTER_KINDS; 7236 } else if (len == 2 && tok[0] == '#') { 7237 *tagchar = tok[1]; 7238 return NDB_FILTER_TAGS; 7239 } else if (len == 5 && !strncmp(tok, "since", 5)) { 7240 return NDB_FILTER_SINCE; 7241 } else if (len == 5 && !strncmp(tok, "until", 5)) { 7242 return NDB_FILTER_UNTIL; 7243 } else if (len == 5 && !strncmp(tok, "limit", 5)) { 7244 return NDB_FILTER_LIMIT; 7245 } else if (len == 6 && !strncmp(tok, "search", 6)) { 7246 return NDB_FILTER_SEARCH; 7247 } 7248 7249 return 0; 7250 } 7251 7252 static int ndb_filter_parse_json_ids(struct ndb_json_parser *parser, 7253 struct ndb_filter *filter) 7254 { 7255 jsmntok_t *tok; 7256 const char *start; 7257 unsigned char hexbuf[32]; 7258 int tok_len, i, size; 7259 7260 tok = &parser->toks[parser->i++]; 7261 7262 if (tok->type != JSMN_ARRAY) { 7263 ndb_debug("parse_json_ids: not an array\n"); 7264 return 0; 7265 } 7266 7267 size = tok->size; 7268 7269 for (i = 0; i < size; parser->i++, i++) { 7270 tok = &parser->toks[parser->i]; 7271 start = parser->json + tok->start; 7272 tok_len = toksize(tok); 7273 7274 if (tok->type != JSMN_STRING) { 7275 ndb_debug("parse_json_ids: not a string '%d'\n", tok->type); 7276 return 0; 7277 } 7278 7279 if (tok_len != 64) { 7280 ndb_debug("parse_json_ids: not len 64: '%.*s'\n", tok_len, start); 7281 return 0; 7282 } 7283 7284 // id 7285 if (!hex_decode(start, tok_len, hexbuf, sizeof(hexbuf))) { 7286 ndb_debug("parse_json_ids: hex decode failed\n"); 7287 return 0; 7288 } 7289 7290 ndb_debug("adding id elem\n"); 7291 if (!ndb_filter_add_id_element(filter, hexbuf)) { 7292 ndb_debug("parse_json_ids: failed to add id element\n"); 7293 return 0; 7294 } 7295 } 7296 7297 parser->i--; 7298 return 1; 7299 } 7300 7301 static int ndb_filter_parse_json_elems(struct ndb_json_parser *parser, 7302 struct ndb_filter *filter) 7303 { 7304 jsmntok_t *tok; 7305 const char *start; 7306 int tok_len; 7307 unsigned char hexbuf[32]; 7308 enum ndb_generic_element_type typ; 7309 tok = NULL; 7310 int i, size; 7311 7312 tok = &parser->toks[parser->i++]; 7313 7314 if (tok->type != JSMN_ARRAY) 7315 return 0; 7316 7317 size = tok->size; 7318 7319 for (i = 0; i < size; i++, parser->i++) { 7320 tok = &parser->toks[parser->i]; 7321 start = parser->json + tok->start; 7322 tok_len = toksize(tok); 7323 7324 if (tok->type != JSMN_STRING) 7325 return 0; 7326 7327 if (i == 0) { 7328 if (tok_len == 64 && hex_decode(start, 64, hexbuf, sizeof(hexbuf))) { 7329 typ = NDB_ELEMENT_ID; 7330 if (!ndb_filter_add_id_element(filter, hexbuf)) { 7331 ndb_debug("failed to push id elem\n"); 7332 return 0; 7333 } 7334 } else { 7335 typ = NDB_ELEMENT_STRING; 7336 if (!ndb_filter_add_str_element_len(filter, start, tok_len)) 7337 return 0; 7338 } 7339 } else if (typ == NDB_ELEMENT_ID) { 7340 if (!hex_decode(start, 64, hexbuf, sizeof(hexbuf))) 7341 return 0; 7342 if (!ndb_filter_add_id_element(filter, hexbuf)) 7343 return 0; 7344 } else if (typ == NDB_ELEMENT_STRING) { 7345 if (!ndb_filter_add_str_element_len(filter, start, tok_len)) 7346 return 0; 7347 } else { 7348 // ??? 7349 return 0; 7350 } 7351 } 7352 7353 parser->i--; 7354 return 1; 7355 } 7356 7357 static int ndb_filter_parse_json_str(struct ndb_json_parser *parser, 7358 struct ndb_filter *filter) 7359 { 7360 jsmntok_t *tok; 7361 const char *start; 7362 int tok_len; 7363 7364 tok = &parser->toks[parser->i]; 7365 start = parser->json + tok->start; 7366 tok_len = toksize(tok); 7367 7368 if (tok->type != JSMN_STRING) 7369 return 0; 7370 7371 if (!ndb_filter_add_str_element_len(filter, start, tok_len)) 7372 return 0; 7373 7374 ndb_debug("added str elem '%.*s'\n", tok_len, start); 7375 7376 return 1; 7377 } 7378 7379 static int ndb_filter_parse_json_int(struct ndb_json_parser *parser, 7380 struct ndb_filter *filter) 7381 { 7382 jsmntok_t *tok; 7383 const char *start; 7384 int tok_len; 7385 unsigned int value; 7386 7387 tok = &parser->toks[parser->i]; 7388 start = parser->json + tok->start; 7389 tok_len = toksize(tok); 7390 7391 if (tok->type != JSMN_PRIMITIVE) 7392 return 0; 7393 7394 if (!parse_unsigned_int(start, tok_len, &value)) 7395 return 0; 7396 7397 if (!ndb_filter_add_int_element(filter, (uint64_t)value)) 7398 return 0; 7399 7400 ndb_debug("added int elem %d\n", value); 7401 7402 return 1; 7403 } 7404 7405 7406 static int ndb_filter_parse_json_ints(struct ndb_json_parser *parser, 7407 struct ndb_filter *filter) 7408 { 7409 jsmntok_t *tok; 7410 int size, i; 7411 7412 tok = &parser->toks[parser->i++]; 7413 7414 if (tok->type != JSMN_ARRAY) 7415 return 0; 7416 7417 size = tok->size; 7418 7419 for (i = 0; i < size; parser->i++, i++) { 7420 if (!ndb_filter_parse_json_int(parser, filter)) 7421 return 0; 7422 } 7423 7424 parser->i--; 7425 return 1; 7426 } 7427 7428 7429 static int ndb_filter_parse_json(struct ndb_json_parser *parser, 7430 struct ndb_filter *filter) 7431 { 7432 jsmntok_t *tok = NULL; 7433 const char *json = parser->json; 7434 const char *start; 7435 char tag; 7436 int tok_len; 7437 enum ndb_filter_fieldtype field; 7438 7439 if (parser->toks[parser->i++].type != JSMN_OBJECT) 7440 return 0; 7441 7442 for (; parser->i < parser->num_tokens; parser->i++) { 7443 tok = &parser->toks[parser->i++]; 7444 start = json + tok->start; 7445 tok_len = toksize(tok); 7446 7447 if (!(field = ndb_filter_parse_field(start, tok_len, &tag))) { 7448 ndb_debug("failed field '%.*s'\n", tok_len, start); 7449 continue; 7450 } 7451 7452 if (tag) { 7453 ndb_debug("starting tag field '%c'\n", tag); 7454 if (!ndb_filter_start_tag_field(filter, tag)) { 7455 ndb_debug("failed to start tag field '%c'\n", tag); 7456 return 0; 7457 } 7458 } else if (!ndb_filter_start_field(filter, field)) { 7459 ndb_debug("field already started\n"); 7460 return 0; 7461 } 7462 7463 // we parsed a top-level field 7464 switch(field) { 7465 case NDB_FILTER_CUSTOM: 7466 // can't really parse these yet 7467 break; 7468 case NDB_FILTER_AUTHORS: 7469 case NDB_FILTER_IDS: 7470 if (!ndb_filter_parse_json_ids(parser, filter)) { 7471 ndb_debug("failed to parse filter ids/authors\n"); 7472 return 0; 7473 } 7474 break; 7475 case NDB_FILTER_SEARCH: 7476 if (!ndb_filter_parse_json_str(parser, filter)) { 7477 ndb_debug("failed to parse filter search str\n"); 7478 return 0; 7479 } 7480 break; 7481 case NDB_FILTER_SINCE: 7482 case NDB_FILTER_UNTIL: 7483 case NDB_FILTER_LIMIT: 7484 if (!ndb_filter_parse_json_int(parser, filter)) { 7485 ndb_debug("failed to parse filter since/until/limit\n"); 7486 return 0; 7487 } 7488 break; 7489 case NDB_FILTER_KINDS: 7490 if (!ndb_filter_parse_json_ints(parser, filter)) { 7491 ndb_debug("failed to parse filter kinds\n"); 7492 return 0; 7493 } 7494 break; 7495 case NDB_FILTER_RELAYS: 7496 case NDB_FILTER_TAGS: 7497 if (!ndb_filter_parse_json_elems(parser, filter)) { 7498 ndb_debug("failed to parse filter tags\n"); 7499 return 0; 7500 } 7501 break; 7502 } 7503 7504 ndb_filter_end_field(filter); 7505 } 7506 7507 return ndb_filter_end(filter); 7508 } 7509 7510 int ndb_parse_json_note(struct ndb_json_parser *parser, struct ndb_note **note) 7511 { 7512 jsmntok_t *tok = NULL; 7513 unsigned char hexbuf[64]; 7514 const char *json = parser->json; 7515 const char *start; 7516 int i, tok_len, parsed; 7517 7518 parsed = 0; 7519 7520 if (parser->toks[parser->i].type != JSMN_OBJECT) 7521 return 0; 7522 7523 // TODO: build id buffer and verify at end 7524 7525 for (i = parser->i + 1; i < parser->num_tokens; i++) { 7526 tok = &parser->toks[i]; 7527 start = json + tok->start; 7528 tok_len = toksize(tok); 7529 7530 //printf("toplevel %.*s %d\n", tok_len, json + tok->start, tok->type); 7531 if (tok_len == 0 || i + 1 >= parser->num_tokens) 7532 continue; 7533 7534 if (start[0] == 'p' && jsoneq(json, tok, tok_len, "pubkey")) { 7535 // pubkey 7536 tok = &parser->toks[i+1]; 7537 hex_decode(json + tok->start, toksize(tok), hexbuf, sizeof(hexbuf)); 7538 parsed |= NDB_PARSED_PUBKEY; 7539 ndb_builder_set_pubkey(&parser->builder, hexbuf); 7540 } else if (tok_len == 2 && start[0] == 'i' && start[1] == 'd') { 7541 // id 7542 tok = &parser->toks[i+1]; 7543 hex_decode(json + tok->start, toksize(tok), hexbuf, sizeof(hexbuf)); 7544 parsed |= NDB_PARSED_ID; 7545 ndb_builder_set_id(&parser->builder, hexbuf); 7546 } else if (tok_len == 3 && start[0] == 's' && start[1] == 'i' && start[2] == 'g') { 7547 // sig 7548 tok = &parser->toks[i+1]; 7549 hex_decode(json + tok->start, toksize(tok), hexbuf, sizeof(hexbuf)); 7550 parsed |= NDB_PARSED_SIG; 7551 ndb_builder_set_sig(&parser->builder, hexbuf); 7552 } else if (start[0] == 'k' && jsoneq(json, tok, tok_len, "kind")) { 7553 // kind 7554 tok = &parser->toks[i+1]; 7555 start = json + tok->start; 7556 if (tok->type != JSMN_PRIMITIVE || tok_len <= 0) 7557 return 0; 7558 if (!parse_unsigned_int(start, toksize(tok), 7559 &parser->builder.note->kind)) 7560 return 0; 7561 parsed |= NDB_PARSED_KIND; 7562 } else if (start[0] == 'c') { 7563 if (jsoneq(json, tok, tok_len, "created_at")) { 7564 // created_at 7565 tok = &parser->toks[i+1]; 7566 start = json + tok->start; 7567 if (tok->type != JSMN_PRIMITIVE || tok_len <= 0) 7568 return 0; 7569 // TODO: update to int64 in 2106 ... xD 7570 unsigned int bigi; 7571 if (!parse_unsigned_int(start, toksize(tok), &bigi)) 7572 return 0; 7573 parser->builder.note->created_at = bigi; 7574 parsed |= NDB_PARSED_CREATED_AT; 7575 } else if (jsoneq(json, tok, tok_len, "content")) { 7576 // content 7577 tok = &parser->toks[i+1]; 7578 union ndb_packed_str pstr; 7579 tok_len = toksize(tok); 7580 int written, pack_ids = 0; 7581 if (!ndb_builder_make_json_str(&parser->builder, 7582 json + tok->start, 7583 tok_len, &pstr, 7584 &written, pack_ids)) { 7585 ndb_debug("ndb_builder_make_json_str failed\n"); 7586 return 0; 7587 } 7588 parser->builder.note->content_length = written; 7589 parser->builder.note->content = pstr; 7590 parsed |= NDB_PARSED_CONTENT; 7591 } 7592 } else if (start[0] == 't' && jsoneq(json, tok, tok_len, "tags")) { 7593 tok = &parser->toks[i+1]; 7594 ndb_builder_process_json_tags(parser, tok); 7595 i += tok->size; 7596 parsed |= NDB_PARSED_TAGS; 7597 } 7598 } 7599 7600 //ndb_debug("parsed %d = %d, &->%d", parsed, NDB_PARSED_ALL, parsed & NDB_PARSED_ALL); 7601 if (parsed != NDB_PARSED_ALL) 7602 return 0; 7603 7604 return ndb_builder_finalize(&parser->builder, note, NULL); 7605 } 7606 7607 int ndb_filter_from_json(const char *json, int len, struct ndb_filter *filter, 7608 unsigned char *buf, int bufsize) 7609 { 7610 struct ndb_json_parser parser; 7611 int res; 7612 7613 if (filter->finalized) 7614 return 0; 7615 7616 ndb_json_parser_init(&parser, json, len, buf, bufsize); 7617 if ((res = ndb_json_parser_parse(&parser, NULL)) < 0) 7618 return res; 7619 7620 if (parser.num_tokens < 1) 7621 return 0; 7622 7623 return ndb_filter_parse_json(&parser, filter); 7624 } 7625 7626 int ndb_note_from_json(const char *json, int len, struct ndb_note **note, 7627 unsigned char *buf, int bufsize) 7628 { 7629 struct ndb_json_parser parser; 7630 int res; 7631 7632 ndb_json_parser_init(&parser, json, len, buf, bufsize); 7633 if ((res = ndb_json_parser_parse(&parser, NULL)) < 0) 7634 return res; 7635 7636 if (parser.num_tokens < 1) 7637 return 0; 7638 7639 return ndb_parse_json_note(&parser, note); 7640 } 7641 7642 void ndb_builder_set_pubkey(struct ndb_builder *builder, unsigned char *pubkey) 7643 { 7644 memcpy(builder->note->pubkey, pubkey, 32); 7645 } 7646 7647 void ndb_builder_set_id(struct ndb_builder *builder, unsigned char *id) 7648 { 7649 memcpy(builder->note->id, id, 32); 7650 } 7651 7652 void ndb_builder_set_sig(struct ndb_builder *builder, unsigned char *sig) 7653 { 7654 memcpy(builder->note->sig, sig, 64); 7655 } 7656 7657 void ndb_builder_set_kind(struct ndb_builder *builder, uint32_t kind) 7658 { 7659 builder->note->kind = kind; 7660 } 7661 7662 void ndb_builder_set_created_at(struct ndb_builder *builder, uint64_t created_at) 7663 { 7664 builder->note->created_at = created_at; 7665 } 7666 7667 int ndb_builder_new_tag(struct ndb_builder *builder) 7668 { 7669 builder->note->tags.count++; 7670 struct ndb_tag tag = {0}; 7671 builder->current_tag = (struct ndb_tag *)builder->note_cur.p; 7672 return cursor_push_tag(&builder->note_cur, &tag); 7673 } 7674 7675 void ndb_stat_counts_init(struct ndb_stat_counts *counts) 7676 { 7677 counts->count = 0; 7678 counts->key_size = 0; 7679 counts->value_size = 0; 7680 } 7681 7682 static void ndb_stat_init(struct ndb_stat *stat) 7683 { 7684 // init stats 7685 int i; 7686 7687 for (i = 0; i < NDB_CKIND_COUNT; i++) { 7688 ndb_stat_counts_init(&stat->common_kinds[i]); 7689 } 7690 7691 for (i = 0; i < NDB_DBS; i++) { 7692 ndb_stat_counts_init(&stat->dbs[i]); 7693 } 7694 7695 ndb_stat_counts_init(&stat->other_kinds); 7696 } 7697 7698 int ndb_stat(struct ndb *ndb, struct ndb_stat *stat) 7699 { 7700 int rc; 7701 MDB_cursor *cur; 7702 MDB_val k, v; 7703 MDB_dbi db; 7704 struct ndb_txn txn; 7705 struct ndb_note *note; 7706 int i; 7707 enum ndb_common_kind common_kind; 7708 7709 // initialize to 0 7710 ndb_stat_init(stat); 7711 7712 if (!ndb_begin_query(ndb, &txn)) { 7713 fprintf(stderr, "ndb_stat failed at ndb_begin_query\n"); 7714 return 0; 7715 } 7716 7717 // stat each dbi in the database 7718 for (i = 0; i < NDB_DBS; i++) 7719 { 7720 db = ndb->lmdb.dbs[i]; 7721 7722 if ((rc = mdb_cursor_open(txn.mdb_txn, db, &cur))) { 7723 fprintf(stderr, "ndb_stat: mdb_cursor_open failed, error '%s'\n", 7724 mdb_strerror(rc)); 7725 return 0; 7726 } 7727 7728 // loop over every entry and count kv sizes 7729 while (mdb_cursor_get(cur, &k, &v, MDB_NEXT) == 0) { 7730 // we gather more detailed per-kind stats if we're in 7731 // the notes db 7732 if (i == NDB_DB_NOTE) { 7733 note = v.mv_data; 7734 common_kind = ndb_kind_to_common_kind(note->kind); 7735 7736 // uncommon kind? just count them in bulk 7737 if ((int)common_kind == -1) { 7738 stat->other_kinds.count++; 7739 stat->other_kinds.key_size += k.mv_size; 7740 stat->other_kinds.value_size += v.mv_size; 7741 } else { 7742 stat->common_kinds[common_kind].count++; 7743 stat->common_kinds[common_kind].key_size += k.mv_size; 7744 stat->common_kinds[common_kind].value_size += v.mv_size; 7745 } 7746 } 7747 7748 stat->dbs[i].count++; 7749 stat->dbs[i].key_size += k.mv_size; 7750 stat->dbs[i].value_size += v.mv_size; 7751 } 7752 7753 // close the cursor, they are per-dbi 7754 mdb_cursor_close(cur); 7755 } 7756 7757 ndb_end_query(&txn); 7758 7759 return 1; 7760 } 7761 7762 /// Push an element to the current tag 7763 /// 7764 /// Basic idea is to call ndb_builder_new_tag 7765 int ndb_builder_push_tag_str(struct ndb_builder *builder, 7766 const char *str, int len) 7767 { 7768 union ndb_packed_str pstr; 7769 int pack_ids = 1; 7770 if (!ndb_builder_make_str(builder, str, len, &pstr, pack_ids)) 7771 return 0; 7772 return ndb_builder_finalize_tag(builder, pstr); 7773 } 7774 7775 /// Push an id element to the current tag. Needs to be 32 bytes 7776 int ndb_builder_push_tag_id(struct ndb_builder *builder, 7777 unsigned char *id) 7778 { 7779 union ndb_packed_str pstr; 7780 if (!ndb_builder_push_packed_id(builder, id, &pstr)) 7781 return 0; 7782 return ndb_builder_finalize_tag(builder, pstr); 7783 } 7784 7785 // 7786 // CONFIG 7787 // 7788 void ndb_default_config(struct ndb_config *config) 7789 { 7790 int cores = get_cpu_cores(); 7791 config->mapsize = 1024UL * 1024UL * 1024UL * 32UL; // 32 GiB 7792 config->ingester_threads = cores == -1 ? 4 : cores; 7793 config->flags = 0; 7794 config->ingest_filter = NULL; 7795 config->filter_context = NULL; 7796 config->sub_cb_ctx = NULL; 7797 config->sub_cb = NULL; 7798 config->writer_scratch_buffer_size = DEFAULT_WRITER_SCRATCH_SIZE; 7799 } 7800 7801 void ndb_config_set_subscription_callback(struct ndb_config *config, ndb_sub_fn fn, void *context) 7802 { 7803 config->sub_cb_ctx = context; 7804 config->sub_cb = fn; 7805 } 7806 7807 void ndb_config_set_writer_scratch_buffer_size(struct ndb_config *config, int scratch_size) 7808 { 7809 config->writer_scratch_buffer_size = scratch_size; 7810 } 7811 7812 void ndb_config_set_ingest_threads(struct ndb_config *config, int threads) 7813 { 7814 config->ingester_threads = threads; 7815 } 7816 7817 void ndb_config_set_flags(struct ndb_config *config, int flags) 7818 { 7819 config->flags = flags; 7820 } 7821 7822 void ndb_config_set_mapsize(struct ndb_config *config, size_t mapsize) 7823 { 7824 config->mapsize = mapsize; 7825 } 7826 7827 void ndb_config_set_ingest_filter(struct ndb_config *config, 7828 ndb_ingest_filter_fn fn, void *filter_ctx) 7829 { 7830 config->ingest_filter = fn; 7831 config->filter_context = filter_ctx; 7832 } 7833 7834 int ndb_print_author_kind_index(struct ndb_txn *txn) 7835 { 7836 MDB_cursor *cur; 7837 struct ndb_id_u64_ts *key; 7838 MDB_val k, v; 7839 int i; 7840 7841 if (mdb_cursor_open(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_NOTE_PUBKEY_KIND], &cur)) 7842 return 0; 7843 7844 i = 1; 7845 printf("author\tkind\tcreated_at\tnote_id\n"); 7846 while (mdb_cursor_get(cur, &k, &v, MDB_NEXT) == 0) { 7847 key = (struct ndb_id_u64_ts *)k.mv_data; 7848 print_hex(key->id, 32); 7849 printf("\t%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\n", 7850 key->u64, key->timestamp, *(uint64_t*)v.mv_data); 7851 i++; 7852 } 7853 7854 mdb_cursor_close(cur); 7855 7856 return i; 7857 } 7858 7859 int ndb_print_relay_kind_index(struct ndb_txn *txn) 7860 { 7861 MDB_cursor *cur; 7862 unsigned char *d; 7863 MDB_val k, v; 7864 int i; 7865 7866 if (mdb_cursor_open(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_NOTE_RELAY_KIND], &cur)) 7867 return 0; 7868 7869 i = 1; 7870 printf("relay\tkind\tcreated_at\tnote_id\n"); 7871 while (mdb_cursor_get(cur, &k, &v, MDB_NEXT) == 0) { 7872 d = (unsigned char *)k.mv_data; 7873 printf("%s\t", (const char *)(d + 25)); 7874 printf("%" PRIu64 "\t", *(uint64_t*)(d + 8)); 7875 printf("%" PRIu64 "\t", *(uint64_t*)(d + 16)); 7876 printf("%" PRIu64 "\n", *(uint64_t*)(d + 0)); 7877 i++; 7878 } 7879 7880 mdb_cursor_close(cur); 7881 7882 return i; 7883 } 7884 7885 int ndb_print_tag_index(struct ndb_txn *txn) 7886 { 7887 MDB_cursor *cur; 7888 MDB_val k, v; 7889 int i; 7890 7891 if (mdb_cursor_open(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_NOTE_TAGS], &cur)) 7892 return 0; 7893 7894 i = 1; 7895 while (mdb_cursor_get(cur, &k, &v, MDB_NEXT) == 0) { 7896 printf("%d ", i); 7897 print_tag_kv(txn, &k, &v); 7898 i++; 7899 } 7900 7901 mdb_cursor_close(cur); 7902 7903 return 1; 7904 } 7905 7906 int ndb_print_kind_keys(struct ndb_txn *txn) 7907 { 7908 MDB_cursor *cur; 7909 MDB_val k, v; 7910 int i; 7911 struct ndb_u64_ts *tsid; 7912 7913 if (mdb_cursor_open(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_NOTE_KIND], &cur)) 7914 return 0; 7915 7916 i = 1; 7917 while (mdb_cursor_get(cur, &k, &v, MDB_NEXT) == 0) { 7918 tsid = k.mv_data; 7919 printf("%d note_kind %" PRIu64 " %" PRIu64 "\n", 7920 i, tsid->u64, tsid->timestamp); 7921 7922 i++; 7923 } 7924 7925 mdb_cursor_close(cur); 7926 7927 return 1; 7928 } 7929 7930 // used by ndb.c 7931 int ndb_print_search_keys(struct ndb_txn *txn) 7932 { 7933 MDB_cursor *cur; 7934 MDB_val k, v; 7935 int i; 7936 struct ndb_text_search_key search_key; 7937 7938 if (mdb_cursor_open(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_NOTE_TEXT], &cur)) 7939 return 0; 7940 7941 i = 1; 7942 while (mdb_cursor_get(cur, &k, &v, MDB_NEXT) == 0) { 7943 if (!ndb_unpack_text_search_key(k.mv_data, k.mv_size, &search_key)) { 7944 fprintf(stderr, "error decoding key %d\n", i); 7945 continue; 7946 } 7947 7948 ndb_print_text_search_key(k.mv_size, &search_key); 7949 printf("\n"); 7950 7951 i++; 7952 } 7953 7954 mdb_cursor_close(cur); 7955 7956 return 1; 7957 } 7958 7959 struct ndb_tags *ndb_note_tags(struct ndb_note *note) 7960 { 7961 return ¬e->tags; 7962 } 7963 7964 struct ndb_str ndb_note_str(struct ndb_note *note, union ndb_packed_str *pstr) 7965 { 7966 struct ndb_str str; 7967 str.flag = pstr->packed.flag; 7968 7969 if (str.flag == NDB_PACKED_STR) { 7970 str.str = pstr->packed.str; 7971 return str; 7972 } 7973 7974 str.str = ((const char *)note) + note->strings + (pstr->offset & 0xFFFFFF); 7975 return str; 7976 } 7977 7978 struct ndb_str ndb_tag_str(struct ndb_note *note, struct ndb_tag *tag, int ind) 7979 { 7980 return ndb_note_str(note, &tag->strs[ind]); 7981 } 7982 7983 int ndb_str_len(struct ndb_str *str) 7984 { 7985 if (str->flag == NDB_PACKED_ID) 7986 return 32; 7987 return strlen(str->str); 7988 } 7989 7990 struct ndb_str ndb_iter_tag_str(struct ndb_iterator *iter, int ind) 7991 { 7992 return ndb_tag_str(iter->note, iter->tag, ind); 7993 } 7994 7995 unsigned char * ndb_note_id(struct ndb_note *note) 7996 { 7997 return note->id; 7998 } 7999 8000 unsigned char * ndb_note_pubkey(struct ndb_note *note) 8001 { 8002 return note->pubkey; 8003 } 8004 8005 unsigned char * ndb_note_sig(struct ndb_note *note) 8006 { 8007 return note->sig; 8008 } 8009 8010 uint32_t ndb_note_created_at(struct ndb_note *note) 8011 { 8012 return note->created_at; 8013 } 8014 8015 uint32_t ndb_note_kind(struct ndb_note *note) 8016 { 8017 return note->kind; 8018 } 8019 8020 void _ndb_note_set_kind(struct ndb_note *note, uint32_t kind) 8021 { 8022 note->kind = kind; 8023 } 8024 8025 const char *ndb_note_content(struct ndb_note *note) 8026 { 8027 return ndb_note_str(note, ¬e->content).str; 8028 } 8029 8030 uint32_t ndb_note_content_length(struct ndb_note *note) 8031 { 8032 return note->content_length; 8033 } 8034 8035 struct ndb_note * ndb_note_from_bytes(unsigned char *bytes) 8036 { 8037 struct ndb_note *note = (struct ndb_note *)bytes; 8038 if (note->version != 1) 8039 return 0; 8040 return note; 8041 } 8042 8043 int ndb_note_relay_iterate_start(struct ndb_txn *txn, 8044 struct ndb_note_relay_iterator *iter, 8045 uint64_t note_key) 8046 { 8047 if (mdb_cursor_open(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_NOTE_RELAYS], 8048 (MDB_cursor**)&iter->mdb_cur)) { 8049 return 0; 8050 } 8051 8052 iter->txn = txn; 8053 iter->cursor_op = MDB_SET_KEY; 8054 iter->note_key = note_key; 8055 8056 return 1; 8057 } 8058 8059 const char *ndb_note_relay_iterate_next(struct ndb_note_relay_iterator *iter) 8060 { 8061 int rc; 8062 MDB_val k, v; 8063 8064 if (iter->mdb_cur == NULL) 8065 return NULL; 8066 8067 k.mv_data = &iter->note_key; 8068 k.mv_size = sizeof(iter->note_key); 8069 8070 if ((rc = mdb_cursor_get((MDB_cursor *)iter->mdb_cur, &k, &v, 8071 (MDB_cursor_op)iter->cursor_op))) 8072 { 8073 //fprintf(stderr, "autoclosing %d '%s'\n", iter->cursor_op, mdb_strerror(rc)); 8074 // autoclose 8075 ndb_note_relay_iterate_close(iter); 8076 return NULL; 8077 } 8078 8079 iter->cursor_op = MDB_NEXT_DUP; 8080 8081 return (const char*)v.mv_data; 8082 } 8083 8084 void ndb_note_relay_iterate_close(struct ndb_note_relay_iterator *iter) 8085 { 8086 if (!iter || iter->mdb_cur == NULL) 8087 return; 8088 8089 mdb_cursor_close((MDB_cursor*)iter->mdb_cur); 8090 8091 iter->mdb_cur = NULL; 8092 } 8093 8094 void ndb_tags_iterate_start(struct ndb_note *note, struct ndb_iterator *iter) 8095 { 8096 iter->note = note; 8097 iter->tag = NULL; 8098 iter->index = -1; 8099 } 8100 8101 // Helper function to get a pointer to the nth tag 8102 static struct ndb_tag *ndb_tags_tag(struct ndb_tags *tags, size_t index) { 8103 return (struct ndb_tag *)((uint8_t *)tags + sizeof(struct ndb_tags) + index * sizeof(struct ndb_tag)); 8104 } 8105 8106 int ndb_tags_iterate_next(struct ndb_iterator *iter) 8107 { 8108 struct ndb_tags *tags; 8109 8110 if (iter->tag == NULL || iter->index == -1) { 8111 iter->tag = ndb_tags_tag(&iter->note->tags, 0); 8112 iter->index = 0; 8113 return iter->note->tags.count != 0; 8114 } 8115 8116 tags = &iter->note->tags; 8117 8118 if (++iter->index < tags->count) { 8119 uint32_t tag_data_size = iter->tag->count * sizeof(iter->tag->strs[0]); 8120 iter->tag = (struct ndb_tag *)(iter->tag->strs[0].bytes + tag_data_size); 8121 return 1; 8122 } 8123 8124 return 0; 8125 } 8126 8127 uint16_t ndb_tags_count(struct ndb_tags *tags) 8128 { 8129 return tags->count; 8130 } 8131 8132 uint16_t ndb_tag_count(struct ndb_tag *tags) 8133 { 8134 return tags->count; 8135 } 8136 8137 enum ndb_common_kind ndb_kind_to_common_kind(int kind) 8138 { 8139 switch (kind) 8140 { 8141 case 0: return NDB_CKIND_PROFILE; 8142 case 1: return NDB_CKIND_TEXT; 8143 case 3: return NDB_CKIND_CONTACTS; 8144 case 4: return NDB_CKIND_DM; 8145 case 5: return NDB_CKIND_DELETE; 8146 case 6: return NDB_CKIND_REPOST; 8147 case 7: return NDB_CKIND_REACTION; 8148 case 9735: return NDB_CKIND_ZAP; 8149 case 9734: return NDB_CKIND_ZAP_REQUEST; 8150 case 23194: return NDB_CKIND_NWC_REQUEST; 8151 case 23195: return NDB_CKIND_NWC_RESPONSE; 8152 case 27235: return NDB_CKIND_HTTP_AUTH; 8153 case 30000: return NDB_CKIND_LIST; 8154 case 30023: return NDB_CKIND_LONGFORM; 8155 case 30315: return NDB_CKIND_STATUS; 8156 } 8157 8158 return -1; 8159 } 8160 8161 const char *ndb_kind_name(enum ndb_common_kind ck) 8162 { 8163 switch (ck) { 8164 case NDB_CKIND_PROFILE: return "profile"; 8165 case NDB_CKIND_TEXT: return "text"; 8166 case NDB_CKIND_CONTACTS: return "contacts"; 8167 case NDB_CKIND_DM: return "dm"; 8168 case NDB_CKIND_DELETE: return "delete"; 8169 case NDB_CKIND_REPOST: return "repost"; 8170 case NDB_CKIND_REACTION: return "reaction"; 8171 case NDB_CKIND_ZAP: return "zap"; 8172 case NDB_CKIND_ZAP_REQUEST: return "zap_request"; 8173 case NDB_CKIND_NWC_REQUEST: return "nwc_request"; 8174 case NDB_CKIND_NWC_RESPONSE: return "nwc_response"; 8175 case NDB_CKIND_HTTP_AUTH: return "http_auth"; 8176 case NDB_CKIND_LIST: return "list"; 8177 case NDB_CKIND_LONGFORM: return "longform"; 8178 case NDB_CKIND_STATUS: return "status"; 8179 case NDB_CKIND_COUNT: return "unknown"; 8180 } 8181 8182 return "unknown"; 8183 } 8184 8185 const char *ndb_db_name(enum ndb_dbs db) 8186 { 8187 switch (db) { 8188 case NDB_DB_NOTE: 8189 return "note"; 8190 case NDB_DB_META: 8191 return "note_metadata"; 8192 case NDB_DB_PROFILE: 8193 return "profile"; 8194 case NDB_DB_NOTE_ID: 8195 return "note_index"; 8196 case NDB_DB_PROFILE_PK: 8197 return "profile_pubkey_index"; 8198 case NDB_DB_NDB_META: 8199 return "nostrdb_metadata"; 8200 case NDB_DB_PROFILE_SEARCH: 8201 return "profile_search"; 8202 case NDB_DB_PROFILE_LAST_FETCH: 8203 return "profile_last_fetch"; 8204 case NDB_DB_NOTE_KIND: 8205 return "note_kind_index"; 8206 case NDB_DB_NOTE_TEXT: 8207 return "note_fulltext"; 8208 case NDB_DB_NOTE_BLOCKS: 8209 return "note_blocks"; 8210 case NDB_DB_NOTE_TAGS: 8211 return "note_tags"; 8212 case NDB_DB_NOTE_PUBKEY: 8213 return "note_pubkey_index"; 8214 case NDB_DB_NOTE_PUBKEY_KIND: 8215 return "note_pubkey_kind_index"; 8216 case NDB_DB_NOTE_RELAY_KIND: 8217 return "note_relay_kind_index"; 8218 case NDB_DB_NOTE_RELAYS: 8219 return "note_relays"; 8220 case NDB_DBS: 8221 return "count"; 8222 } 8223 8224 return "unknown"; 8225 } 8226 8227 static struct ndb_blocks *ndb_note_to_blocks(struct ndb_note *note) 8228 { 8229 const char *content; 8230 size_t content_len; 8231 struct ndb_blocks *blocks; 8232 unsigned char *buffer; 8233 8234 content = ndb_note_content(note); 8235 content_len = ndb_note_content_length(note); 8236 8237 // something weird is going on 8238 if (content_len >= INT32_MAX) 8239 return NULL; 8240 8241 buffer = malloc(2<<18); // Not carefully calculated, but ok because we will not need this once we switch to the local relay model 8242 if (!buffer) 8243 return NULL; 8244 8245 if (!ndb_parse_content(buffer, content_len, content, content_len, &blocks)) { 8246 free(buffer); 8247 return NULL; 8248 } 8249 8250 //blocks = realloc(blocks, ndb_blocks_total_size(blocks)); 8251 //if (blocks == NULL) 8252 //return NULL; 8253 8254 blocks->flags |= NDB_BLOCK_FLAG_OWNED; 8255 8256 return blocks; 8257 } 8258 8259 struct ndb_blocks *ndb_get_blocks_by_key(struct ndb *ndb, struct ndb_txn *txn, uint64_t note_key) 8260 { 8261 struct ndb_blocks *blocks, *blocks_to_writer; 8262 size_t blocks_size; 8263 struct ndb_note *note; 8264 size_t note_len; 8265 8266 if ((blocks = ndb_lookup_by_key(txn, note_key, NDB_DB_NOTE_BLOCKS, ¬e_len))) { 8267 return blocks; 8268 } 8269 8270 // If we don't have note blocks, let's lazily generate them. This is 8271 // migration-friendly instead of doing them all at once 8272 if (!(note = ndb_get_note_by_key(txn, note_key, ¬e_len))) { 8273 // no note found, can't return note blocks 8274 return NULL; 8275 } 8276 8277 if (!(blocks = ndb_note_to_blocks(note))) 8278 return NULL; 8279 8280 // send a copy to the writer 8281 blocks_size = ndb_blocks_total_size(blocks); 8282 blocks_to_writer = malloc(blocks_size); 8283 memcpy(blocks_to_writer, blocks, blocks_size); 8284 assert(blocks->flags & NDB_BLOCK_FLAG_OWNED); 8285 8286 // we generated new blocks, let's store them in the DB 8287 struct ndb_writer_blocks write_blocks = { 8288 .blocks = blocks_to_writer, 8289 .note_key = note_key 8290 }; 8291 8292 assert(write_blocks.blocks != blocks); 8293 8294 struct ndb_writer_msg msg = { .type = NDB_WRITER_BLOCKS }; 8295 msg.blocks = write_blocks; 8296 8297 ndb_writer_queue_msg(&ndb->writer, &msg); 8298 8299 return blocks; 8300 } 8301 8302 // please call ndb_monitor_lock before calling this 8303 static struct ndb_subscription * 8304 ndb_monitor_find_subscription(struct ndb_monitor *monitor, uint64_t subid, int *index) 8305 { 8306 struct ndb_subscription *sub, *tsub; 8307 int i; 8308 8309 for (i = 0, sub = NULL; i < monitor->num_subscriptions; i++) { 8310 tsub = &monitor->subscriptions[i]; 8311 if (tsub->subid == subid) { 8312 sub = tsub; 8313 if (index) 8314 *index = i; 8315 break; 8316 } 8317 } 8318 8319 return sub; 8320 } 8321 8322 int ndb_poll_for_notes(struct ndb *ndb, uint64_t subid, uint64_t *note_ids, 8323 int note_id_capacity) 8324 { 8325 struct ndb_subscription *sub; 8326 int res; 8327 8328 if (subid == 0) 8329 return 0; 8330 8331 ndb_monitor_lock(&ndb->monitor); 8332 8333 if (!(sub = ndb_monitor_find_subscription(&ndb->monitor, subid, NULL))) 8334 res = 0; 8335 else 8336 res = prot_queue_try_pop_all(&sub->inbox, note_ids, note_id_capacity); 8337 8338 ndb_monitor_unlock(&ndb->monitor); 8339 8340 return res; 8341 } 8342 8343 int ndb_wait_for_notes(struct ndb *ndb, uint64_t subid, uint64_t *note_ids, 8344 int note_id_capacity) 8345 { 8346 struct ndb_subscription *sub; 8347 struct prot_queue *queue_inbox; 8348 8349 // this is not a valid subscription id 8350 if (subid == 0) 8351 return 0; 8352 8353 ndb_monitor_lock(&ndb->monitor); 8354 8355 if (!(sub = ndb_monitor_find_subscription(&ndb->monitor, subid, NULL))) { 8356 ndb_monitor_unlock(&ndb->monitor); 8357 return 0; 8358 } 8359 8360 queue_inbox = &sub->inbox; 8361 8362 ndb_monitor_unlock(&ndb->monitor); 8363 8364 // there is technically a race condition if the thread yeilds at this 8365 // comment and a subscription is added/removed. A deadlock in the 8366 // writer queue would be much worse though. This function is dubious 8367 // anyways. 8368 8369 return prot_queue_pop_all(queue_inbox, note_ids, note_id_capacity); 8370 } 8371 8372 int ndb_unsubscribe(struct ndb *ndb, uint64_t subid) 8373 { 8374 struct ndb_subscription *sub; 8375 int index, res, elems_to_move; 8376 8377 ndb_monitor_lock(&ndb->monitor); 8378 8379 if (!(sub = ndb_monitor_find_subscription(&ndb->monitor, subid, &index))) { 8380 res = 0; 8381 goto done; 8382 } 8383 8384 ndb_subscription_destroy(sub); 8385 8386 elems_to_move = (--ndb->monitor.num_subscriptions) - index; 8387 8388 memmove(&ndb->monitor.subscriptions[index], 8389 &ndb->monitor.subscriptions[index+1], 8390 elems_to_move * sizeof(*sub)); 8391 8392 res = 1; 8393 8394 done: 8395 ndb_monitor_unlock(&ndb->monitor); 8396 8397 return res; 8398 } 8399 8400 int ndb_num_subscriptions(struct ndb *ndb) 8401 { 8402 return ndb->monitor.num_subscriptions; 8403 } 8404 8405 uint64_t ndb_subscribe(struct ndb *ndb, struct ndb_filter *filters, int num_filters) 8406 { 8407 static uint64_t subids = 0; 8408 struct ndb_subscription *sub; 8409 size_t buflen; 8410 uint64_t subid; 8411 char *buf; 8412 8413 ndb_monitor_lock(&ndb->monitor); 8414 8415 if (ndb->monitor.num_subscriptions + 1 >= MAX_SUBSCRIPTIONS) { 8416 fprintf(stderr, "too many subscriptions\n"); 8417 subid = 0; 8418 goto done; 8419 } 8420 8421 sub = &ndb->monitor.subscriptions[ndb->monitor.num_subscriptions]; 8422 subid = ++subids; 8423 sub->subid = subid; 8424 8425 ndb_filter_group_init(&sub->group); 8426 if (!ndb_filter_group_add_filters(&sub->group, filters, num_filters)) { 8427 subid = 0; 8428 goto done; 8429 } 8430 8431 // 500k ought to be enough for anyone 8432 buflen = sizeof(uint64_t) * DEFAULT_QUEUE_SIZE; 8433 buf = malloc(buflen); 8434 8435 if (!prot_queue_init(&sub->inbox, buf, buflen, sizeof(uint64_t))) { 8436 fprintf(stderr, "failed to push prot queue\n"); 8437 subid = 0; 8438 goto done; 8439 } 8440 8441 ndb->monitor.num_subscriptions++; 8442 done: 8443 ndb_monitor_unlock(&ndb->monitor); 8444 8445 return subid; 8446 }