nostrdb.h (17717B)
1 #ifndef NOSTRDB_H 2 #define NOSTRDB_H 3 4 #include <inttypes.h> 5 #include "win.h" 6 #include "cursor.h" 7 8 // maximum number of filters allowed in a filter group 9 #define NDB_PACKED_STR 0x1 10 #define NDB_PACKED_ID 0x2 11 12 #define NDB_FLAG_NOMIGRATE (1 << 0) 13 #define NDB_FLAG_SKIP_NOTE_VERIFY (1 << 1) 14 #define NDB_FLAG_NO_FULLTEXT (1 << 2) 15 #define NDB_FLAG_NO_NOTE_BLOCKS (1 << 3) 16 #define NDB_FLAG_NO_STATS (1 << 4) 17 18 //#define DEBUG 1 19 20 #ifdef NDB_LOG 21 #define ndb_debug(...) printf(__VA_ARGS__) 22 #else 23 #define ndb_debug(...) (void)0 24 #endif 25 26 #include "str_block.h" 27 28 struct ndb_json_parser; 29 struct ndb; 30 struct ndb_blocks; 31 struct ndb_block; 32 struct ndb_note; 33 struct ndb_tag; 34 struct ndb_tags; 35 struct ndb_lmdb; 36 union ndb_packed_str; 37 struct bolt11; 38 39 // some bindings like swift needs help with forward declared pointers 40 struct ndb_tag_ptr { struct ndb_tag *ptr; }; 41 struct ndb_tags_ptr { struct ndb_tags *ptr; }; 42 struct ndb_block_ptr { struct ndb_block *ptr; }; 43 struct ndb_blocks_ptr { struct ndb_blocks *ptr; }; 44 struct ndb_note_ptr { struct ndb_note *ptr; }; 45 46 struct ndb_t { 47 struct ndb *ndb; 48 }; 49 50 struct ndb_str { 51 unsigned char flag; 52 union { 53 const char *str; 54 unsigned char *id; 55 }; 56 }; 57 58 struct ndb_keypair { 59 unsigned char pubkey[32]; 60 unsigned char secret[32]; 61 62 // this corresponds to secp256k1's keypair type. it's guaranteed to 63 // be 96 bytes according to their docs. I don't want to depend on 64 // the secp256k1 header here so we just use raw bytes. 65 unsigned char pair[96]; 66 }; 67 68 // function pointer for controlling what to do after we parse an id 69 typedef enum ndb_idres (*ndb_id_fn)(void *, const char *); 70 71 // callback function for when we receive new subscription results 72 typedef void (*ndb_sub_fn)(void *, uint64_t subid); 73 74 // id callback + closure data 75 struct ndb_id_cb { 76 ndb_id_fn fn; 77 void *data; 78 }; 79 80 // required to keep a read 81 struct ndb_txn { 82 struct ndb_lmdb *lmdb; 83 void *mdb_txn; 84 }; 85 86 struct ndb_event { 87 struct ndb_note *note; 88 }; 89 90 struct ndb_command_result { 91 int ok; 92 const char *msg; 93 int msglen; 94 }; 95 96 // From-client event types 97 enum fce_type { 98 NDB_FCE_EVENT = 0x1 99 }; 100 101 // To-client event types 102 enum tce_type { 103 NDB_TCE_EVENT = 0x1, 104 NDB_TCE_OK = 0x2, 105 NDB_TCE_NOTICE = 0x3, 106 NDB_TCE_EOSE = 0x4, 107 NDB_TCE_AUTH = 0x5, 108 }; 109 110 enum ndb_ingest_filter_action { 111 NDB_INGEST_REJECT, 112 NDB_INGEST_ACCEPT, 113 NDB_INGEST_SKIP_VALIDATION 114 }; 115 116 struct ndb_search_key 117 { 118 char search[24]; 119 unsigned char id[32]; 120 uint64_t timestamp; 121 }; 122 123 struct ndb_search { 124 struct ndb_search_key *key; 125 uint64_t profile_key; 126 void *cursor; // MDB_cursor * 127 }; 128 129 // From-client event 130 struct ndb_fce { 131 enum fce_type evtype; 132 union { 133 struct ndb_event event; 134 }; 135 }; 136 137 // To-client event 138 struct ndb_tce { 139 enum tce_type evtype; 140 const char *subid; 141 int subid_len; 142 143 union { 144 struct ndb_event event; 145 struct ndb_command_result command_result; 146 }; 147 }; 148 149 typedef enum ndb_ingest_filter_action (*ndb_ingest_filter_fn)(void *, struct ndb_note *); 150 151 enum ndb_filter_fieldtype { 152 NDB_FILTER_IDS = 1, 153 NDB_FILTER_AUTHORS = 2, 154 NDB_FILTER_KINDS = 3, 155 NDB_FILTER_TAGS = 4, 156 NDB_FILTER_SINCE = 5, 157 NDB_FILTER_UNTIL = 6, 158 NDB_FILTER_LIMIT = 7, 159 }; 160 #define NDB_NUM_FILTERS 7 161 162 // when matching generic tags, we need to know if we're dealing with 163 // a pointer to a 32-byte ID or a null terminated string 164 enum ndb_generic_element_type { 165 NDB_ELEMENT_UNKNOWN = 0, 166 NDB_ELEMENT_STRING = 1, 167 NDB_ELEMENT_ID = 2, 168 NDB_ELEMENT_INT = 3, 169 }; 170 171 enum ndb_search_order { 172 NDB_ORDER_DESCENDING, 173 NDB_ORDER_ASCENDING, 174 }; 175 176 enum ndb_dbs { 177 NDB_DB_NOTE, 178 NDB_DB_META, 179 NDB_DB_PROFILE, 180 NDB_DB_NOTE_ID, 181 NDB_DB_PROFILE_PK, // profile pk index 182 NDB_DB_NDB_META, 183 NDB_DB_PROFILE_SEARCH, 184 NDB_DB_PROFILE_LAST_FETCH, 185 NDB_DB_NOTE_KIND, // note kind index 186 NDB_DB_NOTE_TEXT, // note fulltext index 187 NDB_DB_NOTE_BLOCKS, // parsed note blocks for rendering 188 NDB_DB_NOTE_TAGS, // note tags index 189 NDB_DB_NOTE_PUBKEY, // note pubkey index 190 NDB_DB_NOTE_PUBKEY_KIND, // note pubkey kind index 191 NDB_DBS, 192 }; 193 194 // common kinds. we collect stats on these in ndb_stat. mainly because I don't 195 // want to deal with including a hashtable to the project. 196 enum ndb_common_kind { 197 NDB_CKIND_PROFILE, 198 NDB_CKIND_TEXT, 199 NDB_CKIND_CONTACTS, 200 NDB_CKIND_DM, 201 NDB_CKIND_DELETE, 202 NDB_CKIND_REPOST, 203 NDB_CKIND_REACTION, 204 NDB_CKIND_ZAP, 205 NDB_CKIND_ZAP_REQUEST, 206 NDB_CKIND_NWC_REQUEST, 207 NDB_CKIND_NWC_RESPONSE, 208 NDB_CKIND_HTTP_AUTH, 209 NDB_CKIND_LIST, 210 NDB_CKIND_LONGFORM, 211 NDB_CKIND_STATUS, 212 NDB_CKIND_COUNT, // should always be last 213 }; 214 215 struct ndb_builder { 216 struct cursor mem; 217 struct cursor note_cur; 218 struct cursor strings; 219 struct cursor str_indices; 220 struct ndb_note *note; 221 struct ndb_tag *current_tag; 222 }; 223 224 struct ndb_iterator { 225 struct ndb_note *note; 226 struct ndb_tag *tag; 227 228 // current outer index 229 int index; 230 }; 231 232 struct ndb_filter_string { 233 const char *string; 234 int len; 235 }; 236 237 union ndb_filter_element { 238 struct ndb_filter_string string; 239 const unsigned char *id; 240 uint64_t integer; 241 }; 242 243 struct ndb_filter_field { 244 enum ndb_filter_fieldtype type; 245 enum ndb_generic_element_type elem_type; 246 char tag; // for generic queries like #t 247 }; 248 249 struct ndb_filter_elements { 250 struct ndb_filter_field field; 251 int count; 252 253 // this needs to be pointer size for reasons 254 // FIXME: what about on 32bit systems?? 255 uint64_t elements[0]; 256 }; 257 258 struct ndb_filter { 259 struct cursor elem_buf; 260 struct cursor data_buf; 261 int num_elements; 262 int finalized; 263 int current; 264 265 // struct ndb_filter_elements offsets into elem_buf 266 // 267 // TODO(jb55): this should probably be called fields. elements are 268 // the things within fields 269 int elements[NDB_NUM_FILTERS]; 270 }; 271 272 struct ndb_config { 273 int flags; 274 int ingester_threads; 275 size_t mapsize; 276 void *filter_context; 277 ndb_ingest_filter_fn ingest_filter; 278 void *sub_cb_ctx; 279 ndb_sub_fn sub_cb; 280 }; 281 282 struct ndb_text_search_config { 283 enum ndb_search_order order; 284 int limit; 285 }; 286 287 struct ndb_stat_counts { 288 size_t key_size; 289 size_t value_size; 290 size_t count; 291 }; 292 293 struct ndb_stat { 294 struct ndb_stat_counts dbs[NDB_DBS]; 295 struct ndb_stat_counts common_kinds[NDB_CKIND_COUNT]; 296 struct ndb_stat_counts other_kinds; 297 }; 298 299 #define MAX_TEXT_SEARCH_RESULTS 128 300 #define MAX_TEXT_SEARCH_WORDS 8 301 302 // unpacked form of the actual lmdb fulltext search key 303 // see `ndb_make_text_search_key` for how the packed version is constructed 304 struct ndb_text_search_key 305 { 306 int str_len; 307 const char *str; 308 uint64_t timestamp; 309 uint64_t note_id; 310 uint64_t word_index; 311 }; 312 313 struct ndb_text_search_result { 314 struct ndb_text_search_key key; 315 int prefix_chars; 316 }; 317 318 struct ndb_text_search_results { 319 struct ndb_text_search_result results[MAX_TEXT_SEARCH_RESULTS]; 320 int num_results; 321 }; 322 323 enum ndb_block_type { 324 BLOCK_HASHTAG = 1, 325 BLOCK_TEXT = 2, 326 BLOCK_MENTION_INDEX = 3, 327 BLOCK_MENTION_BECH32 = 4, 328 BLOCK_URL = 5, 329 BLOCK_INVOICE = 6, 330 }; 331 #define NDB_NUM_BLOCK_TYPES 6 332 #define NDB_MAX_RELAYS 24 333 334 struct ndb_relays { 335 struct ndb_str_block relays[NDB_MAX_RELAYS]; 336 int num_relays; 337 }; 338 339 enum nostr_bech32_type { 340 NOSTR_BECH32_NOTE = 1, 341 NOSTR_BECH32_NPUB = 2, 342 NOSTR_BECH32_NPROFILE = 3, 343 NOSTR_BECH32_NEVENT = 4, 344 NOSTR_BECH32_NRELAY = 5, 345 NOSTR_BECH32_NADDR = 6, 346 NOSTR_BECH32_NSEC = 7, 347 }; 348 #define NOSTR_BECH32_KNOWN_TYPES 7 349 350 struct bech32_note { 351 const unsigned char *event_id; 352 }; 353 354 struct bech32_npub { 355 const unsigned char *pubkey; 356 }; 357 358 struct bech32_nsec { 359 const unsigned char *nsec; 360 }; 361 362 struct bech32_nevent { 363 struct ndb_relays relays; 364 const unsigned char *event_id; 365 const unsigned char *pubkey; // optional 366 }; 367 368 struct bech32_nprofile { 369 struct ndb_relays relays; 370 const unsigned char *pubkey; 371 }; 372 373 struct bech32_naddr { 374 struct ndb_relays relays; 375 struct ndb_str_block identifier; 376 const unsigned char *pubkey; 377 }; 378 379 struct bech32_nrelay { 380 struct ndb_str_block relay; 381 }; 382 383 struct nostr_bech32 { 384 enum nostr_bech32_type type; 385 386 union { 387 struct bech32_note note; 388 struct bech32_npub npub; 389 struct bech32_nsec nsec; 390 struct bech32_nevent nevent; 391 struct bech32_nprofile nprofile; 392 struct bech32_naddr naddr; 393 struct bech32_nrelay nrelay; 394 }; 395 }; 396 397 398 struct ndb_mention_bech32_block { 399 struct ndb_str_block str; 400 struct nostr_bech32 bech32; 401 }; 402 403 struct ndb_invoice { 404 unsigned char version; 405 uint64_t amount; 406 uint64_t timestamp; 407 uint64_t expiry; 408 char *description; 409 unsigned char *description_hash; 410 }; 411 412 struct ndb_invoice_block { 413 struct ndb_str_block invstr; 414 struct ndb_invoice invoice; 415 }; 416 417 struct ndb_block { 418 enum ndb_block_type type; 419 union { 420 struct ndb_str_block str; 421 struct ndb_invoice_block invoice; 422 struct ndb_mention_bech32_block mention_bech32; 423 uint32_t mention_index; 424 } block; 425 }; 426 427 struct ndb_block_iterator { 428 const char *content; 429 struct ndb_blocks *blocks; 430 struct ndb_block block; 431 unsigned char *p; 432 }; 433 434 struct ndb_query_result { 435 struct ndb_note *note; 436 uint64_t note_size; 437 uint64_t note_id; 438 }; 439 440 struct ndb_query_results { 441 struct cursor cur; 442 }; 443 444 // CONFIG 445 void ndb_default_config(struct ndb_config *); 446 void ndb_config_set_ingest_threads(struct ndb_config *config, int threads); 447 void ndb_config_set_flags(struct ndb_config *config, int flags); 448 void ndb_config_set_mapsize(struct ndb_config *config, size_t mapsize); 449 void ndb_config_set_ingest_filter(struct ndb_config *config, ndb_ingest_filter_fn fn, void *); 450 void ndb_config_set_subscription_callback(struct ndb_config *config, ndb_sub_fn fn, void *ctx); 451 452 // HELPERS 453 int ndb_calculate_id(struct ndb_note *note, unsigned char *buf, int buflen); 454 int ndb_sign_id(struct ndb_keypair *keypair, unsigned char id[32], unsigned char sig[64]); 455 int ndb_create_keypair(struct ndb_keypair *key); 456 int ndb_decode_key(const char *secstr, struct ndb_keypair *keypair); 457 int ndb_note_verify(void *secp_ctx, unsigned char pubkey[32], unsigned char id[32], unsigned char signature[64]); 458 459 // NDB 460 int ndb_init(struct ndb **ndb, const char *dbdir, const struct ndb_config *); 461 int ndb_db_version(struct ndb_txn *txn); 462 int ndb_process_event(struct ndb *, const char *json, int len); 463 int ndb_process_events(struct ndb *, const char *ldjson, size_t len); 464 #ifndef _WIN32 465 // TODO: fix on windows 466 int ndb_process_events_stream(struct ndb *, FILE* fp); 467 #endif 468 int ndb_process_client_event(struct ndb *, const char *json, int len); 469 int ndb_process_client_events(struct ndb *, const char *json, size_t len); 470 int ndb_begin_query(struct ndb *, struct ndb_txn *); 471 int ndb_search_profile(struct ndb_txn *txn, struct ndb_search *search, const char *query); 472 int ndb_search_profile_next(struct ndb_search *search); 473 void ndb_search_profile_end(struct ndb_search *search); 474 int ndb_end_query(struct ndb_txn *); 475 int ndb_write_last_profile_fetch(struct ndb *ndb, const unsigned char *pubkey, uint64_t fetched_at); 476 uint64_t ndb_read_last_profile_fetch(struct ndb_txn *txn, const unsigned char *pubkey); 477 void *ndb_get_profile_by_pubkey(struct ndb_txn *txn, const unsigned char *pubkey, size_t *len, uint64_t *primkey); 478 void *ndb_get_profile_by_key(struct ndb_txn *txn, uint64_t key, size_t *len); 479 uint64_t ndb_get_notekey_by_id(struct ndb_txn *txn, const unsigned char *id); 480 uint64_t ndb_get_profilekey_by_pubkey(struct ndb_txn *txn, const unsigned char *id); 481 struct ndb_note *ndb_get_note_by_id(struct ndb_txn *txn, const unsigned char *id, size_t *len, uint64_t *primkey); 482 struct ndb_note *ndb_get_note_by_key(struct ndb_txn *txn, uint64_t key, size_t *len); 483 void *ndb_get_note_meta(struct ndb_txn *txn, const unsigned char *id, size_t *len); 484 void ndb_destroy(struct ndb *); 485 486 // BUILDER 487 int ndb_parse_json_note(struct ndb_json_parser *, struct ndb_note **); 488 int ndb_client_event_from_json(const char *json, int len, struct ndb_fce *fce, unsigned char *buf, int bufsize, struct ndb_id_cb *cb); 489 int ndb_ws_event_from_json(const char *json, int len, struct ndb_tce *tce, unsigned char *buf, int bufsize, struct ndb_id_cb *); 490 int ndb_note_from_json(const char *json, int len, struct ndb_note **, unsigned char *buf, int buflen); 491 int ndb_builder_init(struct ndb_builder *builder, unsigned char *buf, size_t bufsize); 492 int ndb_builder_finalize(struct ndb_builder *builder, struct ndb_note **note, struct ndb_keypair *privkey); 493 int ndb_builder_set_content(struct ndb_builder *builder, const char *content, int len); 494 void ndb_builder_set_created_at(struct ndb_builder *builder, uint64_t created_at); 495 void ndb_builder_set_sig(struct ndb_builder *builder, unsigned char *sig); 496 void ndb_builder_set_pubkey(struct ndb_builder *builder, unsigned char *pubkey); 497 void ndb_builder_set_id(struct ndb_builder *builder, unsigned char *id); 498 void ndb_builder_set_kind(struct ndb_builder *builder, uint32_t kind); 499 int ndb_builder_new_tag(struct ndb_builder *builder); 500 int ndb_builder_push_tag_str(struct ndb_builder *builder, const char *str, int len); 501 502 // FILTERS 503 int ndb_filter_init(struct ndb_filter *); 504 int ndb_filter_add_id_element(struct ndb_filter *, const unsigned char *id); 505 int ndb_filter_add_int_element(struct ndb_filter *, uint64_t integer); 506 int ndb_filter_add_str_element(struct ndb_filter *, const char *str); 507 int ndb_filter_eq(const struct ndb_filter *, const struct ndb_filter *); 508 509 /// is `a` a subset of `b` 510 int ndb_filter_is_subset_of(const struct ndb_filter *a, const struct ndb_filter *b); 511 512 // filters from json 513 int ndb_filter_from_json(const char *, int len, struct ndb_filter *filter, unsigned char *buf, int bufsize); 514 515 // getting field elements 516 unsigned char *ndb_filter_get_id_element(const struct ndb_filter *, const struct ndb_filter_elements *, int index); 517 const char *ndb_filter_get_string_element(const struct ndb_filter *, const struct ndb_filter_elements *, int index); 518 uint64_t ndb_filter_get_int_element(const struct ndb_filter_elements *, int index); 519 uint64_t *ndb_filter_get_int_element_ptr(struct ndb_filter_elements *, int index); 520 521 struct ndb_filter_elements *ndb_filter_current_element(const struct ndb_filter *); 522 struct ndb_filter_elements *ndb_filter_get_elements(const struct ndb_filter *, int); 523 int ndb_filter_start_field(struct ndb_filter *, enum ndb_filter_fieldtype); 524 int ndb_filter_start_tag_field(struct ndb_filter *, char tag); 525 int ndb_filter_matches(struct ndb_filter *, struct ndb_note *); 526 int ndb_filter_clone(struct ndb_filter *dst, struct ndb_filter *src); 527 int ndb_filter_end(struct ndb_filter *); 528 void ndb_filter_end_field(struct ndb_filter *); 529 void ndb_filter_destroy(struct ndb_filter *); 530 int ndb_filter_json(const struct ndb_filter *, char *buf, int buflen); 531 532 // SUBSCRIPTIONS 533 uint64_t ndb_subscribe(struct ndb *, struct ndb_filter *, int num_filters); 534 int ndb_wait_for_notes(struct ndb *, uint64_t subid, uint64_t *note_ids, int note_id_capacity); 535 int ndb_poll_for_notes(struct ndb *, uint64_t subid, uint64_t *note_ids, int note_id_capacity); 536 int ndb_unsubscribe(struct ndb *, uint64_t subid); 537 int ndb_num_subscriptions(struct ndb *); 538 539 // FULLTEXT SEARCH 540 int ndb_text_search(struct ndb_txn *txn, const char *query, struct ndb_text_search_results *, struct ndb_text_search_config *); 541 void ndb_default_text_search_config(struct ndb_text_search_config *); 542 void ndb_text_search_config_set_order(struct ndb_text_search_config *, enum ndb_search_order); 543 void ndb_text_search_config_set_limit(struct ndb_text_search_config *, int limit); 544 545 // QUERY 546 int ndb_query(struct ndb_txn *txn, struct ndb_filter *filters, int num_filters, struct ndb_query_result *results, int result_capacity, int *count); 547 548 // STATS 549 int ndb_stat(struct ndb *ndb, struct ndb_stat *stat); 550 void ndb_stat_counts_init(struct ndb_stat_counts *counts); 551 552 // NOTE 553 const char *ndb_note_content(struct ndb_note *note); 554 struct ndb_str ndb_note_str(struct ndb_note *note, union ndb_packed_str *pstr); 555 uint32_t ndb_note_content_length(struct ndb_note *note); 556 uint32_t ndb_note_created_at(struct ndb_note *note); 557 uint32_t ndb_note_kind(struct ndb_note *note); 558 unsigned char *ndb_note_id(struct ndb_note *note); 559 unsigned char *ndb_note_pubkey(struct ndb_note *note); 560 unsigned char *ndb_note_sig(struct ndb_note *note); 561 void _ndb_note_set_kind(struct ndb_note *note, uint32_t kind); 562 struct ndb_tags *ndb_note_tags(struct ndb_note *note); 563 int ndb_str_len(struct ndb_str *str); 564 565 /// write the note as json to a buffer 566 int ndb_note_json(struct ndb_note *, char *buf, int buflen); 567 568 // TAGS 569 void ndb_tags_iterate_start(struct ndb_note *note, struct ndb_iterator *iter); 570 uint16_t ndb_tags_count(struct ndb_tags *); 571 uint16_t ndb_tag_count(struct ndb_tag *); 572 573 // ITER 574 int ndb_tags_iterate_next(struct ndb_iterator *iter); 575 struct ndb_str ndb_iter_tag_str(struct ndb_iterator *iter, int ind); 576 struct ndb_str ndb_tag_str(struct ndb_note *note, struct ndb_tag *tag, int ind); 577 578 // NAMES 579 const char *ndb_db_name(enum ndb_dbs db); 580 const char *ndb_kind_name(enum ndb_common_kind ck); 581 enum ndb_common_kind ndb_kind_to_common_kind(int kind); 582 583 // CONTENT PARSER 584 int ndb_parse_content(unsigned char *buf, int buf_size, 585 const char *content, int content_len, 586 struct ndb_blocks **blocks_p); 587 588 // BLOCKS 589 enum ndb_block_type ndb_get_block_type(struct ndb_block *block); 590 int ndb_blocks_flags(struct ndb_blocks *block); 591 size_t ndb_blocks_total_size(struct ndb_blocks *blocks); 592 int ndb_blocks_word_count(struct ndb_blocks *blocks); 593 594 /// Free blocks if they are owned, safe to call on unowned blocks as well. 595 void ndb_blocks_free(struct ndb_blocks *blocks); 596 597 // BLOCK DB 598 struct ndb_blocks *ndb_get_blocks_by_key(struct ndb *ndb, struct ndb_txn *txn, uint64_t note_key); 599 600 // BLOCK ITERATORS 601 void ndb_blocks_iterate_start(const char *, struct ndb_blocks *, struct ndb_block_iterator *); 602 struct ndb_block *ndb_blocks_iterate_next(struct ndb_block_iterator *); 603 604 // STR BLOCKS 605 struct ndb_str_block *ndb_block_str(struct ndb_block *); 606 const char *ndb_str_block_ptr(struct ndb_str_block *); 607 uint32_t ndb_str_block_len(struct ndb_str_block *); 608 609 // BECH32 BLOCKS 610 struct nostr_bech32 *ndb_bech32_block(struct ndb_block *block); 611 612 #endif