nostrdb.h (17131B)
1 #ifndef NOSTRDB_H 2 #define NOSTRDB_H 3 4 #include <inttypes.h> 5 #include "cursor.h" 6 7 // maximum number of filters allowed in a filter group 8 #define NDB_PACKED_STR 0x1 9 #define NDB_PACKED_ID 0x2 10 11 #define NDB_FLAG_NOMIGRATE (1 << 0) 12 #define NDB_FLAG_SKIP_NOTE_VERIFY (1 << 1) 13 14 //#define DEBUG 1 15 16 #ifdef DEBUG 17 #define ndb_debug(...) printf(__VA_ARGS__) 18 #else 19 #define ndb_debug(...) (void)0 20 #endif 21 22 #include "str_block.h" 23 24 struct ndb_json_parser; 25 struct ndb; 26 struct ndb_blocks; 27 struct ndb_block; 28 struct ndb_note; 29 struct ndb_tag; 30 struct ndb_tags; 31 struct ndb_lmdb; 32 union ndb_packed_str; 33 struct bolt11; 34 35 // some bindings like swift needs help with forward declared pointers 36 struct ndb_tag_ptr { struct ndb_tag *ptr; }; 37 struct ndb_tags_ptr { struct ndb_tags *ptr; }; 38 struct ndb_block_ptr { struct ndb_block *ptr; }; 39 struct ndb_blocks_ptr { struct ndb_blocks *ptr; }; 40 struct ndb_note_ptr { struct ndb_note *ptr; }; 41 42 struct ndb_t { 43 struct ndb *ndb; 44 }; 45 46 struct ndb_str { 47 unsigned char flag; 48 union { 49 const char *str; 50 unsigned char *id; 51 }; 52 }; 53 54 struct ndb_keypair { 55 unsigned char pubkey[32]; 56 unsigned char secret[32]; 57 58 // this corresponds to secp256k1's keypair type. it's guaranteed to 59 // be 96 bytes according to their docs. I don't want to depend on 60 // the secp256k1 header here so we just use raw bytes. 61 unsigned char pair[96]; 62 }; 63 64 // function pointer for controlling what to do after we parse an id 65 typedef enum ndb_idres (*ndb_id_fn)(void *, const char *); 66 67 // callback function for when we receive new subscription results 68 typedef void (*ndb_sub_fn)(void *, uint64_t subid); 69 70 // id callback + closure data 71 struct ndb_id_cb { 72 ndb_id_fn fn; 73 void *data; 74 }; 75 76 // required to keep a read 77 struct ndb_txn { 78 struct ndb_lmdb *lmdb; 79 void *mdb_txn; 80 }; 81 82 struct ndb_event { 83 struct ndb_note *note; 84 }; 85 86 struct ndb_command_result { 87 int ok; 88 const char *msg; 89 int msglen; 90 }; 91 92 // From-client event types 93 enum fce_type { 94 NDB_FCE_EVENT = 0x1 95 }; 96 97 // To-client event types 98 enum tce_type { 99 NDB_TCE_EVENT = 0x1, 100 NDB_TCE_OK = 0x2, 101 NDB_TCE_NOTICE = 0x3, 102 NDB_TCE_EOSE = 0x4, 103 NDB_TCE_AUTH = 0x5, 104 }; 105 106 enum ndb_ingest_filter_action { 107 NDB_INGEST_REJECT, 108 NDB_INGEST_ACCEPT, 109 NDB_INGEST_SKIP_VALIDATION 110 }; 111 112 struct ndb_search_key 113 { 114 char search[24]; 115 unsigned char id[32]; 116 uint64_t timestamp; 117 }; 118 119 struct ndb_search { 120 struct ndb_search_key *key; 121 uint64_t profile_key; 122 void *cursor; // MDB_cursor * 123 }; 124 125 // From-client event 126 struct ndb_fce { 127 enum fce_type evtype; 128 union { 129 struct ndb_event event; 130 }; 131 }; 132 133 // To-client event 134 struct ndb_tce { 135 enum tce_type evtype; 136 const char *subid; 137 int subid_len; 138 139 union { 140 struct ndb_event event; 141 struct ndb_command_result command_result; 142 }; 143 }; 144 145 typedef enum ndb_ingest_filter_action (*ndb_ingest_filter_fn)(void *, struct ndb_note *); 146 147 enum ndb_filter_fieldtype { 148 NDB_FILTER_IDS = 1, 149 NDB_FILTER_AUTHORS = 2, 150 NDB_FILTER_KINDS = 3, 151 NDB_FILTER_TAGS = 4, 152 NDB_FILTER_SINCE = 5, 153 NDB_FILTER_UNTIL = 6, 154 NDB_FILTER_LIMIT = 7, 155 }; 156 #define NDB_NUM_FILTERS 7 157 158 // when matching generic tags, we need to know if we're dealing with 159 // a pointer to a 32-byte ID or a null terminated string 160 enum ndb_generic_element_type { 161 NDB_ELEMENT_UNKNOWN = 0, 162 NDB_ELEMENT_STRING = 1, 163 NDB_ELEMENT_ID = 2, 164 NDB_ELEMENT_INT = 3, 165 }; 166 167 enum ndb_search_order { 168 NDB_ORDER_DESCENDING, 169 NDB_ORDER_ASCENDING, 170 }; 171 172 enum ndb_dbs { 173 NDB_DB_NOTE, 174 NDB_DB_META, 175 NDB_DB_PROFILE, 176 NDB_DB_NOTE_ID, 177 NDB_DB_PROFILE_PK, // profile pk index 178 NDB_DB_NDB_META, 179 NDB_DB_PROFILE_SEARCH, 180 NDB_DB_PROFILE_LAST_FETCH, 181 NDB_DB_NOTE_KIND, // note kind index 182 NDB_DB_NOTE_TEXT, // note fulltext index 183 NDB_DB_NOTE_BLOCKS, // parsed note blocks for rendering 184 NDB_DB_NOTE_TAGS, // note tags index 185 NDB_DBS, 186 }; 187 188 // common kinds. we collect stats on these in ndb_stat. mainly because I don't 189 // want to deal with including a hashtable to the project. 190 enum ndb_common_kind { 191 NDB_CKIND_PROFILE, 192 NDB_CKIND_TEXT, 193 NDB_CKIND_CONTACTS, 194 NDB_CKIND_DM, 195 NDB_CKIND_DELETE, 196 NDB_CKIND_REPOST, 197 NDB_CKIND_REACTION, 198 NDB_CKIND_ZAP, 199 NDB_CKIND_ZAP_REQUEST, 200 NDB_CKIND_NWC_REQUEST, 201 NDB_CKIND_NWC_RESPONSE, 202 NDB_CKIND_HTTP_AUTH, 203 NDB_CKIND_LIST, 204 NDB_CKIND_LONGFORM, 205 NDB_CKIND_STATUS, 206 NDB_CKIND_COUNT, // should always be last 207 }; 208 209 struct ndb_builder { 210 struct cursor mem; 211 struct cursor note_cur; 212 struct cursor strings; 213 struct cursor str_indices; 214 struct ndb_note *note; 215 struct ndb_tag *current_tag; 216 }; 217 218 struct ndb_iterator { 219 struct ndb_note *note; 220 struct ndb_tag *tag; 221 222 // current outer index 223 int index; 224 }; 225 226 struct ndb_filter_string { 227 const char *string; 228 int len; 229 }; 230 231 union ndb_filter_element { 232 struct ndb_filter_string string; 233 const unsigned char *id; 234 uint64_t integer; 235 }; 236 237 struct ndb_filter_field { 238 enum ndb_filter_fieldtype type; 239 enum ndb_generic_element_type elem_type; 240 char tag; // for generic queries like #t 241 }; 242 243 struct ndb_filter_elements { 244 struct ndb_filter_field field; 245 int count; 246 247 // this needs to be pointer size for reasons 248 // FIXME: what about on 32bit systems?? 249 uint64_t elements[0]; 250 }; 251 252 struct ndb_filter { 253 struct cursor elem_buf; 254 struct cursor data_buf; 255 int num_elements; 256 int finalized; 257 int current; 258 259 // struct ndb_filter_elements offsets into elem_buf 260 int elements[NDB_NUM_FILTERS]; 261 }; 262 263 struct ndb_config { 264 int flags; 265 int ingester_threads; 266 size_t mapsize; 267 void *filter_context; 268 ndb_ingest_filter_fn ingest_filter; 269 void *sub_cb_ctx; 270 ndb_sub_fn sub_cb; 271 }; 272 273 struct ndb_text_search_config { 274 enum ndb_search_order order; 275 int limit; 276 }; 277 278 struct ndb_stat_counts { 279 size_t key_size; 280 size_t value_size; 281 size_t count; 282 }; 283 284 struct ndb_stat { 285 struct ndb_stat_counts dbs[NDB_DBS]; 286 struct ndb_stat_counts common_kinds[NDB_CKIND_COUNT]; 287 struct ndb_stat_counts other_kinds; 288 }; 289 290 #define MAX_TEXT_SEARCH_RESULTS 128 291 #define MAX_TEXT_SEARCH_WORDS 8 292 293 // unpacked form of the actual lmdb fulltext search key 294 // see `ndb_make_text_search_key` for how the packed version is constructed 295 struct ndb_text_search_key 296 { 297 int str_len; 298 const char *str; 299 uint64_t timestamp; 300 uint64_t note_id; 301 uint64_t word_index; 302 }; 303 304 struct ndb_text_search_result { 305 struct ndb_text_search_key key; 306 int prefix_chars; 307 }; 308 309 struct ndb_text_search_results { 310 struct ndb_text_search_result results[MAX_TEXT_SEARCH_RESULTS]; 311 int num_results; 312 }; 313 314 enum ndb_block_type { 315 BLOCK_HASHTAG = 1, 316 BLOCK_TEXT = 2, 317 BLOCK_MENTION_INDEX = 3, 318 BLOCK_MENTION_BECH32 = 4, 319 BLOCK_URL = 5, 320 BLOCK_INVOICE = 6, 321 }; 322 #define NDB_NUM_BLOCK_TYPES 6 323 #define NDB_MAX_RELAYS 24 324 325 struct ndb_relays { 326 struct ndb_str_block relays[NDB_MAX_RELAYS]; 327 int num_relays; 328 }; 329 330 enum nostr_bech32_type { 331 NOSTR_BECH32_NOTE = 1, 332 NOSTR_BECH32_NPUB = 2, 333 NOSTR_BECH32_NPROFILE = 3, 334 NOSTR_BECH32_NEVENT = 4, 335 NOSTR_BECH32_NRELAY = 5, 336 NOSTR_BECH32_NADDR = 6, 337 NOSTR_BECH32_NSEC = 7, 338 }; 339 #define NOSTR_BECH32_KNOWN_TYPES 7 340 341 struct bech32_note { 342 const unsigned char *event_id; 343 }; 344 345 struct bech32_npub { 346 const unsigned char *pubkey; 347 }; 348 349 struct bech32_nsec { 350 const unsigned char *nsec; 351 }; 352 353 struct bech32_nevent { 354 struct ndb_relays relays; 355 const unsigned char *event_id; 356 const unsigned char *pubkey; // optional 357 }; 358 359 struct bech32_nprofile { 360 struct ndb_relays relays; 361 const unsigned char *pubkey; 362 }; 363 364 struct bech32_naddr { 365 struct ndb_relays relays; 366 struct ndb_str_block identifier; 367 const unsigned char *pubkey; 368 }; 369 370 struct bech32_nrelay { 371 struct ndb_str_block relay; 372 }; 373 374 struct nostr_bech32 { 375 enum nostr_bech32_type type; 376 377 union { 378 struct bech32_note note; 379 struct bech32_npub npub; 380 struct bech32_nsec nsec; 381 struct bech32_nevent nevent; 382 struct bech32_nprofile nprofile; 383 struct bech32_naddr naddr; 384 struct bech32_nrelay nrelay; 385 }; 386 }; 387 388 389 struct ndb_mention_bech32_block { 390 struct ndb_str_block str; 391 struct nostr_bech32 bech32; 392 }; 393 394 struct ndb_invoice { 395 unsigned char version; 396 uint64_t amount; 397 uint64_t timestamp; 398 uint64_t expiry; 399 char *description; 400 unsigned char *description_hash; 401 }; 402 403 struct ndb_invoice_block { 404 struct ndb_str_block invstr; 405 struct ndb_invoice invoice; 406 }; 407 408 struct ndb_block { 409 enum ndb_block_type type; 410 union { 411 struct ndb_str_block str; 412 struct ndb_invoice_block invoice; 413 struct ndb_mention_bech32_block mention_bech32; 414 uint32_t mention_index; 415 } block; 416 }; 417 418 struct ndb_block_iterator { 419 const char *content; 420 struct ndb_blocks *blocks; 421 struct ndb_block block; 422 unsigned char *p; 423 }; 424 425 struct ndb_query_result { 426 struct ndb_note *note; 427 uint64_t note_size; 428 uint64_t note_id; 429 }; 430 431 struct ndb_query_results { 432 struct cursor cur; 433 }; 434 435 // CONFIG 436 void ndb_default_config(struct ndb_config *); 437 void ndb_config_set_ingest_threads(struct ndb_config *config, int threads); 438 void ndb_config_set_flags(struct ndb_config *config, int flags); 439 void ndb_config_set_mapsize(struct ndb_config *config, size_t mapsize); 440 void ndb_config_set_ingest_filter(struct ndb_config *config, ndb_ingest_filter_fn fn, void *); 441 void ndb_config_set_subscription_callback(struct ndb_config *config, ndb_sub_fn fn, void *ctx); 442 443 // HELPERS 444 int ndb_calculate_id(struct ndb_note *note, unsigned char *buf, int buflen); 445 int ndb_sign_id(struct ndb_keypair *keypair, unsigned char id[32], unsigned char sig[64]); 446 int ndb_create_keypair(struct ndb_keypair *key); 447 int ndb_decode_key(const char *secstr, struct ndb_keypair *keypair); 448 int ndb_note_verify(void *secp_ctx, unsigned char pubkey[32], unsigned char id[32], unsigned char signature[64]); 449 450 // NDB 451 int ndb_init(struct ndb **ndb, const char *dbdir, const struct ndb_config *); 452 int ndb_db_version(struct ndb *ndb); 453 int ndb_process_event(struct ndb *, const char *json, int len); 454 int ndb_process_events(struct ndb *, const char *ldjson, size_t len); 455 int ndb_process_events_stream(struct ndb *, FILE* fp); 456 int ndb_process_client_event(struct ndb *, const char *json, int len); 457 int ndb_process_client_events(struct ndb *, const char *json, size_t len); 458 int ndb_begin_query(struct ndb *, struct ndb_txn *); 459 int ndb_search_profile(struct ndb_txn *txn, struct ndb_search *search, const char *query); 460 int ndb_search_profile_next(struct ndb_search *search); 461 void ndb_search_profile_end(struct ndb_search *search); 462 int ndb_end_query(struct ndb_txn *); 463 int ndb_write_last_profile_fetch(struct ndb *ndb, const unsigned char *pubkey, uint64_t fetched_at); 464 uint64_t ndb_read_last_profile_fetch(struct ndb_txn *txn, const unsigned char *pubkey); 465 void *ndb_get_profile_by_pubkey(struct ndb_txn *txn, const unsigned char *pubkey, size_t *len, uint64_t *primkey); 466 void *ndb_get_profile_by_key(struct ndb_txn *txn, uint64_t key, size_t *len); 467 uint64_t ndb_get_notekey_by_id(struct ndb_txn *txn, const unsigned char *id); 468 uint64_t ndb_get_profilekey_by_pubkey(struct ndb_txn *txn, const unsigned char *id); 469 struct ndb_note *ndb_get_note_by_id(struct ndb_txn *txn, const unsigned char *id, size_t *len, uint64_t *primkey); 470 struct ndb_note *ndb_get_note_by_key(struct ndb_txn *txn, uint64_t key, size_t *len); 471 void *ndb_get_note_meta(struct ndb_txn *txn, const unsigned char *id, size_t *len); 472 void ndb_destroy(struct ndb *); 473 474 // BUILDER 475 int ndb_parse_json_note(struct ndb_json_parser *, struct ndb_note **); 476 int ndb_client_event_from_json(const char *json, int len, struct ndb_fce *fce, unsigned char *buf, int bufsize, struct ndb_id_cb *cb); 477 int ndb_ws_event_from_json(const char *json, int len, struct ndb_tce *tce, unsigned char *buf, int bufsize, struct ndb_id_cb *); 478 int ndb_note_from_json(const char *json, int len, struct ndb_note **, unsigned char *buf, int buflen); 479 int ndb_builder_init(struct ndb_builder *builder, unsigned char *buf, size_t bufsize); 480 int ndb_builder_finalize(struct ndb_builder *builder, struct ndb_note **note, struct ndb_keypair *privkey); 481 int ndb_builder_set_content(struct ndb_builder *builder, const char *content, int len); 482 void ndb_builder_set_created_at(struct ndb_builder *builder, uint64_t created_at); 483 void ndb_builder_set_sig(struct ndb_builder *builder, unsigned char *sig); 484 void ndb_builder_set_pubkey(struct ndb_builder *builder, unsigned char *pubkey); 485 void ndb_builder_set_id(struct ndb_builder *builder, unsigned char *id); 486 void ndb_builder_set_kind(struct ndb_builder *builder, uint32_t kind); 487 int ndb_builder_new_tag(struct ndb_builder *builder); 488 int ndb_builder_push_tag_str(struct ndb_builder *builder, const char *str, int len); 489 490 // FILTERS 491 int ndb_filter_init(struct ndb_filter *); 492 int ndb_filter_add_id_element(struct ndb_filter *, const unsigned char *id); 493 int ndb_filter_add_int_element(struct ndb_filter *, uint64_t integer); 494 int ndb_filter_add_str_element(struct ndb_filter *, const char *str); 495 496 // filters from json 497 int ndb_filter_from_json(const char *, int len, struct ndb_filter *filter, unsigned char *buf, int bufsize); 498 499 // getting field elements 500 unsigned char *ndb_filter_get_id_element(const struct ndb_filter *, const struct ndb_filter_elements *, int index); 501 const char *ndb_filter_get_string_element(const struct ndb_filter *, const struct ndb_filter_elements *, int index); 502 uint64_t ndb_filter_get_int_element(const struct ndb_filter_elements *, int index); 503 uint64_t *ndb_filter_get_int_element_ptr(struct ndb_filter_elements *, int index); 504 505 struct ndb_filter_elements *ndb_filter_current_element(const struct ndb_filter *); 506 struct ndb_filter_elements *ndb_filter_get_elements(const struct ndb_filter *, int); 507 int ndb_filter_start_field(struct ndb_filter *, enum ndb_filter_fieldtype); 508 int ndb_filter_start_tag_field(struct ndb_filter *, char tag); 509 int ndb_filter_matches(struct ndb_filter *, struct ndb_note *); 510 int ndb_filter_clone(struct ndb_filter *dst, struct ndb_filter *src); 511 int ndb_filter_end(struct ndb_filter *); 512 void ndb_filter_end_field(struct ndb_filter *); 513 void ndb_filter_destroy(struct ndb_filter *); 514 int ndb_filter_json(const struct ndb_filter *, char *buf, int buflen); 515 516 // SUBSCRIPTIONS 517 uint64_t ndb_subscribe(struct ndb *, struct ndb_filter *, int num_filters); 518 int ndb_wait_for_notes(struct ndb *, uint64_t subid, uint64_t *note_ids, int note_id_capacity); 519 int ndb_poll_for_notes(struct ndb *, uint64_t subid, uint64_t *note_ids, int note_id_capacity); 520 int ndb_unsubscribe(struct ndb *, uint64_t subid); 521 int ndb_num_subscriptions(struct ndb *); 522 523 // FULLTEXT SEARCH 524 int ndb_text_search(struct ndb_txn *txn, const char *query, struct ndb_text_search_results *, struct ndb_text_search_config *); 525 void ndb_default_text_search_config(struct ndb_text_search_config *); 526 void ndb_text_search_config_set_order(struct ndb_text_search_config *, enum ndb_search_order); 527 void ndb_text_search_config_set_limit(struct ndb_text_search_config *, int limit); 528 529 // QUERY 530 int ndb_query(struct ndb_txn *txn, struct ndb_filter *filters, int num_filters, struct ndb_query_result *results, int result_capacity, int *count); 531 532 // STATS 533 int ndb_stat(struct ndb *ndb, struct ndb_stat *stat); 534 void ndb_stat_counts_init(struct ndb_stat_counts *counts); 535 536 // NOTE 537 const char *ndb_note_content(struct ndb_note *note); 538 struct ndb_str ndb_note_str(struct ndb_note *note, union ndb_packed_str *pstr); 539 uint32_t ndb_note_content_length(struct ndb_note *note); 540 uint32_t ndb_note_created_at(struct ndb_note *note); 541 uint32_t ndb_note_kind(struct ndb_note *note); 542 unsigned char *ndb_note_id(struct ndb_note *note); 543 unsigned char *ndb_note_pubkey(struct ndb_note *note); 544 unsigned char *ndb_note_sig(struct ndb_note *note); 545 void _ndb_note_set_kind(struct ndb_note *note, uint32_t kind); 546 struct ndb_tags *ndb_note_tags(struct ndb_note *note); 547 int ndb_str_len(struct ndb_str *str); 548 549 /// write the note as json to a buffer 550 int ndb_note_json(struct ndb_note *, char *buf, int buflen); 551 552 // TAGS 553 void ndb_tags_iterate_start(struct ndb_note *note, struct ndb_iterator *iter); 554 uint16_t ndb_tags_count(struct ndb_tags *); 555 uint16_t ndb_tag_count(struct ndb_tag *); 556 557 // ITER 558 int ndb_tags_iterate_next(struct ndb_iterator *iter); 559 struct ndb_str ndb_iter_tag_str(struct ndb_iterator *iter, int ind); 560 struct ndb_str ndb_tag_str(struct ndb_note *note, struct ndb_tag *tag, int ind); 561 562 // NAMES 563 const char *ndb_db_name(enum ndb_dbs db); 564 const char *ndb_kind_name(enum ndb_common_kind ck); 565 enum ndb_common_kind ndb_kind_to_common_kind(int kind); 566 567 // CONTENT PARSER 568 int ndb_parse_content(unsigned char *buf, int buf_size, 569 const char *content, int content_len, 570 struct ndb_blocks **blocks_p); 571 572 // BLOCKS 573 enum ndb_block_type ndb_get_block_type(struct ndb_block *block); 574 int ndb_blocks_flags(struct ndb_blocks *block); 575 size_t ndb_blocks_total_size(struct ndb_blocks *blocks); 576 int ndb_blocks_word_count(struct ndb_blocks *blocks); 577 578 /// Free blocks if they are owned, safe to call on unowned blocks as well. 579 void ndb_blocks_free(struct ndb_blocks *blocks); 580 581 // BLOCK DB 582 struct ndb_blocks *ndb_get_blocks_by_key(struct ndb *ndb, struct ndb_txn *txn, uint64_t note_key); 583 584 // BLOCK ITERATORS 585 void ndb_blocks_iterate_start(const char *, struct ndb_blocks *, struct ndb_block_iterator *); 586 struct ndb_block *ndb_blocks_iterate_next(struct ndb_block_iterator *); 587 588 // STR BLOCKS 589 struct ndb_str_block *ndb_block_str(struct ndb_block *); 590 const char *ndb_str_block_ptr(struct ndb_str_block *); 591 uint32_t ndb_str_block_len(struct ndb_str_block *); 592 593 // BECH32 BLOCKS 594 struct nostr_bech32 *ndb_bech32_block(struct ndb_block *block); 595 596 #endif