nostrdb.h (11326B)
1 #ifndef NOSTRDB_H 2 #define NOSTRDB_H 3 4 #include <inttypes.h> 5 #include "cursor.h" 6 7 #define NDB_PACKED_STR 0x1 8 #define NDB_PACKED_ID 0x2 9 10 #define NDB_FLAG_NOMIGRATE (1 << 0) 11 #define NDB_FLAG_SKIP_NOTE_VERIFY (1 << 1) 12 13 //#define DEBUG 1 14 15 #ifdef DEBUG 16 #define ndb_debug(...) printf(__VA_ARGS__) 17 #else 18 #define ndb_debug(...) (void)0 19 #endif 20 21 struct ndb_json_parser; 22 struct ndb; 23 struct ndb_note; 24 struct ndb_tag; 25 struct ndb_tags; 26 struct ndb_lmdb; 27 union ndb_packed_str; 28 29 // sorry, swift needs help with forward declared pointers like this 30 struct ndb_t { 31 struct ndb *ndb; 32 }; 33 34 struct ndb_str { 35 unsigned char flag; 36 union { 37 const char *str; 38 unsigned char *id; 39 }; 40 }; 41 42 struct ndb_keypair { 43 unsigned char pubkey[32]; 44 unsigned char secret[32]; 45 46 // this corresponds to secp256k1's keypair type. it's guaranteed to 47 // be 96 bytes according to their docs. I don't want to depend on 48 // the secp256k1 header here so we just use raw bytes. 49 unsigned char pair[96]; 50 }; 51 52 // function pointer for controlling what to do after we parse an id 53 typedef enum ndb_idres (*ndb_id_fn)(void *, const char *); 54 55 // id callback + closure data 56 struct ndb_id_cb { 57 ndb_id_fn fn; 58 void *data; 59 }; 60 61 // required to keep a read 62 struct ndb_txn { 63 struct ndb_lmdb *lmdb; 64 void *mdb_txn; 65 }; 66 67 struct ndb_event { 68 struct ndb_note *note; 69 }; 70 71 struct ndb_command_result { 72 int ok; 73 const char *msg; 74 int msglen; 75 }; 76 77 // From-client event types 78 enum fce_type { 79 NDB_FCE_EVENT = 0x1 80 }; 81 82 // To-client event types 83 enum tce_type { 84 NDB_TCE_EVENT = 0x1, 85 NDB_TCE_OK = 0x2, 86 NDB_TCE_NOTICE = 0x3, 87 NDB_TCE_EOSE = 0x4, 88 }; 89 90 enum ndb_ingest_filter_action { 91 NDB_INGEST_REJECT, 92 NDB_INGEST_ACCEPT, 93 NDB_INGEST_SKIP_VALIDATION 94 }; 95 96 struct ndb_search_key 97 { 98 char search[24]; 99 unsigned char id[32]; 100 uint64_t timestamp; 101 }; 102 103 struct ndb_search { 104 struct ndb_search_key *key; 105 uint64_t profile_key; 106 void *cursor; // MDB_cursor * 107 }; 108 109 // From-client event 110 struct ndb_fce { 111 enum fce_type evtype; 112 union { 113 struct ndb_event event; 114 }; 115 }; 116 117 // To-client event 118 struct ndb_tce { 119 enum tce_type evtype; 120 const char *subid; 121 int subid_len; 122 123 union { 124 struct ndb_event event; 125 struct ndb_command_result command_result; 126 }; 127 }; 128 129 typedef enum ndb_ingest_filter_action (*ndb_ingest_filter_fn)(void *, struct ndb_note *); 130 131 enum ndb_filter_fieldtype { 132 NDB_FILTER_IDS = 1, 133 NDB_FILTER_AUTHORS = 2, 134 NDB_FILTER_KINDS = 3, 135 NDB_FILTER_GENERIC = 4, 136 NDB_FILTER_SINCE = 5, 137 NDB_FILTER_UNTIL = 6, 138 NDB_FILTER_LIMIT = 7, 139 }; 140 #define NDB_NUM_FILTERS 7 141 142 // when matching generic tags, we need to know if we're dealing with 143 // a pointer to a 32-byte ID or a null terminated string 144 enum ndb_generic_element_type { 145 NDB_ELEMENT_UNKNOWN = 0, 146 NDB_ELEMENT_STRING = 1, 147 NDB_ELEMENT_ID = 2, 148 }; 149 150 enum ndb_search_order { 151 NDB_ORDER_DESCENDING, 152 NDB_ORDER_ASCENDING, 153 }; 154 155 enum ndb_dbs { 156 NDB_DB_NOTE, 157 NDB_DB_META, 158 NDB_DB_PROFILE, 159 NDB_DB_NOTE_ID, 160 NDB_DB_PROFILE_PK, // profile pk index 161 NDB_DB_NDB_META, 162 NDB_DB_PROFILE_SEARCH, 163 NDB_DB_PROFILE_LAST_FETCH, 164 NDB_DB_NOTE_KIND, // note kind index 165 NDB_DB_NOTE_TEXT, // note fulltext index 166 NDB_DBS, 167 }; 168 169 // common kinds. we collect stats on these in ndb_stat. mainly because I don't 170 // want to deal with including a hashtable to the project. 171 enum ndb_common_kind { 172 NDB_CKIND_PROFILE, 173 NDB_CKIND_TEXT, 174 NDB_CKIND_CONTACTS, 175 NDB_CKIND_DM, 176 NDB_CKIND_DELETE, 177 NDB_CKIND_REPOST, 178 NDB_CKIND_REACTION, 179 NDB_CKIND_ZAP, 180 NDB_CKIND_ZAP_REQUEST, 181 NDB_CKIND_NWC_REQUEST, 182 NDB_CKIND_NWC_RESPONSE, 183 NDB_CKIND_HTTP_AUTH, 184 NDB_CKIND_LIST, 185 NDB_CKIND_LONGFORM, 186 NDB_CKIND_STATUS, 187 NDB_CKIND_COUNT, // should always be last 188 }; 189 190 struct ndb_builder { 191 struct cursor mem; 192 struct cursor note_cur; 193 struct cursor strings; 194 struct cursor str_indices; 195 struct ndb_note *note; 196 struct ndb_tag *current_tag; 197 }; 198 199 struct ndb_iterator { 200 struct ndb_note *note; 201 struct ndb_tag *tag; 202 203 // current outer index 204 int index; 205 }; 206 207 union ndb_filter_element { 208 const char *string; 209 const unsigned char *id; 210 uint64_t integer; 211 }; 212 213 struct ndb_filter_field { 214 enum ndb_filter_fieldtype type; 215 enum ndb_generic_element_type elem_type; 216 char generic; // for generic queries like #t 217 }; 218 219 struct ndb_filter_elements { 220 struct ndb_filter_field field; 221 int count; 222 union ndb_filter_element elements[0]; 223 }; 224 225 struct ndb_filter { 226 struct cursor elem_buf; 227 struct cursor data_buf; 228 int num_elements; 229 struct ndb_filter_elements *current; 230 struct ndb_filter_elements *elements[NDB_NUM_FILTERS]; 231 }; 232 233 struct ndb_config { 234 int flags; 235 int ingester_threads; 236 size_t mapsize; 237 void *filter_context; 238 ndb_ingest_filter_fn ingest_filter; 239 }; 240 241 struct ndb_text_search_config { 242 enum ndb_search_order order; 243 int limit; 244 }; 245 246 struct ndb_stat_counts { 247 size_t key_size; 248 size_t value_size; 249 size_t count; 250 }; 251 252 struct ndb_stat { 253 struct ndb_stat_counts dbs[NDB_DBS]; 254 struct ndb_stat_counts common_kinds[NDB_CKIND_COUNT]; 255 struct ndb_stat_counts other_kinds; 256 }; 257 258 #define MAX_TEXT_SEARCH_RESULTS 128 259 #define MAX_TEXT_SEARCH_WORDS 8 260 261 // unpacked form of the actual lmdb fulltext search key 262 // see `ndb_make_text_search_key` for how the packed version is constructed 263 struct ndb_text_search_key 264 { 265 int str_len; 266 const char *str; 267 uint64_t timestamp; 268 uint64_t note_id; 269 int word_index; 270 }; 271 272 struct ndb_text_search_result { 273 struct ndb_text_search_key key; 274 int prefix_chars; 275 }; 276 277 struct ndb_text_search_results { 278 struct ndb_text_search_result results[MAX_TEXT_SEARCH_RESULTS]; 279 int num_results; 280 }; 281 282 283 // CONFIG 284 void ndb_default_config(struct ndb_config *); 285 void ndb_config_set_ingest_threads(struct ndb_config *config, int threads); 286 void ndb_config_set_flags(struct ndb_config *config, int flags); 287 void ndb_config_set_mapsize(struct ndb_config *config, size_t mapsize); 288 void ndb_config_set_ingest_filter(struct ndb_config *config, ndb_ingest_filter_fn fn, void *); 289 290 // HELPERS 291 int ndb_calculate_id(struct ndb_note *note, unsigned char *buf, int buflen); 292 int ndb_sign_id(struct ndb_keypair *keypair, unsigned char id[32], unsigned char sig[64]); 293 int ndb_create_keypair(struct ndb_keypair *key); 294 int ndb_decode_key(const char *secstr, struct ndb_keypair *keypair); 295 int ndb_note_verify(void *secp_ctx, unsigned char pubkey[32], unsigned char id[32], unsigned char signature[64]); 296 297 // NDB 298 int ndb_init(struct ndb **ndb, const char *dbdir, const struct ndb_config *); 299 int ndb_db_version(struct ndb *ndb); 300 int ndb_process_event(struct ndb *, const char *json, int len); 301 int ndb_process_events(struct ndb *, const char *ldjson, size_t len); 302 int ndb_process_events_stream(struct ndb *, FILE* fp); 303 int ndb_process_client_event(struct ndb *, const char *json, int len); 304 int ndb_process_client_events(struct ndb *, const char *json, size_t len); 305 int ndb_begin_query(struct ndb *, struct ndb_txn *); 306 int ndb_search_profile(struct ndb_txn *txn, struct ndb_search *search, const char *query); 307 int ndb_search_profile_next(struct ndb_search *search); 308 void ndb_search_profile_end(struct ndb_search *search); 309 int ndb_end_query(struct ndb_txn *); 310 int ndb_write_last_profile_fetch(struct ndb *ndb, const unsigned char *pubkey, uint64_t fetched_at); 311 uint64_t ndb_read_last_profile_fetch(struct ndb_txn *txn, const unsigned char *pubkey); 312 void *ndb_get_profile_by_pubkey(struct ndb_txn *txn, const unsigned char *pubkey, size_t *len, uint64_t *primkey); 313 void *ndb_get_profile_by_key(struct ndb_txn *txn, uint64_t key, size_t *len); 314 uint64_t ndb_get_notekey_by_id(struct ndb_txn *txn, const unsigned char *id); 315 uint64_t ndb_get_profilekey_by_pubkey(struct ndb_txn *txn, const unsigned char *id); 316 struct ndb_note *ndb_get_note_by_id(struct ndb_txn *txn, const unsigned char *id, size_t *len, uint64_t *primkey); 317 struct ndb_note *ndb_get_note_by_key(struct ndb_txn *txn, uint64_t key, size_t *len); 318 void *ndb_get_note_meta(struct ndb_txn *txn, const unsigned char *id, size_t *len); 319 void ndb_destroy(struct ndb *); 320 321 // BUILDER 322 int ndb_parse_json_note(struct ndb_json_parser *, struct ndb_note **); 323 int ndb_client_event_from_json(const char *json, int len, struct ndb_fce *fce, unsigned char *buf, int bufsize, struct ndb_id_cb *cb); 324 int ndb_ws_event_from_json(const char *json, int len, struct ndb_tce *tce, unsigned char *buf, int bufsize, struct ndb_id_cb *); 325 int ndb_note_from_json(const char *json, int len, struct ndb_note **, unsigned char *buf, int buflen); 326 int ndb_builder_init(struct ndb_builder *builder, unsigned char *buf, size_t bufsize); 327 int ndb_builder_finalize(struct ndb_builder *builder, struct ndb_note **note, struct ndb_keypair *privkey); 328 int ndb_builder_set_content(struct ndb_builder *builder, const char *content, int len); 329 void ndb_builder_set_created_at(struct ndb_builder *builder, uint64_t created_at); 330 void ndb_builder_set_sig(struct ndb_builder *builder, unsigned char *sig); 331 void ndb_builder_set_pubkey(struct ndb_builder *builder, unsigned char *pubkey); 332 void ndb_builder_set_id(struct ndb_builder *builder, unsigned char *id); 333 void ndb_builder_set_kind(struct ndb_builder *builder, uint32_t kind); 334 int ndb_builder_new_tag(struct ndb_builder *builder); 335 int ndb_builder_push_tag_str(struct ndb_builder *builder, const char *str, int len); 336 337 // FILTERS 338 int ndb_filter_init(struct ndb_filter *); 339 int ndb_filter_add_id_element(struct ndb_filter *, const unsigned char *id); 340 int ndb_filter_add_int_element(struct ndb_filter *, uint64_t integer); 341 int ndb_filter_add_str_element(struct ndb_filter *, const char *str); 342 int ndb_filter_start_field(struct ndb_filter *, enum ndb_filter_fieldtype); 343 int ndb_filter_start_generic_field(struct ndb_filter *, char tag); 344 int ndb_filter_matches(struct ndb_filter *, struct ndb_note *); 345 void ndb_filter_reset(struct ndb_filter *); 346 void ndb_filter_end_field(struct ndb_filter *); 347 void ndb_filter_free(struct ndb_filter *); 348 349 // FULLTEXT SEARCH 350 int ndb_text_search(struct ndb_txn *txn, const char *query, struct ndb_text_search_results *, struct ndb_text_search_config *); 351 void ndb_default_text_search_config(struct ndb_text_search_config *); 352 void ndb_text_search_config_set_order(struct ndb_text_search_config *, enum ndb_search_order); 353 void ndb_text_search_config_set_limit(struct ndb_text_search_config *, int limit); 354 355 // STATS 356 int ndb_stat(struct ndb *ndb, struct ndb_stat *stat); 357 void ndb_stat_counts_init(struct ndb_stat_counts *counts); 358 359 // NOTE 360 const char *ndb_note_content(struct ndb_note *note); 361 struct ndb_str ndb_note_str(struct ndb_note *note, union ndb_packed_str *pstr); 362 uint32_t ndb_note_content_length(struct ndb_note *note); 363 uint32_t ndb_note_created_at(struct ndb_note *note); 364 uint32_t ndb_note_kind(struct ndb_note *note); 365 unsigned char *ndb_note_id(struct ndb_note *note); 366 unsigned char *ndb_note_pubkey(struct ndb_note *note); 367 unsigned char *ndb_note_sig(struct ndb_note *note); 368 void _ndb_note_set_kind(struct ndb_note *note, uint32_t kind); 369 struct ndb_tags *ndb_note_tags(struct ndb_note *note); 370 371 // TAGS 372 void ndb_tags_iterate_start(struct ndb_note *note, struct ndb_iterator *iter); 373 uint16_t ndb_tags_count(struct ndb_tags *); 374 uint16_t ndb_tag_count(struct ndb_tag *); 375 376 // ITER 377 int ndb_tags_iterate_next(struct ndb_iterator *iter); 378 struct ndb_str ndb_iter_tag_str(struct ndb_iterator *iter, int ind); 379 struct ndb_str ndb_tag_str(struct ndb_note *note, struct ndb_tag *tag, int ind); 380 381 // NAMES 382 const char *ndb_db_name(enum ndb_dbs db); 383 const char *ndb_kind_name(enum ndb_common_kind ck); 384 enum ndb_common_kind ndb_kind_to_common_kind(int kind); 385 386 #endif