nostrdb

an unfairly fast embedded nostr database backed by lmdb
git clone git://jb55.com/nostrdb
Log | Files | Refs | Submodules | README | LICENSE

nostrdb.c (211299B)


      1 
      2 #include "nostrdb.h"
      3 #include "jsmn.h"
      4 #include "hex.h"
      5 #include "cursor.h"
      6 #include "random.h"
      7 #include "ccan/crypto/sha256/sha256.h"
      8 #include "bolt11/bolt11.h"
      9 #include "bolt11/amount.h"
     10 #include "lmdb.h"
     11 #include "util.h"
     12 #include "cpu.h"
     13 #include "block.h"
     14 #include "threadpool.h"
     15 #include "thread.h"
     16 #include "protected_queue.h"
     17 #include "memchr.h"
     18 #include "print_util.h"
     19 #include <stdlib.h>
     20 #include <limits.h>
     21 #include <assert.h>
     22 
     23 #include "bindings/c/profile_json_parser.h"
     24 #include "bindings/c/profile_builder.h"
     25 #include "bindings/c/meta_builder.h"
     26 #include "bindings/c/meta_reader.h"
     27 #include "bindings/c/profile_verifier.h"
     28 #include "secp256k1.h"
     29 #include "secp256k1_ecdh.h"
     30 #include "secp256k1_schnorrsig.h"
     31 
     32 #define max(a,b) ((a) > (b) ? (a) : (b))
     33 #define min(a,b) ((a) < (b) ? (a) : (b))
     34 
     35 // the maximum number of things threads pop and push in bulk
     36 #define THREAD_QUEUE_BATCH 4096
     37 
     38 // maximum number of active subscriptions
     39 #define MAX_SUBSCRIPTIONS 256
     40 #define MAX_SCAN_CURSORS 12
     41 #define MAX_FILTERS    16
     42 
     43 // the maximum size of inbox queues
     44 static const int DEFAULT_QUEUE_SIZE = 32768;
     45 
     46 // 2mb scratch size for the writer thread
     47 static const int DEFAULT_WRITER_SCRATCH_SIZE = 2097152;
     48 
     49 // increase if we need bigger filters
     50 #define NDB_FILTER_PAGES 64
     51 
     52 #define ndb_flag_set(flags, f) ((flags & f) == f)
     53 
     54 #define NDB_PARSED_ID           (1 << 0)
     55 #define NDB_PARSED_PUBKEY       (1 << 1)
     56 #define NDB_PARSED_SIG          (1 << 2)
     57 #define NDB_PARSED_CREATED_AT   (1 << 3)
     58 #define NDB_PARSED_KIND         (1 << 4)
     59 #define NDB_PARSED_CONTENT      (1 << 5)
     60 #define NDB_PARSED_TAGS         (1 << 6)
     61 #define NDB_PARSED_ALL          (NDB_PARSED_ID|NDB_PARSED_PUBKEY|NDB_PARSED_SIG|NDB_PARSED_CREATED_AT|NDB_PARSED_KIND|NDB_PARSED_CONTENT|NDB_PARSED_TAGS)
     62 
     63 typedef int (*ndb_migrate_fn)(struct ndb_txn *);
     64 typedef int (*ndb_word_parser_fn)(void *, const char *word, int word_len,
     65 				  int word_index);
     66 
     67 // these must be byte-aligned, they are directly accessing the serialized data
     68 // representation
     69 #pragma pack(push, 1)
     70 
     71 union ndb_packed_str {
     72 	struct {
     73 		char str[3];
     74 		// we assume little endian everywhere. sorry not sorry.
     75 		unsigned char flag; // NDB_PACKED_STR, etc
     76 	} packed;
     77 
     78 	uint32_t offset;
     79 	unsigned char bytes[4];
     80 };
     81 
     82 struct ndb_tag {
     83 	uint16_t count;
     84 	union ndb_packed_str strs[0];
     85 };
     86 
     87 struct ndb_tags {
     88 	uint16_t padding;
     89 	uint16_t count;
     90 };
     91 
     92 // v1
     93 struct ndb_note {
     94 	unsigned char version;    // v=1
     95 	unsigned char padding[3]; // keep things aligned
     96 	unsigned char id[32];
     97 	unsigned char pubkey[32];
     98 	unsigned char sig[64];
     99 
    100 	uint64_t created_at;
    101 	uint32_t kind;
    102 	uint32_t content_length;
    103 	union ndb_packed_str content;
    104 	uint32_t strings;
    105 	// nothing can come after tags since it contains variadic data
    106 	struct ndb_tags tags;
    107 };
    108 
    109 #pragma pack(pop)
    110 
    111 
    112 struct ndb_migration {
    113 	ndb_migrate_fn fn;
    114 };
    115 
    116 struct ndb_profile_record_builder {
    117 	flatcc_builder_t *builder;
    118 	void *flatbuf;
    119 };
    120 
    121 // controls whether to continue or stop the json parser
    122 enum ndb_idres {
    123 	NDB_IDRES_CONT,
    124 	NDB_IDRES_STOP,
    125 };
    126 
    127 // closure data for the id-detecting ingest controller
    128 struct ndb_ingest_controller
    129 {
    130 	MDB_txn *read_txn;
    131 	struct ndb_lmdb *lmdb;
    132 	struct ndb_note *note;
    133 	uint64_t note_key;
    134 };
    135 
    136 enum ndb_writer_msgtype {
    137 	NDB_WRITER_QUIT, // kill thread immediately
    138 	NDB_WRITER_NOTE, // write a note to the db
    139 	NDB_WRITER_PROFILE, // write a profile to the db
    140 	NDB_WRITER_DBMETA, // write ndb metadata
    141 	NDB_WRITER_PROFILE_LAST_FETCH, // when profiles were last fetched
    142 	NDB_WRITER_BLOCKS, // write parsed note blocks
    143 	NDB_WRITER_MIGRATE, // migrate the database
    144 	NDB_WRITER_NOTE_RELAY, // we already have the note, but we have more relays to write
    145 };
    146 
    147 // keys used for storing data in the NDB metadata database (NDB_DB_NDB_META)
    148 enum ndb_meta_key {
    149 	NDB_META_KEY_VERSION = 1
    150 };
    151 
    152 struct ndb_json_parser {
    153 	const char *json;
    154 	int json_len;
    155 	struct ndb_builder builder;
    156 	jsmn_parser json_parser;
    157 	jsmntok_t *toks, *toks_end;
    158 	int i;
    159 	int num_tokens;
    160 };
    161 
    162 // useful to pass to threads on its own
    163 struct ndb_lmdb {
    164 	MDB_env *env;
    165 	MDB_dbi dbs[NDB_DBS];
    166 };
    167 
    168 struct ndb_writer {
    169 	struct ndb_lmdb *lmdb;
    170 	struct ndb_monitor *monitor;
    171 
    172 	int scratch_size;
    173 	uint32_t ndb_flags;
    174 	void *queue_buf;
    175 	int queue_buflen;
    176 	pthread_t thread_id;
    177 
    178 	struct prot_queue inbox;
    179 };
    180 
    181 struct ndb_ingester {
    182 	struct ndb_lmdb *lmdb;
    183 	uint32_t flags;
    184 	struct threadpool tp;
    185 	struct prot_queue *writer_inbox;
    186 	void *filter_context;
    187 	ndb_ingest_filter_fn filter;
    188 
    189 	int scratch_size;
    190 };
    191 
    192 struct ndb_filter_group {
    193 	struct ndb_filter filters[MAX_FILTERS];
    194 	int num_filters;
    195 };
    196 
    197 struct ndb_subscription {
    198 	uint64_t subid;
    199 	struct ndb_filter_group group;
    200 	struct prot_queue inbox;
    201 };
    202 
    203 struct ndb_monitor {
    204 	struct ndb_subscription subscriptions[MAX_SUBSCRIPTIONS];
    205 	ndb_sub_fn sub_cb;
    206 	void *sub_cb_ctx;
    207 	int num_subscriptions;
    208 
    209 	// monitor isn't a full inbox. We want pollers to be able to poll
    210 	// subscriptions efficiently without going through a message queue, so
    211 	// we use a simple mutex here.
    212 	pthread_mutex_t mutex;
    213 };
    214 
    215 struct ndb {
    216 	struct ndb_lmdb lmdb;
    217 	struct ndb_ingester ingester;
    218 	struct ndb_monitor monitor;
    219 	struct ndb_writer writer;
    220 	int version;
    221 	uint32_t flags; // setting flags
    222 	// lmdb environ handles, etc
    223 };
    224 
    225 ///
    226 /// Query Plans
    227 ///
    228 /// There are general strategies for performing certain types of query
    229 /// depending on the filter. For example, for large contact list queries
    230 /// with many authors, we simply do a descending scan on created_at
    231 /// instead of doing 1000s of pubkey scans.
    232 ///
    233 /// Query plans are calculated from filters via `ndb_filter_plan`
    234 ///
    235 enum ndb_query_plan {
    236 	NDB_PLAN_KINDS,
    237 	NDB_PLAN_IDS,
    238 	NDB_PLAN_AUTHORS,
    239 	NDB_PLAN_AUTHOR_KINDS,
    240 	NDB_PLAN_CREATED,
    241 	NDB_PLAN_TAGS,
    242 	NDB_PLAN_SEARCH,
    243 	NDB_PLAN_RELAY_KINDS,
    244 	NDB_PLAN_PROFILE_SEARCH,
    245 };
    246 
    247 // A id + u64 + timestamp
    248 struct ndb_id_u64_ts {
    249 	unsigned char id[32]; // pubkey, id, etc
    250 	uint64_t u64; // kind, etc
    251 	uint64_t timestamp;
    252 };
    253 
    254 // A clustered key with an id and a timestamp
    255 struct ndb_tsid {
    256 	unsigned char id[32];
    257 	uint64_t timestamp;
    258 };
    259 
    260 // A u64 + timestamp id. Just using this for kinds at the moment.
    261 struct ndb_u64_ts {
    262 	uint64_t u64; // kind, etc
    263 	uint64_t timestamp;
    264 };
    265 
    266 struct ndb_word
    267 {
    268 	const char *word;
    269 	int word_len;
    270 };
    271 
    272 struct ndb_search_words
    273 {
    274 	struct ndb_word words[MAX_TEXT_SEARCH_WORDS];
    275 	int num_words;
    276 };
    277 
    278 
    279 static inline int is_replaceable_kind(uint64_t kind)
    280 {
    281 	return kind == 0 || kind == 3
    282 		|| (10000 <= kind && kind < 20000)
    283 		|| (30000 <= kind && kind < 40000);
    284 }
    285 
    286 
    287 // ndb_text_search_key
    288 //
    289 // This is compressed when in lmdb:
    290 //
    291 //   note_id:    varint
    292 //   strlen:     varint
    293 //   str:        cstr
    294 //   timestamp:  varint
    295 //   word_index: varint
    296 //
    297 static int ndb_make_text_search_key(unsigned char *buf, int bufsize,
    298 				    int word_index, int word_len, const char *str,
    299 				    uint64_t timestamp, uint64_t note_id,
    300 				    int *keysize)
    301 {
    302 	struct cursor cur;
    303 	make_cursor(buf, buf + bufsize, &cur);
    304 
    305 	// TODO: need update this to uint64_t
    306 	// we push this first because our query function can pull this off
    307 	// quickly to check matches
    308 	if (!cursor_push_varint(&cur, (int32_t)note_id))
    309 		return 0;
    310 
    311 	// string length
    312 	if (!cursor_push_varint(&cur, word_len))
    313 		return 0;
    314 
    315 	// non-null terminated, lowercase string
    316 	if (!cursor_push_lowercase(&cur, str, word_len))
    317 		return 0;
    318 
    319 	// TODO: need update this to uint64_t
    320 	if (!cursor_push_varint(&cur, (int)timestamp))
    321 		return 0;
    322 
    323 	// the index of the word in the content so that we can do more accurate
    324 	// phrase searches
    325 	if (!cursor_push_varint(&cur, word_index))
    326 		return 0;
    327 
    328 	// pad to 8-byte alignment
    329 	if (!cursor_align(&cur, 8))
    330 		return 0;
    331 
    332 	*keysize = cur.p - cur.start;
    333 	assert((*keysize % 8) == 0);
    334 
    335 	return 1;
    336 }
    337 
    338 static int ndb_make_noted_text_search_key(unsigned char *buf, int bufsize,
    339 					  int wordlen, const char *word,
    340 					  uint64_t timestamp, uint64_t note_id,
    341 					  int *keysize)
    342 {
    343 	return ndb_make_text_search_key(buf, bufsize, 0, wordlen, word,
    344 					timestamp, note_id, keysize);
    345 }
    346 
    347 static int ndb_make_text_search_key_low(unsigned char *buf, int bufsize,
    348 					int wordlen, const char *word,
    349 					uint64_t since,
    350 					int *keysize)
    351 {
    352 	uint64_t note_id;
    353 	note_id = 0;
    354 	return ndb_make_text_search_key(buf, bufsize, 0, wordlen, word,
    355 					since, note_id, keysize);
    356 }
    357 
    358 static int ndb_make_text_search_key_high(unsigned char *buf, int bufsize,
    359 					 int wordlen, const char *word,
    360 					 uint64_t until,
    361 					 int *keysize)
    362 {
    363 	uint64_t note_id;
    364 	note_id = INT32_MAX;
    365 	return ndb_make_text_search_key(buf, bufsize, 0, wordlen, word,
    366 					until, note_id, keysize);
    367 }
    368 
    369 typedef int (*ndb_text_search_key_order_fn)(unsigned char *buf, int bufsize, int wordlen, const char *word, uint64_t timestamp, int *keysize);
    370 
    371 /** From LMDB: Compare two items lexically */
    372 static int mdb_cmp_memn(const MDB_val *a, const MDB_val *b) {
    373 	int diff;
    374 	ssize_t len_diff;
    375 	unsigned int len;
    376 
    377 	len = a->mv_size;
    378 	len_diff = (ssize_t) a->mv_size - (ssize_t) b->mv_size;
    379 	if (len_diff > 0) {
    380 		len = b->mv_size;
    381 		len_diff = 1;
    382 	}
    383 
    384 	diff = memcmp(a->mv_data, b->mv_data, len);
    385 	return diff ? diff : len_diff<0 ? -1 : len_diff;
    386 }
    387 
    388 static int ndb_tag_key_compare(const MDB_val *a, const MDB_val *b)
    389 {
    390 	MDB_val va, vb;
    391 	uint64_t ts_a, ts_b;
    392 	int cmp;
    393 
    394 	va.mv_data = a->mv_data;
    395 	va.mv_size = a->mv_size - 8;
    396 
    397 	vb.mv_data = b->mv_data;
    398 	vb.mv_size = b->mv_size - 8;
    399 
    400 	if ((cmp = mdb_cmp_memn(&va, &vb)))
    401 		return cmp;
    402 
    403 	ts_a = *(uint64_t*)((unsigned char *)va.mv_data + va.mv_size);
    404 	ts_b = *(uint64_t*)((unsigned char *)vb.mv_data + vb.mv_size);
    405 
    406 	if (ts_a < ts_b)
    407 		return -1;
    408 	else if (ts_a > ts_b)
    409 		return 1;
    410 
    411 	return 0;
    412 }
    413 
    414 static int ndb_text_search_key_compare(const MDB_val *a, const MDB_val *b)
    415 {
    416 	struct cursor ca, cb;
    417 	uint64_t sa, sb, nid_a, nid_b;
    418 	MDB_val a2, b2;
    419 
    420 	make_cursor(a->mv_data, (unsigned char *)a->mv_data + a->mv_size, &ca);
    421 	make_cursor(b->mv_data, (unsigned char *)b->mv_data + b->mv_size, &cb);
    422 
    423 	// note_id
    424 	if (unlikely(!cursor_pull_varint(&ca, &nid_a) || !cursor_pull_varint(&cb, &nid_b)))
    425 		return 0;
    426 
    427 	// string size
    428 	if (unlikely(!cursor_pull_varint(&ca, &sa) || !cursor_pull_varint(&cb, &sb)))
    429 		return 0;
    430 
    431 	a2.mv_data = ca.p;
    432 	a2.mv_size = sa;
    433 
    434 	b2.mv_data = cb.p;
    435 	b2.mv_size = sb;
    436 
    437 	int cmp = mdb_cmp_memn(&a2, &b2);
    438 	if (cmp) return cmp;
    439 
    440 	// skip over string
    441 	ca.p += sa;
    442 	cb.p += sb;
    443 
    444 	// timestamp
    445 	if (unlikely(!cursor_pull_varint(&ca, &sa) || !cursor_pull_varint(&cb, &sb)))
    446 		return 0;
    447 
    448 	if      (sa < sb) return -1;
    449 	else if (sa > sb) return 1;
    450 
    451 	// note_id
    452 	if      (nid_a < nid_b) return -1;
    453 	else if (nid_a > nid_b) return 1;
    454 
    455 	// word index
    456 	if (unlikely(!cursor_pull_varint(&ca, &sa) || !cursor_pull_varint(&cb, &sb)))
    457 		return 0;
    458 
    459 	if      (sa < sb) return -1;
    460 	else if (sa > sb) return 1;
    461 
    462 	return 0;
    463 }
    464 
    465 static inline int ndb_unpack_text_search_key_noteid(
    466 		struct cursor *cur, uint64_t *note_id)
    467 {
    468 	if (!cursor_pull_varint(cur, note_id))
    469 		return 0;
    470 
    471 	return 1;
    472 }
    473 
    474 // faster peek of just the string instead of unpacking everything
    475 // this is used to quickly discard range query matches if there is no
    476 // common prefix
    477 static inline int ndb_unpack_text_search_key_string(struct cursor *cur,
    478 						    const char **str,
    479 						    int *str_len)
    480 {
    481 	uint64_t len;
    482 
    483 	if (!cursor_pull_varint(cur, &len))
    484 		return 0;
    485 
    486 	*str_len = len;
    487 
    488 	*str = (const char *)cur->p;
    489 
    490 	if (!cursor_skip(cur, *str_len))
    491 		return 0;
    492 
    493 	return 1;
    494 }
    495 
    496 // should be called after ndb_unpack_text_search_key_string. It continues
    497 // the unpacking of a text search key if we've already started it.
    498 static inline int
    499 ndb_unpack_remaining_text_search_key(struct cursor *cur,
    500 				     struct ndb_text_search_key *key)
    501 {
    502 	if (!cursor_pull_varint(cur, &key->timestamp))
    503 		return 0;
    504 
    505 	if (!cursor_pull_varint(cur, &key->word_index))
    506 		return 0;
    507 
    508 	return 1;
    509 }
    510 
    511 // unpack a fulltext search key
    512 //
    513 // full version of string + unpack remaining. This is split up because text
    514 // searching only requires to pull the string for prefix searching, and the
    515 // remaining is optional
    516 static inline int ndb_unpack_text_search_key(unsigned char *p, int len,
    517 				      struct ndb_text_search_key *key)
    518 {
    519 	struct cursor c;
    520 	make_cursor(p, p + len, &c);
    521 
    522 	if (!ndb_unpack_text_search_key_noteid(&c, &key->note_id))
    523 		return 0;
    524 
    525 	if (!ndb_unpack_text_search_key_string(&c, &key->str, &key->str_len))
    526 		return 0;
    527 
    528 	return ndb_unpack_remaining_text_search_key(&c, key);
    529 }
    530 
    531 // Copies only lowercase characters to the destination string and fills the rest with null bytes.
    532 // `dst` and `src` are pointers to the destination and source strings, respectively.
    533 // `n` is the maximum number of characters to copy.
    534 static void lowercase_strncpy(char *dst, const char *src, int n) {
    535 	int j = 0, i = 0;
    536 
    537 	if (!dst || !src || n == 0) {
    538 		return;
    539 	}
    540 
    541 	while (src[i] != '\0' && j < n) {
    542 		dst[j++] = tolower(src[i++]);
    543 	}
    544 
    545 	// Null-terminate and fill the destination string
    546 	while (j < n) {
    547 		dst[j++] = '\0';
    548 	}
    549 }
    550 
    551 static inline int ndb_filter_elem_is_ptr(struct ndb_filter_field *field) {
    552 	return field->elem_type == NDB_ELEMENT_STRING || field->elem_type == NDB_ELEMENT_ID;
    553 }
    554 
    555 // Copy the filter
    556 int ndb_filter_clone(struct ndb_filter *dst, struct ndb_filter *src)
    557 {
    558 	size_t src_size, elem_size, data_size;
    559 
    560 	memcpy(dst, src, sizeof(*src));
    561 
    562 	elem_size = src->elem_buf.end - src->elem_buf.start;
    563 	data_size = src->data_buf.end - src->data_buf.start;
    564 	src_size = data_size + elem_size;
    565 
    566 	// let's only allow finalized filters to be cloned
    567 	if (!src || !src->finalized)
    568 		return 0;
    569 
    570 	dst->elem_buf.start = malloc(src_size);
    571 	dst->elem_buf.end = dst->elem_buf.start + elem_size;
    572 	dst->elem_buf.p = dst->elem_buf.end;
    573 
    574 	dst->data_buf.start = dst->elem_buf.start + elem_size;
    575 	dst->data_buf.end = dst->data_buf.start + data_size;
    576 	dst->data_buf.p = dst->data_buf.end;
    577 
    578 	if (dst->elem_buf.start == NULL)
    579 		return 0;
    580 
    581 	memcpy(dst->elem_buf.start, src->elem_buf.start, src_size);
    582 
    583 	return 1;
    584 }
    585 
    586 // "Finalize" the filter. This resizes the allocated heap buffers so that they
    587 // are as small as possible. This also prevents new fields from being added
    588 int ndb_filter_end(struct ndb_filter *filter)
    589 {
    590 #ifdef NDB_LOG
    591 	size_t orig_size;
    592 #endif
    593 	size_t data_len, elem_len;
    594 	unsigned char *rel;
    595 
    596 	assert(filter);
    597 	assert(filter->elem_buf.start);
    598 
    599 	if (filter->finalized == 1)
    600 		return 0;
    601 
    602 	// move the data buffer to the end of the element buffer and update
    603 	// all of the element pointers accordingly
    604 	data_len = filter->data_buf.p - filter->data_buf.start;
    605 	elem_len = filter->elem_buf.p - filter->elem_buf.start;
    606 #ifdef NDB_LOG
    607 	orig_size = filter->data_buf.end - filter->elem_buf.start;
    608 #endif
    609 
    610 	// cap the elem buff
    611 	filter->elem_buf.end = filter->elem_buf.p;
    612 
    613 	// move the data buffer to the end of the element buffer
    614 	memmove(filter->elem_buf.p, filter->data_buf.start, data_len);
    615 
    616 	// realloc the whole thing
    617 	rel = realloc(filter->elem_buf.start, elem_len + data_len);
    618 	if (rel) 
    619 		filter->elem_buf.start = rel;
    620 	assert(filter->elem_buf.start);
    621 	filter->elem_buf.end = filter->elem_buf.start + elem_len;
    622 	filter->elem_buf.p = filter->elem_buf.end;
    623 
    624 	filter->data_buf.start = filter->elem_buf.end;
    625 	filter->data_buf.end = filter->data_buf.start + data_len;
    626 	filter->data_buf.p = filter->data_buf.end;
    627 
    628 	filter->finalized = 1;
    629 
    630 	ndb_debug("ndb_filter_end: %ld -> %ld\n", orig_size, elem_len + data_len);
    631 
    632 	return 1;
    633 }
    634 
    635 static inline struct ndb_filter_elements *
    636 ndb_filter_get_elements_by_offset(const struct ndb_filter *filter, int offset)
    637 {
    638 	struct ndb_filter_elements *els;
    639 
    640 	if (offset < 0)
    641 		return NULL;
    642 
    643 	els = (struct ndb_filter_elements *)(filter->elem_buf.start + offset);
    644 
    645 	if ((unsigned char *)els > filter->elem_buf.p)
    646 		return NULL;
    647 
    648 	return els;
    649 }
    650 
    651 struct ndb_filter_elements *
    652 ndb_filter_current_element(const struct ndb_filter *filter)
    653 {
    654 	return ndb_filter_get_elements_by_offset(filter, filter->current);
    655 }
    656 
    657 struct ndb_filter_elements *
    658 ndb_filter_get_elements(const struct ndb_filter *filter, int index)
    659 {
    660 	if (filter->num_elements <= 0)
    661 		return NULL;
    662 
    663 	if (index > filter->num_elements-1)
    664 		return NULL;
    665 
    666 	return ndb_filter_get_elements_by_offset(filter, filter->elements[index]);
    667 }
    668 
    669 static inline unsigned char *
    670 ndb_filter_elements_data(const struct ndb_filter *filter, int offset)
    671 {
    672 	unsigned char *data;
    673 
    674 	if (offset < 0)
    675 		return NULL;
    676 
    677 	data = filter->data_buf.start + offset;
    678 	if (data > filter->data_buf.p)
    679 		return NULL;
    680 
    681 	return data;
    682 }
    683 
    684 struct ndb_filter_custom *
    685 ndb_filter_get_custom_element(const struct ndb_filter *filter, const struct ndb_filter_elements *els)
    686 {
    687 	return (struct ndb_filter_custom *)ndb_filter_elements_data(filter, els->elements[0]);
    688 }
    689 
    690 unsigned char *
    691 ndb_filter_get_id_element(const struct ndb_filter *filter, const struct ndb_filter_elements *els, int index)
    692 {
    693 	return ndb_filter_elements_data(filter, els->elements[index]);
    694 }
    695 
    696 const char *
    697 ndb_filter_get_string_element(const struct ndb_filter *filter, const struct ndb_filter_elements *els, int index)
    698 {
    699 	return (const char *)ndb_filter_elements_data(filter, els->elements[index]);
    700 }
    701 
    702 uint64_t *
    703 ndb_filter_get_int_element_ptr(struct ndb_filter_elements *els, int index)
    704 {
    705 	return &els->elements[index];
    706 }
    707 
    708 uint64_t
    709 ndb_filter_get_int_element(const struct ndb_filter_elements *els, int index)
    710 {
    711 	return els->elements[index];
    712 }
    713 
    714 int ndb_filter_init_with(struct ndb_filter *filter, int pages)
    715 {
    716 	struct cursor cur;
    717 	int page_size, elem_size, data_size, buf_size;
    718 
    719 	page_size = 4096; // assuming this, not a big deal if we're wrong
    720 
    721 	buf_size = page_size * pages;
    722 	elem_size = buf_size / 4;
    723 	data_size = buf_size - elem_size;
    724 
    725 	unsigned char *buf = malloc(buf_size);
    726 	if (!buf)
    727 		return 0;
    728 
    729 	// init memory arena for the cursor
    730 	make_cursor(buf, buf + buf_size, &cur);
    731 
    732 	cursor_slice(&cur, &filter->elem_buf, elem_size);
    733 	cursor_slice(&cur, &filter->data_buf, data_size);
    734 
    735 	// make sure we are fully allocated
    736 	assert(cur.p == cur.end);
    737 
    738 	// make sure elem_buf is the start of the buffer
    739 	assert(filter->elem_buf.start == cur.start);
    740 
    741 	filter->num_elements = 0;
    742 	filter->elements[0] = 0;
    743 	filter->current = -1;
    744 	filter->finalized = 0;
    745 
    746 	return 1;
    747 }
    748 
    749 int ndb_filter_init(struct ndb_filter *filter) {
    750 	return ndb_filter_init_with(filter, NDB_FILTER_PAGES);
    751 }
    752 
    753 void ndb_filter_destroy(struct ndb_filter *filter)
    754 {
    755 	if (filter->elem_buf.start)
    756 		free(filter->elem_buf.start);
    757 
    758 	memset(filter, 0, sizeof(*filter));
    759 }
    760 
    761 static const char *ndb_filter_field_name(enum ndb_filter_fieldtype field)
    762 {
    763 	switch (field) {
    764 	case NDB_FILTER_IDS: return "ids";
    765 	case NDB_FILTER_AUTHORS: return "authors";
    766 	case NDB_FILTER_KINDS: return "kinds";
    767 	case NDB_FILTER_TAGS: return "tags";
    768 	case NDB_FILTER_SINCE: return "since";
    769 	case NDB_FILTER_UNTIL: return "until";
    770 	case NDB_FILTER_LIMIT: return "limit";
    771 	case NDB_FILTER_SEARCH: return "search";
    772 	case NDB_FILTER_RELAYS: return "relays";
    773 	case NDB_FILTER_CUSTOM: return "custom";
    774 	}
    775 
    776 	return "unknown";
    777 }
    778 
    779 static int ndb_filter_start_field_impl(struct ndb_filter *filter, enum ndb_filter_fieldtype field, char tag)
    780 {
    781 	int i;
    782 	struct ndb_filter_elements *els, *el;
    783 
    784 	if (ndb_filter_current_element(filter)) {
    785 		fprintf(stderr, "ndb_filter_start_field: filter field already in progress, did you forget to call ndb_filter_end_field?\n");
    786 		return 0;
    787 	}
    788 
    789 	// you can only start and end fields once
    790 	for (i = 0; i < filter->num_elements; i++) {
    791 		el = ndb_filter_get_elements(filter, i);
    792 		assert(el);
    793 		// TODO: fix this tags check to try to find matching tags
    794 		if (el->field.type == field && field != NDB_FILTER_TAGS) {
    795 			fprintf(stderr, "ndb_filter_start_field: field '%s' already exists\n",
    796 					ndb_filter_field_name(field));
    797 			return 0;
    798 		}
    799 	}
    800 
    801 	filter->current = filter->elem_buf.p - filter->elem_buf.start;
    802 	els = ndb_filter_current_element(filter);
    803 	assert(els);
    804 
    805 	// advance elem buffer to the variable data section
    806 	if (!cursor_skip(&filter->elem_buf, sizeof(struct ndb_filter_elements))) {
    807 		fprintf(stderr, "ndb_filter_start_field: '%s' oom (todo: realloc?)\n",
    808 				ndb_filter_field_name(field));
    809 		return 0;
    810 	}
    811 
    812 	els->field.type = field;
    813 	els->field.tag = tag;
    814 	els->field.elem_type = 0;
    815 	els->count = 0;
    816 
    817 	return 1;
    818 }
    819 
    820 int ndb_filter_start_field(struct ndb_filter *filter, enum ndb_filter_fieldtype field)
    821 {
    822 	return ndb_filter_start_field_impl(filter, field, 0);
    823 }
    824 
    825 int ndb_filter_start_tag_field(struct ndb_filter *filter, char tag)
    826 {
    827 	return ndb_filter_start_field_impl(filter, NDB_FILTER_TAGS, tag);
    828 }
    829 
    830 static int ndb_filter_add_element(struct ndb_filter *filter, union ndb_filter_element el)
    831 {
    832 	struct ndb_filter_elements *current;
    833 	uint64_t offset;
    834 
    835 	if (!(current = ndb_filter_current_element(filter)))
    836 		return 0;
    837 
    838 	offset = filter->data_buf.p - filter->data_buf.start;
    839 
    840 	switch (current->field.type) {
    841 	case NDB_FILTER_CUSTOM:
    842 		if (!cursor_push(&filter->data_buf, (unsigned char *)&el, sizeof(el)))
    843 			return 0;
    844 		break;
    845 	case NDB_FILTER_IDS:
    846 	case NDB_FILTER_AUTHORS:
    847 		if (!cursor_push(&filter->data_buf, (unsigned char *)el.id, 32))
    848 			return 0;
    849 		break;
    850 	case NDB_FILTER_KINDS:
    851 		offset = el.integer;
    852 		break;
    853 	case NDB_FILTER_SINCE:
    854 	case NDB_FILTER_UNTIL:
    855 	case NDB_FILTER_LIMIT:
    856 		// only one allowed for since/until
    857 		if (current->count != 0)
    858 			return 0;
    859 		offset = el.integer;
    860 		break;
    861 	case NDB_FILTER_SEARCH:
    862 		if (current->field.elem_type != NDB_ELEMENT_STRING) {
    863 			return 0;
    864 		}
    865 		if (!cursor_push(&filter->data_buf, (unsigned char *)el.string.string, el.string.len))
    866 			return 0;
    867 		if (!cursor_push_byte(&filter->data_buf, 0))
    868 			return 0;
    869 		break;
    870 	case NDB_FILTER_TAGS:
    871 		switch (current->field.elem_type) {
    872 		case NDB_ELEMENT_ID:
    873 			if (!cursor_push(&filter->data_buf, (unsigned char *)el.id, 32))
    874 				return 0;
    875 			break;
    876 		case NDB_ELEMENT_STRING:
    877 			if (!cursor_push(&filter->data_buf, (unsigned char *)el.string.string, el.string.len))
    878 				return 0;
    879 			if (!cursor_push_byte(&filter->data_buf, 0))
    880 				return 0;
    881 			break;
    882 		case NDB_ELEMENT_INT:
    883 			// ints are not allowed in tag filters
    884 		case NDB_ELEMENT_UNKNOWN:
    885 		case NDB_ELEMENT_CUSTOM:
    886 			return 0;
    887 		}
    888 		// push a pointer of the string in the databuf as an element
    889 		break;
    890 	case NDB_FILTER_RELAYS:
    891 		if (current->field.elem_type != NDB_ELEMENT_STRING) {
    892 			return 0;
    893 		}
    894 		if (!cursor_push(&filter->data_buf, (unsigned char *)el.string.string, el.string.len))
    895 			return 0;
    896 		if (!cursor_push_byte(&filter->data_buf, 0))
    897 			return 0;
    898 		break;
    899 	}
    900 
    901 	if (!cursor_push(&filter->elem_buf, (unsigned char *)&offset,
    902 			 sizeof(offset))) {
    903 		return 0;
    904 	}
    905 
    906 	current->count++;
    907 
    908 	return 1;
    909 }
    910 
    911 static int ndb_filter_set_elem_type(struct ndb_filter *filter,
    912 				    enum ndb_generic_element_type elem_type)
    913 {
    914 	enum ndb_generic_element_type current_elem_type;
    915 	struct ndb_filter_elements *current;
    916 
    917 	if (!(current = ndb_filter_current_element(filter)))
    918 		return 0;
    919 
    920 	current_elem_type = current->field.elem_type;
    921 
    922 	// element types must be uniform
    923 	if (current_elem_type != elem_type && current_elem_type != NDB_ELEMENT_UNKNOWN) {
    924 		fprintf(stderr, "ndb_filter_set_elem_type: element types must be uniform\n");
    925 		return 0;
    926 	}
    927 
    928 	current->field.elem_type = elem_type;
    929 
    930 	return 1;
    931 }
    932 
    933 
    934 int ndb_filter_add_str_element_len(struct ndb_filter *filter, const char *str, int len)
    935 {
    936 	union ndb_filter_element el;
    937 	struct ndb_filter_elements *current;
    938 
    939 	if (!(current = ndb_filter_current_element(filter)))
    940 		return 0;
    941 
    942 	// only generic tags and search queries are allowed to have strings
    943 	switch (current->field.type) {
    944 	case NDB_FILTER_SINCE:
    945 	case NDB_FILTER_UNTIL:
    946 	case NDB_FILTER_LIMIT:
    947 	case NDB_FILTER_IDS:
    948 	case NDB_FILTER_AUTHORS:
    949 	case NDB_FILTER_KINDS:
    950 	case NDB_FILTER_CUSTOM:
    951 		return 0;
    952 	case NDB_FILTER_SEARCH:
    953 		if (current->count == 1) {
    954 			// you can't add more than one string to a search
    955 			return 0;
    956 		}
    957 		break;
    958 	case NDB_FILTER_RELAYS:
    959 	case NDB_FILTER_TAGS:
    960 		break;
    961 	}
    962 
    963 	if (!ndb_filter_set_elem_type(filter, NDB_ELEMENT_STRING))
    964 		return 0;
    965 
    966 	el.string.string = str;
    967 	el.string.len = len;
    968 
    969 	return ndb_filter_add_element(filter, el);
    970 }
    971 
    972 int ndb_filter_add_str_element(struct ndb_filter *filter, const char *str)
    973 {
    974 	return ndb_filter_add_str_element_len(filter, str, strlen(str));
    975 }
    976 
    977 int ndb_filter_add_custom_filter_element(struct ndb_filter *filter, ndb_filter_callback_fn *cb, void *ctx)
    978 {
    979 	union ndb_filter_element el;
    980 	struct ndb_filter_elements *current;
    981 	struct ndb_filter_custom custom;
    982 
    983 	custom.cb = cb;
    984 	custom.ctx = ctx;
    985 
    986 	if (!(current = ndb_filter_current_element(filter)))
    987 		return 0;
    988 
    989 	switch (current->field.type) {
    990 	case NDB_FILTER_CUSTOM:
    991 		break;
    992 	case NDB_FILTER_IDS:
    993 	case NDB_FILTER_AUTHORS:
    994 	case NDB_FILTER_TAGS:
    995 	case NDB_FILTER_SEARCH:
    996 	case NDB_FILTER_RELAYS:
    997 	case NDB_FILTER_KINDS:
    998 	case NDB_FILTER_SINCE:
    999 	case NDB_FILTER_UNTIL:
   1000 	case NDB_FILTER_LIMIT:
   1001 		return 0;
   1002 	}
   1003 
   1004 	if (!ndb_filter_set_elem_type(filter, NDB_ELEMENT_CUSTOM))
   1005 		return 0;
   1006 
   1007 	el.custom_filter = custom;
   1008 
   1009 	return ndb_filter_add_element(filter, el);
   1010 }
   1011 
   1012 int ndb_filter_add_int_element(struct ndb_filter *filter, uint64_t integer)
   1013 {
   1014 	union ndb_filter_element el;
   1015 	struct ndb_filter_elements *current;
   1016 	if (!(current = ndb_filter_current_element(filter)))
   1017 		return 0;
   1018 
   1019 	switch (current->field.type) {
   1020 	case NDB_FILTER_IDS:
   1021 	case NDB_FILTER_AUTHORS:
   1022 	case NDB_FILTER_TAGS:
   1023 	case NDB_FILTER_SEARCH:
   1024 	case NDB_FILTER_RELAYS:
   1025 	case NDB_FILTER_CUSTOM:
   1026 		return 0;
   1027 	case NDB_FILTER_KINDS:
   1028 	case NDB_FILTER_SINCE:
   1029 	case NDB_FILTER_UNTIL:
   1030 	case NDB_FILTER_LIMIT:
   1031 		break;
   1032 	}
   1033 
   1034 	if (!ndb_filter_set_elem_type(filter, NDB_ELEMENT_INT))
   1035 		return 0;
   1036 
   1037 	el.integer = integer;
   1038 
   1039 	return ndb_filter_add_element(filter, el);
   1040 }
   1041 
   1042 int ndb_filter_add_id_element(struct ndb_filter *filter, const unsigned char *id)
   1043 {
   1044 	union ndb_filter_element el;
   1045 	struct ndb_filter_elements *current;
   1046 
   1047 	if (!(current = ndb_filter_current_element(filter)))
   1048 		return 0;
   1049 
   1050 	// only certain filter types allow pushing id elements
   1051 	switch (current->field.type) {
   1052 	case NDB_FILTER_SINCE:
   1053 	case NDB_FILTER_UNTIL:
   1054 	case NDB_FILTER_LIMIT:
   1055 	case NDB_FILTER_KINDS:
   1056 	case NDB_FILTER_SEARCH:
   1057 	case NDB_FILTER_RELAYS:
   1058 	case NDB_FILTER_CUSTOM:
   1059 		return 0;
   1060 	case NDB_FILTER_IDS:
   1061 	case NDB_FILTER_AUTHORS:
   1062 	case NDB_FILTER_TAGS:
   1063 		break;
   1064 	}
   1065 
   1066 	if (!ndb_filter_set_elem_type(filter, NDB_ELEMENT_ID))
   1067 		return 0;
   1068 
   1069 	// this is needed so that generic filters know its an id
   1070 	el.id = id;
   1071 
   1072 	return ndb_filter_add_element(filter, el);
   1073 }
   1074 
   1075 static int ndb_tag_filter_matches(struct ndb_filter *filter,
   1076 				  struct ndb_filter_elements *els,
   1077 				  struct ndb_note *note)
   1078 {
   1079 	int i;
   1080 	const unsigned char *id;
   1081 	const char *el_str;
   1082 	struct ndb_iterator iter, *it = &iter;
   1083 	struct ndb_str str;
   1084 
   1085 	ndb_tags_iterate_start(note, it);
   1086 
   1087 	while (ndb_tags_iterate_next(it)) {
   1088 		// we're looking for tags with 2 or more entries: ["p", id], etc
   1089 		if (it->tag->count < 2)
   1090 			continue;
   1091 
   1092 		str = ndb_tag_str(note, it->tag, 0);
   1093 
   1094 		// we only care about packed strings (single char, etc)
   1095 		if (str.flag != NDB_PACKED_STR)
   1096 			continue;
   1097 
   1098 		// do we have #e matching e (or p, etc)
   1099 		if (str.str[0] != els->field.tag || str.str[1] != 0)
   1100 			continue;
   1101 
   1102 		str = ndb_tag_str(note, it->tag, 1);
   1103 
   1104 		switch (els->field.elem_type) {
   1105 		case NDB_ELEMENT_ID:
   1106 			// if our filter element type is an id, then we
   1107 			// expect a packed id in the tag, otherwise skip
   1108 			if (str.flag != NDB_PACKED_ID)
   1109 				continue;
   1110 			break;
   1111 		case NDB_ELEMENT_STRING:
   1112 			// if our filter element type is a string, then
   1113 			// we should not expect an id
   1114 			if (str.flag == NDB_PACKED_ID)
   1115 				continue;
   1116 
   1117 			break;
   1118 		case NDB_ELEMENT_UNKNOWN:
   1119 		default:
   1120 			// For some reason the element type is not set. It's
   1121 			// possible nothing was added to the generic filter?
   1122 			// Let's just fail here and log a note for debugging
   1123 			fprintf(stderr, "UNUSUAL ndb_tag_filter_matches: have unknown element type %d\n", els->field.elem_type);
   1124 			return 0;
   1125 		}
   1126 
   1127 		for (i = 0; i < els->count; i++) {
   1128 			switch (els->field.elem_type) {
   1129 			case NDB_ELEMENT_ID:
   1130 				id = ndb_filter_get_id_element(filter, els, i);
   1131 				if (!memcmp(id, str.id, 32))
   1132 					return 1;
   1133 				break;
   1134 			case NDB_ELEMENT_STRING:
   1135 				el_str = ndb_filter_get_string_element(filter, els, i);
   1136 				if (!strcmp(el_str, str.str))
   1137 					return 1;
   1138 				break;
   1139 			case NDB_ELEMENT_INT:
   1140 				// int elements int tag queries are not supported
   1141 			case NDB_ELEMENT_UNKNOWN:
   1142 			case NDB_ELEMENT_CUSTOM:
   1143 				return 0;
   1144 			}
   1145 		}
   1146 	}
   1147 
   1148 	return 0;
   1149 }
   1150 
   1151 struct search_id_state {
   1152 	struct ndb_filter *filter;
   1153 	struct ndb_filter_elements *els;
   1154 	unsigned char *key;
   1155 };
   1156 
   1157 static int compare_ids(const void *pa, const void *pb)
   1158 {
   1159         unsigned char *a = *(unsigned char **)pa;
   1160         unsigned char *b = *(unsigned char **)pb;
   1161 
   1162         return memcmp(a, b, 32);
   1163 }
   1164 
   1165 static int compare_strs(const void *pa, const void *pb)
   1166 {
   1167         const char *a = *(const char **)pa;
   1168         const char *b = *(const char **)pb;
   1169 
   1170         return strcmp(a, b);
   1171 }
   1172 
   1173 static int search_strs(const void *ctx, const void *mstr_ptr)
   1174 {
   1175 	// we reuse search_id_state here and just cast to (const char *) when
   1176 	// needed
   1177 	struct search_id_state *state;
   1178 	const char *mstr_str;
   1179 	uint32_t mstr;
   1180 
   1181 	state = (struct search_id_state *)ctx;
   1182 	mstr = *(uint32_t *)mstr_ptr;
   1183 
   1184 	mstr_str = (const char *)ndb_filter_elements_data(state->filter, mstr);
   1185 	assert(mstr_str);
   1186 
   1187 	return strcmp((const char *)state->key, mstr_str);
   1188 }
   1189 
   1190 static int search_ids(const void *ctx, const void *mid_ptr)
   1191 {
   1192 	struct search_id_state *state;
   1193 	unsigned char *mid_id;
   1194 	uint32_t mid;
   1195 
   1196 	state = (struct search_id_state *)ctx;
   1197 	mid = *(uint32_t *)mid_ptr;
   1198 
   1199 	mid_id = ndb_filter_elements_data(state->filter, mid);
   1200 	assert(mid_id);
   1201 
   1202 	return memcmp(state->key, mid_id, 32);
   1203 }
   1204 
   1205 static int compare_kinds(const void *pa, const void *pb)
   1206 {
   1207 
   1208 	// NOTE: this should match type in `union ndb_filter_element`
   1209 	uint64_t a = *(uint64_t *)pa;
   1210 	uint64_t b = *(uint64_t *)pb;
   1211 
   1212 	if (a < b) {
   1213 		return -1;
   1214 	} else if (a > b) {
   1215 		return 1;
   1216 	} else {
   1217 		return 0;
   1218 	}
   1219 }
   1220 
   1221 //
   1222 // returns 1 if a filter matches a note
   1223 static int ndb_filter_matches_with(struct ndb_filter *filter,
   1224 				   struct ndb_note *note, int already_matched,
   1225 				   struct ndb_note_relay_iterator *relay_iter)
   1226 {
   1227 	int i, j;
   1228 	struct ndb_filter_elements *els;
   1229 	struct search_id_state state;
   1230 	struct ndb_filter_custom *custom;
   1231 
   1232 	state.filter = filter;
   1233 
   1234 	for (i = 0; i < filter->num_elements; i++) {
   1235 		els = ndb_filter_get_elements(filter, i);
   1236 		state.els = els;
   1237 		assert(els);
   1238 
   1239 		// if we know we already match from a query scan result,
   1240 		// we can skip this check
   1241 		if ((1 << els->field.type) & already_matched)
   1242 			continue;
   1243 
   1244 		switch (els->field.type) {
   1245                 case NDB_FILTER_KINDS:
   1246                         for (j = 0; j < els->count; j++) {
   1247                                 if ((unsigned int)els->elements[j] == note->kind)
   1248                                         goto cont;
   1249 			}
   1250 			break;
   1251 		case NDB_FILTER_RELAYS:
   1252 			// for each relay the note was seen on, see if any match 
   1253 			if (!relay_iter) {
   1254 				assert(!"expected relay iterator...");
   1255 				break;
   1256 			}
   1257 			while ((state.key = (unsigned char *)ndb_note_relay_iterate_next(relay_iter))) {
   1258 				// relays in filters are always sorted
   1259 				if (bsearch(&state, &els->elements[0], els->count,
   1260 					    sizeof(els->elements[0]), search_strs)) {
   1261 					ndb_note_relay_iterate_close(relay_iter);
   1262 					goto cont;
   1263 				}
   1264 			}
   1265 			ndb_note_relay_iterate_close(relay_iter);
   1266 			break;
   1267 		case NDB_FILTER_IDS:
   1268 			state.key = ndb_note_id(note);
   1269 			if (bsearch(&state, &els->elements[0], els->count,
   1270 				    sizeof(els->elements[0]), search_ids)) {
   1271 				continue;
   1272 			}
   1273 			break;
   1274 		case NDB_FILTER_AUTHORS:
   1275 			state.key = ndb_note_pubkey(note);
   1276 			if (bsearch(&state, &els->elements[0], els->count,
   1277 				    sizeof(els->elements[0]), search_ids)) {
   1278 				continue;
   1279 			}
   1280 			break;
   1281 		case NDB_FILTER_TAGS:
   1282 			if (ndb_tag_filter_matches(filter, els, note))
   1283 				continue;
   1284 			break;
   1285 		case NDB_FILTER_SINCE:
   1286 			assert(els->count == 1);
   1287 			if (note->created_at >= els->elements[0])
   1288 				continue;
   1289 			break;
   1290 		case NDB_FILTER_UNTIL:
   1291 			assert(els->count == 1);
   1292 			if (note->created_at < els->elements[0])
   1293 				continue;
   1294 			break;
   1295 		case NDB_FILTER_SEARCH:
   1296 			// TODO: matching search filters will need an accelerated
   1297 			// data structure, like our minimal perfect hashmap
   1298 			// idea for mutewords.
   1299 			//
   1300 			// We'll also want to store tokenized words in the filter
   1301 			// itself, so that we can walk over each word and check
   1302 			// the hashmap to see if the note contains at least
   1303 			// one word.
   1304 			//
   1305 			// For now we always return true, since we assume
   1306 			// the search index will be walked for these kinds
   1307 			// of queries.
   1308 			continue;
   1309 		case NDB_FILTER_CUSTOM:
   1310 			custom = ndb_filter_get_custom_element(filter, els);
   1311 			if (custom->cb(custom->ctx, note))
   1312 				continue;
   1313 			break;
   1314 
   1315 		case NDB_FILTER_LIMIT:
   1316 cont:
   1317 			continue;
   1318 		}
   1319 
   1320 		// all need to match
   1321 		return 0;
   1322 	}
   1323 
   1324 	return 1;
   1325 }
   1326 
   1327 int ndb_filter_matches(struct ndb_filter *filter, struct ndb_note *note)
   1328 {
   1329 	return ndb_filter_matches_with(filter, note, 0, NULL);
   1330 }
   1331 
   1332 int ndb_filter_matches_with_relay(struct ndb_filter *filter,
   1333 				  struct ndb_note *note,
   1334 				  struct ndb_note_relay_iterator *note_relay_iter)
   1335 {
   1336 	return ndb_filter_matches_with(filter, note, 0, note_relay_iter);
   1337 }
   1338 
   1339 // because elements are stored as offsets and qsort doesn't support context,
   1340 // we do a dumb thing where we convert elements to pointers and back if we
   1341 // are doing an id or string sort
   1342 static void sort_filter_elements(struct ndb_filter *filter,
   1343 				 struct ndb_filter_elements *els,
   1344 				 int (*cmp)(const void *, const void *))
   1345 {
   1346 	int i;
   1347 
   1348 	assert(ndb_filter_elem_is_ptr(&els->field));
   1349 
   1350 	for (i = 0; i < els->count; i++)
   1351 		els->elements[i] += (uint64_t)filter->data_buf.start;
   1352 
   1353 	qsort(&els->elements[0], els->count, sizeof(els->elements[0]), cmp);
   1354 
   1355 	for (i = 0; i < els->count; i++)
   1356 		els->elements[i] -= (uint64_t)filter->data_buf.start;
   1357 }
   1358 
   1359 static int ndb_filter_field_eq(struct ndb_filter *a_filt,
   1360 			       struct ndb_filter_elements *a_field,
   1361 			       struct ndb_filter *b_filt,
   1362 			       struct ndb_filter_elements *b_field)
   1363 {
   1364 	int i;
   1365 	const char *a_str, *b_str;
   1366 	unsigned char *a_id, *b_id;
   1367 	uint64_t a_int, b_int;
   1368 	struct ndb_filter_custom *a_custom, *b_custom;
   1369 
   1370 	if (a_field->count != b_field->count)
   1371 		return 0;
   1372 
   1373 	if (a_field->field.type != b_field->field.type) {
   1374 		ndb_debug("UNUSUAL: field types do not match in ndb_filter_field_eq\n");
   1375 		return 0;
   1376 	}
   1377 
   1378 	if (a_field->field.elem_type != b_field->field.elem_type) {
   1379 		ndb_debug("UNUSUAL: field element types do not match in ndb_filter_field_eq\n");
   1380 		return 0;
   1381 	}
   1382 
   1383 	if (a_field->field.elem_type == NDB_ELEMENT_UNKNOWN) {
   1384 		ndb_debug("UNUSUAL: field element types are unknown\n");
   1385 		return 0;
   1386 	}
   1387 
   1388 	for (i = 0; i < a_field->count; i++) {
   1389 		switch (a_field->field.elem_type) {
   1390 		case NDB_ELEMENT_CUSTOM:
   1391 			a_custom = ndb_filter_get_custom_element(a_filt, a_field);
   1392 			b_custom = ndb_filter_get_custom_element(b_filt, b_field);
   1393 			if (memcmp(a_custom, b_custom, sizeof(*a_custom)))
   1394 				return 0;
   1395 			break;
   1396 		case NDB_ELEMENT_UNKNOWN:
   1397 			return 0;
   1398 		case NDB_ELEMENT_STRING:
   1399 			a_str = ndb_filter_get_string_element(a_filt, a_field, i);
   1400 			b_str = ndb_filter_get_string_element(b_filt, b_field, i);
   1401 			if (strcmp(a_str, b_str))
   1402 				return 0;
   1403 			break;
   1404 		case NDB_ELEMENT_ID:
   1405 			a_id = ndb_filter_get_id_element(a_filt, a_field, i);
   1406 			b_id = ndb_filter_get_id_element(b_filt, b_field, i);
   1407 			if (memcmp(a_id, b_id, 32))
   1408 				return 0;
   1409 			break;
   1410 		case NDB_ELEMENT_INT:
   1411 			a_int = ndb_filter_get_int_element(a_field, i);
   1412 			b_int = ndb_filter_get_int_element(b_field, i);
   1413 			if (a_int != b_int)
   1414 				return 0;
   1415 			break;
   1416 		}
   1417 	}
   1418 
   1419 	return 1;
   1420 }
   1421 
   1422 void ndb_filter_end_field(struct ndb_filter *filter)
   1423 {
   1424 	int cur_offset;
   1425 	struct ndb_filter_elements *cur;
   1426 
   1427 	cur_offset = filter->current;
   1428 
   1429 	if (!(cur = ndb_filter_current_element(filter)))
   1430 		return;
   1431 
   1432 	filter->elements[filter->num_elements++] = cur_offset;
   1433 
   1434 	// sort elements for binary search
   1435 	switch (cur->field.type) {
   1436 	case NDB_FILTER_IDS:
   1437 	case NDB_FILTER_AUTHORS:
   1438 		sort_filter_elements(filter, cur, compare_ids);
   1439 		break;
   1440 	case NDB_FILTER_RELAYS:
   1441 		sort_filter_elements(filter, cur, compare_strs);
   1442 		break;
   1443 	case NDB_FILTER_KINDS:
   1444 		qsort(&cur->elements[0], cur->count,
   1445 		      sizeof(cur->elements[0]), compare_kinds);
   1446 		break;
   1447 	case NDB_FILTER_TAGS:
   1448 		// TODO: generic tag search sorting
   1449 		break;
   1450 	case NDB_FILTER_SINCE:
   1451 	case NDB_FILTER_CUSTOM:
   1452 	case NDB_FILTER_UNTIL:
   1453 	case NDB_FILTER_LIMIT:
   1454 	case NDB_FILTER_SEARCH:
   1455 		// don't need to sort these
   1456 		break;
   1457 	}
   1458 
   1459 	filter->current = -1;
   1460 
   1461 }
   1462 
   1463 static void ndb_filter_group_init(struct ndb_filter_group *group)
   1464 {
   1465 	group->num_filters = 0;
   1466 }
   1467 
   1468 static int ndb_filter_group_add(struct ndb_filter_group *group,
   1469 				struct ndb_filter *filter)
   1470 {
   1471 	if (group->num_filters + 1 > MAX_FILTERS)
   1472 		return 0;
   1473 
   1474 	return ndb_filter_clone(&group->filters[group->num_filters++], filter);
   1475 }
   1476 
   1477 static int ndb_filter_group_matches(struct ndb_filter_group *group,
   1478 				    struct ndb_note *note)
   1479 {
   1480 	int i;
   1481 	struct ndb_filter *filter;
   1482 
   1483 	if (group->num_filters == 0)
   1484 		return 1;
   1485 
   1486 	for (i = 0; i < group->num_filters; i++) {
   1487 		filter = &group->filters[i];
   1488 
   1489 		if (ndb_filter_matches(filter, note))
   1490 			return 1;
   1491 	}
   1492 
   1493 	return 0;
   1494 }
   1495 
   1496 static void ndb_make_search_key(struct ndb_search_key *key, unsigned char *id,
   1497 			        uint64_t timestamp, const char *search)
   1498 {
   1499 	memcpy(key->id, id, 32);
   1500 	key->timestamp = timestamp;
   1501 	lowercase_strncpy(key->search, search, sizeof(key->search) - 1);
   1502 	key->search[sizeof(key->search) - 1] = '\0';
   1503 }
   1504 
   1505 static int ndb_write_profile_search_index(struct ndb_txn *txn,
   1506 					  struct ndb_search_key *index_key,
   1507 					  uint64_t profile_key)
   1508 {
   1509 	int rc;
   1510 	MDB_val key, val;
   1511 
   1512 	key.mv_data = index_key;
   1513 	key.mv_size = sizeof(*index_key);
   1514 	val.mv_data = &profile_key;
   1515 	val.mv_size = sizeof(profile_key);
   1516 
   1517 	if ((rc = mdb_put(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_PROFILE_SEARCH],
   1518 			  &key, &val, 0)))
   1519 	{
   1520 		ndb_debug("ndb_write_profile_search_index failed: %s\n",
   1521 			  mdb_strerror(rc));
   1522 		return 0;
   1523 	}
   1524 
   1525 	return 1;
   1526 }
   1527 
   1528 
   1529 // map usernames and display names to profile keys for user searching
   1530 static int ndb_write_profile_search_indices(struct ndb_txn *txn,
   1531 					    struct ndb_note *note,
   1532 					    uint64_t profile_key,
   1533 					    void *profile_root)
   1534 {
   1535 	struct ndb_search_key index;
   1536 	NdbProfileRecord_table_t profile_record;
   1537 	NdbProfile_table_t profile;
   1538 
   1539 	profile_record = NdbProfileRecord_as_root(profile_root);
   1540 	profile = NdbProfileRecord_profile_get(profile_record);
   1541 
   1542 	const char *name = NdbProfile_name_get(profile);
   1543 	const char *display_name = NdbProfile_display_name_get(profile);
   1544 
   1545 	// words + pubkey + created
   1546 	if (name) {
   1547 		ndb_make_search_key(&index, note->pubkey, note->created_at,
   1548 				    name);
   1549 		if (!ndb_write_profile_search_index(txn, &index, profile_key))
   1550 			return 0;
   1551 	}
   1552 
   1553 	if (display_name) {
   1554 		// don't write the same name/display_name twice
   1555 		if (name && !strcmp(display_name, name)) {
   1556 			return 1;
   1557 		}
   1558 		ndb_make_search_key(&index, note->pubkey, note->created_at,
   1559 				    display_name);
   1560 		if (!ndb_write_profile_search_index(txn, &index, profile_key))
   1561 			return 0;
   1562 	}
   1563 
   1564 	return 1;
   1565 }
   1566 
   1567 static inline void ndb_tsid_init(struct ndb_tsid *key, unsigned char *id,
   1568 				 uint64_t timestamp)
   1569 {
   1570 	memcpy(key->id, id, 32);
   1571 	key->timestamp = timestamp;
   1572 }
   1573 
   1574 static inline void ndb_tsid_low(struct ndb_tsid *key, unsigned char *id)
   1575 {
   1576 	memcpy(key->id, id, 32);
   1577 	key->timestamp = 0;
   1578 }
   1579 
   1580 static inline void ndb_u64_ts_init(struct ndb_u64_ts *key, uint64_t integer,
   1581 				     uint64_t timestamp)
   1582 {
   1583 	key->u64 = integer;
   1584 	key->timestamp = timestamp;
   1585 }
   1586 
   1587 // useful for range-searching for the latest key with a clustered created_at timen
   1588 static inline void ndb_tsid_high(struct ndb_tsid *key, const unsigned char *id)
   1589 {
   1590 	memcpy(key->id, id, 32);
   1591 	key->timestamp = UINT64_MAX;
   1592 }
   1593 
   1594 static int _ndb_begin_query(struct ndb *ndb, struct ndb_txn *txn, int flags)
   1595 {
   1596 	txn->lmdb = &ndb->lmdb;
   1597 	MDB_txn **mdb_txn = (MDB_txn **)&txn->mdb_txn;
   1598 	if (!txn->lmdb->env)
   1599 		return 0;
   1600 	return mdb_txn_begin(txn->lmdb->env, NULL, flags, mdb_txn) == 0;
   1601 }
   1602 
   1603 int ndb_begin_query(struct ndb *ndb, struct ndb_txn *txn)
   1604 {
   1605 	return _ndb_begin_query(ndb, txn, MDB_RDONLY);
   1606 }
   1607 
   1608 // this should only be used in migrations, etc
   1609 static int ndb_begin_rw_query(struct ndb *ndb, struct ndb_txn *txn)
   1610 {
   1611 	return _ndb_begin_query(ndb, txn, 0);
   1612 }
   1613 
   1614 static int ndb_db_is_index(enum ndb_dbs index)
   1615 {
   1616 	switch (index) {
   1617 		case NDB_DB_NOTE:
   1618 		case NDB_DB_META:
   1619 		case NDB_DB_PROFILE:
   1620 		case NDB_DB_NOTE_ID:
   1621 		case NDB_DB_NDB_META:
   1622 		case NDB_DB_PROFILE_SEARCH:
   1623 		case NDB_DB_PROFILE_LAST_FETCH:
   1624 		case NDB_DB_NOTE_RELAYS:
   1625 		case NDB_DBS:
   1626 			return 0;
   1627 		case NDB_DB_PROFILE_PK:
   1628 		case NDB_DB_NOTE_KIND:
   1629 		case NDB_DB_NOTE_TEXT:
   1630 		case NDB_DB_NOTE_BLOCKS:
   1631 		case NDB_DB_NOTE_TAGS:
   1632 		case NDB_DB_NOTE_PUBKEY:
   1633 		case NDB_DB_NOTE_PUBKEY_KIND:
   1634 		case NDB_DB_NOTE_RELAY_KIND:
   1635 			return 1;
   1636 	}
   1637 
   1638 	return 0;
   1639 }
   1640 
   1641 static inline void ndb_id_u64_ts_init(struct ndb_id_u64_ts *key,
   1642 				      unsigned char *id, uint64_t iu64,
   1643 				      uint64_t timestamp)
   1644 {
   1645 	memcpy(key->id, id, 32);
   1646 	key->u64 = iu64;
   1647 	key->timestamp = timestamp;
   1648 }
   1649 
   1650 // formats the relay url buffer for the NDB_DB_NOTE_RELAYS value. It's a
   1651 // null terminated string padded to 8 bytes (we must keep the entire database
   1652 // aligned to 8 bytes at all times)
   1653 static int prepare_relay_buf(char *relay_buf, int bufsize, const char *relay,
   1654 			     int relay_len)
   1655 {
   1656 	struct cursor cur;
   1657 
   1658 	// make sure the size of the buffer is aligned
   1659 	assert((bufsize % 8) == 0);
   1660 
   1661 	make_cursor((unsigned char *)relay_buf, (unsigned char *)relay_buf + bufsize, &cur);
   1662 
   1663 	// push the relay string
   1664 	if (!cursor_push(&cur, (unsigned char *)relay, relay_len))
   1665 		return 0;
   1666 
   1667 	// relay urls are null terminated for convenience
   1668 	if (!cursor_push_byte(&cur, 0))
   1669 		return 0;
   1670 
   1671 	// align the buffer
   1672 	if (!cursor_align(&cur, 8))
   1673 		return 0;
   1674 
   1675 	return cur.p - cur.start;
   1676 }
   1677 
   1678 // Write to the note_id -> relay_url database. This records where notes
   1679 // have been seen
   1680 static int ndb_write_note_relay(struct ndb_txn *txn, uint64_t note_key,
   1681 				const char *relay, uint8_t relay_len)
   1682 {
   1683 	char relay_buf[256];
   1684 	int rc, len;
   1685 	MDB_val k, v;
   1686 
   1687 	if (relay == NULL || relay_len == 0) {
   1688 		ndb_debug("relay is NULL in ndb_write_note_relay? '%s' %d\n", relay, relay_len);
   1689 		return 0;
   1690 	}
   1691 
   1692 	ndb_debug("writing note_relay '%s' for notekey:%" PRIu64 "\n", relay, note_key);
   1693 
   1694 	if (!(len = prepare_relay_buf(relay_buf, sizeof(relay_buf), relay, relay_len))) {
   1695 		fprintf(stderr, "relay url '%s' too large when writing note relay index\n", relay);
   1696 		return 0;
   1697 	}
   1698 
   1699 	assert((len % 8) == 0);
   1700 
   1701 	k.mv_data = &note_key;
   1702 	k.mv_size = sizeof(note_key);
   1703 
   1704 	v.mv_data = relay_buf;
   1705 	v.mv_size = len;
   1706 
   1707 	// NODUPDATA is specified so that we don't accidently add duplicate
   1708 	// key/value pairs
   1709 	if ((rc = mdb_put(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_NOTE_RELAYS],
   1710 			  &k, &v, MDB_NODUPDATA)))
   1711 	{
   1712 		ndb_debug("ndb_write_note_relay failed: %s\n", mdb_strerror(rc));
   1713 		return 0;
   1714 	}
   1715 
   1716 	ndb_debug("wrote %d bytes to note relay: '%s'\n", len, relay_buf);
   1717 
   1718 	return 1;
   1719 }
   1720 
   1721 struct ndb_relay_kind_key {
   1722 	uint64_t note_key;
   1723 	uint64_t kind;
   1724 	uint64_t created_at;
   1725 	uint8_t relay_len;
   1726 	const char *relay;
   1727 };
   1728 
   1729 static int ndb_relay_kind_key_init(
   1730 		struct ndb_relay_kind_key *key,
   1731 		uint64_t note_key,
   1732 		uint64_t kind,
   1733 		uint64_t created_at,
   1734 		const char *relay)
   1735 {
   1736 	if (relay == NULL)
   1737 		return 0;
   1738 
   1739 	key->relay = relay;
   1740 	key->relay_len = strlen(relay);
   1741 	if (key->relay_len > 248)
   1742 		return 0;
   1743 
   1744 	key->note_key = note_key;
   1745 	key->kind = kind;
   1746 	key->created_at = created_at;
   1747 	return 1;
   1748 }
   1749 
   1750 
   1751 // create a range key for a relay kind query
   1752 static int ndb_relay_kind_key_init_high(
   1753 		struct ndb_relay_kind_key *key,
   1754 		const char *relay,
   1755 		uint64_t kind,
   1756 		uint64_t until)
   1757 {
   1758 	return ndb_relay_kind_key_init(key, UINT64_MAX, kind, UINT64_MAX, relay);
   1759 }
   1760 
   1761 static void ndb_parse_relay_kind_key(struct ndb_relay_kind_key *key, unsigned char *buf)
   1762 {
   1763 	// WE ARE ASSUMING WE ARE PARSING FROM AN ALIGNED BUFFER HERE
   1764 	assert((uint64_t)buf % 8 == 0);
   1765 	// - note_id:        00 + 8 bytes
   1766 	// - kind:           08 + 8 bytes
   1767 	// - created_at:     16 + 8 bytes
   1768 	// - relay_url_size: 24 + 1 byte
   1769 	// - relay_url:      25 + n byte null-terminated string
   1770 	// - pad to 8 byte alignment
   1771 	key->note_key   =  *(uint64_t*) (buf + 0);
   1772 	key->kind       =  *(uint64_t*) (buf + 8);
   1773 	key->created_at =  *(uint64_t*) (buf + 16);
   1774 	key->relay_len  =   *(uint8_t*) (buf + 24);
   1775 	key->relay      = (const char*) (buf + 25);
   1776 }
   1777 
   1778 static void ndb_debug_relay_kind_key(struct ndb_relay_kind_key *key)
   1779 {
   1780 	ndb_debug("note_key:%" PRIu64 " kind:%" PRIu64 " created_at:%" PRIu64 " '%s'\n",
   1781 			key->note_key, key->kind, key->created_at, key->relay);
   1782 }
   1783 
   1784 static int ndb_build_relay_kind_key(unsigned char *buf, int bufsize, struct ndb_relay_kind_key *key)
   1785 {
   1786 	struct cursor cur;
   1787 	make_cursor(buf, buf + bufsize, &cur);
   1788 
   1789 	if (!cursor_push(&cur, (unsigned char *)&key->note_key, 8))   return 0;
   1790 	if (!cursor_push(&cur, (unsigned char *)&key->kind, 8))       return 0;
   1791 	if (!cursor_push(&cur, (unsigned char *)&key->created_at, 8)) return 0;
   1792 	if (!cursor_push_byte(&cur, key->relay_len)) return 0;
   1793 	if (!cursor_push(&cur, (unsigned char *)key->relay, key->relay_len)) return 0;
   1794 	if (!cursor_push_byte(&cur, 0)) return 0;
   1795 	if (!cursor_align(&cur, 8)) return 0;
   1796 
   1797 	assert(((cur.p-cur.start)%8) == 0);
   1798 
   1799 	return cur.p - cur.start;
   1800 }
   1801 
   1802 static int ndb_write_note_relay_kind_index(
   1803 		struct ndb_txn *txn,
   1804 		struct ndb_relay_kind_key *key)
   1805 {
   1806 	// The relay kind key has a layout like so 
   1807 	//
   1808 	// - note_key:       00 + 8 bytes
   1809 	// - kind:           08 + 8 bytes
   1810 	// - created_at:     16 + 8 bytes
   1811 	// - relay_url_size: 24 + 1 byte
   1812 	// - relay_url:      25 + n byte null-terminated string
   1813 	// - pad to 8 byte alignment
   1814 
   1815 	unsigned char buf[256];
   1816 	int rc, len;
   1817 	MDB_val k, v;
   1818 
   1819 	// come on bro
   1820 	if (key->relay_len > 248 || key->relay == NULL || key->relay_len == 0)
   1821 		return 0;
   1822 
   1823 	ndb_debug("writing note_relay_kind_index '%s' for notekey:%" PRIu64 "\n", key->relay, key->note_key);
   1824 
   1825 	if (!(len = ndb_build_relay_kind_key(buf, sizeof(buf), key)))
   1826 		return 0;
   1827 
   1828 	k.mv_data = buf;
   1829 	k.mv_size = len;
   1830 
   1831 	v.mv_data = NULL;
   1832 	v.mv_size = 0;
   1833 
   1834 	if ((rc = mdb_put(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_NOTE_RELAY_KIND], &k, &v, 0))) {
   1835 		fprintf(stderr, "write note relay kind index failed: %s\n",
   1836 			  mdb_strerror(rc));
   1837 		return 0;
   1838 	}
   1839 
   1840 	return 1;
   1841 }
   1842 
   1843 // writes the relay note kind index and the note_id -> relay db
   1844 static int ndb_write_note_relay_indexes(struct ndb_txn *txn, struct ndb_relay_kind_key *key)
   1845 {
   1846 	ndb_write_note_relay_kind_index(txn, key);
   1847 	ndb_write_note_relay(txn, key->note_key, key->relay, key->relay_len);
   1848 	return 1;
   1849 }
   1850 
   1851 static int ndb_write_note_pubkey_index(struct ndb_txn *txn, struct ndb_note *note,
   1852 				       uint64_t note_key)
   1853 {
   1854 	int rc;
   1855 	struct ndb_tsid key;
   1856 	MDB_val k, v;
   1857 
   1858 	ndb_tsid_init(&key, ndb_note_pubkey(note), ndb_note_created_at(note));
   1859 
   1860 	k.mv_data = &key;
   1861 	k.mv_size = sizeof(key);
   1862 
   1863 	v.mv_data = &note_key;
   1864 	v.mv_size = sizeof(note_key);
   1865 
   1866 	if ((rc = mdb_put(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_NOTE_PUBKEY], &k, &v, 0))) {
   1867 		fprintf(stderr, "write note pubkey index failed: %s\n",
   1868 			  mdb_strerror(rc));
   1869 		return 0;
   1870 	}
   1871 
   1872 	return 1;
   1873 }
   1874 
   1875 static int ndb_write_note_pubkey_kind_index(struct ndb_txn *txn,
   1876 					    struct ndb_note *note,
   1877 					    uint64_t note_key)
   1878 {
   1879 	int rc;
   1880 	struct ndb_id_u64_ts key;
   1881 	MDB_val k, v;
   1882 
   1883 	ndb_id_u64_ts_init(&key, ndb_note_pubkey(note), ndb_note_kind(note),
   1884 			   ndb_note_created_at(note));
   1885 
   1886 	k.mv_data = &key;
   1887 	k.mv_size = sizeof(key);
   1888 
   1889 	v.mv_data = &note_key;
   1890 	v.mv_size = sizeof(note_key);
   1891 
   1892 	if ((rc = mdb_put(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_NOTE_PUBKEY_KIND], &k, &v, 0))) {
   1893 		fprintf(stderr, "write note pubkey_kind index failed: %s\n",
   1894 			  mdb_strerror(rc));
   1895 		return 0;
   1896 	}
   1897 
   1898 	return 1;
   1899 }
   1900 
   1901 
   1902 static int ndb_rebuild_note_indices(struct ndb_txn *txn, enum ndb_dbs *indices, int num_indices)
   1903 {
   1904 	MDB_val k, v;
   1905 	MDB_cursor *cur;
   1906 	int i, drop_dbi, count, rc;
   1907 	uint64_t note_key;
   1908 	struct ndb_note *note;
   1909 	enum ndb_dbs index;
   1910 
   1911 	// 0 means empty, not delete the dbi
   1912 	drop_dbi = 0;
   1913 
   1914 	// ensure they are all index dbs
   1915 	for (i = 0; i < num_indices; i++) {
   1916 		if (!ndb_db_is_index(indices[i])) {
   1917 			fprintf(stderr, "ndb_rebuild_note_index: %s is not an index db\n", ndb_db_name(indices[i]));
   1918 			return -1;
   1919 		}
   1920 	}
   1921 
   1922 	// empty the index dbs before we rebuild
   1923 	for (i = 0; i < num_indices; i++) {
   1924 		index = indices[i];
   1925 		if (mdb_drop(txn->mdb_txn, index, drop_dbi)) {
   1926 			fprintf(stderr, "ndb_rebuild_pubkey_index: mdb_drop failed for %s\n", ndb_db_name(index));
   1927 			return -1;
   1928 		}
   1929 	}
   1930 
   1931 	if ((rc = mdb_cursor_open(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_NOTE], &cur))) {
   1932 		fprintf(stderr, "ndb_migrate_user_search_indices: mdb_cursor_open failed, error %d\n", rc);
   1933 		return -1;
   1934 	}
   1935 
   1936 	count = 0;
   1937 
   1938 	// loop through all notes and write search indices
   1939 	while (mdb_cursor_get(cur, &k, &v, MDB_NEXT) == 0) {
   1940 		note = v.mv_data;
   1941 		note_key = *((uint64_t*)k.mv_data);
   1942 
   1943 		for (i = 0; i < num_indices; i++) {
   1944 			index = indices[i];
   1945 			switch (index) {
   1946 			case NDB_DB_NOTE:
   1947 			case NDB_DB_META:
   1948 			case NDB_DB_PROFILE:
   1949 			case NDB_DB_NOTE_ID:
   1950 			case NDB_DB_NDB_META:
   1951 			case NDB_DB_PROFILE_SEARCH:
   1952 			case NDB_DB_PROFILE_LAST_FETCH:
   1953 			case NDB_DB_NOTE_RELAYS:
   1954 			case NDB_DBS:
   1955 				// this should never happen since we check at
   1956 				// the start
   1957 				count = -1;
   1958 				goto cleanup;
   1959 			case NDB_DB_PROFILE_PK:
   1960 			case NDB_DB_NOTE_KIND:
   1961 			case NDB_DB_NOTE_TEXT:
   1962 			case NDB_DB_NOTE_BLOCKS:
   1963 			case NDB_DB_NOTE_TAGS:
   1964 				fprintf(stderr, "%s index rebuild not supported yet. sorry.\n", ndb_db_name(index));
   1965 				count = -1;
   1966 				goto cleanup;
   1967 			case NDB_DB_NOTE_PUBKEY:
   1968 				if (!ndb_write_note_pubkey_index(txn, note, note_key)) {
   1969 					count = -1;
   1970 					goto cleanup;
   1971 				}
   1972 				break;
   1973 			case NDB_DB_NOTE_RELAY_KIND:
   1974 				fprintf(stderr, "it doesn't make sense to rebuild note relay kind index\n");
   1975 				return 0;
   1976 			case NDB_DB_NOTE_PUBKEY_KIND:
   1977 				if (!ndb_write_note_pubkey_kind_index(txn, note, note_key)) {
   1978 					count = -1;
   1979 					goto cleanup;
   1980 				}
   1981 				break;
   1982 			}
   1983 		}
   1984 
   1985 		count++;
   1986 	}
   1987 
   1988 cleanup:
   1989 	mdb_cursor_close(cur);
   1990 
   1991 	return count;
   1992 }
   1993 
   1994 
   1995 // Migrations
   1996 //
   1997 
   1998 // This was before we had note_profile_pubkey{,_kind} indices. Let's create them.
   1999 static int ndb_migrate_profile_indices(struct ndb_txn *txn)
   2000 {
   2001 	int count;
   2002 
   2003 	enum ndb_dbs indices[] = {NDB_DB_NOTE_PUBKEY, NDB_DB_NOTE_PUBKEY_KIND};
   2004 	if ((count = ndb_rebuild_note_indices(txn, indices, 2)) != -1) {
   2005 		fprintf(stderr, "migrated %d notes to have pubkey and pubkey_kind indices\n", count);
   2006 		return 1;
   2007 	} else {
   2008 		fprintf(stderr, "error migrating notes to have pubkey and pubkey_kind indices, aborting.\n");
   2009 		return 0;
   2010 	}
   2011 }
   2012 
   2013 static int ndb_migrate_user_search_indices(struct ndb_txn *txn)
   2014 {
   2015 	int rc;
   2016 	MDB_cursor *cur;
   2017 	MDB_val k, v;
   2018 	void *profile_root;
   2019 	NdbProfileRecord_table_t record;
   2020 	struct ndb_note *note;
   2021 	uint64_t note_key, profile_key;
   2022 	size_t len;
   2023 	int count;
   2024 
   2025 	if ((rc = mdb_cursor_open(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_PROFILE], &cur))) {
   2026 		fprintf(stderr, "ndb_migrate_user_search_indices: mdb_cursor_open failed, error %d\n", rc);
   2027 		return 0;
   2028 	}
   2029 
   2030 	count = 0;
   2031 
   2032 	// loop through all profiles and write search indices
   2033 	while (mdb_cursor_get(cur, &k, &v, MDB_NEXT) == 0) {
   2034 		profile_root = v.mv_data;
   2035 		profile_key = *((uint64_t*)k.mv_data);
   2036 		record = NdbProfileRecord_as_root(profile_root);
   2037 		note_key = NdbProfileRecord_note_key(record);
   2038 		note = ndb_get_note_by_key(txn, note_key, &len);
   2039 
   2040 		if (note == NULL) {
   2041 			continue;
   2042 		}
   2043 
   2044 		if (!ndb_write_profile_search_indices(txn, note, profile_key,
   2045 						      profile_root)) {
   2046 			fprintf(stderr, "ndb_migrate_user_search_indices: ndb_write_profile_search_indices failed\n");
   2047 			continue;
   2048 		}
   2049 
   2050 		count++;
   2051 	}
   2052 
   2053 	fprintf(stderr, "migrated %d profiles to include search indices\n", count);
   2054 
   2055 	mdb_cursor_close(cur);
   2056 
   2057 	return 1;
   2058 }
   2059 
   2060 static int ndb_migrate_lower_user_search_indices(struct ndb_txn *txn)
   2061 {
   2062 	// just drop the search db so we can rebuild it
   2063 	if (mdb_drop(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_PROFILE_SEARCH], 0)) {
   2064 		fprintf(stderr, "ndb_migrate_lower_user_search_indices: mdb_drop failed\n");
   2065 		return 0;
   2066 	}
   2067 
   2068 	return ndb_migrate_user_search_indices(txn);
   2069 }
   2070 
   2071 int ndb_process_profile_note(struct ndb_note *note, struct ndb_profile_record_builder *profile);
   2072 
   2073 
   2074 int ndb_db_version(struct ndb_txn *txn)
   2075 {
   2076 	uint64_t version, version_key;
   2077 	MDB_val k, v;
   2078 
   2079 	version_key = NDB_META_KEY_VERSION;
   2080 	k.mv_data = &version_key;
   2081 	k.mv_size = sizeof(version_key);
   2082 
   2083 	if (mdb_get(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_NDB_META], &k, &v)) {
   2084 		version = -1;
   2085 	} else {
   2086 		if (v.mv_size != 8) {
   2087 			fprintf(stderr, "run_migrations: invalid version size?");
   2088 			return 0;
   2089 		}
   2090 		version = *((uint64_t*)v.mv_data);
   2091 	}
   2092 
   2093 	return version;
   2094 }
   2095 
   2096 // custom pubkey+kind+timestamp comparison function. This is used by lmdb to
   2097 // perform b+ tree searches over the pubkey+kind+timestamp index
   2098 static int ndb_id_u64_ts_compare(const MDB_val *a, const MDB_val *b)
   2099 {
   2100 	struct ndb_id_u64_ts *tsa, *tsb;
   2101 	MDB_val a2 = *a, b2 = *b;
   2102 
   2103 	a2.mv_size = sizeof(tsa->id);
   2104 	b2.mv_size = sizeof(tsb->id);
   2105 
   2106 	int cmp = mdb_cmp_memn(&a2, &b2);
   2107 	if (cmp) return cmp;
   2108 
   2109 	tsa = a->mv_data;
   2110 	tsb = b->mv_data;
   2111 
   2112 	if (tsa->u64 < tsb->u64)
   2113 		return -1;
   2114 	else if (tsa->u64 > tsb->u64)
   2115 		return 1;
   2116 
   2117 	if (tsa->timestamp < tsb->timestamp)
   2118 		return -1;
   2119 	else if (tsa->timestamp > tsb->timestamp)
   2120 		return 1;
   2121 
   2122 	return 0;
   2123 }
   2124 
   2125 // custom kind+timestamp comparison function. This is used by lmdb to perform
   2126 // b+ tree searches over the kind+timestamp index
   2127 static int ndb_u64_ts_compare(const MDB_val *a, const MDB_val *b)
   2128 {
   2129 	struct ndb_u64_ts *tsa, *tsb;
   2130 	tsa = a->mv_data;
   2131 	tsb = b->mv_data;
   2132 
   2133 	if (tsa->u64 < tsb->u64)
   2134 		return -1;
   2135 	else if (tsa->u64 > tsb->u64)
   2136 		return 1;
   2137 
   2138 	if (tsa->timestamp < tsb->timestamp)
   2139 		return -1;
   2140 	else if (tsa->timestamp > tsb->timestamp)
   2141 		return 1;
   2142 
   2143 	return 0;
   2144 }
   2145 
   2146 static int ndb_tsid_compare(const MDB_val *a, const MDB_val *b)
   2147 {
   2148 	struct ndb_tsid *tsa, *tsb;
   2149 	MDB_val a2 = *a, b2 = *b;
   2150 
   2151 	a2.mv_size = sizeof(tsa->id);
   2152 	b2.mv_size = sizeof(tsb->id);
   2153 
   2154 	int cmp = mdb_cmp_memn(&a2, &b2);
   2155 	if (cmp) return cmp;
   2156 
   2157 	tsa = a->mv_data;
   2158 	tsb = b->mv_data;
   2159 
   2160 	if (tsa->timestamp < tsb->timestamp)
   2161 		return -1;
   2162 	else if (tsa->timestamp > tsb->timestamp)
   2163 		return 1;
   2164 	return 0;
   2165 }
   2166 
   2167 enum ndb_ingester_msgtype {
   2168 	NDB_INGEST_EVENT, // write json to the ingester queue for processing
   2169 	NDB_INGEST_QUIT,  // kill ingester thread immediately
   2170 };
   2171 
   2172 struct ndb_ingester_event {
   2173 	const char *relay;
   2174 	char *json;
   2175 	unsigned client : 1; // ["EVENT", {...}] messages
   2176 	unsigned len : 31;
   2177 };
   2178 
   2179 struct ndb_writer_note_relay {
   2180 	const char *relay;
   2181 	uint64_t note_key;
   2182 	uint64_t kind;
   2183 	uint64_t created_at;
   2184 };
   2185 
   2186 struct ndb_writer_note {
   2187 	struct ndb_note *note;
   2188 	size_t note_len;
   2189 	const char *relay;
   2190 };
   2191 
   2192 static void ndb_writer_note_init(struct ndb_writer_note *writer_note, struct ndb_note *note, size_t note_len, const char *relay)
   2193 {
   2194 	writer_note->note = note;
   2195 	writer_note->note_len = note_len;
   2196 	writer_note->relay = relay;
   2197 }
   2198 
   2199 struct ndb_writer_profile {
   2200 	struct ndb_writer_note note;
   2201 	struct ndb_profile_record_builder record;
   2202 };
   2203 
   2204 struct ndb_ingester_msg {
   2205 	enum ndb_ingester_msgtype type;
   2206 	union {
   2207 		struct ndb_ingester_event event;
   2208 	};
   2209 };
   2210 
   2211 struct ndb_writer_ndb_meta {
   2212 	// these are 64 bit because I'm paranoid of db-wide alignment issues
   2213 	uint64_t version;
   2214 };
   2215 
   2216 // Used in the writer thread when writing ndb_profile_fetch_record's
   2217 //   kv = pubkey: recor
   2218 struct ndb_writer_last_fetch {
   2219 	unsigned char pubkey[32];
   2220 	uint64_t fetched_at;
   2221 };
   2222 
   2223 // write note blocks
   2224 struct ndb_writer_blocks {
   2225 	struct ndb_blocks *blocks;
   2226 	uint64_t note_key;
   2227 };
   2228 
   2229 // The different types of messages that the writer thread can write to the
   2230 // database
   2231 struct ndb_writer_msg {
   2232 	enum ndb_writer_msgtype type;
   2233 	union {
   2234 		struct ndb_writer_note_relay note_relay;
   2235 		struct ndb_writer_note note;
   2236 		struct ndb_writer_profile profile;
   2237 		struct ndb_writer_ndb_meta ndb_meta;
   2238 		struct ndb_writer_last_fetch last_fetch;
   2239 		struct ndb_writer_blocks blocks;
   2240 	};
   2241 };
   2242 
   2243 static inline int ndb_writer_queue_msg(struct ndb_writer *writer,
   2244 				       struct ndb_writer_msg *msg)
   2245 {
   2246 	return prot_queue_push(&writer->inbox, msg);
   2247 }
   2248 
   2249 static uint64_t ndb_write_note_and_profile(struct ndb_txn *txn, struct ndb_writer_profile *profile, unsigned char *scratch, size_t scratch_size, uint32_t ndb_flags);
   2250 static int ndb_migrate_utf8_profile_names(struct ndb_txn *txn)
   2251 {
   2252 	int rc;
   2253 	MDB_cursor *cur;
   2254 	MDB_val k, v;
   2255 	void *profile_root;
   2256 	NdbProfileRecord_table_t record;
   2257 	struct ndb_note *note, *copied_note;
   2258 	uint64_t note_key;
   2259 	size_t len;
   2260 	int count, failed, ret;
   2261 	struct ndb_writer_profile profile;
   2262 
   2263 	if ((rc = mdb_cursor_open(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_PROFILE], &cur))) {
   2264 		fprintf(stderr, "ndb_migrate_utf8_profile_names: mdb_cursor_open failed, error %d\n", rc);
   2265 		return 0;
   2266 	}
   2267 
   2268 	size_t scratch_size = 8 * 1024 * 1024;
   2269 	unsigned char *scratch = malloc(scratch_size);
   2270 
   2271 	ret = 1;
   2272 	count = 0;
   2273 	failed = 0;
   2274 
   2275 	// loop through all profiles and write search indices
   2276 	while (mdb_cursor_get(cur, &k, &v, MDB_NEXT) == 0) {
   2277 		profile_root = v.mv_data;
   2278 		record = NdbProfileRecord_as_root(profile_root);
   2279 		note_key = NdbProfileRecord_note_key(record);
   2280 		note = ndb_get_note_by_key(txn, note_key, &len);
   2281 
   2282 		if (note == NULL) {
   2283 			failed++;
   2284 			continue;
   2285 		}
   2286 
   2287 		struct ndb_profile_record_builder *b = &profile.record;
   2288 
   2289 		// reprocess profile
   2290 		if (!ndb_process_profile_note(note, b)) {
   2291 			failed++;
   2292 			continue;
   2293 		}
   2294 
   2295 		// the writer needs to own this note, and its expected to free it
   2296 		copied_note = malloc(len);
   2297 		memcpy(copied_note, note, len);
   2298 
   2299 		ndb_writer_note_init(&profile.note, copied_note, len, NULL);
   2300 
   2301 		// we don't pass in flags when migrating... a bit sketchy but
   2302 		// whatever. noone is using this to customize nostrdb atm
   2303 		if (ndb_write_note_and_profile(txn, &profile, scratch, scratch_size, 0)) {
   2304 			count++;
   2305 		}
   2306 	}
   2307 
   2308 	fprintf(stderr, "migrated %d profiles to fix utf8 profile names\n", count);
   2309 
   2310 	if (failed != 0) {
   2311 		fprintf(stderr, "failed to migrate %d profiles to fix utf8 profile names\n", failed);
   2312 	}
   2313 
   2314 	free(scratch);
   2315 	mdb_cursor_close(cur);
   2316 
   2317 	return ret;
   2318 }
   2319 
   2320 static struct ndb_migration MIGRATIONS[] = {
   2321 	{ .fn = ndb_migrate_user_search_indices },
   2322 	{ .fn = ndb_migrate_lower_user_search_indices },
   2323 	{ .fn = ndb_migrate_utf8_profile_names },
   2324 	{ .fn = ndb_migrate_profile_indices },
   2325 };
   2326 
   2327 
   2328 int ndb_end_query(struct ndb_txn *txn)
   2329 {
   2330 	// this works on read or write queries.
   2331 	return mdb_txn_commit(txn->mdb_txn) == 0;
   2332 }
   2333 
   2334 int ndb_note_verify(void *ctx, unsigned char *scratch, size_t scratch_size,
   2335 		    struct ndb_note *note)
   2336 {
   2337 	unsigned char id[32];
   2338 	secp256k1_xonly_pubkey xonly_pubkey;
   2339 	int ok;
   2340 
   2341 	// first, we ensure the id is valid by calculating the id independently
   2342 	// from what is given to us
   2343 	if (!ndb_calculate_id(note, scratch, scratch_size, id)) {
   2344 		ndb_debug("ndb_note_verify: scratch buffer size too small");
   2345 		return 0;
   2346 	}
   2347 
   2348 	if (memcmp(id, note->id, 32)) {
   2349 		ndb_debug("ndb_note_verify: note id does not match!");
   2350 		return 0;
   2351 	}
   2352 
   2353         // id is ok, let's check signature
   2354 
   2355 	ok = secp256k1_xonly_pubkey_parse((secp256k1_context*)ctx,
   2356 					  &xonly_pubkey,
   2357 					  ndb_note_pubkey(note)) != 0;
   2358 	if (!ok) return 0;
   2359 
   2360 	ok = secp256k1_schnorrsig_verify((secp256k1_context*)ctx,
   2361 					 ndb_note_sig(note), id, 32,
   2362 					 &xonly_pubkey) > 0;
   2363 	if (!ok) return 0;
   2364 
   2365 	return 1;
   2366 }
   2367 
   2368 static void ndb_writer_last_profile_fetch(struct ndb_txn *txn,
   2369 					  const unsigned char *pubkey,
   2370 					  uint64_t fetched_at)
   2371 {
   2372 	int rc;
   2373 	MDB_val key, val;
   2374 
   2375 	key.mv_data = (unsigned char*)pubkey;
   2376 	key.mv_size = 32;
   2377 	val.mv_data = &fetched_at;
   2378 	val.mv_size = sizeof(fetched_at);
   2379 
   2380 	if ((rc = mdb_put(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_PROFILE_LAST_FETCH],
   2381 			  &key, &val, 0)))
   2382 	{
   2383 		ndb_debug("write version to ndb_meta failed: %s\n",
   2384 				mdb_strerror(rc));
   2385 		return;
   2386 	}
   2387 
   2388 	//fprintf(stderr, "writing version %" PRIu64 "\n", version);
   2389 }
   2390 
   2391 
   2392 // We just received a profile that we haven't processed yet, but it could
   2393 // be an older one! Make sure we only write last fetched profile if it's a new
   2394 // one
   2395 //
   2396 // To do this, we first check the latest profile in the database. If the
   2397 // created_date for this profile note is newer, then we write a
   2398 // last_profile_fetch record, otherwise we do not.
   2399 //
   2400 // WARNING: This function is only valid when called from the writer thread
   2401 static int ndb_maybe_write_last_profile_fetch(struct ndb_txn *txn,
   2402 					       struct ndb_note *note)
   2403 {
   2404 	size_t len;
   2405 	uint64_t profile_key, note_key;
   2406 	void *root;
   2407 	struct ndb_note *last_profile;
   2408 	NdbProfileRecord_table_t record;
   2409 
   2410 	if ((root = ndb_get_profile_by_pubkey(txn, note->pubkey, &len, &profile_key))) {
   2411 		record = NdbProfileRecord_as_root(root);
   2412 		note_key = NdbProfileRecord_note_key(record);
   2413 		last_profile = ndb_get_note_by_key(txn, note_key, &len);
   2414 		if (last_profile == NULL) {
   2415 			return 0;
   2416 		}
   2417 
   2418 		// found profile, let's see if it's newer than ours
   2419 		if (note->created_at > last_profile->created_at) {
   2420 			// this is a new profile note, record last fetched time
   2421 			ndb_writer_last_profile_fetch(txn, note->pubkey, time(NULL));
   2422 		}
   2423 	} else {
   2424 		// couldn't fetch profile. record last fetched time
   2425 		ndb_writer_last_profile_fetch(txn, note->pubkey, time(NULL));
   2426 	}
   2427 
   2428 	return 1;
   2429 }
   2430 
   2431 int ndb_write_last_profile_fetch(struct ndb *ndb, const unsigned char *pubkey,
   2432 				 uint64_t fetched_at)
   2433 {
   2434 	struct ndb_writer_msg msg;
   2435 	msg.type = NDB_WRITER_PROFILE_LAST_FETCH;
   2436 	memcpy(&msg.last_fetch.pubkey[0], pubkey, 32);
   2437 	msg.last_fetch.fetched_at = fetched_at;
   2438 
   2439 	return ndb_writer_queue_msg(&ndb->writer, &msg);
   2440 }
   2441 
   2442 
   2443 // When doing cursor scans from greatest to lowest, this function positions the
   2444 // cursor at the first element before descending. MDB_SET_RANGE puts us right
   2445 // after the first element, so we have to go back one.
   2446 static int ndb_cursor_start(MDB_cursor *cur, MDB_val *k, MDB_val *v)
   2447 {
   2448 	int rc;
   2449 	// Position cursor at the next key greater than or equal to the
   2450 	// specified key
   2451 
   2452 	if ((rc = mdb_cursor_get(cur, k, v, MDB_SET_RANGE))) {
   2453 		ndb_debug("MDB_SET_RANGE failed: '%s'\n", mdb_strerror(rc));
   2454 		// Failed :(. It could be the last element?
   2455 		if ((rc = mdb_cursor_get(cur, k, v, MDB_LAST))) {
   2456 			ndb_debug("MDB_LAST failed: '%s'\n", mdb_strerror(rc));
   2457 			return 0;
   2458 		}
   2459 	} else {
   2460 		// if set range worked and our key exists, it should be
   2461 		// the one right before this one
   2462 		if ((rc = mdb_cursor_get(cur, k, v, MDB_PREV))) {
   2463 			ndb_debug("moving back failed: '%s'\n", mdb_strerror(rc));
   2464 			return 0;
   2465 		}
   2466 	}
   2467 
   2468 	return 1;
   2469 }
   2470 
   2471 
   2472 // get some value based on a clustered id key
   2473 int ndb_get_tsid(struct ndb_txn *txn, enum ndb_dbs db, const unsigned char *id,
   2474 		 MDB_val *val)
   2475 {
   2476 	MDB_val k, v;
   2477 	MDB_cursor *cur;
   2478 	int success = 0, rc;
   2479 	struct ndb_tsid tsid;
   2480 
   2481 	// position at the most recent
   2482 	ndb_tsid_high(&tsid, id);
   2483 
   2484 	k.mv_data = &tsid;
   2485 	k.mv_size = sizeof(tsid);
   2486 
   2487 	if ((rc = mdb_cursor_open(txn->mdb_txn, txn->lmdb->dbs[db], &cur))) {
   2488 		ndb_debug("ndb_get_tsid: failed to open cursor: '%s'\n", mdb_strerror(rc));
   2489 		return 0;
   2490 	}
   2491 
   2492 	if (!ndb_cursor_start(cur, &k, &v))
   2493 		goto cleanup;
   2494 
   2495 	if (memcmp(k.mv_data, id, 32) == 0) {
   2496 		*val = v;
   2497 		success = 1;
   2498 	}
   2499 
   2500 cleanup:
   2501 	mdb_cursor_close(cur);
   2502 	return success;
   2503 }
   2504 
   2505 static void *ndb_lookup_by_key(struct ndb_txn *txn, uint64_t key,
   2506 			       enum ndb_dbs store, size_t *len)
   2507 {
   2508 	MDB_val k, v;
   2509 
   2510 	k.mv_data = &key;
   2511 	k.mv_size = sizeof(key);
   2512 
   2513 	if (mdb_get(txn->mdb_txn, txn->lmdb->dbs[store], &k, &v)) {
   2514 		ndb_debug("ndb_lookup_by_key: mdb_get note failed\n");
   2515 		return NULL;
   2516 	}
   2517 
   2518 	if (len)
   2519 		*len = v.mv_size;
   2520 
   2521 	return v.mv_data;
   2522 }
   2523 
   2524 static void *ndb_lookup_tsid(struct ndb_txn *txn, enum ndb_dbs ind,
   2525 			     enum ndb_dbs store, const unsigned char *pk,
   2526 			     size_t *len, uint64_t *primkey)
   2527 {
   2528 	MDB_val k, v;
   2529 	void *res = NULL;
   2530 	if (len)
   2531 		*len = 0;
   2532 
   2533 	if (!ndb_get_tsid(txn, ind, pk, &k)) {
   2534 		//ndb_debug("ndb_get_profile_by_pubkey: ndb_get_tsid failed\n");
   2535 		return 0;
   2536 	}
   2537 
   2538 	if (primkey)
   2539 		*primkey = *(uint64_t*)k.mv_data;
   2540 
   2541 	if (mdb_get(txn->mdb_txn, txn->lmdb->dbs[store], &k, &v)) {
   2542 		ndb_debug("ndb_get_profile_by_pubkey: mdb_get note failed\n");
   2543 		return 0;
   2544 	}
   2545 
   2546 	res = v.mv_data;
   2547 	assert(((uint64_t)res % 4) == 0);
   2548 	if (len)
   2549 		*len = v.mv_size;
   2550 	return res;
   2551 }
   2552 
   2553 void *ndb_get_profile_by_pubkey(struct ndb_txn *txn, const unsigned char *pk, size_t *len, uint64_t *key)
   2554 {
   2555 	return ndb_lookup_tsid(txn, NDB_DB_PROFILE_PK, NDB_DB_PROFILE, pk, len, key);
   2556 }
   2557 
   2558 struct ndb_note *ndb_get_note_by_id(struct ndb_txn *txn, const unsigned char *id, size_t *len, uint64_t *key)
   2559 {
   2560 	return ndb_lookup_tsid(txn, NDB_DB_NOTE_ID, NDB_DB_NOTE, id, len, key);
   2561 }
   2562 
   2563 static inline uint64_t ndb_get_indexkey_by_id(struct ndb_txn *txn,
   2564 					      enum ndb_dbs db,
   2565 					      const unsigned char *id)
   2566 {
   2567 	MDB_val k;
   2568 
   2569 	if (!ndb_get_tsid(txn, db, id, &k))
   2570 		return 0;
   2571 
   2572 	return *(uint32_t*)k.mv_data;
   2573 }
   2574 
   2575 uint64_t ndb_get_notekey_by_id(struct ndb_txn *txn, const unsigned char *id)
   2576 {
   2577 	return ndb_get_indexkey_by_id(txn, NDB_DB_NOTE_ID, id);
   2578 }
   2579 
   2580 uint64_t ndb_get_profilekey_by_pubkey(struct ndb_txn *txn, const unsigned char *id)
   2581 {
   2582 	return ndb_get_indexkey_by_id(txn, NDB_DB_PROFILE_PK, id);
   2583 }
   2584 
   2585 struct ndb_note *ndb_get_note_by_key(struct ndb_txn *txn, uint64_t key, size_t *len)
   2586 {
   2587 	return ndb_lookup_by_key(txn, key, NDB_DB_NOTE, len);
   2588 }
   2589 
   2590 void *ndb_get_profile_by_key(struct ndb_txn *txn, uint64_t key, size_t *len)
   2591 {
   2592 	return ndb_lookup_by_key(txn, key, NDB_DB_PROFILE, len);
   2593 }
   2594 
   2595 uint64_t
   2596 ndb_read_last_profile_fetch(struct ndb_txn *txn, const unsigned char *pubkey)
   2597 {
   2598 	MDB_val k, v;
   2599 
   2600 	k.mv_data = (unsigned char*)pubkey;
   2601 	k.mv_size = 32;
   2602 
   2603 	if (mdb_get(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_PROFILE_LAST_FETCH], &k, &v)) {
   2604 		//ndb_debug("ndb_read_last_profile_fetch: mdb_get note failed\n");
   2605 		return 0;
   2606 	}
   2607 
   2608 	return *((uint64_t*)v.mv_data);
   2609 }
   2610 
   2611 
   2612 static int ndb_has_note(struct ndb_txn *txn, const unsigned char *id)
   2613 {
   2614 	MDB_val val;
   2615 
   2616 	if (!ndb_get_tsid(txn, NDB_DB_NOTE_ID, id, &val))
   2617 		return 0;
   2618 
   2619 	return 1;
   2620 }
   2621 
   2622 static void ndb_txn_from_mdb(struct ndb_txn *txn, struct ndb_lmdb *lmdb,
   2623 			     MDB_txn *mdb_txn)
   2624 {
   2625 	txn->lmdb = lmdb;
   2626 	txn->mdb_txn = mdb_txn;
   2627 }
   2628 
   2629 static enum ndb_idres ndb_ingester_json_controller(void *data, const char *hexid)
   2630 {
   2631 	unsigned char id[32];
   2632 	struct ndb_ingest_controller *c = data;
   2633 	struct ndb_txn txn;
   2634 
   2635 	hex_decode(hexid, 64, id, sizeof(id));
   2636 
   2637 	// let's see if we already have it
   2638 	ndb_txn_from_mdb(&txn, c->lmdb, c->read_txn);
   2639 	c->note = ndb_get_note_by_id(&txn, id, NULL, &c->note_key);
   2640 
   2641 	if (c->note == NULL)
   2642 		return NDB_IDRES_CONT;
   2643 
   2644 	return NDB_IDRES_STOP;
   2645 }
   2646 
   2647 static int ndbprofile_parse_json(flatcc_builder_t *B,
   2648         const char *buf, size_t bufsiz, int flags, NdbProfile_ref_t *profile)
   2649 {
   2650 	flatcc_json_parser_t parser, *ctx = &parser;
   2651 	flatcc_json_parser_init(ctx, B, buf, buf + bufsiz, flags);
   2652 
   2653 	if (flatcc_builder_start_buffer(B, 0, 0, 0))
   2654 		return 0;
   2655 
   2656 	NdbProfile_parse_json_table(ctx, buf, buf + bufsiz, profile);
   2657 	if (ctx->error)
   2658 		return 0;
   2659 
   2660 	if (!flatcc_builder_end_buffer(B, *profile))
   2661 		return 0;
   2662 
   2663 	ctx->end_loc = buf;
   2664 
   2665 
   2666 	return 1;
   2667 }
   2668 
   2669 void ndb_profile_record_builder_init(struct ndb_profile_record_builder *b)
   2670 {
   2671 	b->builder = malloc(sizeof(*b->builder));
   2672 	b->flatbuf = NULL;
   2673 }
   2674 
   2675 void ndb_profile_record_builder_free(struct ndb_profile_record_builder *b)
   2676 {
   2677 	if (b->flatbuf)
   2678 		flatcc_builder_aligned_free(b->flatbuf);
   2679 	if (b->builder) {
   2680 		flatcc_builder_clear(b->builder);
   2681 		free(b->builder);
   2682 	}
   2683 
   2684 	b->builder = NULL;
   2685 	b->flatbuf = NULL;
   2686 
   2687 }
   2688 
   2689 int ndb_process_profile_note(struct ndb_note *note,
   2690 			     struct ndb_profile_record_builder *profile)
   2691 {
   2692 	int res;
   2693 
   2694 	NdbProfile_ref_t profile_table;
   2695 	flatcc_builder_t *builder;
   2696 
   2697 	ndb_profile_record_builder_init(profile);
   2698 	builder = profile->builder;
   2699 	flatcc_builder_init(builder);
   2700 
   2701 	NdbProfileRecord_start_as_root(builder);
   2702 
   2703 	//printf("parsing profile '%.*s'\n", note->content_length, ndb_note_content(note));
   2704 	if (!(res = ndbprofile_parse_json(builder, ndb_note_content(note),
   2705 					  note->content_length,
   2706 					  flatcc_json_parser_f_skip_unknown,
   2707 					  &profile_table)))
   2708 	{
   2709 		ndb_debug("profile_parse_json failed %d '%.*s'\n", res,
   2710 			  note->content_length, ndb_note_content(note));
   2711 		ndb_profile_record_builder_free(profile);
   2712 		return 0;
   2713 	}
   2714 
   2715 	uint64_t received_at = time(NULL);
   2716 	const char *lnurl = "fixme";
   2717 
   2718 	NdbProfileRecord_profile_add(builder, profile_table);
   2719 	NdbProfileRecord_received_at_add(builder, received_at);
   2720 
   2721 	flatcc_builder_ref_t lnurl_off;
   2722 	lnurl_off = flatcc_builder_create_string_str(builder, lnurl);
   2723 
   2724 	NdbProfileRecord_lnurl_add(builder, lnurl_off);
   2725 
   2726 	//*profile = flatcc_builder_finalize_aligned_buffer(builder, profile_len);
   2727 	return 1;
   2728 }
   2729 
   2730 static int ndb_ingester_queue_event(struct ndb_ingester *ingester,
   2731 				    char *json, unsigned len,
   2732 				    unsigned client, const char *relay)
   2733 {
   2734 	struct ndb_ingester_msg msg;
   2735 	msg.type = NDB_INGEST_EVENT;
   2736 
   2737 	msg.event.json = json;
   2738 	msg.event.len = len;
   2739 	msg.event.client = client;
   2740 	msg.event.relay = relay;
   2741 
   2742 	return threadpool_dispatch(&ingester->tp, &msg);
   2743 }
   2744 
   2745 void ndb_ingest_meta_init(struct ndb_ingest_meta *meta, unsigned client, const char *relay)
   2746 {
   2747 	meta->client = client;
   2748 	meta->relay = relay;
   2749 }
   2750 
   2751 static int ndb_ingest_event(struct ndb_ingester *ingester, const char *json,
   2752 			    int len, struct ndb_ingest_meta *meta)
   2753 {
   2754 	const char *relay = meta->relay;
   2755 
   2756 	// Without this, we get bus errors in the json parser inside when
   2757 	// trying to ingest empty kind 6 reposts... we should probably do fuzz
   2758 	// testing on inputs to the json parser
   2759 	if (len == 0)
   2760 		return 0;
   2761 
   2762 	// Since we need to return as soon as possible, and we're not
   2763 	// making any assumptions about the lifetime of the string, we
   2764 	// definitely need to copy the json here. In the future once we
   2765 	// have our thread that manages a websocket connection, we can
   2766 	// avoid the copy and just use the buffer we get from that
   2767 	// thread.
   2768 	char *json_copy = strdupn(json, len);
   2769 	if (json_copy == NULL)
   2770 		return 0;
   2771 
   2772 	if (relay != NULL) {
   2773 		relay = strdup(meta->relay);
   2774 		if (relay == NULL)
   2775 			return 0;
   2776 	}
   2777 
   2778 	return ndb_ingester_queue_event(ingester, json_copy, len, meta->client, relay);
   2779 }
   2780 
   2781 
   2782 static int ndb_ingester_process_note(secp256k1_context *ctx,
   2783 				     struct ndb_note *note,
   2784 				     size_t note_size,
   2785 				     struct ndb_writer_msg *out,
   2786 				     struct ndb_ingester *ingester,
   2787 				     unsigned char *scratch,
   2788 				     const char *relay)
   2789 {
   2790 	enum ndb_ingest_filter_action action;
   2791 	struct ndb_ingest_meta meta;
   2792 
   2793 	action = NDB_INGEST_ACCEPT;
   2794 
   2795 	if (ingester->filter)
   2796 		action = ingester->filter(ingester->filter_context, note);
   2797 
   2798 	if (action == NDB_INGEST_REJECT)
   2799 		return 0;
   2800 
   2801 	// some special situations we might want to skip sig validation,
   2802 	// like during large imports
   2803 	if (action == NDB_INGEST_SKIP_VALIDATION || (ingester->flags & NDB_FLAG_SKIP_NOTE_VERIFY)) {
   2804 		// if we're skipping validation we don't need to verify
   2805 	} else {
   2806 		// verify! If it's an invalid note we don't need to
   2807 		// bother writing it to the database
   2808 		if (!ndb_note_verify(ctx, scratch, ingester->scratch_size, note)) {
   2809 			ndb_debug("note verification failed\n");
   2810 			return 0;
   2811 		}
   2812 	}
   2813 
   2814 	// we didn't find anything. let's send it
   2815 	// to the writer thread
   2816 	note = realloc(note, note_size);
   2817 	assert(((uint64_t)note % 4) == 0);
   2818 
   2819 	if (note->kind == 0) {
   2820 		struct ndb_profile_record_builder *b =
   2821 			&out->profile.record;
   2822 
   2823 		ndb_process_profile_note(note, b);
   2824 
   2825 		out->type = NDB_WRITER_PROFILE;
   2826 		ndb_writer_note_init(&out->profile.note, note, note_size, relay);
   2827 		return 1;
   2828 	} else if (note->kind == 6) {
   2829 		// process the repost if we have a repost event
   2830 		ndb_debug("processing kind 6 repost\n");
   2831 		// dup the relay string
   2832 		ndb_ingest_meta_init(&meta, 0, relay);
   2833 		ndb_ingest_event(ingester, ndb_note_content(note),
   2834 					   ndb_note_content_length(note),
   2835 					   &meta);
   2836 	}
   2837 
   2838 	out->type = NDB_WRITER_NOTE;
   2839 	ndb_writer_note_init(&out->note, note, note_size, relay);
   2840 
   2841 	return 1;
   2842 }
   2843 
   2844 int ndb_note_seen_on_relay(struct ndb_txn *txn, uint64_t note_key, const char *relay)
   2845 {
   2846 	MDB_val k, v;
   2847 	MDB_cursor *cur;
   2848 	int rc, len;
   2849 	char relay_buf[256];
   2850 
   2851 	if (relay == NULL)
   2852 		return 0;
   2853 
   2854 	len = strlen(relay);
   2855 
   2856 	if (!(len = prepare_relay_buf(relay_buf, sizeof(relay_buf), relay, len)))
   2857 		return 0;
   2858 
   2859 	assert((len % 8) == 0);
   2860 
   2861 	k.mv_data = &note_key;
   2862 	k.mv_size = sizeof(note_key);
   2863 
   2864 	v.mv_data = relay_buf;
   2865 	v.mv_size = len;
   2866 
   2867 	if ((rc = mdb_cursor_open(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_NOTE_RELAYS], &cur)) != MDB_SUCCESS)
   2868 		return 0;
   2869 
   2870 	rc = mdb_cursor_get(cur, &k, &v, MDB_GET_BOTH);
   2871 	ndb_debug("seen_on_relay result: %s\n", mdb_strerror(rc));
   2872 	mdb_cursor_close(cur);
   2873 
   2874 	return rc == MDB_SUCCESS;
   2875 }
   2876 
   2877 // process the relay for the note. this is called when we already have the
   2878 // note in the database but still need to check if the relay needs to be
   2879 // written to the relay indexes for corresponding note
   2880 static int ndb_process_note_relay(struct ndb_txn *txn, struct ndb_writer_msg *out,
   2881 				  uint64_t note_key, struct ndb_note *note,
   2882 				  const char *relay)
   2883 {
   2884 	// query to see if we already have the relay on this note
   2885 	if (ndb_note_seen_on_relay(txn, note_key, relay)) {
   2886 		return 0;
   2887 	}
   2888 
   2889 	// if not, tell the writer thread to emit a NOTE_RELAY event
   2890 	out->type = NDB_WRITER_NOTE_RELAY;
   2891 
   2892 	ndb_debug("pushing NDB_WRITER_NOTE_RELAY with note_key %" PRIu64 "\n", note_key);
   2893 	out->note_relay.relay = relay;
   2894 	out->note_relay.note_key = note_key;
   2895 	out->note_relay.kind = ndb_note_kind(note);
   2896 	out->note_relay.created_at = ndb_note_created_at(note);
   2897 
   2898 	return 1;
   2899 }
   2900 
   2901 static int ndb_ingester_process_event(secp256k1_context *ctx,
   2902 				      struct ndb_ingester *ingester,
   2903 				      struct ndb_ingester_event *ev,
   2904 				      struct ndb_writer_msg *out,
   2905 				      unsigned char *scratch,
   2906 				      MDB_txn *read_txn)
   2907 {
   2908 	struct ndb_tce tce;
   2909 	struct ndb_fce fce;
   2910 	struct ndb_note *note;
   2911 	struct ndb_ingest_controller controller;
   2912 	struct ndb_id_cb cb;
   2913 	void *buf;
   2914 	int ok;
   2915 	size_t bufsize, note_size;
   2916 
   2917 	ok = 0;
   2918 
   2919 	// we will use this to check if we already have it in the DB during
   2920 	// ID parsing
   2921 	controller.read_txn = read_txn;
   2922 	controller.lmdb = ingester->lmdb;
   2923 	cb.fn = ndb_ingester_json_controller;
   2924 	cb.data = &controller;
   2925 
   2926 	// since we're going to be passing this allocated note to a different
   2927 	// thread, we can't use thread-local buffers. just allocate a block
   2928         bufsize = max(ev->len * 8.0, 4096);
   2929 	buf = malloc(bufsize);
   2930 	if (!buf) {
   2931 		ndb_debug("couldn't malloc buf\n");
   2932 		return 0;
   2933 	}
   2934 
   2935 	note_size =
   2936 		ev->client ?
   2937 		ndb_client_event_from_json(ev->json, ev->len, &fce, buf, bufsize, &cb) :
   2938 		ndb_ws_event_from_json(ev->json, ev->len, &tce, buf, bufsize, &cb);
   2939 
   2940 	// This is a result from our special json parser. It parsed the id
   2941 	// and found that we already have it in the database
   2942 	if ((int)note_size == -42) {
   2943 		assert(controller.note != NULL);
   2944 		assert(controller.note_key != 0);
   2945 		struct ndb_txn txn;
   2946 		ndb_txn_from_mdb(&txn, ingester->lmdb, read_txn);
   2947 
   2948 		// we still need to process the relays on the note even
   2949 		// if we already have it
   2950 	 	if (ev->relay && ndb_process_note_relay(&txn, out,
   2951 							controller.note_key,
   2952 							controller.note,
   2953 							ev->relay))
   2954 		{
   2955 			// free note buf here since we don't pass the note to the writer thread
   2956 			free(buf);
   2957 			goto success;
   2958 		} else {
   2959 			// we already have the note and there are no new
   2960 			// relays to process. nothing to write.
   2961 			goto cleanup;
   2962 		}
   2963 	} else if (note_size == 0) {
   2964 		ndb_debug("failed to parse '%.*s'\n", ev->len, ev->json);
   2965 		goto cleanup;
   2966 	}
   2967 
   2968 	//ndb_debug("parsed evtype:%d '%.*s'\n", tce.evtype, ev->len, ev->json);
   2969 
   2970 	if (ev->client) {
   2971 		switch (fce.evtype) {
   2972 		case NDB_FCE_EVENT:
   2973 			note = fce.event.note;
   2974 			if (note != buf) {
   2975 				ndb_debug("note buffer not equal to malloc'd buffer\n");
   2976 				goto cleanup;
   2977 			}
   2978 
   2979 			if (!ndb_ingester_process_note(ctx, note, note_size,
   2980 						       out, ingester, scratch,
   2981 						       ev->relay)) {
   2982 				ndb_debug("failed to process note\n");
   2983 				goto cleanup;
   2984 			} else {
   2985 				goto success;
   2986 			}
   2987 		}
   2988 	} else {
   2989 		switch (tce.evtype) {
   2990 		case NDB_TCE_AUTH:   goto cleanup;
   2991 		case NDB_TCE_NOTICE: goto cleanup;
   2992 		case NDB_TCE_EOSE:   goto cleanup;
   2993 		case NDB_TCE_OK:     goto cleanup;
   2994 		case NDB_TCE_EVENT:
   2995 			note = tce.event.note;
   2996 			if (note != buf) {
   2997 				ndb_debug("note buffer not equal to malloc'd buffer\n");
   2998 				goto cleanup;
   2999 			}
   3000 
   3001 			if (!ndb_ingester_process_note(ctx, note, note_size,
   3002 						       out, ingester, scratch,
   3003 						       ev->relay)) {
   3004 				ndb_debug("failed to process note\n");
   3005 				goto cleanup;
   3006 			} else {
   3007 				goto success;
   3008 			}
   3009 		}
   3010 	}
   3011 
   3012 
   3013 success:
   3014 	free(ev->json);
   3015 	// we don't free relay or buf since those are passed to the writer thread
   3016 	return 1;
   3017 
   3018 cleanup:
   3019 	free(ev->json);
   3020 	if (ev->relay)
   3021 		free((void*)ev->relay);
   3022 	free(buf);
   3023 
   3024 	return ok;
   3025 }
   3026 
   3027 static uint64_t ndb_get_last_key(MDB_txn *txn, MDB_dbi db)
   3028 {
   3029 	MDB_cursor *mc;
   3030 	MDB_val key, val;
   3031 
   3032 	if (mdb_cursor_open(txn, db, &mc))
   3033 		return 0;
   3034 
   3035 	if (mdb_cursor_get(mc, &key, &val, MDB_LAST)) {
   3036 		mdb_cursor_close(mc);
   3037 		return 0;
   3038 	}
   3039 
   3040 	mdb_cursor_close(mc);
   3041 
   3042 	assert(key.mv_size == 8);
   3043         return *((uint64_t*)key.mv_data);
   3044 }
   3045 
   3046 //
   3047 // make a search key meant for user queries without any other note info
   3048 static void ndb_make_search_key_low(struct ndb_search_key *key, const char *search)
   3049 {
   3050 	memset(key->id, 0, sizeof(key->id));
   3051 	key->timestamp = 0;
   3052 	lowercase_strncpy(key->search, search, sizeof(key->search) - 1);
   3053 	key->search[sizeof(key->search) - 1] = '\0';
   3054 }
   3055 
   3056 int ndb_search_profile(struct ndb_txn *txn, struct ndb_search *search, const char *query)
   3057 {
   3058 	int rc;
   3059 	struct ndb_search_key s;
   3060 	MDB_val k, v;
   3061 	search->cursor = NULL;
   3062 
   3063 	MDB_cursor **cursor = (MDB_cursor **)&search->cursor;
   3064 
   3065 	ndb_make_search_key_low(&s, query);
   3066 
   3067 	k.mv_data = &s;
   3068 	k.mv_size = sizeof(s);
   3069 
   3070 	if ((rc = mdb_cursor_open(txn->mdb_txn,
   3071 				  txn->lmdb->dbs[NDB_DB_PROFILE_SEARCH],
   3072 				  cursor))) {
   3073 		printf("search_profile: cursor opened failed: %s\n",
   3074 				mdb_strerror(rc));
   3075 		return 0;
   3076 	}
   3077 
   3078 	// Position cursor at the next key greater than or equal to the specified key
   3079 	if (mdb_cursor_get(search->cursor, &k, &v, MDB_SET_RANGE)) {
   3080 		printf("search_profile: cursor get failed\n");
   3081 		goto cleanup;
   3082 	} else {
   3083 		search->key = k.mv_data;
   3084 		assert(v.mv_size == 8);
   3085 		search->profile_key = *((uint64_t*)v.mv_data);
   3086 		return 1;
   3087 	}
   3088 
   3089 cleanup:
   3090 	mdb_cursor_close(search->cursor);
   3091 	search->cursor = NULL;
   3092 	return 0;
   3093 }
   3094 
   3095 void ndb_search_profile_end(struct ndb_search *search)
   3096 {
   3097 	if (search->cursor)
   3098 		mdb_cursor_close(search->cursor);
   3099 }
   3100 
   3101 int ndb_search_profile_next(struct ndb_search *search)
   3102 {
   3103 	int rc;
   3104 	MDB_val k, v;
   3105 	unsigned char *init_id;
   3106 
   3107 	init_id = search->key->id;
   3108 	k.mv_data = search->key;
   3109 	k.mv_size = sizeof(*search->key);
   3110 
   3111 retry:
   3112 	if ((rc = mdb_cursor_get(search->cursor, &k, &v, MDB_NEXT))) {
   3113 		ndb_debug("ndb_search_profile_next: %s\n",
   3114 				mdb_strerror(rc));
   3115 		return 0;
   3116 	} else {
   3117 		search->key = k.mv_data;
   3118 		assert(v.mv_size == 8);
   3119 		search->profile_key = *((uint64_t*)v.mv_data);
   3120 
   3121 		// skip duplicate pubkeys
   3122 		if (!memcmp(init_id, search->key->id, 32))
   3123 			goto retry;
   3124 	}
   3125 
   3126 	return 1;
   3127 }
   3128 
   3129 //
   3130 // The relay kind index has a layout like so (so we don't need dupsort)
   3131 //
   3132 // - note_id:        00 + 8 bytes
   3133 // - kind:           08 + 8 bytes
   3134 // - created_at:     16 + 8 bytes
   3135 // - relay_url_size: 24 + 1 byte
   3136 // - relay_url:      25 + n byte null-terminated string
   3137 // - pad to 8 byte alignment
   3138 //
   3139 // The key sort order is:
   3140 //
   3141 // relay_url, kind, created_at
   3142 //
   3143 static int ndb_relay_kind_cmp(const MDB_val *a, const MDB_val *b)
   3144 {
   3145 	int cmp;
   3146 	MDB_val va, vb;
   3147 	uint64_t iva, ivb;
   3148 	unsigned char *ad = (unsigned char *)a->mv_data;
   3149 	unsigned char *bd = (unsigned char *)b->mv_data;
   3150 	assert(((uint64_t)a->mv_data % 8) == 0);
   3151 
   3152 	va.mv_size = *(ad + 24);
   3153 	va.mv_data =   ad + 25;
   3154 
   3155 	vb.mv_size = *(bd + 24);
   3156 	vb.mv_data =   bd + 25;
   3157 
   3158 	cmp = mdb_cmp_memn(&va, &vb);
   3159 	if (cmp) return cmp;
   3160 
   3161 	// kind
   3162 	iva = *(uint64_t*)(ad + 8);
   3163 	ivb = *(uint64_t*)(bd + 8);
   3164 
   3165 	if (iva < ivb)
   3166 		return -1;
   3167 	else if (iva > ivb)
   3168 		return 1;
   3169 
   3170 	// created_at
   3171 	iva = *(uint64_t*)(ad + 16);
   3172 	ivb = *(uint64_t*)(bd + 16);
   3173 
   3174 	if (iva < ivb)
   3175 		return -1;
   3176 	else if (iva > ivb)
   3177 		return 1;
   3178 
   3179 	// note_id (so we don't need dupsort logic)
   3180 	iva = *(uint64_t*)ad;
   3181 	ivb = *(uint64_t*)bd;
   3182 
   3183 	if (iva < ivb)
   3184 		return -1;
   3185 	else if (iva > ivb)
   3186 		return 1;
   3187 
   3188 	return 0;
   3189 }
   3190 
   3191 static int ndb_search_key_cmp(const MDB_val *a, const MDB_val *b)
   3192 {
   3193 	int cmp;
   3194 	struct ndb_search_key *ska, *skb;
   3195 
   3196 	ska = a->mv_data;
   3197 	skb = b->mv_data;
   3198 
   3199 	MDB_val a2 = *a;
   3200 	MDB_val b2 = *b;
   3201 
   3202 	a2.mv_data = ska->search;
   3203 	a2.mv_size = sizeof(ska->search) + sizeof(ska->id);
   3204 
   3205 	cmp = mdb_cmp_memn(&a2, &b2);
   3206 	if (cmp) return cmp;
   3207 
   3208 	if (ska->timestamp < skb->timestamp)
   3209 		return -1;
   3210 	else if (ska->timestamp > skb->timestamp)
   3211 		return 1;
   3212 
   3213 	return 0;
   3214 }
   3215 
   3216 static int ndb_write_profile_pk_index(struct ndb_txn *txn, struct ndb_note *note, uint64_t profile_key)
   3217 
   3218 {
   3219 	MDB_val key, val;
   3220 	int rc;
   3221 	struct ndb_tsid tsid;
   3222 	MDB_dbi pk_db;
   3223 
   3224 	pk_db = txn->lmdb->dbs[NDB_DB_PROFILE_PK];
   3225 
   3226 	// write profile_pk + created_at index
   3227 	ndb_tsid_init(&tsid, note->pubkey, note->created_at);
   3228 
   3229 	key.mv_data = &tsid;
   3230 	key.mv_size = sizeof(tsid);
   3231 	val.mv_data = &profile_key;
   3232 	val.mv_size = sizeof(profile_key);
   3233 
   3234 	if ((rc = mdb_put(txn->mdb_txn, pk_db, &key, &val, 0))) {
   3235 		ndb_debug("write profile_pk(%" PRIu64 ") to db failed: %s\n",
   3236 				profile_key, mdb_strerror(rc));
   3237 		return 0;
   3238 	}
   3239 
   3240 	return 1;
   3241 }
   3242 
   3243 static int ndb_write_profile(struct ndb_txn *txn,
   3244 			     struct ndb_writer_profile *profile,
   3245 			     uint64_t note_key)
   3246 {
   3247 	uint64_t profile_key;
   3248 	struct ndb_note *note;
   3249 	void *flatbuf;
   3250 	size_t flatbuf_len;
   3251 	int rc;
   3252 
   3253 	MDB_val key, val;
   3254 	MDB_dbi profile_db;
   3255 
   3256 	note = profile->note.note;
   3257 
   3258 	// add note_key to profile record
   3259 	NdbProfileRecord_note_key_add(profile->record.builder, note_key);
   3260 	NdbProfileRecord_end_as_root(profile->record.builder);
   3261 
   3262 	flatbuf = profile->record.flatbuf =
   3263 		flatcc_builder_finalize_aligned_buffer(profile->record.builder, &flatbuf_len);
   3264 
   3265 	assert(((uint64_t)flatbuf % 8) == 0);
   3266 
   3267 	// TODO: this may not be safe!?
   3268 	flatbuf_len = (flatbuf_len + 7) & ~7;
   3269 
   3270 	//assert(NdbProfileRecord_verify_as_root(flatbuf, flatbuf_len) == 0);
   3271 
   3272 	// get dbs
   3273 	profile_db = txn->lmdb->dbs[NDB_DB_PROFILE];
   3274 
   3275 	// get new key
   3276 	profile_key = ndb_get_last_key(txn->mdb_txn, profile_db) + 1;
   3277 
   3278 	// write profile to profile store
   3279 	key.mv_data = &profile_key;
   3280 	key.mv_size = sizeof(profile_key);
   3281 	val.mv_data = flatbuf;
   3282 	val.mv_size = flatbuf_len;
   3283 
   3284 	if ((rc = mdb_put(txn->mdb_txn, profile_db, &key, &val, 0))) {
   3285 		ndb_debug("write profile to db failed: %s\n", mdb_strerror(rc));
   3286 		return 0;
   3287 	}
   3288 
   3289 	// write last fetched record
   3290 	if (!ndb_maybe_write_last_profile_fetch(txn, note)) {
   3291 		ndb_debug("failed to write last profile fetched record\n");
   3292 	}
   3293 
   3294 	// write profile pubkey index
   3295 	if (!ndb_write_profile_pk_index(txn, note, profile_key)) {
   3296 		ndb_debug("failed to write profile pubkey index\n");
   3297 		return 0;
   3298 	}
   3299 
   3300 	// write name, display_name profile search indices
   3301 	if (!ndb_write_profile_search_indices(txn, note, profile_key,
   3302 					      flatbuf)) {
   3303 		ndb_debug("failed to write profile search indices\n");
   3304 		return 0;
   3305 	}
   3306 
   3307 	return 1;
   3308 }
   3309 
   3310 // find the last id tag in a note (e, p, etc)
   3311 static unsigned char *ndb_note_last_id_tag(struct ndb_note *note, char type)
   3312 {
   3313 	unsigned char *last = NULL;
   3314 	struct ndb_iterator iter;
   3315 	struct ndb_str str;
   3316 
   3317 	// get the liked event id (last id)
   3318 	ndb_tags_iterate_start(note, &iter);
   3319 
   3320 	while (ndb_tags_iterate_next(&iter)) {
   3321 		if (iter.tag->count < 2)
   3322 			continue;
   3323 
   3324 		str = ndb_tag_str(note, iter.tag, 0);
   3325 
   3326 		// assign liked to the last e tag
   3327 		if (str.flag == NDB_PACKED_STR && str.str[0] == type) {
   3328 			str = ndb_tag_str(note, iter.tag, 1);
   3329 			if (str.flag == NDB_PACKED_ID)
   3330 				last = str.id;
   3331 		}
   3332 	}
   3333 
   3334 	return last;
   3335 }
   3336 
   3337 void *ndb_get_note_meta(struct ndb_txn *txn, const unsigned char *id, size_t *len)
   3338 {
   3339 	MDB_val k, v;
   3340 
   3341 	k.mv_data = (unsigned char*)id;
   3342 	k.mv_size = 32;
   3343 
   3344 	if (mdb_get(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_META], &k, &v)) {
   3345 		//ndb_debug("ndb_get_note_meta: mdb_get note failed\n");
   3346 		return NULL;
   3347 	}
   3348 
   3349 	if (len)
   3350 		*len = v.mv_size;
   3351 
   3352 	return v.mv_data;
   3353 }
   3354 
   3355 // When receiving a reaction note, look for the liked id and increase the
   3356 // reaction counter in the note metadata database
   3357 static int ndb_write_reaction_stats(struct ndb_txn *txn, struct ndb_note *note)
   3358 {
   3359 	size_t len;
   3360 	void *root;
   3361 	int reactions, rc;
   3362 	MDB_val key, val;
   3363 	NdbEventMeta_table_t meta;
   3364 	unsigned char *liked = ndb_note_last_id_tag(note, 'e');
   3365 
   3366 	if (liked == NULL)
   3367 		return 0;
   3368 
   3369 	root = ndb_get_note_meta(txn, liked, &len);
   3370 
   3371 	flatcc_builder_t builder;
   3372 	flatcc_builder_init(&builder);
   3373 	NdbEventMeta_start_as_root(&builder);
   3374 
   3375 	// no meta record, let's make one
   3376 	if (root == NULL) {
   3377 		NdbEventMeta_reactions_add(&builder, 1);
   3378 	} else {
   3379 		// clone existing and add to it
   3380 		meta = NdbEventMeta_as_root(root);
   3381 
   3382 		reactions = NdbEventMeta_reactions_get(meta);
   3383 		NdbEventMeta_clone(&builder, meta);
   3384 		NdbEventMeta_reactions_add(&builder, reactions + 1);
   3385 	}
   3386 
   3387 	NdbProfileRecord_end_as_root(&builder);
   3388 	root = flatcc_builder_finalize_aligned_buffer(&builder, &len);
   3389 	assert(((uint64_t)root % 8) == 0);
   3390 
   3391 	if (root == NULL) {
   3392 		ndb_debug("failed to create note metadata record\n");
   3393 		goto fail;
   3394 	}
   3395 
   3396 	// metadata is keyed on id because we want to collect stats regardless
   3397 	// if we have the note yet or not
   3398 	key.mv_data = liked;
   3399 	key.mv_size = 32;
   3400 
   3401 	val.mv_data = root;
   3402 	val.mv_size = len;
   3403 
   3404 	// write the new meta record
   3405 	//ndb_debug("writing stats record for ");
   3406 	//print_hex(liked, 32);
   3407 	//ndb_debug("\n");
   3408 
   3409 	if ((rc = mdb_put(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_META], &key, &val, 0))) {
   3410 		ndb_debug("write reaction stats to db failed: %s\n", mdb_strerror(rc));
   3411 		goto fail;
   3412 	}
   3413 
   3414 	free(root);
   3415 	flatcc_builder_clear(&builder);
   3416 
   3417 	return 1;
   3418 
   3419 fail:
   3420 	free(root);
   3421 	flatcc_builder_clear(&builder);
   3422 	return 0;
   3423 }
   3424 
   3425 
   3426 static int ndb_write_note_id_index(struct ndb_txn *txn, struct ndb_note *note,
   3427 				   uint64_t note_key)
   3428 
   3429 {
   3430 	struct ndb_tsid tsid;
   3431 	int rc;
   3432 	MDB_val key, val;
   3433 	MDB_dbi id_db;
   3434 
   3435 	ndb_tsid_init(&tsid, note->id, note->created_at);
   3436 
   3437 	key.mv_data = &tsid;
   3438 	key.mv_size = sizeof(tsid);
   3439 	val.mv_data = &note_key;
   3440 	val.mv_size = sizeof(note_key);
   3441 
   3442 	id_db = txn->lmdb->dbs[NDB_DB_NOTE_ID];
   3443 
   3444 	if ((rc = mdb_put(txn->mdb_txn, id_db, &key, &val, 0))) {
   3445 		ndb_debug("write note id index to db failed: %s\n",
   3446 				mdb_strerror(rc));
   3447 		return 0;
   3448 	}
   3449 
   3450 	return 1;
   3451 }
   3452 
   3453 static int ndb_filter_group_add_filters(struct ndb_filter_group *group,
   3454 					struct ndb_filter *filters,
   3455 					int num_filters)
   3456 {
   3457 	int i;
   3458 
   3459 	for (i = 0; i < num_filters; i++) {
   3460 		if (!ndb_filter_group_add(group, &filters[i]))
   3461 			return 0;
   3462 	}
   3463 
   3464 	return 1;
   3465 }
   3466 
   3467 
   3468 static struct ndb_filter_elements *
   3469 ndb_filter_find_elements(struct ndb_filter *filter, enum ndb_filter_fieldtype typ)
   3470 {
   3471 	int i;
   3472 	struct ndb_filter_elements *els;
   3473 
   3474 	for (i = 0; i < filter->num_elements; i++) {
   3475 		els = ndb_filter_get_elements(filter, i);
   3476 		assert(els);
   3477 		if (els->field.type == typ) {
   3478 			return els;
   3479 		}
   3480 	}
   3481 
   3482 	return NULL;
   3483 }
   3484 
   3485 static const char *ndb_filter_find_search(struct ndb_filter *filter)
   3486 {
   3487 	struct ndb_filter_elements *els;
   3488 
   3489 	if (!(els = ndb_filter_find_elements(filter, NDB_FILTER_SEARCH)))
   3490 		return NULL;
   3491 
   3492 	return ndb_filter_get_string_element(filter, els, 0);
   3493 }
   3494 
   3495 int ndb_filter_is_subset_of(const struct ndb_filter *a, const struct ndb_filter *b)
   3496 {
   3497 	int i;
   3498 	struct ndb_filter_elements *b_field, *a_field;
   3499 
   3500         // Everything is always a subset of {}
   3501 	if (b->num_elements == 0)
   3502 		return 1;
   3503 
   3504         // We can't be a subset if the number of elements in the other
   3505         // filter is larger then the number of elements we have.
   3506 	if (b->num_elements > a->num_elements)
   3507 		return 0;
   3508 
   3509         // If our filter count matches, we can only be a subset if we are
   3510         // equal
   3511 	if (b->num_elements == a->num_elements)
   3512 		return ndb_filter_eq(a, b);
   3513 
   3514         // If our element count is larger than the other filter, then we
   3515         // must see if every element in the other filter exists in ours. If
   3516         // so, then we are a subset of the other.
   3517         //
   3518         // eg: B={k:1, a:b} <- A={t:x, k:1, a:b}
   3519         //
   3520         // A is a subset of B because `k:1` and `a:b` both exist in A
   3521 
   3522 	for (i = 0; i < b->num_elements; i++) {
   3523 		b_field = ndb_filter_get_elements((struct ndb_filter*)b, i);
   3524 		a_field = ndb_filter_find_elements((struct ndb_filter*)a,
   3525 						   b_field->field.type);
   3526 
   3527 		if (a_field == NULL)
   3528 			return 0;
   3529 
   3530 		if (!ndb_filter_field_eq((struct ndb_filter*)a, a_field,
   3531 					 (struct ndb_filter*)b, b_field))
   3532 			return 0;
   3533 	}
   3534 
   3535 	return 1;
   3536 }
   3537 
   3538 int ndb_filter_eq(const struct ndb_filter *a, const struct ndb_filter *b)
   3539 {
   3540 	int i;
   3541 	struct ndb_filter_elements *a_els, *b_els;
   3542 
   3543 	if (a->num_elements != b->num_elements)
   3544 		return 0;
   3545 
   3546 	for (i = 0; i < a->num_elements; i++) {
   3547 		a_els = ndb_filter_get_elements((struct ndb_filter*)a, i);
   3548 		b_els = ndb_filter_find_elements((struct ndb_filter *)b,
   3549 						 a_els->field.type);
   3550 
   3551 		if (b_els == NULL)
   3552 			return 0;
   3553 
   3554 		if (!ndb_filter_field_eq((struct ndb_filter*)a, a_els,
   3555 					 (struct ndb_filter*)b, b_els))
   3556 			return 0;
   3557 	}
   3558 
   3559 	return 1;
   3560 }
   3561 
   3562 
   3563 static uint64_t *
   3564 ndb_filter_get_elem(struct ndb_filter *filter, enum ndb_filter_fieldtype typ)
   3565 {
   3566 	struct ndb_filter_elements *els;
   3567 	if ((els = ndb_filter_find_elements(filter, typ)))
   3568 		return &els->elements[0];
   3569 	return NULL;
   3570 }
   3571 
   3572 static uint64_t *ndb_filter_get_int(struct ndb_filter *filter,
   3573 				    enum ndb_filter_fieldtype typ)
   3574 {
   3575 	uint64_t *el;
   3576 	if (NULL == (el = ndb_filter_get_elem(filter, typ)))
   3577 		return NULL;
   3578 	return el;
   3579 }
   3580 
   3581 static inline int push_query_result(struct ndb_query_results *results,
   3582 				    struct ndb_query_result *result)
   3583 {
   3584 	return cursor_push(&results->cur, (unsigned char*)result, sizeof(*result));
   3585 }
   3586 
   3587 static int compare_query_results(const void *pa, const void *pb)
   3588 {
   3589 	struct ndb_query_result *a, *b;
   3590 
   3591 	a = (struct ndb_query_result *)pa;
   3592 	b = (struct ndb_query_result *)pb;
   3593 
   3594 	if (a->note->created_at == b->note->created_at) {
   3595 		return 0;
   3596 	} else if (a->note->created_at > b->note->created_at) {
   3597 		return -1;
   3598 	} else {
   3599 		return 1;
   3600 	}
   3601 }
   3602 
   3603 static void ndb_query_result_init(struct ndb_query_result *res,
   3604 				  struct ndb_note *note,
   3605 				  uint64_t note_size,
   3606 				  uint64_t note_id)
   3607 {
   3608 	*res = (struct ndb_query_result){
   3609 		.note_id = note_id,
   3610 		.note_size = note_size,
   3611 		.note = note,
   3612 	};
   3613 }
   3614 
   3615 static int query_is_full(struct ndb_query_results *results, int limit)
   3616 {
   3617 	if (results->cur.p >= results->cur.end)
   3618 		return 1;
   3619 
   3620 	return cursor_count(&results->cur, sizeof(struct ndb_query_result)) >= limit;
   3621 }
   3622 
   3623 static int ndb_query_plan_execute_search(struct ndb_txn *txn,
   3624 					 struct ndb_filter *filter,
   3625 					 struct ndb_query_results *results,
   3626 					 int limit)
   3627 {
   3628 	const char *search;
   3629 	int i;
   3630 	struct ndb_text_search_results text_results;
   3631 	struct ndb_text_search_result *text_result;
   3632 	struct ndb_text_search_config config;
   3633 	struct ndb_query_result result;
   3634 
   3635 	ndb_default_text_search_config(&config);
   3636 
   3637 	if (!(search = ndb_filter_find_search(filter)))
   3638 		return 0;
   3639 
   3640 	if (!ndb_text_search_with(txn, search, &text_results, &config, filter))
   3641 		return 0;
   3642 
   3643 	for (i = 0; i < text_results.num_results; i++) {
   3644 		if (query_is_full(results, limit))
   3645 			break;
   3646 
   3647 		text_result = &text_results.results[i];
   3648 
   3649 		result.note = text_result->note;
   3650 		result.note_size = text_result->note_size;
   3651 		result.note_id = text_result->key.note_id;
   3652 
   3653 		if (!push_query_result(results, &result))
   3654 			break;
   3655 	}
   3656 
   3657 	return 1;
   3658 }
   3659 
   3660 static int ndb_query_plan_execute_ids(struct ndb_txn *txn,
   3661 				      struct ndb_filter *filter,
   3662 				      struct ndb_query_results *results,
   3663 				      int limit)
   3664 {
   3665 	MDB_cursor *cur;
   3666 	MDB_dbi db;
   3667 	MDB_val k, v;
   3668 	int rc, i, need_relays = 0;
   3669 	struct ndb_filter_elements *ids;
   3670 	struct ndb_note *note;
   3671 	struct ndb_query_result res;
   3672 	struct ndb_tsid tsid, *ptsid;
   3673 	uint64_t note_id, until, *pint;
   3674 	size_t note_size;
   3675 	unsigned char *id;
   3676 	struct ndb_note_relay_iterator note_relay_iter = {0};
   3677 	struct ndb_note_relay_iterator *relay_iter = NULL;
   3678 
   3679 	until = UINT64_MAX;
   3680 
   3681 	if (!(ids = ndb_filter_find_elements(filter, NDB_FILTER_IDS)))
   3682 		return 0;
   3683 
   3684 	if ((pint = ndb_filter_get_int(filter, NDB_FILTER_UNTIL)))
   3685 		until = *pint;
   3686 
   3687 	if (ndb_filter_find_elements(filter, NDB_FILTER_RELAYS))
   3688 		need_relays = 1;
   3689 
   3690 	db = txn->lmdb->dbs[NDB_DB_NOTE_ID];
   3691 	if ((rc = mdb_cursor_open(txn->mdb_txn, db, &cur)))
   3692 		return 0;
   3693 
   3694 	// for each id in our ids filter, find in the db
   3695 	for (i = 0; i < ids->count; i++) {
   3696 		if (query_is_full(results, limit))
   3697 			break;
   3698 
   3699 		id = ndb_filter_get_id_element(filter, ids, i);
   3700 		ndb_tsid_init(&tsid, (unsigned char *)id, until);
   3701 
   3702 		k.mv_data = &tsid;
   3703 		k.mv_size = sizeof(tsid);
   3704 
   3705 		if (!ndb_cursor_start(cur, &k, &v))
   3706 			continue;
   3707 
   3708 		ptsid = (struct ndb_tsid *)k.mv_data;
   3709 		note_id = *(uint64_t*)v.mv_data;
   3710 
   3711 		if (memcmp(id, ptsid->id, 32))
   3712 			continue;
   3713 
   3714 		// get the note because we need it to match against the filter
   3715 		if (!(note = ndb_get_note_by_key(txn, note_id, &note_size)))
   3716 			continue;
   3717 
   3718 		relay_iter = need_relays ? &note_relay_iter : NULL;
   3719 		if (relay_iter)
   3720 			ndb_note_relay_iterate_start(txn, relay_iter, note_id);
   3721 
   3722 		// Sure this particular lookup matched the index query, but
   3723 		// does it match the entire filter? Check! We also pass in
   3724 		// things we've already matched via the filter so we don't have
   3725 		// to check again. This can be pretty important for filters
   3726 		// with a large number of entries.
   3727 		if (!ndb_filter_matches_with(filter, note, 1 << NDB_FILTER_IDS, relay_iter)) {
   3728 			ndb_note_relay_iterate_close(relay_iter);
   3729 			continue;
   3730 		}
   3731 		ndb_note_relay_iterate_close(relay_iter);
   3732 
   3733 		ndb_query_result_init(&res, note, note_size, note_id);
   3734 		if (!push_query_result(results, &res))
   3735 			break;
   3736 	}
   3737 
   3738 	mdb_cursor_close(cur);
   3739 	return 1;
   3740 }
   3741 
   3742 //
   3743 // encode a tag index key
   3744 //
   3745 // consists of:
   3746 //
   3747 //   u8   tag
   3748 //   u8   tag_val_len
   3749 //   [u8] tag_val_bytes
   3750 //   u64  created_at
   3751 //
   3752 static int ndb_encode_tag_key(unsigned char *buf, int buf_size,
   3753 			      char tag, const unsigned char *val,
   3754 			      unsigned char val_len,
   3755 			      uint64_t timestamp)
   3756 {
   3757 	struct cursor writer;
   3758 	int ok;
   3759 
   3760 	// quick exit for obvious case where it will be too big. There can be
   3761 	// values of val_len that still fail, but we just let the writer handle
   3762 	// those failure cases
   3763 	if (val_len >= buf_size)
   3764 		return 0;
   3765 
   3766 	make_cursor(buf, buf + buf_size, &writer);
   3767 
   3768 	ok =
   3769 		cursor_push_byte(&writer, tag) &&
   3770 		cursor_push(&writer, (unsigned char*)val, val_len) &&
   3771 		cursor_push(&writer, (unsigned char*)&timestamp, sizeof(timestamp));
   3772 
   3773 	if (!ok)
   3774 		return 0;
   3775 
   3776 	return writer.p - writer.start;
   3777 }
   3778 
   3779 static int ndb_query_plan_execute_authors(struct ndb_txn *txn,
   3780 					  struct ndb_filter *filter,
   3781 					  struct ndb_query_results *results,
   3782 					  int limit)
   3783 {
   3784 	MDB_val k, v;
   3785 	MDB_cursor *cur;
   3786 	int rc, i, need_relays = 0;
   3787 	uint64_t *pint, until, since, note_key;
   3788 	unsigned char *author;
   3789 	struct ndb_note *note;
   3790 	size_t note_size;
   3791 	struct ndb_filter_elements *authors;
   3792 	struct ndb_query_result res;
   3793 	struct ndb_tsid tsid, *ptsid;
   3794 	struct ndb_note_relay_iterator note_relay_iter;
   3795 	enum ndb_dbs db;
   3796 
   3797 	db = txn->lmdb->dbs[NDB_DB_NOTE_PUBKEY];
   3798 
   3799 	if (!(authors = ndb_filter_find_elements(filter, NDB_FILTER_AUTHORS)))
   3800 		return 0;
   3801 
   3802 	until = UINT64_MAX;
   3803 	if ((pint = ndb_filter_get_int(filter, NDB_FILTER_UNTIL)))
   3804 		until = *pint;
   3805 
   3806 	since = 0;
   3807 	if ((pint = ndb_filter_get_int(filter, NDB_FILTER_SINCE)))
   3808 		since = *pint;
   3809 
   3810 	if (ndb_filter_find_elements(filter, NDB_FILTER_RELAYS))
   3811 		need_relays = 1;
   3812 
   3813 	if ((rc = mdb_cursor_open(txn->mdb_txn, db, &cur)))
   3814 		return 0;
   3815 
   3816 	for (i = 0; i < authors->count; i++) {
   3817 		author = ndb_filter_get_id_element(filter, authors, i);
   3818 
   3819 		ndb_tsid_init(&tsid, author, until);
   3820 
   3821 		k.mv_data = &tsid;
   3822 		k.mv_size = sizeof(tsid);
   3823 
   3824 		if (!ndb_cursor_start(cur, &k, &v))
   3825 			continue;
   3826 
   3827 		// for each id in our ids filter, find in the db
   3828 		while (!query_is_full(results, limit)) {
   3829 			ptsid = (struct ndb_tsid *)k.mv_data;
   3830 			note_key = *(uint64_t*)v.mv_data;
   3831 
   3832 			// don't continue the scan if we're below `since`
   3833 			if (ptsid->timestamp < since)
   3834 				break;
   3835 
   3836 			// our author should match, if not bail
   3837 			if (memcmp(author, ptsid->id, 32))
   3838 				break;
   3839 
   3840 			// fetch the note, we need it for our query results
   3841 			// and to match further against the filter
   3842 			if (!(note = ndb_get_note_by_key(txn, note_key, &note_size)))
   3843 				goto next;
   3844 
   3845 			if (need_relays)
   3846 				ndb_note_relay_iterate_start(txn, &note_relay_iter, note_key);
   3847 
   3848 			if (!ndb_filter_matches_with(filter, note,
   3849 						     1 << NDB_FILTER_AUTHORS,
   3850 						     need_relays ? &note_relay_iter : NULL))
   3851 			{
   3852 				goto next;
   3853 			}
   3854 
   3855 			ndb_query_result_init(&res, note, note_size, note_key);
   3856 			if (!push_query_result(results, &res))
   3857 				break;
   3858 
   3859 next:
   3860 			if (mdb_cursor_get(cur, &k, &v, MDB_PREV))
   3861 				break;
   3862 		}
   3863 	}
   3864 
   3865 	mdb_cursor_close(cur);
   3866 	return 1;
   3867 }
   3868 
   3869 static int ndb_query_plan_execute_created_at(struct ndb_txn *txn,
   3870 					     struct ndb_filter *filter,
   3871 					     struct ndb_query_results *results,
   3872 					     int limit)
   3873 {
   3874 	MDB_dbi db;
   3875 	MDB_val k, v;
   3876 	MDB_cursor *cur;
   3877 	int rc, need_relays = 0;
   3878 	struct ndb_note *note;
   3879 	struct ndb_tsid key, *pkey;
   3880 	uint64_t *pint, until, since, note_id;
   3881 	size_t note_size;
   3882 	struct ndb_query_result res;
   3883 	struct ndb_note_relay_iterator note_relay_iter;
   3884 	unsigned char high_key[32] = {0xFF};
   3885 
   3886 	db = txn->lmdb->dbs[NDB_DB_NOTE_ID];
   3887 
   3888 	until = UINT64_MAX;
   3889 	if ((pint = ndb_filter_get_int(filter, NDB_FILTER_UNTIL)))
   3890 		until = *pint;
   3891 
   3892 	since = 0;
   3893 	if ((pint = ndb_filter_get_int(filter, NDB_FILTER_SINCE)))
   3894 		since = *pint;
   3895 
   3896 	if (ndb_filter_find_elements(filter, NDB_FILTER_RELAYS))
   3897 		need_relays = 1;
   3898 
   3899 	if ((rc = mdb_cursor_open(txn->mdb_txn, db, &cur)))
   3900 		return 0;
   3901 
   3902 	// if we have until, start there, otherwise just use max
   3903 	ndb_tsid_init(&key, high_key, until);
   3904 	k.mv_data = &key;
   3905 	k.mv_size = sizeof(key);
   3906 
   3907 	if (!ndb_cursor_start(cur, &k, &v))
   3908 		return 1;
   3909 
   3910 	while (!query_is_full(results, limit)) {
   3911 		pkey = (struct ndb_tsid *)k.mv_data;
   3912 		note_id = *(uint64_t*)v.mv_data;
   3913 		assert(v.mv_size == 8);
   3914 
   3915 		// don't continue the scan if we're below `since`
   3916 		if (pkey->timestamp < since)
   3917 			break;
   3918 
   3919 		if (!(note = ndb_get_note_by_key(txn, note_id, &note_size)))
   3920 			goto next;
   3921 
   3922 		if (need_relays)
   3923 			ndb_note_relay_iterate_start(txn, &note_relay_iter, note_id);
   3924 
   3925 		// does this entry match our filter?
   3926 		if (!ndb_filter_matches_with(filter, note, 0, need_relays ? &note_relay_iter : NULL))
   3927 			goto next;
   3928 
   3929 		ndb_query_result_init(&res, note, (uint64_t)note_size, note_id);
   3930 		if (!push_query_result(results, &res))
   3931 			break;
   3932 next:
   3933 		if (mdb_cursor_get(cur, &k, &v, MDB_PREV))
   3934 			break;
   3935 	}
   3936 
   3937 	mdb_cursor_close(cur);
   3938 	return 1;
   3939 }
   3940 
   3941 static int ndb_query_plan_execute_tags(struct ndb_txn *txn,
   3942 				       struct ndb_filter *filter,
   3943 				       struct ndb_query_results *results,
   3944 				       int limit)
   3945 {
   3946 	MDB_cursor *cur;
   3947 	MDB_dbi db;
   3948 	MDB_val k, v;
   3949 	int len, taglen, rc, i, need_relays = 0;
   3950 	uint64_t *pint, until, note_id;
   3951 	size_t note_size;
   3952 	unsigned char key_buffer[255];
   3953 	struct ndb_note *note;
   3954 	struct ndb_filter_elements *tags;
   3955 	unsigned char *tag;
   3956 	struct ndb_query_result res;
   3957 	struct ndb_note_relay_iterator note_relay_iter;
   3958 
   3959 	db = txn->lmdb->dbs[NDB_DB_NOTE_TAGS];
   3960 
   3961 	if (!(tags = ndb_filter_find_elements(filter, NDB_FILTER_TAGS)))
   3962 		return 0;
   3963 
   3964 	if (ndb_filter_find_elements(filter, NDB_FILTER_RELAYS))
   3965 		need_relays = 1;
   3966 
   3967 	until = UINT64_MAX;
   3968 	if ((pint = ndb_filter_get_int(filter, NDB_FILTER_UNTIL)))
   3969 		until = *pint;
   3970 
   3971 	if ((rc = mdb_cursor_open(txn->mdb_txn, db, &cur)))
   3972 		return 0;
   3973 
   3974 	for (i = 0; i < tags->count; i++) {
   3975 		tag = ndb_filter_get_id_element(filter, tags, i);
   3976 
   3977 		taglen = tags->field.elem_type == NDB_ELEMENT_ID
   3978 		       ? 32 : strlen((const char*)tag);
   3979 
   3980 		if (!(len = ndb_encode_tag_key(key_buffer, sizeof(key_buffer),
   3981 					       tags->field.tag, tag, taglen,
   3982 					       until))) {
   3983 			goto fail;
   3984 		}
   3985 
   3986 		k.mv_data = key_buffer;
   3987 		k.mv_size = len;
   3988 
   3989 		if (!ndb_cursor_start(cur, &k, &v))
   3990 			continue;
   3991 
   3992 		// for each id in our ids filter, find in the db
   3993 		while (!query_is_full(results, limit)) {
   3994 			// check if tag value matches, bail if not
   3995 			if (((unsigned char *)k.mv_data)[0] != tags->field.tag)
   3996 				break;
   3997 
   3998 			// check if tag value matches, bail if not
   3999 			if (taglen != k.mv_size - 9)
   4000 				break;
   4001 
   4002 			if (memcmp((unsigned char *)k.mv_data+1, tag, k.mv_size-9))
   4003 				break;
   4004 
   4005 			note_id = *(uint64_t*)v.mv_data;
   4006 
   4007 			if (!(note = ndb_get_note_by_key(txn, note_id, &note_size)))
   4008 				goto next;
   4009 
   4010 			if (need_relays)
   4011 				ndb_note_relay_iterate_start(txn, &note_relay_iter, note_id);
   4012 
   4013 			if (!ndb_filter_matches_with(filter, note,
   4014 						     1 << NDB_FILTER_TAGS,
   4015 						     need_relays ? &note_relay_iter : NULL))
   4016 				goto next;
   4017 
   4018 			ndb_query_result_init(&res, note, note_size, note_id);
   4019 			if (!push_query_result(results, &res))
   4020 				break;
   4021 
   4022 next:
   4023 			if (mdb_cursor_get(cur, &k, &v, MDB_PREV))
   4024 				break;
   4025 		}
   4026 	}
   4027 
   4028 	mdb_cursor_close(cur);
   4029 	return 1;
   4030 fail:
   4031 	mdb_cursor_close(cur);
   4032 	return 0;
   4033 }
   4034 
   4035 static int ndb_query_plan_execute_author_kinds(
   4036 		struct ndb_txn *txn,
   4037 		struct ndb_filter *filter,
   4038 		struct ndb_query_results *results,
   4039 		int limit)
   4040 {
   4041 	MDB_cursor *cur;
   4042 	MDB_dbi db;
   4043 	MDB_val k, v;
   4044 	struct ndb_note *note;
   4045 	struct ndb_filter_elements *kinds, *relays, *authors;
   4046 	struct ndb_query_result res;
   4047 	uint64_t kind, note_id, until, since, *pint;
   4048 	size_t note_size;
   4049 	unsigned char *author;
   4050 	int i, j, rc;
   4051 	struct ndb_id_u64_ts key, *pkey;
   4052 	struct ndb_note_relay_iterator note_relay_iter;
   4053 
   4054 	// we should have kinds in a kinds filter!
   4055 	if (!(kinds = ndb_filter_find_elements(filter, NDB_FILTER_KINDS)))
   4056 		return 0;
   4057 	//
   4058 	// we should have kinds in a kinds filter!
   4059 	if (!(authors = ndb_filter_find_elements(filter, NDB_FILTER_AUTHORS)))
   4060 		return 0;
   4061 
   4062 	relays = ndb_filter_find_elements(filter, NDB_FILTER_RELAYS);
   4063 
   4064 	until = UINT64_MAX;
   4065 	if ((pint = ndb_filter_get_int(filter, NDB_FILTER_UNTIL)))
   4066 		until = *pint;
   4067 
   4068 	since = 0;
   4069 	if ((pint = ndb_filter_get_int(filter, NDB_FILTER_SINCE)))
   4070 		since = *pint;
   4071 
   4072 	db = txn->lmdb->dbs[NDB_DB_NOTE_PUBKEY_KIND];
   4073 
   4074 	if ((rc = mdb_cursor_open(txn->mdb_txn, db, &cur)))
   4075 		return 0;
   4076 
   4077 	for (j = 0; j < authors->count; j++) {
   4078 		if (query_is_full(results, limit))
   4079 			break;
   4080 
   4081 		if (!(author = ndb_filter_get_id_element(filter, authors, j)))
   4082 			continue;
   4083 
   4084 	for (i = 0; i < kinds->count; i++) {
   4085 		if (query_is_full(results, limit))
   4086 			break;
   4087 
   4088 		kind = kinds->elements[i];
   4089 
   4090 		ndb_debug("finding kind %"PRIu64"\n", kind);
   4091 
   4092 		ndb_id_u64_ts_init(&key, author, kind, until);
   4093 		
   4094 		k.mv_data = &key;
   4095 		k.mv_size = sizeof(key);
   4096 
   4097 		if (!ndb_cursor_start(cur, &k, &v))
   4098 			continue;
   4099 
   4100 		// scan the kind subindex
   4101 		while (!query_is_full(results, limit)) {
   4102 			pkey = (struct ndb_id_u64_ts*)k.mv_data;
   4103 
   4104 			ndb_debug("scanning subindex kind:%"PRIu64" created_at:%"PRIu64" pubkey:",
   4105 					pkey->u64,
   4106 					pkey->timestamp);
   4107 
   4108 			if (pkey->u64 != kind)
   4109 				break;
   4110 
   4111 			// don't continue the scan if we're below `since`
   4112 			if (pkey->timestamp < since)
   4113 				break;
   4114 
   4115 			if (memcmp(pkey->id, author, 32))
   4116 				break;
   4117 
   4118 			note_id = *(uint64_t*)v.mv_data;
   4119 			if (!(note = ndb_get_note_by_key(txn, note_id, &note_size)))
   4120 				goto next;
   4121 
   4122 			if (relays)
   4123 				ndb_note_relay_iterate_start(txn, &note_relay_iter, note_id);
   4124 
   4125 			if (!ndb_filter_matches_with(filter, note,
   4126 						     (1 << NDB_FILTER_KINDS) | (1 << NDB_FILTER_AUTHORS),
   4127 						     relays? &note_relay_iter : NULL))
   4128 				goto next;
   4129 
   4130 			ndb_query_result_init(&res, note, note_size, note_id);
   4131 			if (!push_query_result(results, &res))
   4132 				break;
   4133 
   4134 next:
   4135 			if (mdb_cursor_get(cur, &k, &v, MDB_PREV))
   4136 				break;
   4137 		}
   4138 	}
   4139 	}
   4140 
   4141 	mdb_cursor_close(cur);
   4142 	return 1;
   4143 }
   4144 
   4145 static int ndb_query_plan_execute_profile_search(
   4146 		struct ndb_txn *txn,
   4147 		struct ndb_filter *filter,
   4148 		struct ndb_query_results *results,
   4149 		int limit)
   4150 {
   4151 	const char *search;
   4152 	int i;
   4153 
   4154 	// The filter pubkey is updated inplace for each note search
   4155 	unsigned char *filter_pubkey;
   4156 	unsigned char pubkey[32] = {0};
   4157 	struct ndb_filter_elements *els;
   4158 	struct ndb_search profile_search;
   4159 	struct ndb_filter note_filter, *f = &note_filter;
   4160 
   4161 	if (!(search = ndb_filter_find_search(filter)))
   4162 		return 0;
   4163 
   4164 	if (!ndb_filter_init_with(f, 1))
   4165 		return 0;
   4166 
   4167 	ndb_filter_start_field(f, NDB_FILTER_KINDS);
   4168 	ndb_filter_add_int_element(f, 0);
   4169 	ndb_filter_end_field(f);
   4170 
   4171 	ndb_filter_start_field(f, NDB_FILTER_AUTHORS);
   4172 	ndb_filter_add_id_element(f, pubkey);
   4173 	ndb_filter_end_field(f);
   4174 	ndb_filter_end(f);
   4175 
   4176 	// get the authors element after we finalize the filter, since
   4177 	// the data could have moved
   4178 	if (!(els = ndb_filter_find_elements(f, NDB_FILTER_AUTHORS)))
   4179 		goto fail;
   4180 
   4181 	// grab pointer to pubkey in the filter so that we can
   4182 	// update the filter as we go
   4183 	if (!(filter_pubkey = ndb_filter_get_id_element(f, els, 0)))
   4184 		goto fail;
   4185 
   4186 	for (i = 0; !query_is_full(results, limit); i++) {
   4187 		if (i == 0) {
   4188 			if (!ndb_search_profile(txn, &profile_search, search))
   4189 				break;
   4190 		} else {
   4191 			if (!ndb_search_profile_next(&profile_search))
   4192 				break;
   4193 		}
   4194 
   4195 		// Copy pubkey into filter
   4196 		memcpy(filter_pubkey, profile_search.key->id, 32);
   4197 
   4198 		// Look up the corresponding note associated with that pubkey
   4199 		if (!ndb_query_plan_execute_author_kinds(txn, f, results, limit))
   4200 			goto fail;
   4201 	}
   4202 
   4203 	ndb_search_profile_end(&profile_search);
   4204 	ndb_filter_destroy(f);
   4205 	return 1;
   4206 
   4207 fail:
   4208 	ndb_filter_destroy(f);
   4209 	return 0;
   4210 }
   4211 
   4212 static int ndb_query_plan_execute_relay_kinds(
   4213 		struct ndb_txn *txn,
   4214 		struct ndb_filter *filter,
   4215 		struct ndb_query_results *results,
   4216 		int limit)
   4217 {
   4218 	MDB_cursor *cur;
   4219 	MDB_dbi db;
   4220 	MDB_val k, v;
   4221 	struct ndb_note *note;
   4222 	struct ndb_filter_elements *kinds, *relays;
   4223 	struct ndb_query_result res;
   4224 	uint64_t kind, note_id, until, since, *pint;
   4225 	size_t note_size;
   4226 	const char *relay;
   4227 	int i, j, rc, len;
   4228 	struct ndb_relay_kind_key relay_key;
   4229 	unsigned char keybuf[256];
   4230 
   4231 	// we should have kinds in a kinds filter!
   4232 	if (!(kinds = ndb_filter_find_elements(filter, NDB_FILTER_KINDS)))
   4233 		return 0;
   4234 
   4235 	if (!(relays = ndb_filter_find_elements(filter, NDB_FILTER_RELAYS)))
   4236 		return 0;
   4237 
   4238 	until = UINT64_MAX;
   4239 	if ((pint = ndb_filter_get_int(filter, NDB_FILTER_UNTIL)))
   4240 		until = *pint;
   4241 
   4242 	since = 0;
   4243 	if ((pint = ndb_filter_get_int(filter, NDB_FILTER_SINCE)))
   4244 		since = *pint;
   4245 
   4246 	db = txn->lmdb->dbs[NDB_DB_NOTE_RELAY_KIND];
   4247 
   4248 	if ((rc = mdb_cursor_open(txn->mdb_txn, db, &cur)))
   4249 		return 0;
   4250 
   4251 	for (j = 0; j < relays->count; j++) {
   4252 		if (query_is_full(results, limit))
   4253 			break;
   4254 
   4255 		if (!(relay = ndb_filter_get_string_element(filter, relays, j)))
   4256 			continue;
   4257 
   4258 	for (i = 0; i < kinds->count; i++) {
   4259 		if (query_is_full(results, limit))
   4260 			break;
   4261 
   4262 		kind = kinds->elements[i];
   4263 		ndb_debug("kind %" PRIu64 "\n", kind);
   4264 		
   4265 		if (!ndb_relay_kind_key_init_high(&relay_key, relay, kind, until)) {
   4266 			ndb_debug("ndb_relay_kind_key_init_high failed in relay query\n");
   4267 			continue;
   4268 		}
   4269 
   4270 		if (!(len = ndb_build_relay_kind_key(keybuf, sizeof(keybuf), &relay_key))) {
   4271 			ndb_debug("ndb_build_relay_kind_key failed in relay query\n");
   4272 			ndb_debug_relay_kind_key(&relay_key);
   4273 			continue;
   4274 		}
   4275 
   4276 		k.mv_data = keybuf;
   4277 		k.mv_size = len;
   4278 
   4279 		ndb_debug("starting with key ");
   4280 		ndb_debug_relay_kind_key(&relay_key);
   4281 
   4282 		if (!ndb_cursor_start(cur, &k, &v))
   4283 			continue;
   4284 
   4285 		// scan the kind subindex
   4286 		while (!query_is_full(results, limit)) {
   4287 			ndb_parse_relay_kind_key(&relay_key, k.mv_data);
   4288 
   4289 			ndb_debug("inside kind subindex ");
   4290 			ndb_debug_relay_kind_key(&relay_key);
   4291 
   4292 			if (relay_key.kind != kind)
   4293 				break;
   4294 
   4295 			if (strcmp(relay_key.relay, relay))
   4296 				break;
   4297 
   4298 			// don't continue the scan if we're below `since`
   4299 			if (relay_key.created_at < since)
   4300 				break;
   4301 
   4302 			note_id = relay_key.note_key;
   4303 			if (!(note = ndb_get_note_by_key(txn, note_id, &note_size)))
   4304 				goto next;
   4305 
   4306 			if (!ndb_filter_matches_with(filter, note,
   4307 						     (1 << NDB_FILTER_KINDS) | (1 << NDB_FILTER_RELAYS),
   4308 						     NULL))
   4309 				goto next;
   4310 
   4311 			ndb_query_result_init(&res, note, note_size, note_id);
   4312 			if (!push_query_result(results, &res))
   4313 				break;
   4314 
   4315 next:
   4316 			if (mdb_cursor_get(cur, &k, &v, MDB_PREV))
   4317 				break;
   4318 		}
   4319 	}
   4320 	}
   4321 
   4322 	mdb_cursor_close(cur);
   4323 	return 1;
   4324 }
   4325 
   4326 static int ndb_query_plan_execute_kinds(struct ndb_txn *txn,
   4327 					struct ndb_filter *filter,
   4328 					struct ndb_query_results *results,
   4329 					int limit)
   4330 {
   4331 	MDB_cursor *cur;
   4332 	MDB_dbi db;
   4333 	MDB_val k, v;
   4334 	struct ndb_note *note;
   4335 	struct ndb_u64_ts tsid, *ptsid;
   4336 	struct ndb_filter_elements *kinds;
   4337 	struct ndb_query_result res;
   4338 	uint64_t kind, note_id, until, since, *pint;
   4339 	size_t note_size;
   4340 	int i, rc, need_relays = 0;
   4341 	struct ndb_note_relay_iterator note_relay_iter;
   4342 
   4343 	// we should have kinds in a kinds filter!
   4344 	if (!(kinds = ndb_filter_find_elements(filter, NDB_FILTER_KINDS)))
   4345 		return 0;
   4346 
   4347 	if (ndb_filter_find_elements(filter, NDB_FILTER_RELAYS))
   4348 		need_relays = 1;
   4349 
   4350 	until = UINT64_MAX;
   4351 	if ((pint = ndb_filter_get_int(filter, NDB_FILTER_UNTIL)))
   4352 		until = *pint;
   4353 
   4354 	since = 0;
   4355 	if ((pint = ndb_filter_get_int(filter, NDB_FILTER_SINCE)))
   4356 		since = *pint;
   4357 
   4358 	db = txn->lmdb->dbs[NDB_DB_NOTE_KIND];
   4359 
   4360 	if ((rc = mdb_cursor_open(txn->mdb_txn, db, &cur)))
   4361 		return 0;
   4362 
   4363 	for (i = 0; i < kinds->count; i++) {
   4364 		if (query_is_full(results, limit))
   4365 			break;
   4366 
   4367 		kind = kinds->elements[i];
   4368 		ndb_debug("kind %" PRIu64 "\n", kind);
   4369 		ndb_u64_ts_init(&tsid, kind, until);
   4370 
   4371 		k.mv_data = &tsid;
   4372 		k.mv_size = sizeof(tsid);
   4373 
   4374 		if (!ndb_cursor_start(cur, &k, &v))
   4375 			continue;
   4376 
   4377 		// for each id in our ids filter, find in the db
   4378 		while (!query_is_full(results, limit)) {
   4379 			ptsid = (struct ndb_u64_ts *)k.mv_data;
   4380 			if (ptsid->u64 != kind)
   4381 				break;
   4382 
   4383 			// don't continue the scan if we're below `since`
   4384 			if (ptsid->timestamp < since)
   4385 				break;
   4386 
   4387 			note_id = *(uint64_t*)v.mv_data;
   4388 			if (!(note = ndb_get_note_by_key(txn, note_id, &note_size)))
   4389 				goto next;
   4390 
   4391 			if (need_relays)
   4392 				ndb_note_relay_iterate_start(txn, &note_relay_iter, note_id);
   4393 
   4394 			if (!ndb_filter_matches_with(filter, note,
   4395 						     1 << NDB_FILTER_KINDS,
   4396 						     need_relays ? &note_relay_iter : NULL))
   4397 				goto next;
   4398 
   4399 			ndb_query_result_init(&res, note, note_size, note_id);
   4400 			if (!push_query_result(results, &res))
   4401 				break;
   4402 
   4403 next:
   4404 			if (mdb_cursor_get(cur, &k, &v, MDB_PREV))
   4405 				break;
   4406 		}
   4407 	}
   4408 
   4409 	mdb_cursor_close(cur);
   4410 	return 1;
   4411 }
   4412 
   4413 static enum ndb_query_plan ndb_filter_plan(struct ndb_filter *filter)
   4414 {
   4415 	struct ndb_filter_elements *ids, *kinds, *authors, *tags, *search, *relays;
   4416 
   4417 	ids = ndb_filter_find_elements(filter, NDB_FILTER_IDS);
   4418 	search = ndb_filter_find_elements(filter, NDB_FILTER_SEARCH);
   4419 	kinds = ndb_filter_find_elements(filter, NDB_FILTER_KINDS);
   4420 	authors = ndb_filter_find_elements(filter, NDB_FILTER_AUTHORS);
   4421 	tags = ndb_filter_find_elements(filter, NDB_FILTER_TAGS);
   4422 	relays = ndb_filter_find_elements(filter, NDB_FILTER_RELAYS);
   4423 
   4424 	// profile search
   4425 	if (kinds && kinds->count == 1 && kinds->elements[0] == 0 && search) {
   4426 		return NDB_PLAN_PROFILE_SEARCH;
   4427 	}
   4428 
   4429 	// TODO: fix multi-author queries
   4430 	if (search) {
   4431 		return NDB_PLAN_SEARCH;
   4432 	} else if (ids) {
   4433 		return NDB_PLAN_IDS;
   4434 	} else if (relays && kinds && !authors) {
   4435 		return NDB_PLAN_RELAY_KINDS;
   4436 	} else if (kinds && authors && authors->count == 1) {
   4437 		return NDB_PLAN_AUTHOR_KINDS;
   4438 	} else if (authors && authors->count == 1) {
   4439 		return NDB_PLAN_AUTHORS;
   4440 	} else if (tags && tags->count == 1) {
   4441 		return NDB_PLAN_TAGS;
   4442 	} else if (kinds) {
   4443 		return NDB_PLAN_KINDS;
   4444 	}
   4445 
   4446 	return NDB_PLAN_CREATED;
   4447 }
   4448 
   4449 static const char *ndb_query_plan_name(enum ndb_query_plan plan_id)
   4450 {
   4451 	switch (plan_id) {
   4452 		case NDB_PLAN_IDS:     return "ids";
   4453 		case NDB_PLAN_SEARCH:  return "search";
   4454 		case NDB_PLAN_KINDS:   return "kinds";
   4455 		case NDB_PLAN_TAGS:    return "tags";
   4456 		case NDB_PLAN_CREATED: return "created";
   4457 		case NDB_PLAN_AUTHORS: return "authors";
   4458 		case NDB_PLAN_RELAY_KINDS: return "relay_kinds";
   4459 		case NDB_PLAN_AUTHOR_KINDS: return "author_kinds";
   4460 		case NDB_PLAN_PROFILE_SEARCH: return "profile_search";
   4461 	}
   4462 
   4463 	return "unknown";
   4464 }
   4465 
   4466 static int ndb_query_filter(struct ndb_txn *txn, struct ndb_filter *filter,
   4467 			    struct ndb_query_result *res, int capacity,
   4468 			    int *results_out)
   4469 {
   4470 	struct ndb_query_results results;
   4471 	uint64_t limit, *pint;
   4472 	enum ndb_query_plan plan;
   4473 	limit = capacity;
   4474 
   4475 	if ((pint = ndb_filter_get_int(filter, NDB_FILTER_LIMIT)))
   4476 		limit = *pint;
   4477 
   4478 	limit = min(capacity, limit);
   4479 	make_cursor((unsigned char *)res,
   4480 		    ((unsigned char *)res) + limit * sizeof(*res),
   4481 		    &results.cur);
   4482 
   4483 	plan = ndb_filter_plan(filter);
   4484 	ndb_debug("using query plan '%s'\n", ndb_query_plan_name(plan));
   4485 	switch (plan) {
   4486 	// We have a list of ids, just open a cursor and jump to each once
   4487 	case NDB_PLAN_IDS:
   4488 		if (!ndb_query_plan_execute_ids(txn, filter, &results, limit))
   4489 			return 0;
   4490 		break;
   4491 	case NDB_PLAN_RELAY_KINDS:
   4492 		if (!ndb_query_plan_execute_relay_kinds(txn, filter, &results, limit))
   4493 			return 0;
   4494 		break;
   4495 	case NDB_PLAN_SEARCH:
   4496 		if (!ndb_query_plan_execute_search(txn, filter, &results, limit))
   4497 			return 0;
   4498 		break;
   4499 
   4500 	case NDB_PLAN_PROFILE_SEARCH:
   4501 		if (!ndb_query_plan_execute_profile_search(txn, filter, &results, limit))
   4502 			return 0;
   4503 		break;
   4504 
   4505 	// We have just kinds, just scan the kind index
   4506 	case NDB_PLAN_KINDS:
   4507 		if (!ndb_query_plan_execute_kinds(txn, filter, &results, limit))
   4508 			return 0;
   4509 		break;
   4510 	case NDB_PLAN_TAGS:
   4511 		if (!ndb_query_plan_execute_tags(txn, filter, &results, limit))
   4512 			return 0;
   4513 		break;
   4514 	case NDB_PLAN_CREATED:
   4515 		if (!ndb_query_plan_execute_created_at(txn, filter, &results, limit))
   4516 			return 0;
   4517 		break;
   4518 	case NDB_PLAN_AUTHORS:
   4519 		if (!ndb_query_plan_execute_authors(txn, filter, &results, limit))
   4520 			return 0;
   4521 		break;
   4522 	case NDB_PLAN_AUTHOR_KINDS:
   4523 		if (!ndb_query_plan_execute_author_kinds(txn, filter, &results, limit))
   4524 			return 0;
   4525 		break;
   4526 	}
   4527 
   4528 	*results_out = cursor_count(&results.cur, sizeof(*res));
   4529 	return 1;
   4530 }
   4531 
   4532 int ndb_query(struct ndb_txn *txn, struct ndb_filter *filters, int num_filters,
   4533 	      struct ndb_query_result *results, int result_capacity, int *count)
   4534 {
   4535 	int i, out;
   4536 	struct ndb_query_result *p = results;
   4537 
   4538 	out = 0;
   4539 	*count = 0;
   4540 
   4541 	for (i = 0; i < num_filters; i++) {
   4542 		if (!ndb_query_filter(txn, &filters[i], p,
   4543 				      result_capacity, &out)) {
   4544 			return 0;
   4545 		}
   4546 
   4547 		*count += out;
   4548 		p += out;
   4549 		result_capacity -= out;
   4550 		if (result_capacity <= 0)
   4551 			break;
   4552 	}
   4553 
   4554 	// sort results
   4555 	qsort(results, *count, sizeof(*results), compare_query_results);
   4556 	return 1;
   4557 }
   4558 
   4559 static int ndb_write_note_tag_index(struct ndb_txn *txn, struct ndb_note *note,
   4560 				    uint64_t note_key)
   4561 {
   4562 	unsigned char key_buffer[255];
   4563 	struct ndb_iterator iter;
   4564 	struct ndb_str tkey, tval;
   4565 	char tchar;
   4566 	int len, rc;
   4567 	MDB_val key, val;
   4568 	MDB_dbi tags_db;
   4569 
   4570 	tags_db = txn->lmdb->dbs[NDB_DB_NOTE_TAGS];
   4571 
   4572 	ndb_tags_iterate_start(note, &iter);
   4573 
   4574 	while (ndb_tags_iterate_next(&iter)) {
   4575 		if (iter.tag->count < 2)
   4576 			continue;
   4577 
   4578 		tkey = ndb_tag_str(note, iter.tag, 0);
   4579 
   4580 		// we only write indices for 1-char tags.
   4581 		tchar = tkey.str[0];
   4582 		if (tchar == 0 || tkey.str[1] != 0)
   4583 			continue;
   4584 
   4585 		tval = ndb_tag_str(note, iter.tag, 1);
   4586 		len = ndb_str_len(&tval);
   4587 
   4588 		if (!(len = ndb_encode_tag_key(key_buffer, sizeof(key_buffer),
   4589 					       tchar, tval.id, (unsigned char)len,
   4590 					       ndb_note_created_at(note)))) {
   4591 			// this will fail when we try to encode a key that is
   4592 			// too big
   4593 			continue;
   4594 		}
   4595 
   4596 		//ndb_debug("writing tag '%c':'data:%d' to index\n", tchar, len);
   4597 
   4598 		key.mv_data = key_buffer;
   4599 		key.mv_size = len;
   4600 
   4601 		val.mv_data = &note_key;
   4602 		val.mv_size = sizeof(note_key);
   4603 
   4604 		if ((rc = mdb_put(txn->mdb_txn, tags_db, &key, &val, 0))) {
   4605 			ndb_debug("write note tag index to db failed: %s\n",
   4606 					mdb_strerror(rc));
   4607 			return 0;
   4608 		}
   4609 	}
   4610 
   4611 	return 1;
   4612 }
   4613 
   4614 static int ndb_write_note_kind_index(struct ndb_txn *txn, struct ndb_note *note,
   4615 				     uint64_t note_key)
   4616 {
   4617 	struct ndb_u64_ts tsid;
   4618 	int rc;
   4619 	MDB_val key, val;
   4620 	MDB_dbi kind_db;
   4621 
   4622 	ndb_u64_ts_init(&tsid, note->kind, note->created_at);
   4623 
   4624 	key.mv_data = &tsid;
   4625 	key.mv_size = sizeof(tsid);
   4626 	val.mv_data = &note_key;
   4627 	val.mv_size = sizeof(note_key);
   4628 
   4629 	kind_db = txn->lmdb->dbs[NDB_DB_NOTE_KIND];
   4630 
   4631 	if ((rc = mdb_put(txn->mdb_txn, kind_db, &key, &val, 0))) {
   4632 		ndb_debug("write note kind index to db failed: %s\n",
   4633 				mdb_strerror(rc));
   4634 		return 0;
   4635 	}
   4636 
   4637 	return 1;
   4638 }
   4639 
   4640 static int ndb_write_word_to_index(struct ndb_txn *txn, const char *word,
   4641 				   int word_len, int word_index,
   4642 				   uint64_t timestamp, uint64_t note_id)
   4643 {
   4644 	// cap to some reasonable key size
   4645 	unsigned char buffer[1024];
   4646 	int keysize, rc;
   4647 	MDB_val k, v;
   4648 	MDB_dbi text_db;
   4649 
   4650 	// build our compressed text index key
   4651 	if (!ndb_make_text_search_key(buffer, sizeof(buffer), word_index,
   4652 				      word_len, word, timestamp, note_id,
   4653 				      &keysize)) {
   4654 		// probably too big
   4655 
   4656 		return 0;
   4657 	}
   4658 
   4659 	k.mv_data = buffer;
   4660 	k.mv_size = keysize;
   4661 
   4662 	v.mv_data = NULL;
   4663 	v.mv_size = 0;
   4664 
   4665 	text_db = txn->lmdb->dbs[NDB_DB_NOTE_TEXT];
   4666 
   4667 	if ((rc = mdb_put(txn->mdb_txn, text_db, &k, &v, 0))) {
   4668 		ndb_debug("write note text index to db failed: %s\n",
   4669 				mdb_strerror(rc));
   4670 		return 0;
   4671 	}
   4672 
   4673 	return 1;
   4674 }
   4675 
   4676 
   4677 
   4678 // break a string into individual words for querying or for building the
   4679 // fulltext search index. This is callback based so we don't need to
   4680 // build up an intermediate structure
   4681 static int ndb_parse_words(struct cursor *cur, void *ctx, ndb_word_parser_fn fn)
   4682 {
   4683 	int word_len, words;
   4684 	const char *word;
   4685 
   4686 	words = 0;
   4687 
   4688 	while (cur->p < cur->end) {
   4689 		consume_whitespace_or_punctuation(cur);
   4690 		if (cur->p >= cur->end)
   4691 			break;
   4692 		word = (const char *)cur->p;
   4693 
   4694 		if (!consume_until_boundary(cur))
   4695 			break;
   4696 
   4697 		// start of word or end
   4698 		word_len = cur->p - (unsigned char *)word;
   4699 		if (word_len == 0 && cur->p >= cur->end)
   4700 			break;
   4701 
   4702 		if (word_len == 0) {
   4703 			if (!cursor_skip(cur, 1))
   4704 				break;
   4705 			continue;
   4706 		}
   4707 
   4708 		//ndb_debug("writing word index '%.*s'\n", word_len, word);
   4709 
   4710 		if (!fn(ctx, word, word_len, words))
   4711 			continue;
   4712 
   4713 		words++;
   4714 	}
   4715 
   4716 	return 1;
   4717 }
   4718 
   4719 struct ndb_word_writer_ctx
   4720 {
   4721 	struct ndb_txn *txn;
   4722 	struct ndb_note *note;
   4723 	uint64_t note_id;
   4724 };
   4725 
   4726 static int ndb_fulltext_word_writer(void *ctx,
   4727 		const char *word, int word_len, int words)
   4728 {
   4729 	struct ndb_word_writer_ctx *wctx = ctx;
   4730 
   4731 	if (!ndb_write_word_to_index(wctx->txn, word, word_len, words,
   4732 				     wctx->note->created_at, wctx->note_id)) {
   4733 		// too big to write this one, just skip it
   4734 		ndb_debug("failed to write word '%.*s' to index\n", word_len, word);
   4735 
   4736 		return 0;
   4737 	}
   4738 
   4739 	//fprintf(stderr, "wrote '%.*s' to note text index\n", word_len, word);
   4740 	return 1;
   4741 }
   4742 
   4743 static int ndb_write_note_fulltext_index(struct ndb_txn *txn,
   4744 					 struct ndb_note *note,
   4745 					 uint64_t note_id)
   4746 {
   4747 	struct cursor cur;
   4748 	unsigned char *content;
   4749 	struct ndb_str str;
   4750 	struct ndb_word_writer_ctx ctx;
   4751 
   4752 	str = ndb_note_str(note, &note->content);
   4753 	// I don't think this should happen?
   4754 	if (unlikely(str.flag == NDB_PACKED_ID))
   4755 		return 0;
   4756 
   4757 	content = (unsigned char *)str.str;
   4758 
   4759 	make_cursor(content, content + note->content_length, &cur);
   4760 
   4761 	ctx.txn = txn;
   4762 	ctx.note = note;
   4763 	ctx.note_id = note_id;
   4764 
   4765 	ndb_parse_words(&cur, &ctx, ndb_fulltext_word_writer);
   4766 
   4767 	return 1;
   4768 }
   4769 
   4770 static int ndb_parse_search_words(void *ctx, const char *word_str, int word_len, int word_index)
   4771 {
   4772 	(void)word_index;
   4773 	struct ndb_search_words *words = ctx;
   4774 	struct ndb_word *word;
   4775 
   4776 	if (words->num_words + 1 > MAX_TEXT_SEARCH_WORDS)
   4777 		return 0;
   4778 
   4779 	word = &words->words[words->num_words++];
   4780 	word->word = word_str;
   4781 	word->word_len = word_len;
   4782 
   4783 	return 1;
   4784 }
   4785 
   4786 static void ndb_search_words_init(struct ndb_search_words *words)
   4787 {
   4788 	words->num_words = 0;
   4789 }
   4790 
   4791 static int prefix_count(const char *str1, int len1, const char *str2, int len2) {
   4792 	int i, count = 0;
   4793 	int min_len = len1 < len2 ? len1 : len2;
   4794 
   4795 	for (i = 0; i < min_len; i++) {
   4796 		// case insensitive
   4797 		if (tolower(str1[i]) == tolower(str2[i]))
   4798 			count++;
   4799 		else
   4800 			break;
   4801 	}
   4802 
   4803 	return count;
   4804 }
   4805 
   4806 static int ndb_prefix_matches(struct ndb_text_search_result *result,
   4807 			      struct ndb_word *search_word)
   4808 {
   4809 	// Empty strings shouldn't happen but let's
   4810 	if (result->key.str_len < 2 || search_word->word_len < 2)
   4811 		return 0;
   4812 
   4813 	// make sure we at least have two matching prefix characters. exact
   4814 	// matches are nice but range searches allow us to match prefixes as
   4815 	// well. A double-char prefix is suffient, but maybe we could up this
   4816 	// in the future.
   4817 	//
   4818 	// TODO: How are we handling utf-8 prefix matches like
   4819 	// japanese?
   4820 	//
   4821 	if (   result->key.str[0] != tolower(search_word->word[0])
   4822 	    && result->key.str[1] != tolower(search_word->word[1])
   4823 	    )
   4824 		return 0;
   4825 
   4826 	// count the number of prefix-matched characters. This will be used
   4827 	// for ranking search results
   4828 	result->prefix_chars = prefix_count(result->key.str,
   4829 					    result->key.str_len,
   4830 					    search_word->word,
   4831 					    search_word->word_len);
   4832 
   4833 	if (result->prefix_chars <= (int)((double)search_word->word_len / 1.5))
   4834 		return 0;
   4835 
   4836 	return 1;
   4837 }
   4838 
   4839 // This is called when scanning the full text search index. Scanning stops
   4840 // when we no longer have a prefix match for the word
   4841 static int ndb_text_search_next_word(MDB_cursor *cursor, MDB_cursor_op op,
   4842 	MDB_val *k, struct ndb_word *search_word,
   4843 	struct ndb_text_search_result *last_result,
   4844 	struct ndb_text_search_result *result,
   4845 	MDB_cursor_op order_op)
   4846 {
   4847 	struct cursor key_cursor;
   4848 	//struct ndb_text_search_key search_key;
   4849 	MDB_val v;
   4850 	int retries;
   4851 	retries = -1;
   4852 
   4853 	make_cursor(k->mv_data, (unsigned char *)k->mv_data + k->mv_size, &key_cursor);
   4854 
   4855 	// When op is MDB_SET_RANGE, this initializes the search. Position
   4856 	// the cursor at the next key greater than or equal to the specified
   4857 	// key.
   4858 	//
   4859 	// Subsequent searches should use MDB_NEXT
   4860 	if (mdb_cursor_get(cursor, k, &v, op)) {
   4861 		// we should only do this if we're going in reverse
   4862 		if (op == MDB_SET_RANGE && order_op == MDB_PREV) {
   4863 			// if set range worked and our key exists, it should be
   4864 			// the one right before this one
   4865 			if (mdb_cursor_get(cursor, k, &v, MDB_PREV))
   4866 				return 0;
   4867 		} else {
   4868 			return 0;
   4869 		}
   4870 	}
   4871 
   4872 retry:
   4873 	retries++;
   4874 	/*
   4875 	printf("continuing from ");
   4876 	if (ndb_unpack_text_search_key(k->mv_data, k->mv_size, &search_key)) {
   4877 		ndb_print_text_search_key(&search_key);
   4878 	} else { printf("??"); }
   4879 	printf("\n");
   4880 	*/
   4881 
   4882 	make_cursor(k->mv_data, (unsigned char *)k->mv_data + k->mv_size, &key_cursor);
   4883 
   4884 	if (unlikely(!ndb_unpack_text_search_key_noteid(&key_cursor, &result->key.note_id))) {
   4885 		fprintf(stderr, "UNUSUAL: failed to unpack text search key note_id\n");
   4886 		return 0;
   4887 	}
   4888 
   4889 	// if the note id doesn't match the last result, then we stop trying
   4890 	// each search word
   4891 	if (last_result && last_result->key.note_id != result->key.note_id) {
   4892 		return 0;
   4893 	}
   4894 
   4895 	// On success, this could still be not related at all.
   4896 	// It could just be adjacent to the word. Let's check
   4897 	// if we have a matching prefix at least.
   4898 
   4899 	// Before we unpack the entire key, let's quickly
   4900 	// unpack just the string to check the prefix. We don't
   4901 	// need to unpack the entire key if the prefix doesn't
   4902 	// match
   4903 	if (!ndb_unpack_text_search_key_string(&key_cursor,
   4904 					       &result->key.str,
   4905 					       &result->key.str_len)) {
   4906 		// this should never happen
   4907 		fprintf(stderr, "UNUSUAL: failed to unpack text search key string\n");
   4908 		return 0;
   4909 	}
   4910 
   4911 	if (!ndb_prefix_matches(result, search_word)) {
   4912 		/*
   4913 		printf("result prefix '%.*s' didn't match search word '%.*s'\n",
   4914 			result->key.str_len, result->key.str,
   4915 			search_word->word_len, search_word->word);
   4916 			*/
   4917 		// we should only do this if we're going in reverse
   4918 		if (retries == 0 && op == MDB_SET_RANGE && order_op == MDB_PREV) {
   4919 			// if set range worked and our key exists, it should be
   4920 			// the one right before this one
   4921 			mdb_cursor_get(cursor, k, &v, MDB_PREV);
   4922 			goto retry;
   4923 		} else {
   4924 			return 0;
   4925 		}
   4926 	}
   4927 
   4928 	// Unpack the remaining text search key, we will need this information
   4929 	// when building up our search results.
   4930 	if (!ndb_unpack_remaining_text_search_key(&key_cursor, &result->key)) {
   4931 		// This should never happen
   4932 		fprintf(stderr, "UNUSUAL: failed to unpack text search key\n");
   4933 		return 0;
   4934 	}
   4935 
   4936 			/*
   4937 	if (last_result) {
   4938 		if (result->key.word_index < last_result->key.word_index) {
   4939 			fprintf(stderr, "skipping '%.*s' because it is before last result '%.*s'\n",
   4940 					result->key.str_len, result->key.str,
   4941 					last_result->key.str_len, last_result->key.str);
   4942 			return 0;
   4943 		}
   4944 	}
   4945 					*/
   4946 
   4947 	return 1;
   4948 }
   4949 
   4950 static void ndb_text_search_results_init(
   4951 		struct ndb_text_search_results *results) {
   4952 	results->num_results = 0;
   4953 }
   4954 
   4955 void ndb_default_text_search_config(struct ndb_text_search_config *cfg)
   4956 {
   4957 	cfg->order = NDB_ORDER_DESCENDING;
   4958 	cfg->limit = MAX_TEXT_SEARCH_RESULTS;
   4959 }
   4960 
   4961 void ndb_text_search_config_set_order(struct ndb_text_search_config *cfg,
   4962 				     enum ndb_search_order order)
   4963 {
   4964 	cfg->order = order;
   4965 }
   4966 
   4967 void ndb_text_search_config_set_limit(struct ndb_text_search_config *cfg, int limit)
   4968 {
   4969 	cfg->limit = limit;
   4970 }
   4971 
   4972 static int compare_search_words(const void *pa, const void *pb)
   4973 {
   4974 	struct ndb_word *a, *b;
   4975 
   4976 	a = (struct ndb_word *)pa;
   4977 	b = (struct ndb_word *)pb;
   4978 
   4979 	if (a->word_len == b->word_len) {
   4980 		return 0;
   4981 	} else if (a->word_len > b->word_len) {
   4982 		// biggest words should be at the front of the list,
   4983 		// so we say it's "smaller" here
   4984 		return -1;
   4985 	} else {
   4986 		return 1;
   4987 	}
   4988 }
   4989 
   4990 // Sort search words from largest to smallest. Larger words are less likely
   4991 // in the index, allowing our scan to walk fewer words at the root when
   4992 // recursively matching.
   4993 void sort_largest_to_smallest(struct ndb_search_words *words)
   4994 {
   4995 	qsort(words->words, words->num_words, sizeof(words->words[0]), compare_search_words);
   4996 }
   4997 
   4998 
   4999 int ndb_text_search_with(struct ndb_txn *txn, const char *query,
   5000 		    struct ndb_text_search_results *results,
   5001 		    struct ndb_text_search_config *config,
   5002 		    struct ndb_filter *filter)
   5003 {
   5004 	unsigned char buffer[1024], *buf;
   5005 	unsigned char saved_buf[1024], *saved;
   5006 	struct ndb_text_search_result *result, *last_result;
   5007 	struct ndb_text_search_result candidate, last_candidate;
   5008 	struct ndb_search_words search_words;
   5009 	//struct ndb_text_search_key search_key;
   5010 	struct ndb_word *search_word;
   5011 	struct ndb_note *note;
   5012 	struct cursor cur;
   5013 	uint64_t since, until, timestamp_op, *pint;
   5014 	size_t note_size;
   5015 	ndb_text_search_key_order_fn key_order_fn;
   5016 	MDB_dbi text_db;
   5017 	MDB_cursor *cursor;
   5018 	MDB_val k, v;
   5019 	int i, j, keysize, saved_size, limit;
   5020 	MDB_cursor_op op, order_op;
   5021 
   5022 	note_size = 0;
   5023 	note = 0;
   5024 	saved = NULL;
   5025 	ndb_text_search_results_init(results);
   5026 	ndb_search_words_init(&search_words);
   5027 
   5028 	until = UINT64_MAX;
   5029 	since = 0;
   5030 	limit = MAX_TEXT_SEARCH_RESULTS;
   5031 
   5032 	// until, since from filter
   5033 	if (filter != NULL) {
   5034 		if ((pint = ndb_filter_get_int(filter, NDB_FILTER_UNTIL)))
   5035 			until = *pint;
   5036 
   5037 		if ((pint = ndb_filter_get_int(filter, NDB_FILTER_SINCE)))
   5038 			since = *pint;
   5039 
   5040 		if ((pint = ndb_filter_get_int(filter, NDB_FILTER_LIMIT)))
   5041 			limit = *pint;
   5042 	}
   5043 
   5044 	order_op = MDB_PREV;
   5045 	key_order_fn = ndb_make_text_search_key_high;
   5046 	timestamp_op = until;
   5047 	if (config) {
   5048 		if (config->order == NDB_ORDER_ASCENDING) {
   5049 			order_op = MDB_NEXT;
   5050 			// set the min timestamp value to since when ascending
   5051 			timestamp_op = since;
   5052 			key_order_fn = ndb_make_text_search_key_low;
   5053 		}
   5054 		limit = min(limit, config->limit);
   5055 	}
   5056 	// end search config
   5057 
   5058 	text_db = txn->lmdb->dbs[NDB_DB_NOTE_TEXT];
   5059 	make_cursor((unsigned char *)query, (unsigned char *)query + strlen(query), &cur);
   5060 
   5061 	ndb_parse_words(&cur, &search_words, ndb_parse_search_words);
   5062 	if (search_words.num_words == 0)
   5063 		return 0;
   5064 
   5065 	if ((i = mdb_cursor_open(txn->mdb_txn, text_db, &cursor))) {
   5066 		fprintf(stderr, "nd_text_search: mdb_cursor_open failed, error %d\n", i);
   5067 		return 0;
   5068 	}
   5069 
   5070 	// This should complete the query quicker because the larger words are
   5071 	// likely to have fewer entries in the search index. This is not always
   5072 	// true. Words with higher frequency (like bitcoin on nostr in 2024)
   5073 	// may be slower. TODO: Skip word recursion by leveraging a minimal
   5074 	// perfect hashmap of parsed words on a note
   5075 	sort_largest_to_smallest(&search_words);
   5076 
   5077 	// for each word, we recursively find all of the submatches
   5078 	while (results->num_results < limit) {
   5079 		last_result = NULL;
   5080 		result = &results->results[results->num_results];
   5081 
   5082 		// if we have saved, then we continue from the last root search
   5083 		// sequence
   5084 		if (saved) {
   5085 			buf = saved_buf;
   5086 			saved = NULL;
   5087 			keysize = saved_size;
   5088 
   5089 			k.mv_data = buf;
   5090 			k.mv_size = saved_size;
   5091 
   5092 			// reposition the cursor so we can continue
   5093 			if (mdb_cursor_get(cursor, &k, &v, MDB_SET_RANGE))
   5094 				break;
   5095 
   5096 			op = order_op;
   5097 		} else {
   5098 			// construct a packed fulltext search key using this
   5099 			// word. This key doesn't contain any timestamp or index
   5100 			// info, so it should range match instead of exact
   5101 			// match
   5102 			if (!key_order_fn(buffer, sizeof(buffer),
   5103 					  search_words.words[0].word_len,
   5104 					  search_words.words[0].word,
   5105 					  timestamp_op,
   5106 					  &keysize))
   5107 			{
   5108 				// word is too big to fit in 1024-sized key
   5109 				continue;
   5110 			}
   5111 
   5112 			buf = buffer;
   5113 			op = MDB_SET_RANGE;
   5114 		}
   5115 
   5116 		for (j = 0; j < search_words.num_words; j++) {
   5117 			search_word = &search_words.words[j];
   5118 
   5119 			// shouldn't happen but let's be defensive a bit
   5120 			if (search_word->word_len == 0)
   5121 				continue;
   5122 
   5123 			// if we already matched a note in this phrase, make
   5124 			// sure we're including the note id in the query
   5125 			if (last_result) {
   5126 				// we are narrowing down a search.
   5127 				// if we already have this note id, just continue
   5128 				for (i = 0; i < results->num_results; i++) {
   5129 					if (results->results[i].key.note_id == last_result->key.note_id)
   5130 						// we can't break here to
   5131 						// leave the word loop so
   5132 						// have to use a goto
   5133 						goto cont;
   5134 				}
   5135 
   5136 				if (!ndb_make_noted_text_search_key(
   5137 					buffer, sizeof(buffer),
   5138 					search_word->word_len,
   5139 					search_word->word,
   5140 					last_result->key.timestamp,
   5141 					last_result->key.note_id,
   5142 					&keysize))
   5143 				{
   5144 					continue;
   5145 				}
   5146 
   5147 				buf = buffer;
   5148 			}
   5149 
   5150 			k.mv_data = buf;
   5151 			k.mv_size = keysize;
   5152 
   5153 			// TODO: we can speed this up with the minimal perfect
   5154 			// hashmap by quickly rejecting the remaining words
   5155 			// by looking in the word hashmap on the note. This
   5156 			// would allow us to skip the recursive word lookup
   5157 			// thing
   5158 			if (!ndb_text_search_next_word(cursor, op, &k,
   5159 						       search_word,
   5160 						       last_result,
   5161 						       &candidate,
   5162 						       order_op)) {
   5163 				// we didn't find a match for this note_id
   5164 				if (j == 0)
   5165 					// if we're at one of the root words,
   5166 					// this means that there are no further
   5167 					// root word matches for any note, so
   5168 					// we know we're done
   5169 					goto done;
   5170 				else
   5171 					break;
   5172 			}
   5173 
   5174 			*result = candidate;
   5175 			op = MDB_SET_RANGE;
   5176 
   5177 			// save the first key match, since we will continue from
   5178 			// this on the next root word result
   5179 			if (j == 0) {
   5180 				if (!saved) {
   5181 					memcpy(saved_buf, k.mv_data, k.mv_size);
   5182 					saved = saved_buf;
   5183 					saved_size = k.mv_size;
   5184 				}
   5185 
   5186 				// since we will be trying to match the same
   5187 				// note_id on all subsequent word matches,
   5188 				// let's lookup this note and make sure it
   5189 				// matches the filter if we have one. If it
   5190 				// doesn't match, we can quickly skip the
   5191 				// remaining word queries
   5192 				if (filter) {
   5193 					if ((note = ndb_get_note_by_key(txn,
   5194 							result->key.note_id,
   5195 							&note_size)))
   5196 					{
   5197 						if (!ndb_filter_matches(filter, note)) {
   5198 							break;
   5199 						}
   5200 						result->note = note;
   5201 						result->note_size = note_size;
   5202 					}
   5203 				}
   5204 			}
   5205 
   5206 			result->note = note;
   5207 			result->note_size = note_size;
   5208 			last_candidate = *result;
   5209 			last_result = &last_candidate;
   5210 		}
   5211 
   5212 		// we matched all of the queries!
   5213 		if (j == search_words.num_words) {
   5214 			results->num_results++;
   5215 		}
   5216 
   5217 cont:
   5218 		;
   5219 	}
   5220 
   5221 done:
   5222 	mdb_cursor_close(cursor);
   5223 
   5224 	return 1;
   5225 }
   5226 
   5227 int ndb_text_search(struct ndb_txn *txn, const char *query,
   5228 		    struct ndb_text_search_results *results,
   5229 		    struct ndb_text_search_config *config)
   5230 {
   5231 	return ndb_text_search_with(txn, query, results, config, NULL);
   5232 }
   5233 
   5234 static void ndb_write_blocks(struct ndb_txn *txn, uint64_t note_key,
   5235 			     struct ndb_blocks *blocks)
   5236 {
   5237 	int rc;
   5238 	MDB_val key, val;
   5239 
   5240 	// make sure we're not writing the owned flag to the db
   5241 	blocks->flags &= ~NDB_BLOCK_FLAG_OWNED;
   5242 
   5243 	key.mv_data = &note_key;
   5244 	key.mv_size = sizeof(note_key);
   5245 	val.mv_data = blocks;
   5246 	val.mv_size = ndb_blocks_total_size(blocks);
   5247 	assert((val.mv_size % 8) == 0);
   5248 
   5249 	if ((rc = mdb_put(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_NOTE_BLOCKS], &key, &val, 0))) {
   5250 		ndb_debug("write version to note_blocks failed: %s\n",
   5251 				mdb_strerror(rc));
   5252 		return;
   5253 	}
   5254 }
   5255 
   5256 static int ndb_write_new_blocks(struct ndb_txn *txn, struct ndb_note *note,
   5257 				uint64_t note_key, unsigned char *scratch,
   5258 				size_t scratch_size)
   5259 {
   5260 	size_t content_len;
   5261 	const char *content;
   5262 	struct ndb_blocks *blocks;
   5263 
   5264 	content_len = ndb_note_content_length(note);
   5265 	content = ndb_note_content(note);
   5266 
   5267 	if (!ndb_parse_content(scratch, scratch_size, content, content_len, &blocks)) {
   5268 		//ndb_debug("failed to parse content '%.*s'\n", content_len, content);
   5269 		return 0;
   5270 	}
   5271 
   5272 	ndb_write_blocks(txn, note_key, blocks);
   5273 	return 1;
   5274 }
   5275 
   5276 static uint64_t ndb_write_note(struct ndb_txn *txn,
   5277 			       struct ndb_writer_note *note,
   5278 			       unsigned char *scratch, size_t scratch_size,
   5279 			       uint32_t ndb_flags)
   5280 {
   5281 	int rc;
   5282 	uint64_t note_key, kind;
   5283 	struct ndb_relay_kind_key relay_key;
   5284 	MDB_dbi note_db;
   5285 	MDB_val key, val;
   5286 
   5287 	kind = note->note->kind;
   5288 
   5289 	// let's quickly sanity check if we already have this note
   5290 	if ((note_key = ndb_get_notekey_by_id(txn, note->note->id))) {
   5291 		if (ndb_relay_kind_key_init(&relay_key, note_key, kind, ndb_note_created_at(note->note), note->relay))
   5292 			ndb_write_note_relay_indexes(txn, &relay_key);
   5293 		return 0;
   5294 	}
   5295 
   5296 	// get dbs
   5297 	note_db = txn->lmdb->dbs[NDB_DB_NOTE];
   5298 
   5299 	// get new key
   5300 	note_key = ndb_get_last_key(txn->mdb_txn, note_db) + 1;
   5301 
   5302 	// write note to event store
   5303 	key.mv_data = &note_key;
   5304 	key.mv_size = sizeof(note_key);
   5305 	val.mv_data = note->note;
   5306 	val.mv_size = note->note_len;
   5307 
   5308 	if ((rc = mdb_put(txn->mdb_txn, note_db, &key, &val, 0))) {
   5309 		ndb_debug("write note to db failed: %s\n", mdb_strerror(rc));
   5310 		return 0;
   5311 	}
   5312 
   5313 	ndb_write_note_id_index(txn, note->note, note_key);
   5314 	ndb_write_note_kind_index(txn, note->note, note_key);
   5315 	ndb_write_note_tag_index(txn, note->note, note_key);
   5316 	ndb_write_note_pubkey_index(txn, note->note, note_key);
   5317 	ndb_write_note_pubkey_kind_index(txn, note->note, note_key);
   5318 
   5319 	if (ndb_relay_kind_key_init(&relay_key, note_key, kind, ndb_note_created_at(note->note), note->relay))
   5320 		ndb_write_note_relay_indexes(txn, &relay_key);
   5321 
   5322 	// only parse content and do fulltext index on text and longform notes
   5323 	if (kind == 1 || kind == 30023) {
   5324 		if (!ndb_flag_set(ndb_flags, NDB_FLAG_NO_FULLTEXT)) {
   5325 			if (!ndb_write_note_fulltext_index(txn, note->note, note_key))
   5326 				return 0;
   5327 		}
   5328 
   5329 		// write note blocks
   5330 		if (!ndb_flag_set(ndb_flags, NDB_FLAG_NO_NOTE_BLOCKS)) {
   5331 			ndb_write_new_blocks(txn, note->note, note_key, scratch, scratch_size);
   5332 		}
   5333 	} else if (kind == 7 && !ndb_flag_set(ndb_flags, NDB_FLAG_NO_STATS)) {
   5334 		ndb_write_reaction_stats(txn, note->note);
   5335 	}
   5336 
   5337 	return note_key;
   5338 }
   5339 
   5340 static void ndb_monitor_lock(struct ndb_monitor *mon) {
   5341 	pthread_mutex_lock(&mon->mutex);
   5342 }
   5343 
   5344 static void ndb_monitor_unlock(struct ndb_monitor *mon) {
   5345 	pthread_mutex_unlock(&mon->mutex);
   5346 }
   5347 
   5348 struct written_note {
   5349 	uint64_t note_id;
   5350 	struct ndb_writer_note *note;
   5351 };
   5352 
   5353 // When the data has been committed to the database, take all of the written
   5354 // notes, check them against subscriptions, and then write to the subscription
   5355 // inbox for all matching notes
   5356 static void ndb_notify_subscriptions(struct ndb_monitor *monitor,
   5357 				     struct written_note *wrote, int num_notes)
   5358 {
   5359 	int i, k;
   5360 	int pushed;
   5361 	struct written_note *written;
   5362 	struct ndb_note *note;
   5363 	struct ndb_subscription *sub;
   5364 
   5365 	ndb_monitor_lock(monitor);
   5366 
   5367 	for (i = 0; i < monitor->num_subscriptions; i++) {
   5368 		sub = &monitor->subscriptions[i];
   5369 		ndb_debug("checking subscription %d, %d notes\n", i, num_notes);
   5370 
   5371 		pushed = 0;
   5372 		for (k = 0; k < num_notes; k++) {
   5373 			written = &wrote[k];
   5374 			note = written->note->note;
   5375 
   5376 			if (ndb_filter_group_matches(&sub->group, note)) {
   5377 				ndb_debug("pushing note\n");
   5378 
   5379 				if (!prot_queue_push(&sub->inbox, &written->note_id)) {
   5380 					ndb_debug("couldn't push note to subscriber");
   5381 				} else {
   5382 					pushed++;
   5383 				}
   5384 			} else {
   5385 				ndb_debug("not pushing note\n");
   5386 			}
   5387 		}
   5388 
   5389 		// After pushing all of the matching notes, check to see if we
   5390 		// have a registered subscription callback. If so, we call it.
   5391 		// The callback needs to call ndb_poll_for_notes to pull data
   5392 		// that was just pushed to the queue in the for loop above.
   5393 		if (monitor->sub_cb != NULL && pushed > 0) {
   5394 			monitor->sub_cb(monitor->sub_cb_ctx, sub->subid);
   5395 		}
   5396 	}
   5397 
   5398 	ndb_monitor_unlock(monitor);
   5399 }
   5400 
   5401 uint64_t ndb_write_note_and_profile(
   5402 		struct ndb_txn *txn,
   5403 		struct ndb_writer_profile *profile,
   5404 		unsigned char *scratch,
   5405 		size_t scratch_size,
   5406 		uint32_t ndb_flags)
   5407 {
   5408 	uint64_t note_nkey;
   5409 
   5410 	note_nkey = ndb_write_note(txn, &profile->note, scratch, scratch_size, ndb_flags);
   5411 
   5412 	if (profile->record.builder) {
   5413 		// only write if parsing didn't fail
   5414 		ndb_write_profile(txn, profile, note_nkey);
   5415 	}
   5416 
   5417 	return note_nkey;
   5418 }
   5419 
   5420 // only to be called from the writer thread
   5421 static int ndb_write_version(struct ndb_txn *txn, uint64_t version)
   5422 {
   5423 	int rc;
   5424 	MDB_val key, val;
   5425 	uint64_t version_key;
   5426 
   5427 	version_key = NDB_META_KEY_VERSION;
   5428 
   5429 	key.mv_data = &version_key;
   5430 	key.mv_size = sizeof(version_key);
   5431 	val.mv_data = &version;
   5432 	val.mv_size = sizeof(version);
   5433 
   5434 	if ((rc = mdb_put(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_NDB_META], &key, &val, 0))) {
   5435 		ndb_debug("write version to ndb_meta failed: %s\n",
   5436 				mdb_strerror(rc));
   5437 		return 0;
   5438 	}
   5439 
   5440 	//fprintf(stderr, "writing version %" PRIu64 "\n", version);
   5441 	return 1;
   5442 }
   5443 
   5444 
   5445 static int ndb_run_migrations(struct ndb_txn *txn)
   5446 {
   5447 	int64_t version, latest_version, i;
   5448 
   5449 	latest_version = sizeof(MIGRATIONS) / sizeof(MIGRATIONS[0]);
   5450 
   5451 	if ((version = ndb_db_version(txn)) == -1) {
   5452 		ndb_debug("run_migrations: no version found, assuming new db\n");
   5453 		version = latest_version;
   5454 
   5455 		// no version found. fresh db?
   5456 		if (!ndb_write_version(txn, version)) {
   5457 			fprintf(stderr, "run_migrations: failed writing db version");
   5458 			return 0;
   5459 		}
   5460 
   5461 		return 1;
   5462 	} else {
   5463 		ndb_debug("ndb: version %" PRIu64 " found\n", version);
   5464 	}
   5465 
   5466 	if (version < latest_version)
   5467 		fprintf(stderr, "nostrdb: migrating v%d -> v%d\n",
   5468 				(int)version, (int)latest_version);
   5469 
   5470 	for (i = version; i < latest_version; i++) {
   5471 		if (!MIGRATIONS[i].fn(txn)) {
   5472 			fprintf(stderr, "run_migrations: migration v%d -> v%d failed\n", (int)i, (int)(i+1));
   5473 			return 0;
   5474 		}
   5475 
   5476 		if (!ndb_write_version(txn, i+1)) {
   5477 			fprintf(stderr, "run_migrations: failed writing db version");
   5478 			return 0;
   5479 		}
   5480 
   5481 		version = i+1;
   5482 	}
   5483 
   5484 	return 1;
   5485 }
   5486 
   5487 
   5488 static void *ndb_writer_thread(void *data)
   5489 {
   5490 	ndb_debug("started writer thread\n");
   5491 	struct ndb_writer *writer = data;
   5492 	struct ndb_writer_msg msgs[THREAD_QUEUE_BATCH], *msg;
   5493 	struct written_note written_notes[THREAD_QUEUE_BATCH];
   5494 	int i, popped, done, needs_commit, num_notes;
   5495 	uint64_t note_nkey;
   5496 	struct ndb_txn txn;
   5497 	unsigned char *scratch;
   5498 	struct ndb_relay_kind_key relay_key;
   5499 
   5500 	// 2MB scratch buffer for parsing note content
   5501 	scratch = malloc(writer->scratch_size);
   5502 	MDB_txn *mdb_txn = NULL;
   5503 	ndb_txn_from_mdb(&txn, writer->lmdb, mdb_txn);
   5504 
   5505 	done = 0;
   5506 	while (!done) {
   5507 		txn.mdb_txn = NULL;
   5508 		num_notes = 0;
   5509 		ndb_debug("writer waiting for items\n");
   5510 		popped = prot_queue_pop_all(&writer->inbox, msgs, THREAD_QUEUE_BATCH);
   5511 		ndb_debug("writer popped %d items\n", popped);
   5512 
   5513 		needs_commit = 0;
   5514 		for (i = 0 ; i < popped; i++) {
   5515 			msg = &msgs[i];
   5516 			switch (msg->type) {
   5517 			case NDB_WRITER_NOTE: needs_commit = 1; break;
   5518 			case NDB_WRITER_PROFILE: needs_commit = 1; break;
   5519 			case NDB_WRITER_DBMETA: needs_commit = 1; break;
   5520 			case NDB_WRITER_PROFILE_LAST_FETCH: needs_commit = 1; break;
   5521 			case NDB_WRITER_BLOCKS: needs_commit = 1; break;
   5522 			case NDB_WRITER_MIGRATE: needs_commit = 1; break;
   5523 			case NDB_WRITER_NOTE_RELAY: needs_commit = 1; break;
   5524 			case NDB_WRITER_QUIT: break;
   5525 			}
   5526 		}
   5527 
   5528 		if (needs_commit && mdb_txn_begin(txn.lmdb->env, NULL, 0, (MDB_txn **)&txn.mdb_txn))
   5529 		{
   5530 			fprintf(stderr, "writer thread txn_begin failed");
   5531 			// should definitely not happen unless DB is full
   5532 			// or something ?
   5533 			continue;
   5534 		}
   5535 
   5536 		for (i = 0; i < popped; i++) {
   5537 			msg = &msgs[i];
   5538 
   5539 			switch (msg->type) {
   5540 			case NDB_WRITER_QUIT:
   5541 				// quits are handled before this
   5542 				ndb_debug("writer thread got quit message\n");
   5543 				done = 1;
   5544 				continue;
   5545 			case NDB_WRITER_PROFILE:
   5546 				note_nkey =
   5547 					ndb_write_note_and_profile(
   5548 						&txn,
   5549 						&msg->profile,
   5550 						scratch,
   5551 						writer->scratch_size,
   5552 						writer->ndb_flags);
   5553 
   5554 				if (note_nkey > 0) {
   5555 					written_notes[num_notes++] =
   5556 					(struct written_note){
   5557 						.note_id = note_nkey,
   5558 						.note = &msg->profile.note,
   5559 					};
   5560 				} else {
   5561 					ndb_debug("failed to write note\n");
   5562 				}
   5563 				break;
   5564 			case NDB_WRITER_NOTE:
   5565 				note_nkey = ndb_write_note(&txn, &msg->note,
   5566 							   scratch,
   5567 							   writer->scratch_size,
   5568 							   writer->ndb_flags);
   5569 
   5570 				if (note_nkey > 0) {
   5571 					written_notes[num_notes++] = (struct written_note){
   5572 						.note_id = note_nkey,
   5573 						.note = &msg->note,
   5574 					};
   5575 				}
   5576 				break;
   5577 			case NDB_WRITER_NOTE_RELAY:
   5578 				if (ndb_relay_kind_key_init(&relay_key,
   5579 							    msg->note_relay.note_key,
   5580 							    msg->note_relay.kind,
   5581 							    msg->note_relay.created_at,
   5582 							    msg->note_relay.relay))
   5583 				{
   5584 					ndb_write_note_relay_indexes(&txn, &relay_key);
   5585 				}
   5586 				break;
   5587 			case NDB_WRITER_DBMETA:
   5588 				ndb_write_version(&txn, msg->ndb_meta.version);
   5589 				break;
   5590 			case NDB_WRITER_BLOCKS:
   5591 				ndb_write_blocks(&txn, msg->blocks.note_key,
   5592 						       msg->blocks.blocks);
   5593 				break;
   5594 			case NDB_WRITER_MIGRATE:
   5595 				if (!ndb_run_migrations(&txn)) {
   5596 					mdb_txn_abort(txn.mdb_txn);
   5597 					goto bail;
   5598 				}
   5599 				break;
   5600 			case NDB_WRITER_PROFILE_LAST_FETCH:
   5601 				ndb_writer_last_profile_fetch(&txn,
   5602 						msg->last_fetch.pubkey,
   5603 						msg->last_fetch.fetched_at
   5604 						);
   5605 				break;
   5606 			}
   5607 		}
   5608 
   5609 		// commit writes
   5610 		if (needs_commit) {
   5611 			if (!ndb_end_query(&txn)) {
   5612 				ndb_debug("writer thread txn commit failed\n");
   5613 			} else {
   5614 				ndb_debug("notifying subscriptions, %d notes\n", num_notes);
   5615 				ndb_notify_subscriptions(writer->monitor,
   5616 							 written_notes,
   5617 							 num_notes);
   5618 				// update subscriptions
   5619 			}
   5620 		}
   5621 
   5622 		// free notes
   5623 		for (i = 0; i < popped; i++) {
   5624 			msg = &msgs[i];
   5625 			if (msg->type == NDB_WRITER_NOTE) {
   5626 				free(msg->note.note);
   5627 				if (msg->note.relay)
   5628 					free((void*)msg->note.relay);
   5629 			} else if (msg->type == NDB_WRITER_PROFILE) {
   5630 				free(msg->profile.note.note);
   5631 				ndb_profile_record_builder_free(&msg->profile.record);
   5632 			} else if (msg->type == NDB_WRITER_BLOCKS) {
   5633 				ndb_blocks_free(msg->blocks.blocks);
   5634 			} else if (msg->type == NDB_WRITER_NOTE_RELAY) {
   5635 				free((void*)msg->note_relay.relay);
   5636 			}
   5637 		}
   5638 	}
   5639 
   5640 bail:
   5641 	free(scratch);
   5642 	ndb_debug("quitting writer thread\n");
   5643 	return NULL;
   5644 }
   5645 
   5646 static void *ndb_ingester_thread(void *data)
   5647 {
   5648 	secp256k1_context *ctx;
   5649 	struct thread *thread = data;
   5650 	struct ndb_ingester *ingester = (struct ndb_ingester *)thread->ctx;
   5651 	struct ndb_lmdb *lmdb = ingester->lmdb;
   5652 	struct ndb_ingester_msg msgs[THREAD_QUEUE_BATCH], *msg;
   5653 	struct ndb_writer_msg outs[THREAD_QUEUE_BATCH], *out;
   5654 	int i, to_write, popped, done, any_event;
   5655 	MDB_txn *read_txn = NULL;
   5656 	unsigned char *scratch;
   5657 	int rc;
   5658 
   5659 	// this is used in note verification and anything else that
   5660 	// needs a temporary buffer
   5661 	scratch = malloc(ingester->scratch_size);
   5662 
   5663 	ctx = secp256k1_context_create(SECP256K1_CONTEXT_VERIFY);
   5664 	//ndb_debug("started ingester thread\n");
   5665 
   5666 	done = 0;
   5667 	while (!done) {
   5668 		to_write = 0;
   5669 		any_event = 0;
   5670 
   5671 		popped = prot_queue_pop_all(&thread->inbox, msgs, THREAD_QUEUE_BATCH);
   5672 		ndb_debug("ingester popped %d items\n", popped);
   5673 
   5674 		for (i = 0; i < popped; i++) {
   5675 			msg = &msgs[i];
   5676 			if (msg->type == NDB_INGEST_EVENT) {
   5677 				any_event = 1;
   5678 				break;
   5679 			}
   5680 		}
   5681 
   5682 		if (any_event && (rc = mdb_txn_begin(lmdb->env, NULL, MDB_RDONLY, &read_txn))) {
   5683 			// this is bad
   5684 			fprintf(stderr, "UNUSUAL ndb_ingester: mdb_txn_begin failed: '%s'\n",
   5685 					mdb_strerror(rc));
   5686 			continue;
   5687 		}
   5688 
   5689 		for (i = 0; i < popped; i++) {
   5690 			msg = &msgs[i];
   5691 			switch (msg->type) {
   5692 			case NDB_INGEST_QUIT:
   5693 				done = 1;
   5694 				break;
   5695 
   5696 			case NDB_INGEST_EVENT:
   5697 				out = &outs[to_write];
   5698 				if (ndb_ingester_process_event(ctx, ingester,
   5699 							       &msg->event, out,
   5700 							       scratch,
   5701 							       read_txn)) {
   5702 					to_write++;
   5703 				}
   5704 			}
   5705 		}
   5706 
   5707 		if (any_event)
   5708 			mdb_txn_abort(read_txn);
   5709 
   5710 		if (to_write > 0) {
   5711 			ndb_debug("pushing %d events to write queue\n", to_write);
   5712 			if (!prot_queue_push_all(ingester->writer_inbox, outs, to_write)) {
   5713 				ndb_debug("failed pushing %d events to write queue\n", to_write);
   5714 			}
   5715 		}
   5716 	}
   5717 
   5718 	ndb_debug("quitting ingester thread\n");
   5719 	secp256k1_context_destroy(ctx);
   5720 	free(scratch);
   5721 	return NULL;
   5722 }
   5723 
   5724 
   5725 static int ndb_writer_init(struct ndb_writer *writer, struct ndb_lmdb *lmdb,
   5726 			   struct ndb_monitor *monitor, uint32_t ndb_flags,
   5727 			   int scratch_size)
   5728 {
   5729 	writer->lmdb = lmdb;
   5730 	writer->monitor = monitor;
   5731 	writer->ndb_flags = ndb_flags;
   5732 	writer->scratch_size = scratch_size;
   5733 	writer->queue_buflen = sizeof(struct ndb_writer_msg) * DEFAULT_QUEUE_SIZE;
   5734 	writer->queue_buf = malloc(writer->queue_buflen);
   5735 	if (writer->queue_buf == NULL) {
   5736 		fprintf(stderr, "ndb: failed to allocate space for writer queue");
   5737 		return 0;
   5738 	}
   5739 
   5740 	// init the writer queue.
   5741 	prot_queue_init(&writer->inbox, writer->queue_buf,
   5742 			writer->queue_buflen, sizeof(struct ndb_writer_msg));
   5743 
   5744 	// spin up the writer thread
   5745 	if (THREAD_CREATE(writer->thread_id, ndb_writer_thread, writer))
   5746 	{
   5747 		fprintf(stderr, "ndb writer thread failed to create\n");
   5748 		return 0;
   5749 	}
   5750 
   5751 	return 1;
   5752 }
   5753 
   5754 // initialize the ingester queue and then spawn the thread
   5755 static int ndb_ingester_init(struct ndb_ingester *ingester,
   5756 			     struct ndb_lmdb *lmdb,
   5757 			     struct prot_queue *writer_inbox,
   5758 			     int scratch_size,
   5759 			     const struct ndb_config *config)
   5760 {
   5761 	int elem_size, num_elems;
   5762 	static struct ndb_ingester_msg quit_msg = { .type = NDB_INGEST_QUIT };
   5763 
   5764 	// TODO: configurable queue sizes
   5765 	elem_size = sizeof(struct ndb_ingester_msg);
   5766 	num_elems = DEFAULT_QUEUE_SIZE;
   5767 
   5768 	ingester->scratch_size = scratch_size;
   5769 	ingester->writer_inbox = writer_inbox;
   5770 	ingester->lmdb = lmdb;
   5771 	ingester->flags = config->flags;
   5772 	ingester->filter = config->ingest_filter;
   5773 	ingester->filter_context = config->filter_context;
   5774 
   5775 	if (!threadpool_init(&ingester->tp, config->ingester_threads,
   5776 			     elem_size, num_elems, &quit_msg, ingester,
   5777 			     ndb_ingester_thread))
   5778 	{
   5779 		fprintf(stderr, "ndb ingester threadpool failed to init\n");
   5780 		return 0;
   5781 	}
   5782 
   5783 	return 1;
   5784 }
   5785 
   5786 static int ndb_writer_destroy(struct ndb_writer *writer)
   5787 {
   5788 	struct ndb_writer_msg msg;
   5789 
   5790 	// kill thread
   5791 	msg.type = NDB_WRITER_QUIT;
   5792 	ndb_debug("writer: pushing quit message\n");
   5793 	if (!prot_queue_push(&writer->inbox, &msg)) {
   5794 		// queue is too full to push quit message. just kill it.
   5795 		ndb_debug("writer: terminating thread\n");
   5796 		THREAD_TERMINATE(writer->thread_id);
   5797 	} else {
   5798 		ndb_debug("writer: joining thread\n");
   5799 		THREAD_FINISH(writer->thread_id);
   5800 	}
   5801 
   5802 	// cleanup
   5803 	ndb_debug("writer: cleaning up protected queue\n");
   5804 	prot_queue_destroy(&writer->inbox);
   5805 
   5806 	free(writer->queue_buf);
   5807 
   5808 	return 1;
   5809 }
   5810 
   5811 static int ndb_ingester_destroy(struct ndb_ingester *ingester)
   5812 {
   5813 	threadpool_destroy(&ingester->tp);
   5814 	return 1;
   5815 }
   5816 
   5817 static int ndb_init_lmdb(const char *filename, struct ndb_lmdb *lmdb, size_t mapsize)
   5818 {
   5819 	int rc;
   5820 	MDB_txn *txn;
   5821 
   5822 	if ((rc = mdb_env_create(&lmdb->env))) {
   5823 		fprintf(stderr, "mdb_env_create failed, error %d\n", rc);
   5824 		return 0;
   5825 	}
   5826 
   5827 	if ((rc = mdb_env_set_mapsize(lmdb->env, mapsize))) {
   5828 		fprintf(stderr, "mdb_env_set_mapsize failed, error %d\n", rc);
   5829 		return 0;
   5830 	}
   5831 
   5832 	if ((rc = mdb_env_set_maxdbs(lmdb->env, NDB_DBS))) {
   5833 		fprintf(stderr, "mdb_env_set_maxdbs failed, error %d\n", rc);
   5834 		return 0;
   5835 	}
   5836 
   5837 	if ((rc = mdb_env_open(lmdb->env, filename, 0, 0664))) {
   5838 		fprintf(stderr, "mdb_env_open failed, error %d\n", rc);
   5839 		return 0;
   5840 	}
   5841 
   5842 	// Initialize DBs
   5843 	if ((rc = mdb_txn_begin(lmdb->env, NULL, 0, &txn))) {
   5844 		fprintf(stderr, "mdb_txn_begin failed, error %d\n", rc);
   5845 		return 0;
   5846 	}
   5847 
   5848 	// note flatbuffer db
   5849 	if ((rc = mdb_dbi_open(txn, "note", MDB_CREATE | MDB_INTEGERKEY, &lmdb->dbs[NDB_DB_NOTE]))) {
   5850 		fprintf(stderr, "mdb_dbi_open event failed, error %d\n", rc);
   5851 		return 0;
   5852 	}
   5853 
   5854 	// note metadata db
   5855 	if ((rc = mdb_dbi_open(txn, "meta", MDB_CREATE, &lmdb->dbs[NDB_DB_META]))) {
   5856 		fprintf(stderr, "mdb_dbi_open meta failed, error %d\n", rc);
   5857 		return 0;
   5858 	}
   5859 
   5860 	// profile flatbuffer db
   5861 	if ((rc = mdb_dbi_open(txn, "profile", MDB_CREATE | MDB_INTEGERKEY, &lmdb->dbs[NDB_DB_PROFILE]))) {
   5862 		fprintf(stderr, "mdb_dbi_open profile failed, error %d\n", rc);
   5863 		return 0;
   5864 	}
   5865 
   5866 	// profile search db
   5867 	if ((rc = mdb_dbi_open(txn, "profile_search", MDB_CREATE, &lmdb->dbs[NDB_DB_PROFILE_SEARCH]))) {
   5868 		fprintf(stderr, "mdb_dbi_open profile_search failed, error %d\n", rc);
   5869 		return 0;
   5870 	}
   5871 	mdb_set_compare(txn, lmdb->dbs[NDB_DB_PROFILE_SEARCH], ndb_search_key_cmp);
   5872 
   5873 	// ndb metadata (db version, etc)
   5874 	if ((rc = mdb_dbi_open(txn, "ndb_meta", MDB_CREATE | MDB_INTEGERKEY, &lmdb->dbs[NDB_DB_NDB_META]))) {
   5875 		fprintf(stderr, "mdb_dbi_open ndb_meta failed, error %d\n", rc);
   5876 		return 0;
   5877 	}
   5878 
   5879 	// profile last fetches
   5880 	if ((rc = mdb_dbi_open(txn, "profile_last_fetch", MDB_CREATE, &lmdb->dbs[NDB_DB_PROFILE_LAST_FETCH]))) {
   5881 		fprintf(stderr, "mdb_dbi_open profile last fetch, error %d\n", rc);
   5882 		return 0;
   5883 	}
   5884 
   5885 	// relay kind index. maps <relay_url><kind><created><note_id> primary keys to relay records
   5886 	// see ndb_relay_kind_cmp function for more details on the key format
   5887 	if ((rc = mdb_dbi_open(txn, "relay_kind", MDB_CREATE, &lmdb->dbs[NDB_DB_NOTE_RELAY_KIND]))) {
   5888 		fprintf(stderr, "mdb_dbi_open profile last fetch, error %d\n", rc);
   5889 		return 0;
   5890 	}
   5891 	mdb_set_compare(txn, lmdb->dbs[NDB_DB_NOTE_RELAY_KIND], ndb_relay_kind_cmp);
   5892 
   5893 	// note_id -> relay index
   5894 	if ((rc = mdb_dbi_open(txn, "note_relays", MDB_CREATE | MDB_DUPSORT, &lmdb->dbs[NDB_DB_NOTE_RELAYS]))) {
   5895 		fprintf(stderr, "mdb_dbi_open profile last fetch, error %d\n", rc);
   5896 		return 0;
   5897 	}
   5898 
   5899 	// id+ts index flags
   5900 	unsigned int tsid_flags = MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED;
   5901 
   5902 	// index dbs
   5903 	if ((rc = mdb_dbi_open(txn, "note_id", tsid_flags, &lmdb->dbs[NDB_DB_NOTE_ID]))) {
   5904 		fprintf(stderr, "mdb_dbi_open id failed: %s\n", mdb_strerror(rc));
   5905 		return 0;
   5906 	}
   5907 	mdb_set_compare(txn, lmdb->dbs[NDB_DB_NOTE_ID], ndb_tsid_compare);
   5908 
   5909 	if ((rc = mdb_dbi_open(txn, "profile_pk", tsid_flags, &lmdb->dbs[NDB_DB_PROFILE_PK]))) {
   5910 		fprintf(stderr, "mdb_dbi_open profile_pk failed: %s\n", mdb_strerror(rc));
   5911 		return 0;
   5912 	}
   5913 	mdb_set_compare(txn, lmdb->dbs[NDB_DB_PROFILE_PK], ndb_tsid_compare);
   5914 
   5915 	if ((rc = mdb_dbi_open(txn, "note_kind",
   5916 			       MDB_CREATE | MDB_DUPSORT | MDB_INTEGERDUP | MDB_DUPFIXED,
   5917 			       &lmdb->dbs[NDB_DB_NOTE_KIND]))) {
   5918 		fprintf(stderr, "mdb_dbi_open note_kind failed: %s\n", mdb_strerror(rc));
   5919 		return 0;
   5920 	}
   5921 	mdb_set_compare(txn, lmdb->dbs[NDB_DB_NOTE_KIND], ndb_u64_ts_compare);
   5922 
   5923 	if ((rc = mdb_dbi_open(txn, "note_pubkey",
   5924 			       MDB_CREATE | MDB_DUPSORT | MDB_INTEGERDUP | MDB_DUPFIXED,
   5925 			       &lmdb->dbs[NDB_DB_NOTE_PUBKEY]))) {
   5926 		fprintf(stderr, "mdb_dbi_open note_pubkey failed: %s\n", mdb_strerror(rc));
   5927 		return 0;
   5928 	}
   5929 	mdb_set_compare(txn, lmdb->dbs[NDB_DB_NOTE_PUBKEY], ndb_tsid_compare);
   5930 
   5931 	if ((rc = mdb_dbi_open(txn, "note_pubkey_kind",
   5932 			       MDB_CREATE | MDB_DUPSORT | MDB_INTEGERDUP | MDB_DUPFIXED,
   5933 			       &lmdb->dbs[NDB_DB_NOTE_PUBKEY_KIND]))) {
   5934 		fprintf(stderr, "mdb_dbi_open note_pubkey_kind failed: %s\n", mdb_strerror(rc));
   5935 		return 0;
   5936 	}
   5937 	mdb_set_compare(txn, lmdb->dbs[NDB_DB_NOTE_PUBKEY_KIND], ndb_id_u64_ts_compare);
   5938 
   5939 	if ((rc = mdb_dbi_open(txn, "note_text", MDB_CREATE | MDB_DUPSORT,
   5940 			       &lmdb->dbs[NDB_DB_NOTE_TEXT]))) {
   5941 		fprintf(stderr, "mdb_dbi_open note_text failed: %s\n", mdb_strerror(rc));
   5942 		return 0;
   5943 	}
   5944 	mdb_set_compare(txn, lmdb->dbs[NDB_DB_NOTE_TEXT], ndb_text_search_key_compare);
   5945 
   5946 	if ((rc = mdb_dbi_open(txn, "note_blocks", MDB_CREATE | MDB_INTEGERKEY,
   5947 			       &lmdb->dbs[NDB_DB_NOTE_BLOCKS]))) {
   5948 		fprintf(stderr, "mdb_dbi_open note_blocks failed: %s\n", mdb_strerror(rc));
   5949 		return 0;
   5950 	}
   5951 
   5952 	if ((rc = mdb_dbi_open(txn, "note_tags", MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED,
   5953 			       &lmdb->dbs[NDB_DB_NOTE_TAGS]))) {
   5954 		fprintf(stderr, "mdb_dbi_open note_tags failed: %s\n", mdb_strerror(rc));
   5955 		return 0;
   5956 	}
   5957 	mdb_set_compare(txn, lmdb->dbs[NDB_DB_NOTE_TAGS], ndb_tag_key_compare);
   5958 
   5959 	// Commit the transaction
   5960 	if ((rc = mdb_txn_commit(txn))) {
   5961 		fprintf(stderr, "mdb_txn_commit failed, error %d\n", rc);
   5962 		return 0;
   5963 	}
   5964 
   5965 	return 1;
   5966 }
   5967 
   5968 static int ndb_queue_write_version(struct ndb *ndb, uint64_t version)
   5969 {
   5970 	struct ndb_writer_msg msg;
   5971 	msg.type = NDB_WRITER_DBMETA;
   5972 	msg.ndb_meta.version = version;
   5973 	return ndb_writer_queue_msg(&ndb->writer, &msg);
   5974 }
   5975 
   5976 static void ndb_monitor_init(struct ndb_monitor *monitor, ndb_sub_fn cb,
   5977 			     void *sub_cb_ctx)
   5978 {
   5979 	monitor->num_subscriptions = 0;
   5980 	monitor->sub_cb = cb;
   5981 	monitor->sub_cb_ctx = sub_cb_ctx;
   5982 	pthread_mutex_init(&monitor->mutex, NULL);
   5983 }
   5984 
   5985 void ndb_filter_group_destroy(struct ndb_filter_group *group)
   5986 {
   5987 	struct ndb_filter *filter;
   5988 	int i;
   5989 	for (i = 0; i < group->num_filters; i++) {
   5990 		filter = &group->filters[i];
   5991 		ndb_filter_destroy(filter);
   5992 	}
   5993 }
   5994 
   5995 static void ndb_subscription_destroy(struct ndb_subscription *sub)
   5996 {
   5997 	ndb_filter_group_destroy(&sub->group);
   5998 	// this was malloc'd inside ndb_subscribe
   5999 	free(sub->inbox.buf);
   6000 	prot_queue_destroy(&sub->inbox);
   6001 	sub->subid = 0;
   6002 }
   6003 
   6004 static void ndb_monitor_destroy(struct ndb_monitor *monitor)
   6005 {
   6006 	int i;
   6007 
   6008 	ndb_monitor_lock(monitor);
   6009 
   6010 	for (i = 0; i < monitor->num_subscriptions; i++) {
   6011 		ndb_subscription_destroy(&monitor->subscriptions[i]);
   6012 	}
   6013 
   6014 	monitor->num_subscriptions = 0;
   6015 
   6016 	ndb_monitor_unlock(monitor);
   6017 
   6018 	pthread_mutex_destroy(&monitor->mutex);
   6019 }
   6020 
   6021 int ndb_init(struct ndb **pndb, const char *filename, const struct ndb_config *config)
   6022 {
   6023 	struct ndb *ndb;
   6024 	//MDB_dbi ind_id; // TODO: ind_pk, etc
   6025 
   6026 	ndb = *pndb = calloc(1, sizeof(struct ndb));
   6027 	ndb->flags = config->flags;
   6028 
   6029 	if (ndb == NULL) {
   6030 		fprintf(stderr, "ndb_init: malloc failed\n");
   6031 		return 0;
   6032 	}
   6033 
   6034 	if (!ndb_init_lmdb(filename, &ndb->lmdb, config->mapsize))
   6035 		return 0;
   6036 
   6037 	ndb_monitor_init(&ndb->monitor, config->sub_cb, config->sub_cb_ctx);
   6038 
   6039 	if (!ndb_writer_init(&ndb->writer, &ndb->lmdb, &ndb->monitor, ndb->flags,
   6040 			     config->writer_scratch_buffer_size)) {
   6041 		fprintf(stderr, "ndb_writer_init failed\n");
   6042 		return 0;
   6043 	}
   6044 
   6045 	if (!ndb_ingester_init(&ndb->ingester, &ndb->lmdb, &ndb->writer.inbox,
   6046 			       config->writer_scratch_buffer_size, config)) {
   6047 		fprintf(stderr, "failed to initialize %d ingester thread(s)\n",
   6048 				config->ingester_threads);
   6049 		return 0;
   6050 	}
   6051 
   6052 	if (!ndb_flag_set(config->flags, NDB_FLAG_NOMIGRATE)) {
   6053 		struct ndb_writer_msg msg = { .type = NDB_WRITER_MIGRATE };
   6054 		ndb_writer_queue_msg(&ndb->writer, &msg);
   6055 	}
   6056 
   6057 	// Initialize LMDB environment and spin up threads
   6058 	return 1;
   6059 }
   6060 
   6061 void ndb_destroy(struct ndb *ndb)
   6062 {
   6063 	if (ndb == NULL)
   6064 		return;
   6065 
   6066 	// ingester depends on writer and must be destroyed first
   6067 	ndb_debug("destroying ingester\n");
   6068 	ndb_ingester_destroy(&ndb->ingester);
   6069 	ndb_debug("destroying writer\n");
   6070 	ndb_writer_destroy(&ndb->writer);
   6071 	ndb_debug("destroying monitor\n");
   6072 	ndb_monitor_destroy(&ndb->monitor);
   6073 
   6074 	ndb_debug("closing env\n");
   6075 	mdb_env_close(ndb->lmdb.env);
   6076 
   6077 	ndb_debug("ndb destroyed\n");
   6078 	free(ndb);
   6079 }
   6080 
   6081 
   6082 // Process a nostr event from a client
   6083 //
   6084 // ie: ["EVENT", {"content":"..."} ...]
   6085 //
   6086 // The client-sent variation of ndb_process_event
   6087 int ndb_process_client_event(struct ndb *ndb, const char *json, int len)
   6088 {
   6089 	struct ndb_ingest_meta meta;
   6090 	ndb_ingest_meta_init(&meta, 1, NULL);
   6091 
   6092 	return ndb_ingest_event(&ndb->ingester, json, len, &meta);
   6093 }
   6094 
   6095 // Process anostr event from a relay,
   6096 //
   6097 // ie: ["EVENT", "subid", {"content":"..."}...]
   6098 //
   6099 // This function returns as soon as possible, first copying the passed
   6100 // json and then queueing it up for processing. Worker threads then take
   6101 // the json and process it.
   6102 //
   6103 // Processing:
   6104 //
   6105 // 1. The event is parsed into ndb_notes and the signature is validated
   6106 // 2. A quick lookup is made on the database to see if we already have
   6107 //    the note id, if we do we don't need to waste time on json parsing
   6108 //    or note validation.
   6109 // 3. Once validation is done we pass it to the writer queue for writing
   6110 //    to LMDB.
   6111 //
   6112 int ndb_process_event(struct ndb *ndb, const char *json, int json_len)
   6113 {
   6114 	struct ndb_ingest_meta meta;
   6115 	ndb_ingest_meta_init(&meta, 0, NULL);
   6116 
   6117 	return ndb_ingest_event(&ndb->ingester, json, json_len, &meta);
   6118 }
   6119 
   6120 int ndb_process_event_with(struct ndb *ndb, const char *json, int json_len,
   6121 			   struct ndb_ingest_meta *meta)
   6122 {
   6123 	return ndb_ingest_event(&ndb->ingester, json, json_len, meta);
   6124 }
   6125 
   6126 int _ndb_process_events(struct ndb *ndb, const char *ldjson, size_t json_len,
   6127 			struct ndb_ingest_meta *meta)
   6128 {
   6129 	const char *start, *end, *very_end;
   6130 	start = ldjson;
   6131 	end = start + json_len;
   6132 	very_end = ldjson + json_len;
   6133 #if DEBUG
   6134 	int processed = 0;
   6135 #endif
   6136 
   6137 	while ((end = fast_strchr(start, '\n', very_end - start))) {
   6138 		//printf("processing '%.*s'\n", (int)(end-start), start);
   6139 		if (!ndb_process_event_with(ndb, start, end - start, meta)) {
   6140 			ndb_debug("ndb_process_client_event failed\n");
   6141 			return 0;
   6142 		}
   6143 		start = end + 1;
   6144 #if DEBUG
   6145 		processed++;
   6146 #endif
   6147 	}
   6148 
   6149 #if DEBUG
   6150 	ndb_debug("ndb_process_events: processed %d events\n", processed);
   6151 #endif
   6152 
   6153 	return 1;
   6154 }
   6155 
   6156 #ifndef _WIN32
   6157 // TODO: windows
   6158 int ndb_process_events_stream(struct ndb *ndb, FILE* fp)
   6159 {
   6160 	char *line = NULL;
   6161 	size_t len = 0;
   6162 	ssize_t nread;
   6163 
   6164 	while ((nread = getline(&line, &len, fp)) != -1) {
   6165 		if (line == NULL)
   6166 			break;
   6167 		ndb_process_event(ndb, line, len);
   6168 	}
   6169 
   6170 	if (line)
   6171 		free(line);
   6172 
   6173 	return 1;
   6174 }
   6175 #endif
   6176 
   6177 int ndb_process_events_with(struct ndb *ndb, const char *ldjson, size_t json_len,
   6178 			    struct ndb_ingest_meta *meta)
   6179 {
   6180 	return _ndb_process_events(ndb, ldjson, json_len, meta);
   6181 }
   6182 
   6183 int ndb_process_client_events(struct ndb *ndb, const char *ldjson, size_t json_len)
   6184 {
   6185 	struct ndb_ingest_meta meta;
   6186 	ndb_ingest_meta_init(&meta, 1, NULL);
   6187 
   6188 	return _ndb_process_events(ndb, ldjson, json_len, &meta);
   6189 }
   6190 
   6191 int ndb_process_events(struct ndb *ndb, const char *ldjson, size_t json_len)
   6192 {
   6193 	struct ndb_ingest_meta meta;
   6194 	ndb_ingest_meta_init(&meta, 0, NULL);
   6195 
   6196 	return _ndb_process_events(ndb, ldjson, json_len, &meta);
   6197 }
   6198 
   6199 static inline int cursor_push_tag(struct cursor *cur, struct ndb_tag *tag)
   6200 {
   6201 	return cursor_push_u16(cur, tag->count);
   6202 }
   6203 
   6204 int ndb_builder_init(struct ndb_builder *builder, unsigned char *buf,
   6205 		     size_t bufsize)
   6206 {
   6207 	struct ndb_note *note;
   6208 	int half, size, str_indices_size;
   6209 
   6210 	// come on bruh
   6211 	if (bufsize < sizeof(struct ndb_note) * 2)
   6212 		return 0;
   6213 
   6214 	str_indices_size = bufsize / 32;
   6215 	size = bufsize - str_indices_size;
   6216 	half = size / 2;
   6217 
   6218 	//debug("size %d half %d str_indices %d\n", size, half, str_indices_size);
   6219 
   6220 	// make a safe cursor of our available memory
   6221 	make_cursor(buf, buf + bufsize, &builder->mem);
   6222 
   6223 	note = builder->note = (struct ndb_note *)buf;
   6224 
   6225 	// take slices of the memory into subcursors
   6226 	if (!(cursor_slice(&builder->mem, &builder->note_cur, half) &&
   6227 	      cursor_slice(&builder->mem, &builder->strings, half) &&
   6228 	      cursor_slice(&builder->mem, &builder->str_indices, str_indices_size))) {
   6229 		return 0;
   6230 	}
   6231 
   6232 	memset(note, 0, sizeof(*note));
   6233 	builder->note_cur.p += sizeof(*note);
   6234 
   6235 	note->strings = builder->strings.start - buf;
   6236 	note->version = 1;
   6237 
   6238 	return 1;
   6239 }
   6240 
   6241 
   6242 
   6243 static inline int ndb_json_parser_init(struct ndb_json_parser *p,
   6244 				       const char *json, int json_len,
   6245 				       unsigned char *buf, int bufsize)
   6246 {
   6247 	int half = bufsize / 2;
   6248 
   6249 	unsigned char *tok_start = buf + half;
   6250 	unsigned char *tok_end = buf + bufsize;
   6251 
   6252 	p->toks = (jsmntok_t*)tok_start;
   6253 	p->toks_end = (jsmntok_t*)tok_end;
   6254 	p->num_tokens = 0;
   6255 	p->json = json;
   6256 	p->json_len = json_len;
   6257 
   6258 	// ndb_builder gets the first half of the buffer, and jsmn gets the
   6259 	// second half. I like this way of alloating memory (without actually
   6260 	// dynamically allocating memory). You get one big chunk upfront and
   6261 	// then submodules can recursively subdivide it. Maybe you could do
   6262 	// something even more clever like golden-ratio style subdivision where
   6263 	// the more important stuff gets a larger chunk and then it spirals
   6264 	// downward into smaller chunks. Thanks for coming to my TED talk.
   6265 
   6266 	if (!ndb_builder_init(&p->builder, buf, half))
   6267 		return 0;
   6268 
   6269 	jsmn_init(&p->json_parser);
   6270 
   6271 	return 1;
   6272 }
   6273 
   6274 static inline int ndb_json_parser_parse(struct ndb_json_parser *p,
   6275 					struct ndb_id_cb *cb)
   6276 {
   6277 	jsmntok_t *tok;
   6278 	int cap = ((unsigned char *)p->toks_end - (unsigned char*)p->toks)/sizeof(*p->toks);
   6279 	int res =
   6280 		jsmn_parse(&p->json_parser, p->json, p->json_len, p->toks, cap, cb != NULL);
   6281 
   6282 	// got an ID!
   6283 	if (res == -42) {
   6284 		tok = &p->toks[p->json_parser.toknext-1];
   6285 
   6286 		switch (cb->fn(cb->data, p->json + tok->start)) {
   6287 		case NDB_IDRES_CONT:
   6288 			res = jsmn_parse(&p->json_parser, p->json, p->json_len,
   6289 					 p->toks, cap, 0);
   6290 			break;
   6291 		case NDB_IDRES_STOP:
   6292 			return -42;
   6293 		}
   6294 	} else if (res == 0) {
   6295 		return 0;
   6296 	}
   6297 
   6298 	p->num_tokens = res;
   6299 	p->i = 0;
   6300 
   6301 	return 1;
   6302 }
   6303 
   6304 static inline int toksize(jsmntok_t *tok)
   6305 {
   6306 	return tok->end - tok->start;
   6307 }
   6308 
   6309 
   6310 
   6311 static int cursor_push_unescaped_char(struct cursor *cur, char c1, char c2)
   6312 {
   6313 	switch (c2) {
   6314 	case 't':  return cursor_push_byte(cur, '\t');
   6315 	case 'n':  return cursor_push_byte(cur, '\n');
   6316 	case 'r':  return cursor_push_byte(cur, '\r');
   6317 	case 'b':  return cursor_push_byte(cur, '\b');
   6318 	case 'f':  return cursor_push_byte(cur, '\f');
   6319 	case '\\': return cursor_push_byte(cur, '\\');
   6320 	case '/':  return cursor_push_byte(cur, '/');
   6321 	case '"':  return cursor_push_byte(cur, '"');
   6322 	case 'u':
   6323 		// these aren't handled yet
   6324 		return 0;
   6325 	default:
   6326 		return cursor_push_byte(cur, c1) && cursor_push_byte(cur, c2);
   6327 	}
   6328 }
   6329 
   6330 static int cursor_push_escaped_char(struct cursor *cur, char c)
   6331 {
   6332         switch (c) {
   6333         case '"':  return cursor_push_str(cur, "\\\"");
   6334         case '\\': return cursor_push_str(cur, "\\\\");
   6335         case '\b': return cursor_push_str(cur, "\\b");
   6336         case '\f': return cursor_push_str(cur, "\\f");
   6337         case '\n': return cursor_push_str(cur, "\\n");
   6338         case '\r': return cursor_push_str(cur, "\\r");
   6339         case '\t': return cursor_push_str(cur, "\\t");
   6340         // TODO: \u hex hex hex hex
   6341         }
   6342         return cursor_push_byte(cur, c);
   6343 }
   6344 
   6345 static int cursor_push_hex_str(struct cursor *cur, unsigned char *buf, int len)
   6346 {
   6347 	int i;
   6348 
   6349 	if (len % 2 != 0)
   6350 		return 0;
   6351 
   6352         if (!cursor_push_byte(cur, '"'))
   6353                 return 0;
   6354 
   6355 	for (i = 0; i < len; i++) {
   6356 		unsigned int c = ((const unsigned char *)buf)[i];
   6357 		if (!cursor_push_byte(cur, hexchar(c >> 4)))
   6358 			return 0;
   6359 		if (!cursor_push_byte(cur, hexchar(c & 0xF)))
   6360 			return 0;
   6361 	}
   6362 
   6363         if (!cursor_push_byte(cur, '"'))
   6364                 return 0;
   6365 
   6366 	return 1;
   6367 }
   6368 
   6369 static int cursor_push_jsonstr(struct cursor *cur, const char *str)
   6370 {
   6371 	int i;
   6372         int len;
   6373 
   6374 	len = strlen(str);
   6375 
   6376         if (!cursor_push_byte(cur, '"'))
   6377                 return 0;
   6378 
   6379         for (i = 0; i < len; i++) {
   6380                 if (!cursor_push_escaped_char(cur, str[i]))
   6381                         return 0;
   6382         }
   6383 
   6384         if (!cursor_push_byte(cur, '"'))
   6385                 return 0;
   6386 
   6387         return 1;
   6388 }
   6389 
   6390 
   6391 static inline int cursor_push_json_tag_str(struct cursor *cur, struct ndb_str str)
   6392 {
   6393 	if (str.flag == NDB_PACKED_ID)
   6394 		return cursor_push_hex_str(cur, str.id, 32);
   6395 
   6396 	return cursor_push_jsonstr(cur, str.str);
   6397 }
   6398 
   6399 static int cursor_push_json_tag(struct cursor *cur, struct ndb_note *note,
   6400 				struct ndb_tag *tag)
   6401 {
   6402         int i;
   6403 
   6404         if (!cursor_push_byte(cur, '['))
   6405                 return 0;
   6406 
   6407         for (i = 0; i < tag->count; i++) {
   6408                 if (!cursor_push_json_tag_str(cur, ndb_tag_str(note, tag, i)))
   6409                         return 0;
   6410                 if (i != tag->count-1 && !cursor_push_byte(cur, ','))
   6411 			return 0;
   6412         }
   6413 
   6414         return cursor_push_byte(cur, ']');
   6415 }
   6416 
   6417 static int cursor_push_json_tags(struct cursor *cur, struct ndb_note *note)
   6418 {
   6419 	int i;
   6420 	struct ndb_iterator iter, *it = &iter;
   6421 	ndb_tags_iterate_start(note, it);
   6422 
   6423         if (!cursor_push_byte(cur, '['))
   6424                 return 0;
   6425 
   6426 	i = 0;
   6427 	while (ndb_tags_iterate_next(it)) {
   6428 		if (!cursor_push_json_tag(cur, note, it->tag))
   6429 			return 0;
   6430                 if (i != note->tags.count-1 && !cursor_push_str(cur, ","))
   6431 			return 0;
   6432 		i++;
   6433 	}
   6434 
   6435         if (!cursor_push_byte(cur, ']'))
   6436                 return 0;
   6437 
   6438 	return 1;
   6439 }
   6440 
   6441 static int ndb_event_commitment(struct ndb_note *ev, unsigned char *buf, int buflen)
   6442 {
   6443 	char timebuf[16] = {0};
   6444 	char kindbuf[16] = {0};
   6445 	char pubkey[65];
   6446 	struct cursor cur;
   6447 	int ok;
   6448 
   6449 	if (!hex_encode(ev->pubkey, sizeof(ev->pubkey), pubkey))
   6450 		return 0;
   6451 
   6452 	make_cursor(buf, buf + buflen, &cur);
   6453 
   6454 	// TODO: update in 2106 ...
   6455 	snprintf(timebuf, sizeof(timebuf), "%d", (uint32_t)ev->created_at);
   6456 	snprintf(kindbuf, sizeof(kindbuf), "%d", ev->kind);
   6457 
   6458 	ok =
   6459 		cursor_push_str(&cur, "[0,\"") &&
   6460 		cursor_push_str(&cur, pubkey) &&
   6461 		cursor_push_str(&cur, "\",") &&
   6462 		cursor_push_str(&cur, timebuf) &&
   6463 		cursor_push_str(&cur, ",") &&
   6464 		cursor_push_str(&cur, kindbuf) &&
   6465 		cursor_push_str(&cur, ",") &&
   6466 		cursor_push_json_tags(&cur, ev) &&
   6467 		cursor_push_str(&cur, ",") &&
   6468 		cursor_push_jsonstr(&cur, ndb_note_str(ev, &ev->content).str) &&
   6469 		cursor_push_str(&cur, "]");
   6470 
   6471 	if (!ok)
   6472 		return 0;
   6473 
   6474 	return cur.p - cur.start;
   6475 }
   6476 
   6477 static int cursor_push_hex(struct cursor *c, unsigned char *bytes, int len)
   6478 {
   6479 	int i;
   6480 	unsigned char chr;
   6481 	if (c->p + (len * 2) >= c->end)
   6482 		return 0;
   6483 
   6484 	for (i = 0; i < len; i++) {
   6485 		chr = bytes[i];
   6486 
   6487 		*(c->p++) = hexchar(chr >> 4);
   6488 		*(c->p++) = hexchar(chr & 0xF);
   6489 	}
   6490 
   6491 	return 1;
   6492 }
   6493 
   6494 static int cursor_push_int_str(struct cursor *c, uint64_t num)
   6495 {
   6496 	char timebuf[16] = {0};
   6497 	snprintf(timebuf, sizeof(timebuf), "%" PRIu64, num);
   6498 	return cursor_push_str(c, timebuf);
   6499 }
   6500 
   6501 int ndb_note_json(struct ndb_note *note, char *buf, int buflen)
   6502 {
   6503 	struct cursor cur, *c = &cur;
   6504 
   6505 	make_cursor((unsigned char *)buf, (unsigned char*)buf + buflen, &cur);
   6506 
   6507 	int ok = cursor_push_str(c, "{\"id\":\"") &&
   6508 	       cursor_push_hex(c, ndb_note_id(note), 32) &&
   6509 	       cursor_push_str(c, "\",\"pubkey\":\"") &&
   6510 	       cursor_push_hex(c, ndb_note_pubkey(note), 32) &&
   6511 	       cursor_push_str(c, "\",\"created_at\":") &&
   6512 	       cursor_push_int_str(c, ndb_note_created_at(note)) &&
   6513 	       cursor_push_str(c, ",\"kind\":") &&
   6514 	       cursor_push_int_str(c, ndb_note_kind(note)) &&
   6515 	       cursor_push_str(c, ",\"tags\":") &&
   6516 	       cursor_push_json_tags(c, note) &&
   6517 	       cursor_push_str(c, ",\"content\":") &&
   6518 	       cursor_push_jsonstr(c, ndb_note_content(note)) &&
   6519 	       cursor_push_str(c, ",\"sig\":\"") &&
   6520 	       cursor_push_hex(c, ndb_note_sig(note), 64) &&
   6521 	       cursor_push_c_str(c, "\"}");
   6522 
   6523 	if (!ok) {
   6524 		return 0;
   6525 	}
   6526 
   6527 	return cur.p - cur.start;
   6528 }
   6529 
   6530 static int cursor_push_json_elem_array(struct cursor *cur,
   6531 				       const struct ndb_filter *filter,
   6532 				       struct ndb_filter_elements *elems)
   6533 {
   6534 	int i;
   6535 	unsigned char *id;
   6536 	const char *str;
   6537 	uint64_t val;
   6538 
   6539 	if (!cursor_push_byte(cur, '['))
   6540 		return 0;
   6541 
   6542 	for (i = 0; i < elems->count; i++) {
   6543 
   6544 		switch (elems->field.elem_type)  {
   6545 		case NDB_ELEMENT_CUSTOM:
   6546 			// can't serialize custom functions
   6547 			break;
   6548 		case NDB_ELEMENT_STRING:
   6549 			str = ndb_filter_get_string_element(filter, elems, i);
   6550 			if (!cursor_push_jsonstr(cur, str))
   6551 				return 0;
   6552 			break;
   6553 		case NDB_ELEMENT_ID:
   6554 			id = ndb_filter_get_id_element(filter, elems, i);
   6555 			if (!cursor_push_hex_str(cur, id, 32))
   6556 				return 0;
   6557 			break;
   6558 		case NDB_ELEMENT_INT:
   6559 			val = ndb_filter_get_int_element(elems, i);
   6560 			if (!cursor_push_int_str(cur, val))
   6561 				return 0;
   6562 			break;
   6563 		case NDB_ELEMENT_UNKNOWN:
   6564 			ndb_debug("unknown element in cursor_push_json_elem_array");
   6565 			return 0;
   6566 		}
   6567 
   6568 		if (i != elems->count-1) {
   6569 			if (!cursor_push_byte(cur, ','))
   6570 				return 0;
   6571 		}
   6572 	}
   6573 
   6574 	if (!cursor_push_str(cur, "]"))
   6575 		return 0;
   6576 
   6577 	return 1;
   6578 }
   6579 
   6580 int ndb_filter_json(const struct ndb_filter *filter, char *buf, int buflen)
   6581 {
   6582 	const char *str;
   6583 	struct cursor cur, *c = &cur;
   6584 	struct ndb_filter_elements *elems;
   6585 	int i;
   6586 
   6587 	if (!filter->finalized) {
   6588 		ndb_debug("filter not finalized in ndb_filter_json\n");
   6589 		return 0;
   6590 	}
   6591 
   6592 	make_cursor((unsigned char *)buf, (unsigned char*)buf + buflen, c);
   6593 
   6594 	if (!cursor_push_str(c, "{"))
   6595 		return 0;
   6596 
   6597 	for (i = 0; i < filter->num_elements; i++) {
   6598 		elems = ndb_filter_get_elements(filter, i);
   6599 		switch (elems->field.type) {
   6600 		case NDB_FILTER_CUSTOM:
   6601 			// nothing to encode these as
   6602 			break;
   6603 		case NDB_FILTER_IDS:
   6604 			if (!cursor_push_str(c, "\"ids\":"))
   6605 				return 0;
   6606 			if (!cursor_push_json_elem_array(c, filter, elems))
   6607 				return 0;
   6608 			break;
   6609 		case NDB_FILTER_SEARCH:
   6610 			if (!cursor_push_str(c, "\"search\":"))
   6611 				return 0;
   6612 			if (!(str = ndb_filter_get_string_element(filter, elems, 0)))
   6613 				return 0;
   6614 			if (!cursor_push_jsonstr(c, str))
   6615 				return 0;
   6616 			break;
   6617 		case NDB_FILTER_AUTHORS:
   6618 			if (!cursor_push_str(c, "\"authors\":"))
   6619 				return 0;
   6620 			if (!cursor_push_json_elem_array(c, filter, elems))
   6621 				return 0;
   6622 			break;
   6623 		case NDB_FILTER_KINDS:
   6624 			if (!cursor_push_str(c, "\"kinds\":"))
   6625 				return 0;
   6626 			if (!cursor_push_json_elem_array(c, filter, elems))
   6627 				return 0;
   6628 			break;
   6629 		case NDB_FILTER_TAGS:
   6630 			if (!cursor_push_str(c, "\"#"))
   6631 				return 0;
   6632 			if (!cursor_push_byte(c, elems->field.tag))
   6633 				return 0;
   6634 			if (!cursor_push_str(c, "\":"))
   6635 				return 0;
   6636 			if (!cursor_push_json_elem_array(c, filter, elems))
   6637 				return 0;
   6638 			break;
   6639 		case NDB_FILTER_SINCE:
   6640 			if (!cursor_push_str(c, "\"since\":"))
   6641 				return 0;
   6642 			if (!cursor_push_int_str(c, ndb_filter_get_int_element(elems, 0)))
   6643 				return 0;
   6644 			break;
   6645 		case NDB_FILTER_UNTIL:
   6646 			if (!cursor_push_str(c, "\"until\":"))
   6647 				return 0;
   6648 			if (!cursor_push_int_str(c, ndb_filter_get_int_element(elems, 0)))
   6649 				return 0;
   6650 			break;
   6651 		case NDB_FILTER_LIMIT:
   6652 			if (!cursor_push_str(c, "\"limit\":"))
   6653 				return 0;
   6654 			if (!cursor_push_int_str(c, ndb_filter_get_int_element(elems, 0)))
   6655 				return 0;
   6656 			break;
   6657 		case NDB_FILTER_RELAYS:
   6658 			if (!cursor_push_str(c, "\"relays\":"))
   6659 				return 0;
   6660 			if (!cursor_push_json_elem_array(c, filter, elems))
   6661 				return 0;
   6662 			break;
   6663 		}
   6664 
   6665 		if (i != filter->num_elements-1) {
   6666 			if (!cursor_push_byte(c, ',')) {
   6667 				return 0;
   6668 			}
   6669 		}
   6670 
   6671 	}
   6672 
   6673 	if (!cursor_push_c_str(c, "}"))
   6674 		return 0;
   6675 
   6676 	return cur.p - cur.start;
   6677 }
   6678 
   6679 int ndb_calculate_id(struct ndb_note *note, unsigned char *buf, int buflen, unsigned char *id) {
   6680 	int len;
   6681 
   6682 	if (!(len = ndb_event_commitment(note, buf, buflen)))
   6683 		return 0;
   6684 
   6685 	//fprintf(stderr, "%.*s\n", len, buf);
   6686 
   6687 	sha256((struct sha256*)id, buf, len);
   6688 
   6689 	return 1;
   6690 }
   6691 
   6692 int ndb_sign_id(struct ndb_keypair *keypair, unsigned char id[32],
   6693 		unsigned char sig[64])
   6694 {
   6695 	unsigned char aux[32];
   6696 	secp256k1_keypair *pair = (secp256k1_keypair*) keypair->pair;
   6697 
   6698 	if (!fill_random(aux, sizeof(aux)))
   6699 		return 0;
   6700 
   6701 	secp256k1_context *ctx =
   6702 		secp256k1_context_create(SECP256K1_CONTEXT_NONE);
   6703 
   6704 	return secp256k1_schnorrsig_sign32(ctx, sig, id, pair, aux);
   6705 }
   6706 
   6707 int ndb_create_keypair(struct ndb_keypair *kp)
   6708 {
   6709 	secp256k1_keypair *keypair = (secp256k1_keypair*)kp->pair;
   6710 	secp256k1_xonly_pubkey pubkey;
   6711 
   6712 	secp256k1_context *ctx =
   6713 		secp256k1_context_create(SECP256K1_CONTEXT_NONE);;
   6714 
   6715 	/* Try to create a keypair with a valid context, it should only
   6716 	 * fail if the secret key is zero or out of range. */
   6717 	if (!secp256k1_keypair_create(ctx, keypair, kp->secret))
   6718 		return 0;
   6719 
   6720 	if (!secp256k1_keypair_xonly_pub(ctx, &pubkey, NULL, keypair))
   6721 		return 0;
   6722 
   6723 	/* Serialize the public key. Should always return 1 for a valid public key. */
   6724 	return secp256k1_xonly_pubkey_serialize(ctx, kp->pubkey, &pubkey);
   6725 }
   6726 
   6727 int ndb_decode_key(const char *secstr, struct ndb_keypair *keypair)
   6728 {
   6729 	if (!hex_decode(secstr, strlen(secstr), keypair->secret, 32)) {
   6730 		fprintf(stderr, "could not hex decode secret key\n");
   6731 		return 0;
   6732 	}
   6733 
   6734 	return ndb_create_keypair(keypair);
   6735 }
   6736 
   6737 int ndb_builder_finalize(struct ndb_builder *builder, struct ndb_note **note,
   6738 			 struct ndb_keypair *keypair)
   6739 {
   6740 	int strings_len = builder->strings.p - builder->strings.start;
   6741 	unsigned char *note_end = builder->note_cur.p + strings_len;
   6742 	int total_size = note_end - builder->note_cur.start;
   6743 
   6744 	// move the strings buffer next to the end of our ndb_note
   6745 	memmove(builder->note_cur.p, builder->strings.start, strings_len);
   6746 
   6747 	// set the strings location
   6748 	builder->note->strings = builder->note_cur.p - builder->note_cur.start;
   6749 
   6750 	// record the total size
   6751 	//builder->note->size = total_size;
   6752 
   6753 	*note = builder->note;
   6754 
   6755 	// generate id and sign if we're building this manually
   6756 	if (keypair) {
   6757 		// use the remaining memory for building our id buffer
   6758 		unsigned char *end   = builder->mem.end;
   6759 		unsigned char *start = (unsigned char*)(*note) + total_size;
   6760 
   6761 		ndb_builder_set_pubkey(builder, keypair->pubkey);
   6762 
   6763 		if (!ndb_calculate_id(builder->note, start, end - start,
   6764 				      builder->note->id))
   6765 			return 0;
   6766 
   6767 		if (!ndb_sign_id(keypair, (*note)->id, (*note)->sig))
   6768 			return 0;
   6769 	}
   6770 
   6771 	// make sure we're aligned as a whole
   6772 	total_size = (total_size + 7) & ~7;
   6773 	assert((total_size % 8) == 0);
   6774 	return total_size;
   6775 }
   6776 
   6777 struct ndb_note * ndb_builder_note(struct ndb_builder *builder)
   6778 {
   6779 	return builder->note;
   6780 }
   6781 
   6782 static union ndb_packed_str ndb_offset_str(uint32_t offset)
   6783 {
   6784 	// ensure accidents like -1 don't corrupt our packed_str
   6785 	union ndb_packed_str str;
   6786 	// most significant byte is reserved for ndb_packtype
   6787 	str.offset = offset & 0xFFFFFF;
   6788 	return str;
   6789 }
   6790 
   6791 
   6792 /// find an existing string via str_indices. these indices only exist in the
   6793 /// builder phase just for this purpose.
   6794 static inline int ndb_builder_find_str(struct ndb_builder *builder,
   6795 				       const char *str, int len,
   6796 				       union ndb_packed_str *pstr)
   6797 {
   6798 	// find existing matching string to avoid duplicate strings
   6799 	int indices = cursor_count(&builder->str_indices, sizeof(uint32_t));
   6800 	for (int i = 0; i < indices; i++) {
   6801 		uint32_t index = ((uint32_t*)builder->str_indices.start)[i];
   6802 		const char *some_str = (const char*)builder->strings.start + index;
   6803 
   6804 		if (!memcmp(some_str, str, len) && some_str[len] == '\0') {
   6805 			// found an existing matching str, use that index
   6806 			*pstr = ndb_offset_str(index);
   6807 			return 1;
   6808 		}
   6809 	}
   6810 
   6811 	return 0;
   6812 }
   6813 
   6814 static int ndb_builder_push_str(struct ndb_builder *builder, const char *str,
   6815 				int len, union ndb_packed_str *pstr)
   6816 {
   6817 	uint32_t loc;
   6818 
   6819 	// no string found, push a new one
   6820 	loc = builder->strings.p - builder->strings.start;
   6821 	if (!(cursor_push(&builder->strings, (unsigned char*)str, len) &&
   6822 	      cursor_push_byte(&builder->strings, '\0'))) {
   6823 		return 0;
   6824 	}
   6825 
   6826 	*pstr = ndb_offset_str(loc);
   6827 
   6828 	// record in builder indices. ignore return value, if we can't cache it
   6829 	// then whatever
   6830 	cursor_push_u32(&builder->str_indices, loc);
   6831 
   6832 	return 1;
   6833 }
   6834 
   6835 static int ndb_builder_push_packed_id(struct ndb_builder *builder,
   6836 				      unsigned char *id,
   6837 				      union ndb_packed_str *pstr)
   6838 {
   6839 	// Don't both find id duplicates. very rarely are they duplicated
   6840 	// and it slows things down quite a bit. If we really care about this
   6841 	// We can switch to a hash table.
   6842 	//if (ndb_builder_find_str(builder, (const char*)id, 32, pstr)) {
   6843 	//	pstr->packed.flag = NDB_PACKED_ID;
   6844 	//	return 1;
   6845 	//}
   6846 
   6847 	if (ndb_builder_push_str(builder, (const char*)id, 32, pstr)) {
   6848 		pstr->packed.flag = NDB_PACKED_ID;
   6849 		return 1;
   6850 	}
   6851 
   6852 	return 0;
   6853 }
   6854 
   6855 union ndb_packed_str ndb_chars_to_packed_str(char c1, char c2)
   6856 {
   6857 	union ndb_packed_str str;
   6858 	str.packed.flag = NDB_PACKED_STR;
   6859 	str.packed.str[0] = c1;
   6860 	str.packed.str[1] = c2;
   6861 	str.packed.str[2] = '\0';
   6862 	return str;
   6863 }
   6864 
   6865 static union ndb_packed_str ndb_char_to_packed_str(char c)
   6866 {
   6867 	union ndb_packed_str str;
   6868 	str.packed.flag = NDB_PACKED_STR;
   6869 	str.packed.str[0] = c;
   6870 	str.packed.str[1] = '\0';
   6871 	return str;
   6872 }
   6873 
   6874 
   6875 /// Check for small strings to pack
   6876 static inline int ndb_builder_try_compact_str(struct ndb_builder *builder,
   6877 					      const char *str, int len,
   6878 					      union ndb_packed_str *pstr,
   6879 					      int pack_ids)
   6880 {
   6881 	unsigned char id_buf[32];
   6882 
   6883 	if (len == 0) {
   6884 		*pstr = ndb_char_to_packed_str(0);
   6885 		return 1;
   6886 	} else if (len == 1) {
   6887 		*pstr = ndb_char_to_packed_str(str[0]);
   6888 		return 1;
   6889 	} else if (len == 2) {
   6890 		*pstr = ndb_chars_to_packed_str(str[0], str[1]);
   6891 		return 1;
   6892 	} else if (pack_ids && len == 64 && hex_decode(str, 64, id_buf, 32)) {
   6893 		return ndb_builder_push_packed_id(builder, id_buf, pstr);
   6894 	}
   6895 
   6896 	return 0;
   6897 }
   6898 
   6899 
   6900 static int ndb_builder_push_unpacked_str(struct ndb_builder *builder,
   6901 					 const char *str, int len,
   6902 					 union ndb_packed_str *pstr)
   6903 {
   6904 	if (ndb_builder_find_str(builder, str, len, pstr))
   6905 		return 1;
   6906 
   6907 	return ndb_builder_push_str(builder, str, len, pstr);
   6908 }
   6909 
   6910 int ndb_builder_make_str(struct ndb_builder *builder, const char *str, int len,
   6911 			 union ndb_packed_str *pstr, int pack_ids)
   6912 {
   6913 	if (ndb_builder_try_compact_str(builder, str, len, pstr, pack_ids))
   6914 		return 1;
   6915 
   6916 	return ndb_builder_push_unpacked_str(builder, str, len, pstr);
   6917 }
   6918 
   6919 int ndb_builder_set_content(struct ndb_builder *builder, const char *content,
   6920 			    int len)
   6921 {
   6922 	int pack_ids = 0;
   6923 	builder->note->content_length = len;
   6924 	return ndb_builder_make_str(builder, content, len,
   6925 				    &builder->note->content, pack_ids);
   6926 }
   6927 
   6928 
   6929 static inline int jsoneq(const char *json, jsmntok_t *tok, int tok_len,
   6930 			 const char *s)
   6931 {
   6932 	if (tok->type == JSMN_STRING && (int)strlen(s) == tok_len &&
   6933 	    memcmp(json + tok->start, s, tok_len) == 0) {
   6934 		return 1;
   6935 	}
   6936 	return 0;
   6937 }
   6938 
   6939 static int ndb_builder_finalize_tag(struct ndb_builder *builder,
   6940 				    union ndb_packed_str offset)
   6941 {
   6942 	if (!cursor_push_u32(&builder->note_cur, offset.offset))
   6943 		return 0;
   6944 	builder->current_tag->count++;
   6945 	return 1;
   6946 }
   6947 
   6948 /// Unescape and push json strings
   6949 static int ndb_builder_make_json_str(struct ndb_builder *builder,
   6950 				     const char *str, int len,
   6951 				     union ndb_packed_str *pstr,
   6952 				     int *written, int pack_ids)
   6953 {
   6954 	// let's not care about de-duping these. we should just unescape
   6955 	// in-place directly into the strings table.
   6956 	if (written)
   6957 		*written = len;
   6958 
   6959 	const char *p, *end, *start;
   6960 	unsigned char *builder_start;
   6961 
   6962 	// always try compact strings first
   6963 	if (ndb_builder_try_compact_str(builder, str, len, pstr, pack_ids))
   6964 		return 1;
   6965 
   6966 	end = str + len;
   6967 	start = str; // Initialize start to the beginning of the string
   6968 
   6969 	*pstr = ndb_offset_str(builder->strings.p - builder->strings.start);
   6970 	builder_start = builder->strings.p;
   6971 
   6972 	for (p = str; p < end; p++) {
   6973 		if (*p == '\\' && p+1 < end) {
   6974 			// Push the chunk of unescaped characters before this escape sequence
   6975 			if (start < p && !cursor_push(&builder->strings,
   6976 						(unsigned char *)start,
   6977 						p - start)) {
   6978 				return 0;
   6979 			}
   6980 
   6981 			if (!cursor_push_unescaped_char(&builder->strings, *p, *(p+1)))
   6982 				return 0;
   6983 
   6984 			p++; // Skip the character following the backslash
   6985 			start = p + 1; // Update the start pointer to the next character
   6986 		}
   6987 	}
   6988 
   6989 	// Handle the last chunk after the last escape sequence (or if there are no escape sequences at all)
   6990 	if (start < p && !cursor_push(&builder->strings, (unsigned char *)start,
   6991 				      p - start)) {
   6992 		return 0;
   6993 	}
   6994 
   6995 	if (written)
   6996 		*written = builder->strings.p - builder_start;
   6997 
   6998 	// TODO: dedupe these!?
   6999 	return cursor_push_byte(&builder->strings, '\0');
   7000 }
   7001 
   7002 static int ndb_builder_push_json_tag(struct ndb_builder *builder,
   7003 				     const char *str, int len)
   7004 {
   7005 	union ndb_packed_str pstr;
   7006 	int pack_ids = 1;
   7007 	if (!ndb_builder_make_json_str(builder, str, len, &pstr, NULL, pack_ids))
   7008 		return 0;
   7009 	return ndb_builder_finalize_tag(builder, pstr);
   7010 }
   7011 
   7012 // Push a json array into an ndb tag ["p", "abcd..."] -> struct ndb_tag
   7013 static int ndb_builder_tag_from_json_array(struct ndb_json_parser *p,
   7014 					   jsmntok_t *array)
   7015 {
   7016 	jsmntok_t *str_tok;
   7017 	const char *str;
   7018 
   7019 	if (array->size == 0)
   7020 		return 0;
   7021 
   7022 	if (!ndb_builder_new_tag(&p->builder))
   7023 		return 0;
   7024 
   7025 	for (int i = 0; i < array->size; i++) {
   7026 		str_tok = &array[i+1];
   7027 		str = p->json + str_tok->start;
   7028 
   7029 		if (!ndb_builder_push_json_tag(&p->builder, str,
   7030 					       toksize(str_tok))) {
   7031 			return 0;
   7032 		}
   7033 	}
   7034 
   7035 	return 1;
   7036 }
   7037 
   7038 // Push json tags into ndb data
   7039 //   [["t", "hashtag"], ["p", "abcde..."]] -> struct ndb_tags
   7040 static inline int ndb_builder_process_json_tags(struct ndb_json_parser *p,
   7041 						jsmntok_t *array)
   7042 {
   7043 	jsmntok_t *tag = array;
   7044 
   7045 	if (array->size == 0)
   7046 		return 1;
   7047 
   7048 	for (int i = 0; i < array->size; i++) {
   7049 		if (!ndb_builder_tag_from_json_array(p, &tag[i+1]))
   7050 			return 0;
   7051 		tag += tag[i+1].size;
   7052 	}
   7053 
   7054 	return 1;
   7055 }
   7056 
   7057 static int parse_unsigned_int(const char *start, int len, unsigned int *num)
   7058 {
   7059 	unsigned int number = 0;
   7060 	const char *p = start, *end = start + len;
   7061 	int digits = 0;
   7062 
   7063 	while (p < end) {
   7064 		char c = *p;
   7065 
   7066 		if (c < '0' || c > '9')
   7067 			break;
   7068 
   7069 		// Check for overflow
   7070 		char digit = c - '0';
   7071 		if (number > (UINT_MAX - digit) / 10)
   7072 			return 0; // Overflow detected
   7073 
   7074 		number = number * 10 + digit;
   7075 
   7076 		p++;
   7077 		digits++;
   7078 	}
   7079 
   7080 	if (digits == 0)
   7081 		return 0;
   7082 
   7083 	*num = number;
   7084 	return 1;
   7085 }
   7086 
   7087 int ndb_client_event_from_json(const char *json, int len, struct ndb_fce *fce,
   7088 			       unsigned char *buf, int bufsize, struct ndb_id_cb *cb)
   7089 {
   7090 	jsmntok_t *tok = NULL;
   7091 	int tok_len, res;
   7092 	struct ndb_json_parser parser;
   7093 	struct ndb_event *ev = &fce->event;
   7094 
   7095 	ndb_json_parser_init(&parser, json, len, buf, bufsize);
   7096 
   7097 	if ((res = ndb_json_parser_parse(&parser, cb)) < 0)
   7098 		return res;
   7099 
   7100 	if (parser.toks[0].type == JSMN_OBJECT) {
   7101 		ndb_debug("got raw json in client_event_from_json\n");
   7102 		fce->evtype = NDB_FCE_EVENT;
   7103 		return ndb_parse_json_note(&parser, &ev->note);
   7104 	}
   7105 
   7106 	if (parser.num_tokens <= 3 || parser.toks[0].type != JSMN_ARRAY)
   7107 		return 0;
   7108 
   7109 	parser.i = 1;
   7110 	tok = &parser.toks[parser.i++];
   7111 	tok_len = toksize(tok);
   7112 	if (tok->type != JSMN_STRING)
   7113 		return 0;
   7114 
   7115 	if (tok_len == 5 && !memcmp("EVENT", json + tok->start, 5)) {
   7116 		fce->evtype = NDB_FCE_EVENT;
   7117 		return ndb_parse_json_note(&parser, &ev->note);
   7118 	}
   7119 
   7120 	return 0;
   7121 }
   7122 
   7123 
   7124 int ndb_ws_event_from_json(const char *json, int len, struct ndb_tce *tce,
   7125 			   unsigned char *buf, int bufsize,
   7126 			   struct ndb_id_cb *cb)
   7127 {
   7128 	jsmntok_t *tok = NULL;
   7129 	int tok_len, res;
   7130 	struct ndb_json_parser parser;
   7131 	struct ndb_event *ev = &tce->event;
   7132 
   7133 	tce->subid_len = 0;
   7134 	tce->subid = "";
   7135 
   7136 	ndb_json_parser_init(&parser, json, len, buf, bufsize);
   7137 
   7138 	if ((res = ndb_json_parser_parse(&parser, cb)) < 0)
   7139 		return res;
   7140 
   7141 	if (parser.toks[0].type == JSMN_OBJECT) {
   7142 		ndb_debug("got raw json in ws_event_from_json\n");
   7143 		tce->evtype = NDB_TCE_EVENT;
   7144 		return ndb_parse_json_note(&parser, &ev->note);
   7145 	}
   7146 
   7147 	if (parser.num_tokens < 3 || parser.toks[0].type != JSMN_ARRAY)
   7148 		return 0;
   7149 
   7150 	parser.i = 1;
   7151 	tok = &parser.toks[parser.i++];
   7152 	tok_len = toksize(tok);
   7153 	if (tok->type != JSMN_STRING)
   7154 		return 0;
   7155 
   7156 	if (tok_len == 5 && !memcmp("EVENT", json + tok->start, 5)) {
   7157 		tce->evtype = NDB_TCE_EVENT;
   7158 
   7159 		tok = &parser.toks[parser.i++];
   7160 		if (tok->type != JSMN_STRING)
   7161 			return 0;
   7162 
   7163 		tce->subid = json + tok->start;
   7164 		tce->subid_len = toksize(tok);
   7165 
   7166 		return ndb_parse_json_note(&parser, &ev->note);
   7167 	} else if (tok_len == 4 && !memcmp("EOSE", json + tok->start, 4)) {
   7168 		tce->evtype = NDB_TCE_EOSE;
   7169 
   7170 		tok = &parser.toks[parser.i++];
   7171 		if (tok->type != JSMN_STRING)
   7172 			return 0;
   7173 
   7174 		tce->subid = json + tok->start;
   7175 		tce->subid_len = toksize(tok);
   7176 		return 1;
   7177 	} else if (tok_len == 2 && !memcmp("OK", json + tok->start, 2)) {
   7178 		if (parser.num_tokens != 5)
   7179 			return 0;
   7180 
   7181 		struct ndb_command_result *cr = &tce->command_result;
   7182 
   7183 		tce->evtype = NDB_TCE_OK;
   7184 
   7185 		tok = &parser.toks[parser.i++];
   7186 		if (tok->type != JSMN_STRING)
   7187 			return 0;
   7188 
   7189 		tce->subid = json + tok->start;
   7190 		tce->subid_len = toksize(tok);
   7191 
   7192 		tok = &parser.toks[parser.i++];
   7193 		if (tok->type != JSMN_PRIMITIVE || toksize(tok) == 0)
   7194 			return 0;
   7195 
   7196 		cr->ok = (json + tok->start)[0] == 't';
   7197 
   7198 		tok = &parser.toks[parser.i++];
   7199 		if (tok->type != JSMN_STRING)
   7200 			return 0;
   7201 
   7202 		tce->command_result.msg = json + tok->start;
   7203 		tce->command_result.msglen = toksize(tok);
   7204 
   7205 		return 1;
   7206 	} else if (tok_len == 4 && !memcmp("AUTH", json + tok->start, 4)) {
   7207 		tce->evtype = NDB_TCE_AUTH;
   7208 
   7209 		tok = &parser.toks[parser.i++];
   7210 		if (tok->type != JSMN_STRING)
   7211 			return 0;
   7212 
   7213 		tce->subid = json + tok->start;
   7214 		tce->subid_len = toksize(tok);
   7215 
   7216 		return 1;
   7217 	}
   7218 
   7219 	return 0;
   7220 }
   7221 
   7222 static enum ndb_filter_fieldtype
   7223 ndb_filter_parse_field(const char *tok, int len, char *tagchar)
   7224 {
   7225 	*tagchar = 0;
   7226 
   7227 	if (len == 0)
   7228 		return 0;
   7229 
   7230 	if (len == 7 && !strncmp(tok, "authors", 7)) {
   7231 		return NDB_FILTER_AUTHORS;
   7232 	} else if (len == 3 && !strncmp(tok, "ids", 3)) {
   7233 		return NDB_FILTER_IDS;
   7234 	} else if (len == 5 && !strncmp(tok, "kinds", 5)) {
   7235 		return NDB_FILTER_KINDS;
   7236 	} else if (len == 2 && tok[0] == '#') {
   7237 		*tagchar = tok[1];
   7238 		return NDB_FILTER_TAGS;
   7239 	} else if (len == 5 && !strncmp(tok, "since", 5)) {
   7240 		return NDB_FILTER_SINCE;
   7241 	} else if (len == 5 && !strncmp(tok, "until", 5)) {
   7242 		return NDB_FILTER_UNTIL;
   7243 	} else if (len == 5 && !strncmp(tok, "limit", 5)) {
   7244 		return NDB_FILTER_LIMIT;
   7245 	} else if (len == 6 && !strncmp(tok, "search", 6)) {
   7246 		return NDB_FILTER_SEARCH;
   7247 	}
   7248 
   7249 	return 0;
   7250 }
   7251 
   7252 static int ndb_filter_parse_json_ids(struct ndb_json_parser *parser,
   7253 				     struct ndb_filter *filter)
   7254 {
   7255 	jsmntok_t *tok;
   7256 	const char *start;
   7257 	unsigned char hexbuf[32];
   7258 	int tok_len, i, size;
   7259 
   7260 	tok = &parser->toks[parser->i++];
   7261 
   7262 	if (tok->type != JSMN_ARRAY) {
   7263 		ndb_debug("parse_json_ids: not an array\n");
   7264 		return 0;
   7265 	}
   7266 
   7267 	size = tok->size;
   7268 
   7269 	for (i = 0; i < size; parser->i++, i++) {
   7270 		tok = &parser->toks[parser->i];
   7271 		start = parser->json + tok->start;
   7272 		tok_len = toksize(tok);
   7273 
   7274 		if (tok->type != JSMN_STRING) {
   7275 			ndb_debug("parse_json_ids: not a string '%d'\n", tok->type);
   7276 			return 0;
   7277 		}
   7278 
   7279 		if (tok_len != 64) {
   7280 			ndb_debug("parse_json_ids: not len 64: '%.*s'\n", tok_len, start);
   7281 			return 0;
   7282 		}
   7283 
   7284 		// id
   7285 		if (!hex_decode(start, tok_len, hexbuf, sizeof(hexbuf))) {
   7286 			ndb_debug("parse_json_ids: hex decode failed\n");
   7287 			return 0;
   7288 		}
   7289 
   7290 		ndb_debug("adding id elem\n");
   7291 		if (!ndb_filter_add_id_element(filter, hexbuf)) {
   7292 			ndb_debug("parse_json_ids: failed to add id element\n");
   7293 			return 0;
   7294 		}
   7295 	}
   7296 
   7297 	parser->i--;
   7298 	return 1;
   7299 }
   7300 
   7301 static int ndb_filter_parse_json_elems(struct ndb_json_parser *parser,
   7302 				       struct ndb_filter *filter)
   7303 {
   7304 	jsmntok_t *tok;
   7305 	const char *start;
   7306 	int tok_len;
   7307 	unsigned char hexbuf[32];
   7308 	enum ndb_generic_element_type typ;
   7309 	tok = NULL;
   7310 	int i, size;
   7311 
   7312 	tok = &parser->toks[parser->i++];
   7313 
   7314 	if (tok->type != JSMN_ARRAY)
   7315 		return 0;
   7316 
   7317 	size = tok->size;
   7318 
   7319 	for (i = 0; i < size; i++, parser->i++) {
   7320 		tok = &parser->toks[parser->i];
   7321 		start = parser->json + tok->start;
   7322 		tok_len = toksize(tok);
   7323 
   7324 		if (tok->type != JSMN_STRING)
   7325 			return 0;
   7326 
   7327 		if (i == 0) {
   7328 			if (tok_len == 64 && hex_decode(start, 64, hexbuf, sizeof(hexbuf))) {
   7329 				typ = NDB_ELEMENT_ID;
   7330 				if (!ndb_filter_add_id_element(filter, hexbuf)) {
   7331 					ndb_debug("failed to push id elem\n");
   7332 					return 0;
   7333 				}
   7334 			} else {
   7335 				typ = NDB_ELEMENT_STRING;
   7336 				if (!ndb_filter_add_str_element_len(filter, start, tok_len))
   7337 					return 0;
   7338 			}
   7339 		} else if (typ == NDB_ELEMENT_ID) {
   7340 			if (!hex_decode(start, 64, hexbuf, sizeof(hexbuf)))
   7341 				return 0;
   7342 			if (!ndb_filter_add_id_element(filter, hexbuf))
   7343 				return 0;
   7344 		} else if (typ == NDB_ELEMENT_STRING) {
   7345 			if (!ndb_filter_add_str_element_len(filter, start, tok_len))
   7346 				return 0;
   7347 		} else {
   7348 			// ???
   7349 			return 0;
   7350 		}
   7351 	}
   7352 
   7353 	parser->i--;
   7354 	return 1;
   7355 }
   7356 
   7357 static int ndb_filter_parse_json_str(struct ndb_json_parser *parser,
   7358 				     struct ndb_filter *filter)
   7359 {
   7360 	jsmntok_t *tok;
   7361 	const char *start;
   7362 	int tok_len;
   7363 
   7364 	tok = &parser->toks[parser->i];
   7365 	start = parser->json + tok->start;
   7366 	tok_len = toksize(tok);
   7367 
   7368 	if (tok->type != JSMN_STRING)
   7369 		return 0;
   7370 
   7371 	if (!ndb_filter_add_str_element_len(filter, start, tok_len))
   7372 		return 0;
   7373 
   7374 	ndb_debug("added str elem '%.*s'\n", tok_len, start);
   7375 
   7376 	return 1;
   7377 }
   7378 
   7379 static int ndb_filter_parse_json_int(struct ndb_json_parser *parser,
   7380 				     struct ndb_filter *filter)
   7381 {
   7382 	jsmntok_t *tok;
   7383 	const char *start;
   7384 	int tok_len;
   7385 	unsigned int value;
   7386 
   7387 	tok = &parser->toks[parser->i];
   7388 	start = parser->json + tok->start;
   7389 	tok_len = toksize(tok);
   7390 
   7391 	if (tok->type != JSMN_PRIMITIVE)
   7392 		return 0;
   7393 
   7394 	if (!parse_unsigned_int(start, tok_len, &value))
   7395 		return 0;
   7396 
   7397 	if (!ndb_filter_add_int_element(filter, (uint64_t)value))
   7398 		return 0;
   7399 
   7400 	ndb_debug("added int elem %d\n", value);
   7401 
   7402 	return 1;
   7403 }
   7404 
   7405 
   7406 static int ndb_filter_parse_json_ints(struct ndb_json_parser *parser,
   7407 				      struct ndb_filter *filter)
   7408 {
   7409 	jsmntok_t *tok;
   7410 	int size, i;
   7411 
   7412 	tok = &parser->toks[parser->i++];
   7413 
   7414 	if (tok->type != JSMN_ARRAY)
   7415 		return 0;
   7416 
   7417 	size = tok->size;
   7418 
   7419 	for (i = 0; i < size; parser->i++, i++) {
   7420 		if (!ndb_filter_parse_json_int(parser, filter))
   7421 			return 0;
   7422 	}
   7423 
   7424 	parser->i--;
   7425 	return 1;
   7426 }
   7427 
   7428 
   7429 static int ndb_filter_parse_json(struct ndb_json_parser *parser,
   7430 				 struct ndb_filter *filter)
   7431 {
   7432 	jsmntok_t *tok = NULL;
   7433 	const char *json = parser->json;
   7434 	const char *start;
   7435 	char tag;
   7436 	int tok_len;
   7437 	enum ndb_filter_fieldtype field;
   7438 
   7439 	if (parser->toks[parser->i++].type != JSMN_OBJECT)
   7440 		return 0;
   7441 
   7442 	for (; parser->i < parser->num_tokens; parser->i++) {
   7443 		tok = &parser->toks[parser->i++];
   7444 		start = json + tok->start;
   7445 		tok_len = toksize(tok);
   7446 
   7447 		if (!(field = ndb_filter_parse_field(start, tok_len, &tag))) {
   7448 			ndb_debug("failed field '%.*s'\n", tok_len, start);
   7449 			continue;
   7450 		}
   7451 
   7452 		if (tag) {
   7453 			ndb_debug("starting tag field '%c'\n", tag);
   7454 			if (!ndb_filter_start_tag_field(filter, tag)) {
   7455 				ndb_debug("failed to start tag field '%c'\n", tag);
   7456 				return 0;
   7457 			}
   7458 		} else if (!ndb_filter_start_field(filter, field)) {
   7459 			ndb_debug("field already started\n");
   7460 			return 0;
   7461 		}
   7462 
   7463 		// we parsed a top-level field
   7464 		switch(field) {
   7465 		case NDB_FILTER_CUSTOM:
   7466 			// can't really parse these yet
   7467 			break;
   7468 		case NDB_FILTER_AUTHORS:
   7469 		case NDB_FILTER_IDS:
   7470 			if (!ndb_filter_parse_json_ids(parser, filter)) {
   7471 				ndb_debug("failed to parse filter ids/authors\n");
   7472 				return 0;
   7473 			}
   7474 			break;
   7475 		case NDB_FILTER_SEARCH:
   7476 			if (!ndb_filter_parse_json_str(parser, filter)) {
   7477 				ndb_debug("failed to parse filter search str\n");
   7478 				return 0;
   7479 			}
   7480 			break;
   7481 		case NDB_FILTER_SINCE:
   7482 		case NDB_FILTER_UNTIL:
   7483 		case NDB_FILTER_LIMIT:
   7484 			if (!ndb_filter_parse_json_int(parser, filter)) {
   7485 				ndb_debug("failed to parse filter since/until/limit\n");
   7486 				return 0;
   7487 			}
   7488 			break;
   7489 		case NDB_FILTER_KINDS:
   7490 			if (!ndb_filter_parse_json_ints(parser, filter)) {
   7491 				ndb_debug("failed to parse filter kinds\n");
   7492 				return 0;
   7493 			}
   7494 			break;
   7495 		case NDB_FILTER_RELAYS:
   7496 		case NDB_FILTER_TAGS:
   7497 			if (!ndb_filter_parse_json_elems(parser, filter)) {
   7498 				ndb_debug("failed to parse filter tags\n");
   7499 				return 0;
   7500 			}
   7501 			break;
   7502 		}
   7503 
   7504 		ndb_filter_end_field(filter);
   7505 	}
   7506 
   7507 	return ndb_filter_end(filter);
   7508 }
   7509 
   7510 int ndb_parse_json_note(struct ndb_json_parser *parser, struct ndb_note **note)
   7511 {
   7512 	jsmntok_t *tok = NULL;
   7513 	unsigned char hexbuf[64];
   7514 	const char *json = parser->json;
   7515 	const char *start;
   7516 	int i, tok_len, parsed;
   7517 
   7518 	parsed = 0;
   7519 
   7520 	if (parser->toks[parser->i].type != JSMN_OBJECT)
   7521 		return 0;
   7522 
   7523 	// TODO: build id buffer and verify at end
   7524 
   7525 	for (i = parser->i + 1; i < parser->num_tokens; i++) {
   7526 		tok = &parser->toks[i];
   7527 		start = json + tok->start;
   7528 		tok_len = toksize(tok);
   7529 
   7530 		//printf("toplevel %.*s %d\n", tok_len, json + tok->start, tok->type);
   7531 		if (tok_len == 0 || i + 1 >= parser->num_tokens)
   7532 			continue;
   7533 
   7534 		if (start[0] == 'p' && jsoneq(json, tok, tok_len, "pubkey")) {
   7535 			// pubkey
   7536 			tok = &parser->toks[i+1];
   7537 			hex_decode(json + tok->start, toksize(tok), hexbuf, sizeof(hexbuf));
   7538 			parsed |= NDB_PARSED_PUBKEY;
   7539 			ndb_builder_set_pubkey(&parser->builder, hexbuf);
   7540 		} else if (tok_len == 2 && start[0] == 'i' && start[1] == 'd') {
   7541 			// id
   7542 			tok = &parser->toks[i+1];
   7543 			hex_decode(json + tok->start, toksize(tok), hexbuf, sizeof(hexbuf));
   7544 			parsed |= NDB_PARSED_ID;
   7545 			ndb_builder_set_id(&parser->builder, hexbuf);
   7546 		} else if (tok_len == 3 && start[0] == 's' && start[1] == 'i' && start[2] == 'g') {
   7547 			// sig
   7548 			tok = &parser->toks[i+1];
   7549 			hex_decode(json + tok->start, toksize(tok), hexbuf, sizeof(hexbuf));
   7550 			parsed |= NDB_PARSED_SIG;
   7551 			ndb_builder_set_sig(&parser->builder, hexbuf);
   7552 		} else if (start[0] == 'k' && jsoneq(json, tok, tok_len, "kind")) {
   7553 			// kind
   7554 			tok = &parser->toks[i+1];
   7555 			start = json + tok->start;
   7556 			if (tok->type != JSMN_PRIMITIVE || tok_len <= 0)
   7557 				return 0;
   7558 			if (!parse_unsigned_int(start, toksize(tok),
   7559 						&parser->builder.note->kind))
   7560 					return 0;
   7561 			parsed |= NDB_PARSED_KIND;
   7562 		} else if (start[0] == 'c') {
   7563 			if (jsoneq(json, tok, tok_len, "created_at")) {
   7564 				// created_at
   7565 				tok = &parser->toks[i+1];
   7566 				start = json + tok->start;
   7567 				if (tok->type != JSMN_PRIMITIVE || tok_len <= 0)
   7568 					return 0;
   7569 				// TODO: update to int64 in 2106 ... xD
   7570 				unsigned int bigi;
   7571 				if (!parse_unsigned_int(start, toksize(tok), &bigi))
   7572 					return 0;
   7573 				parser->builder.note->created_at = bigi;
   7574 				parsed |= NDB_PARSED_CREATED_AT;
   7575 			} else if (jsoneq(json, tok, tok_len, "content")) {
   7576 				// content
   7577 				tok = &parser->toks[i+1];
   7578 				union ndb_packed_str pstr;
   7579 				tok_len = toksize(tok);
   7580 				int written, pack_ids = 0;
   7581 				if (!ndb_builder_make_json_str(&parser->builder,
   7582 							json + tok->start,
   7583 							tok_len, &pstr,
   7584 							&written, pack_ids)) {
   7585 					ndb_debug("ndb_builder_make_json_str failed\n");
   7586 					return 0;
   7587 				}
   7588 				parser->builder.note->content_length = written;
   7589 				parser->builder.note->content = pstr;
   7590 				parsed |= NDB_PARSED_CONTENT;
   7591 			}
   7592 		} else if (start[0] == 't' && jsoneq(json, tok, tok_len, "tags")) {
   7593 			tok = &parser->toks[i+1];
   7594 			ndb_builder_process_json_tags(parser, tok);
   7595 			i += tok->size;
   7596 			parsed |= NDB_PARSED_TAGS;
   7597 		}
   7598 	}
   7599 
   7600 	//ndb_debug("parsed %d = %d, &->%d", parsed, NDB_PARSED_ALL, parsed & NDB_PARSED_ALL);
   7601 	if (parsed != NDB_PARSED_ALL)
   7602 		return 0;
   7603 
   7604 	return ndb_builder_finalize(&parser->builder, note, NULL);
   7605 }
   7606 
   7607 int ndb_filter_from_json(const char *json, int len, struct ndb_filter *filter,
   7608 			 unsigned char *buf, int bufsize)
   7609 {
   7610 	struct ndb_json_parser parser;
   7611 	int res;
   7612 
   7613 	if (filter->finalized)
   7614 		return 0;
   7615 
   7616 	ndb_json_parser_init(&parser, json, len, buf, bufsize);
   7617 	if ((res = ndb_json_parser_parse(&parser, NULL)) < 0)
   7618 		return res;
   7619 
   7620 	if (parser.num_tokens < 1)
   7621 		return 0;
   7622 
   7623 	return ndb_filter_parse_json(&parser, filter);
   7624 }
   7625 
   7626 int ndb_note_from_json(const char *json, int len, struct ndb_note **note,
   7627 		       unsigned char *buf, int bufsize)
   7628 {
   7629 	struct ndb_json_parser parser;
   7630 	int res;
   7631 
   7632 	ndb_json_parser_init(&parser, json, len, buf, bufsize);
   7633 	if ((res = ndb_json_parser_parse(&parser, NULL)) < 0)
   7634 		return res;
   7635 
   7636 	if (parser.num_tokens < 1)
   7637 		return 0;
   7638 
   7639 	return ndb_parse_json_note(&parser, note);
   7640 }
   7641 
   7642 void ndb_builder_set_pubkey(struct ndb_builder *builder, unsigned char *pubkey)
   7643 {
   7644 	memcpy(builder->note->pubkey, pubkey, 32);
   7645 }
   7646 
   7647 void ndb_builder_set_id(struct ndb_builder *builder, unsigned char *id)
   7648 {
   7649 	memcpy(builder->note->id, id, 32);
   7650 }
   7651 
   7652 void ndb_builder_set_sig(struct ndb_builder *builder, unsigned char *sig)
   7653 {
   7654 	memcpy(builder->note->sig, sig, 64);
   7655 }
   7656 
   7657 void ndb_builder_set_kind(struct ndb_builder *builder, uint32_t kind)
   7658 {
   7659 	builder->note->kind = kind;
   7660 }
   7661 
   7662 void ndb_builder_set_created_at(struct ndb_builder *builder, uint64_t created_at)
   7663 {
   7664 	builder->note->created_at = created_at;
   7665 }
   7666 
   7667 int ndb_builder_new_tag(struct ndb_builder *builder)
   7668 {
   7669 	builder->note->tags.count++;
   7670 	struct ndb_tag tag = {0};
   7671 	builder->current_tag = (struct ndb_tag *)builder->note_cur.p;
   7672 	return cursor_push_tag(&builder->note_cur, &tag);
   7673 }
   7674 
   7675 void ndb_stat_counts_init(struct ndb_stat_counts *counts)
   7676 {
   7677 	counts->count = 0;
   7678 	counts->key_size = 0;
   7679 	counts->value_size = 0;
   7680 }
   7681 
   7682 static void ndb_stat_init(struct ndb_stat *stat)
   7683 {
   7684 	// init stats
   7685 	int i;
   7686 
   7687 	for (i = 0; i < NDB_CKIND_COUNT; i++) {
   7688 		ndb_stat_counts_init(&stat->common_kinds[i]);
   7689 	}
   7690 
   7691 	for (i = 0; i < NDB_DBS; i++) {
   7692 		ndb_stat_counts_init(&stat->dbs[i]);
   7693 	}
   7694 
   7695 	ndb_stat_counts_init(&stat->other_kinds);
   7696 }
   7697 
   7698 int ndb_stat(struct ndb *ndb, struct ndb_stat *stat)
   7699 {
   7700 	int rc;
   7701 	MDB_cursor *cur;
   7702 	MDB_val k, v;
   7703 	MDB_dbi db;
   7704 	struct ndb_txn txn;
   7705 	struct ndb_note *note;
   7706 	int i;
   7707 	enum ndb_common_kind common_kind;
   7708 
   7709 	// initialize to 0
   7710 	ndb_stat_init(stat);
   7711 
   7712 	if (!ndb_begin_query(ndb, &txn)) {
   7713 		fprintf(stderr, "ndb_stat failed at ndb_begin_query\n");
   7714 		return 0;
   7715 	}
   7716 
   7717 	// stat each dbi in the database
   7718 	for (i = 0; i < NDB_DBS; i++)
   7719 	{
   7720 		db = ndb->lmdb.dbs[i];
   7721 
   7722 		if ((rc = mdb_cursor_open(txn.mdb_txn, db, &cur))) {
   7723 			fprintf(stderr, "ndb_stat: mdb_cursor_open failed, error '%s'\n",
   7724 					mdb_strerror(rc));
   7725 			return 0;
   7726 		}
   7727 
   7728 		// loop over every entry and count kv sizes
   7729 		while (mdb_cursor_get(cur, &k, &v, MDB_NEXT) == 0) {
   7730 			// we gather more detailed per-kind stats if we're in
   7731 			// the notes db
   7732 			if (i == NDB_DB_NOTE) {
   7733 				note = v.mv_data;
   7734 				common_kind = ndb_kind_to_common_kind(note->kind);
   7735 
   7736 				// uncommon kind? just count them in bulk
   7737 				if ((int)common_kind == -1) {
   7738 					stat->other_kinds.count++;
   7739 					stat->other_kinds.key_size += k.mv_size;
   7740 					stat->other_kinds.value_size += v.mv_size;
   7741 				} else {
   7742 					stat->common_kinds[common_kind].count++;
   7743 					stat->common_kinds[common_kind].key_size += k.mv_size;
   7744 					stat->common_kinds[common_kind].value_size += v.mv_size;
   7745 				}
   7746 			}
   7747 
   7748 			stat->dbs[i].count++;
   7749 			stat->dbs[i].key_size += k.mv_size;
   7750 			stat->dbs[i].value_size += v.mv_size;
   7751 		}
   7752 
   7753 		// close the cursor, they are per-dbi
   7754 		mdb_cursor_close(cur);
   7755 	}
   7756 
   7757 	ndb_end_query(&txn);
   7758 
   7759 	return 1;
   7760 }
   7761 
   7762 /// Push an element to the current tag
   7763 ///
   7764 /// Basic idea is to call ndb_builder_new_tag
   7765 int ndb_builder_push_tag_str(struct ndb_builder *builder,
   7766 				    const char *str, int len)
   7767 {
   7768 	union ndb_packed_str pstr;
   7769 	int pack_ids = 1;
   7770 	if (!ndb_builder_make_str(builder, str, len, &pstr, pack_ids))
   7771 		return 0;
   7772 	return ndb_builder_finalize_tag(builder, pstr);
   7773 }
   7774 
   7775 /// Push an id element to the current tag. Needs to be 32 bytes
   7776 int ndb_builder_push_tag_id(struct ndb_builder *builder,
   7777 			    unsigned char *id)
   7778 {
   7779 	union ndb_packed_str pstr;
   7780 	if (!ndb_builder_push_packed_id(builder, id, &pstr))
   7781 		return 0;
   7782 	return ndb_builder_finalize_tag(builder, pstr);
   7783 }
   7784 
   7785 //
   7786 // CONFIG
   7787 //
   7788 void ndb_default_config(struct ndb_config *config)
   7789 {
   7790 	int cores = get_cpu_cores();
   7791 	config->mapsize = 1024UL * 1024UL * 1024UL * 32UL; // 32 GiB
   7792 	config->ingester_threads = cores == -1 ? 4 : cores;
   7793 	config->flags = 0;
   7794 	config->ingest_filter = NULL;
   7795 	config->filter_context = NULL;
   7796 	config->sub_cb_ctx = NULL;
   7797 	config->sub_cb = NULL;
   7798 	config->writer_scratch_buffer_size = DEFAULT_WRITER_SCRATCH_SIZE;
   7799 }
   7800 
   7801 void ndb_config_set_subscription_callback(struct ndb_config *config, ndb_sub_fn fn, void *context)
   7802 {
   7803 	config->sub_cb_ctx = context;
   7804 	config->sub_cb = fn;
   7805 }
   7806 
   7807 void ndb_config_set_writer_scratch_buffer_size(struct ndb_config *config, int scratch_size)
   7808 {
   7809 	config->writer_scratch_buffer_size = scratch_size;
   7810 }
   7811 
   7812 void ndb_config_set_ingest_threads(struct ndb_config *config, int threads)
   7813 {
   7814 	config->ingester_threads = threads;
   7815 }
   7816 
   7817 void ndb_config_set_flags(struct ndb_config *config, int flags)
   7818 {
   7819 	config->flags = flags;
   7820 }
   7821 
   7822 void ndb_config_set_mapsize(struct ndb_config *config, size_t mapsize)
   7823 {
   7824 	config->mapsize = mapsize;
   7825 }
   7826 
   7827 void ndb_config_set_ingest_filter(struct ndb_config *config,
   7828 				  ndb_ingest_filter_fn fn, void *filter_ctx)
   7829 {
   7830 	config->ingest_filter = fn;
   7831 	config->filter_context = filter_ctx;
   7832 }
   7833 
   7834 int ndb_print_author_kind_index(struct ndb_txn *txn)
   7835 {
   7836 	MDB_cursor *cur;
   7837 	struct ndb_id_u64_ts *key;
   7838 	MDB_val k, v;
   7839 	int i;
   7840 
   7841 	if (mdb_cursor_open(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_NOTE_PUBKEY_KIND], &cur))
   7842 		return 0;
   7843 
   7844 	i = 1;
   7845 	printf("author\tkind\tcreated_at\tnote_id\n");
   7846 	while (mdb_cursor_get(cur, &k, &v, MDB_NEXT) == 0) {
   7847 		key = (struct ndb_id_u64_ts *)k.mv_data;
   7848 		print_hex(key->id, 32);
   7849 		printf("\t%" PRIu64 "\t%" PRIu64 "\t%" PRIu64 "\n",
   7850 				key->u64, key->timestamp, *(uint64_t*)v.mv_data);
   7851 		i++;
   7852 	}
   7853 
   7854 	mdb_cursor_close(cur);
   7855 
   7856 	return i;
   7857 }
   7858 
   7859 int ndb_print_relay_kind_index(struct ndb_txn *txn)
   7860 {
   7861 	MDB_cursor *cur;
   7862 	unsigned char *d;
   7863 	MDB_val k, v;
   7864 	int i;
   7865 
   7866 	if (mdb_cursor_open(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_NOTE_RELAY_KIND], &cur))
   7867 		return 0;
   7868 
   7869 	i = 1;
   7870 	printf("relay\tkind\tcreated_at\tnote_id\n");
   7871 	while (mdb_cursor_get(cur, &k, &v, MDB_NEXT) == 0) {
   7872 		d = (unsigned char *)k.mv_data;
   7873 		printf("%s\t", (const char *)(d + 25));
   7874 		printf("%" PRIu64 "\t", *(uint64_t*)(d + 8));
   7875 		printf("%" PRIu64 "\t", *(uint64_t*)(d + 16));
   7876 		printf("%" PRIu64 "\n", *(uint64_t*)(d + 0));
   7877 		i++;
   7878 	}
   7879 
   7880 	mdb_cursor_close(cur);
   7881 
   7882 	return i;
   7883 }
   7884 
   7885 int ndb_print_tag_index(struct ndb_txn *txn)
   7886 {
   7887 	MDB_cursor *cur;
   7888 	MDB_val k, v;
   7889 	int i;
   7890 
   7891 	if (mdb_cursor_open(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_NOTE_TAGS], &cur))
   7892 		return 0;
   7893 
   7894 	i = 1;
   7895 	while (mdb_cursor_get(cur, &k, &v, MDB_NEXT) == 0) {
   7896 		printf("%d ", i);
   7897 		print_tag_kv(txn, &k, &v);
   7898 		i++;
   7899 	}
   7900 
   7901 	mdb_cursor_close(cur);
   7902 
   7903 	return 1;
   7904 }
   7905 
   7906 int ndb_print_kind_keys(struct ndb_txn *txn)
   7907 {
   7908 	MDB_cursor *cur;
   7909 	MDB_val k, v;
   7910 	int i;
   7911 	struct ndb_u64_ts *tsid;
   7912 
   7913 	if (mdb_cursor_open(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_NOTE_KIND], &cur))
   7914 		return 0;
   7915 
   7916 	i = 1;
   7917 	while (mdb_cursor_get(cur, &k, &v, MDB_NEXT) == 0) {
   7918 		tsid = k.mv_data;
   7919 		printf("%d note_kind %" PRIu64 " %" PRIu64 "\n",
   7920 			i, tsid->u64, tsid->timestamp);
   7921 
   7922 		i++;
   7923 	}
   7924 
   7925 	mdb_cursor_close(cur);
   7926 
   7927 	return 1;
   7928 }
   7929 
   7930 // used by ndb.c
   7931 int ndb_print_search_keys(struct ndb_txn *txn)
   7932 {
   7933 	MDB_cursor *cur;
   7934 	MDB_val k, v;
   7935 	int i;
   7936 	struct ndb_text_search_key search_key;
   7937 
   7938 	if (mdb_cursor_open(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_NOTE_TEXT], &cur))
   7939 		return 0;
   7940 
   7941 	i = 1;
   7942 	while (mdb_cursor_get(cur, &k, &v, MDB_NEXT) == 0) {
   7943 		if (!ndb_unpack_text_search_key(k.mv_data, k.mv_size, &search_key)) {
   7944 			fprintf(stderr, "error decoding key %d\n", i);
   7945 			continue;
   7946 		}
   7947 
   7948 		ndb_print_text_search_key(k.mv_size, &search_key);
   7949 		printf("\n");
   7950 
   7951 		i++;
   7952 	}
   7953 
   7954 	mdb_cursor_close(cur);
   7955 
   7956 	return 1;
   7957 }
   7958 
   7959 struct ndb_tags *ndb_note_tags(struct ndb_note *note)
   7960 {
   7961 	return &note->tags;
   7962 }
   7963 
   7964 struct ndb_str ndb_note_str(struct ndb_note *note, union ndb_packed_str *pstr)
   7965 {
   7966 	struct ndb_str str;
   7967 	str.flag = pstr->packed.flag;
   7968 
   7969 	if (str.flag == NDB_PACKED_STR) {
   7970 		str.str = pstr->packed.str;
   7971 		return str;
   7972 	}
   7973 
   7974 	str.str = ((const char *)note) + note->strings + (pstr->offset & 0xFFFFFF);
   7975 	return str;
   7976 }
   7977 
   7978 struct ndb_str ndb_tag_str(struct ndb_note *note, struct ndb_tag *tag, int ind)
   7979 {
   7980 	return ndb_note_str(note, &tag->strs[ind]);
   7981 }
   7982 
   7983 int ndb_str_len(struct ndb_str *str)
   7984 {
   7985 	if (str->flag == NDB_PACKED_ID)
   7986 		return 32;
   7987 	return strlen(str->str);
   7988 }
   7989 
   7990 struct ndb_str ndb_iter_tag_str(struct ndb_iterator *iter, int ind)
   7991 {
   7992 	return ndb_tag_str(iter->note, iter->tag, ind);
   7993 }
   7994 
   7995 unsigned char * ndb_note_id(struct ndb_note *note)
   7996 {
   7997 	return note->id;
   7998 }
   7999 
   8000 unsigned char * ndb_note_pubkey(struct ndb_note *note)
   8001 {
   8002 	return note->pubkey;
   8003 }
   8004 
   8005 unsigned char * ndb_note_sig(struct ndb_note *note)
   8006 {
   8007 	return note->sig;
   8008 }
   8009 
   8010 uint32_t ndb_note_created_at(struct ndb_note *note)
   8011 {
   8012 	return note->created_at;
   8013 }
   8014 
   8015 uint32_t ndb_note_kind(struct ndb_note *note)
   8016 {
   8017 	return note->kind;
   8018 }
   8019 
   8020 void _ndb_note_set_kind(struct ndb_note *note, uint32_t kind)
   8021 {
   8022 	note->kind = kind;
   8023 }
   8024 
   8025 const char *ndb_note_content(struct ndb_note *note)
   8026 {
   8027 	return ndb_note_str(note, &note->content).str;
   8028 }
   8029 
   8030 uint32_t ndb_note_content_length(struct ndb_note *note)
   8031 {
   8032 	return note->content_length;
   8033 }
   8034 
   8035 struct ndb_note * ndb_note_from_bytes(unsigned char *bytes)
   8036 {
   8037 	struct ndb_note *note = (struct ndb_note *)bytes;
   8038 	if (note->version != 1)
   8039 		return 0;
   8040 	return note;
   8041 }
   8042 
   8043 int ndb_note_relay_iterate_start(struct ndb_txn *txn,
   8044 				 struct ndb_note_relay_iterator *iter,
   8045 				 uint64_t note_key)
   8046 {
   8047 	if (mdb_cursor_open(txn->mdb_txn, txn->lmdb->dbs[NDB_DB_NOTE_RELAYS],
   8048 			    (MDB_cursor**)&iter->mdb_cur)) {
   8049 		return 0;
   8050 	}
   8051 
   8052 	iter->txn = txn;
   8053 	iter->cursor_op = MDB_SET_KEY;
   8054 	iter->note_key = note_key;
   8055 
   8056 	return 1;
   8057 }
   8058 
   8059 const char *ndb_note_relay_iterate_next(struct ndb_note_relay_iterator *iter)
   8060 {
   8061 	int rc;
   8062 	MDB_val k, v;
   8063 
   8064 	if (iter->mdb_cur == NULL)
   8065 		return NULL;
   8066 
   8067 	k.mv_data = &iter->note_key;
   8068 	k.mv_size = sizeof(iter->note_key);
   8069 
   8070 	if ((rc = mdb_cursor_get((MDB_cursor *)iter->mdb_cur, &k, &v,
   8071 				 (MDB_cursor_op)iter->cursor_op)))
   8072 	{
   8073 		//fprintf(stderr, "autoclosing %d '%s'\n", iter->cursor_op, mdb_strerror(rc));
   8074 		// autoclose
   8075 		ndb_note_relay_iterate_close(iter);
   8076 		return NULL;
   8077 	}
   8078 
   8079 	iter->cursor_op = MDB_NEXT_DUP;
   8080 
   8081 	return (const char*)v.mv_data;
   8082 }
   8083 
   8084 void ndb_note_relay_iterate_close(struct ndb_note_relay_iterator *iter)
   8085 {
   8086 	if (!iter || iter->mdb_cur == NULL)
   8087 		return;
   8088 
   8089 	mdb_cursor_close((MDB_cursor*)iter->mdb_cur);
   8090 
   8091 	iter->mdb_cur = NULL;
   8092 }
   8093 
   8094 void ndb_tags_iterate_start(struct ndb_note *note, struct ndb_iterator *iter)
   8095 {
   8096 	iter->note = note;
   8097 	iter->tag = NULL;
   8098 	iter->index = -1;
   8099 }
   8100 
   8101 // Helper function to get a pointer to the nth tag
   8102 static struct ndb_tag *ndb_tags_tag(struct ndb_tags *tags, size_t index) {
   8103     return (struct ndb_tag *)((uint8_t *)tags + sizeof(struct ndb_tags) + index * sizeof(struct ndb_tag));
   8104 }
   8105 
   8106 int ndb_tags_iterate_next(struct ndb_iterator *iter)
   8107 {
   8108 	struct ndb_tags *tags;
   8109 
   8110 	if (iter->tag == NULL || iter->index == -1) {
   8111 		iter->tag = ndb_tags_tag(&iter->note->tags, 0);
   8112 		iter->index = 0;
   8113 		return iter->note->tags.count != 0;
   8114 	}
   8115 
   8116 	tags = &iter->note->tags;
   8117 
   8118 	if (++iter->index < tags->count) {
   8119 		uint32_t tag_data_size = iter->tag->count * sizeof(iter->tag->strs[0]);
   8120 		iter->tag = (struct ndb_tag *)(iter->tag->strs[0].bytes + tag_data_size);
   8121 		return 1;
   8122 	}
   8123 
   8124 	return 0;
   8125 }
   8126 
   8127 uint16_t ndb_tags_count(struct ndb_tags *tags)
   8128 {
   8129 	return tags->count;
   8130 }
   8131 
   8132 uint16_t ndb_tag_count(struct ndb_tag *tags)
   8133 {
   8134 	return tags->count;
   8135 }
   8136 
   8137 enum ndb_common_kind ndb_kind_to_common_kind(int kind)
   8138 {
   8139 	switch (kind)
   8140 	{
   8141 		case 0:     return NDB_CKIND_PROFILE;
   8142 		case 1:     return NDB_CKIND_TEXT;
   8143 		case 3:     return NDB_CKIND_CONTACTS;
   8144 		case 4:     return NDB_CKIND_DM;
   8145 		case 5:     return NDB_CKIND_DELETE;
   8146 		case 6:     return NDB_CKIND_REPOST;
   8147 		case 7:     return NDB_CKIND_REACTION;
   8148 		case 9735:  return NDB_CKIND_ZAP;
   8149 		case 9734:  return NDB_CKIND_ZAP_REQUEST;
   8150 		case 23194: return NDB_CKIND_NWC_REQUEST;
   8151 		case 23195: return NDB_CKIND_NWC_RESPONSE;
   8152 		case 27235: return NDB_CKIND_HTTP_AUTH;
   8153 		case 30000: return NDB_CKIND_LIST;
   8154 		case 30023: return NDB_CKIND_LONGFORM;
   8155 		case 30315: return NDB_CKIND_STATUS;
   8156 	}
   8157 
   8158 	return -1;
   8159 }
   8160 
   8161 const char *ndb_kind_name(enum ndb_common_kind ck)
   8162 {
   8163 	switch (ck) {
   8164 		case NDB_CKIND_PROFILE:      return "profile";
   8165 		case NDB_CKIND_TEXT:         return "text";
   8166 		case NDB_CKIND_CONTACTS:     return "contacts";
   8167 		case NDB_CKIND_DM:           return "dm";
   8168 		case NDB_CKIND_DELETE:       return "delete";
   8169 		case NDB_CKIND_REPOST:       return "repost";
   8170 		case NDB_CKIND_REACTION:     return "reaction";
   8171 		case NDB_CKIND_ZAP:          return "zap";
   8172 		case NDB_CKIND_ZAP_REQUEST:  return "zap_request";
   8173 		case NDB_CKIND_NWC_REQUEST:  return "nwc_request";
   8174 		case NDB_CKIND_NWC_RESPONSE: return "nwc_response";
   8175 		case NDB_CKIND_HTTP_AUTH:    return "http_auth";
   8176 		case NDB_CKIND_LIST:         return "list";
   8177 		case NDB_CKIND_LONGFORM:     return "longform";
   8178 		case NDB_CKIND_STATUS:       return "status";
   8179 		case NDB_CKIND_COUNT:        return "unknown";
   8180 	}
   8181 
   8182 	return "unknown";
   8183 }
   8184 
   8185 const char *ndb_db_name(enum ndb_dbs db)
   8186 {
   8187 	switch (db) {
   8188 		case NDB_DB_NOTE:
   8189 			return "note";
   8190 		case NDB_DB_META:
   8191 			return "note_metadata";
   8192 		case NDB_DB_PROFILE:
   8193 			return "profile";
   8194 		case NDB_DB_NOTE_ID:
   8195 			return "note_index";
   8196 		case NDB_DB_PROFILE_PK:
   8197 			return "profile_pubkey_index";
   8198 		case NDB_DB_NDB_META:
   8199 			return "nostrdb_metadata";
   8200 		case NDB_DB_PROFILE_SEARCH:
   8201 			return "profile_search";
   8202 		case NDB_DB_PROFILE_LAST_FETCH:
   8203 			return "profile_last_fetch";
   8204 		case NDB_DB_NOTE_KIND:
   8205 			return "note_kind_index";
   8206 		case NDB_DB_NOTE_TEXT:
   8207 			return "note_fulltext";
   8208 		case NDB_DB_NOTE_BLOCKS:
   8209 			return "note_blocks";
   8210 		case NDB_DB_NOTE_TAGS:
   8211 			return "note_tags";
   8212 		case NDB_DB_NOTE_PUBKEY:
   8213 			return "note_pubkey_index";
   8214 		case NDB_DB_NOTE_PUBKEY_KIND:
   8215 			return "note_pubkey_kind_index";
   8216 		case NDB_DB_NOTE_RELAY_KIND:
   8217 			return "note_relay_kind_index";
   8218 		case NDB_DB_NOTE_RELAYS:
   8219 			return "note_relays";
   8220 		case NDB_DBS:
   8221 			return "count";
   8222 	}
   8223 
   8224 	return "unknown";
   8225 }
   8226 
   8227 static struct ndb_blocks *ndb_note_to_blocks(struct ndb_note *note)
   8228 {
   8229 	const char *content;
   8230 	size_t content_len;
   8231 	struct ndb_blocks *blocks;
   8232 	unsigned char *buffer;
   8233 
   8234 	content = ndb_note_content(note);
   8235 	content_len = ndb_note_content_length(note);
   8236 
   8237 	// something weird is going on
   8238 	if (content_len >= INT32_MAX)
   8239 		return NULL;
   8240 
   8241 	buffer = malloc(2<<18);  // Not carefully calculated, but ok because we will not need this once we switch to the local relay model
   8242 	if (!buffer)
   8243 		return NULL;
   8244 
   8245 	if (!ndb_parse_content(buffer, content_len, content, content_len, &blocks)) {
   8246 		free(buffer);
   8247 		return NULL;
   8248 	}
   8249 
   8250 	//blocks = realloc(blocks, ndb_blocks_total_size(blocks));
   8251 	//if (blocks == NULL)
   8252 		//return NULL;
   8253 
   8254 	blocks->flags |= NDB_BLOCK_FLAG_OWNED;
   8255 
   8256 	return blocks;
   8257 }
   8258 
   8259 struct ndb_blocks *ndb_get_blocks_by_key(struct ndb *ndb, struct ndb_txn *txn, uint64_t note_key)
   8260 {
   8261 	struct ndb_blocks *blocks, *blocks_to_writer;
   8262 	size_t blocks_size;
   8263 	struct ndb_note *note;
   8264 	size_t note_len;
   8265 
   8266 	if ((blocks = ndb_lookup_by_key(txn, note_key, NDB_DB_NOTE_BLOCKS, &note_len))) {
   8267 		return blocks;
   8268 	}
   8269 
   8270 	// If we don't have note blocks, let's lazily generate them. This is
   8271 	// migration-friendly instead of doing them all at once
   8272 	if (!(note = ndb_get_note_by_key(txn, note_key, &note_len))) {
   8273 		// no note found, can't return note blocks
   8274 		return NULL;
   8275 	}
   8276 
   8277 	 if (!(blocks = ndb_note_to_blocks(note)))
   8278 		 return NULL;
   8279 
   8280 	 // send a copy to the writer
   8281 	 blocks_size = ndb_blocks_total_size(blocks);
   8282 	 blocks_to_writer = malloc(blocks_size);
   8283 	 memcpy(blocks_to_writer, blocks, blocks_size);
   8284 	 assert(blocks->flags & NDB_BLOCK_FLAG_OWNED);
   8285 
   8286 	 // we generated new blocks, let's store them in the DB
   8287 	 struct ndb_writer_blocks write_blocks = {
   8288 		 .blocks = blocks_to_writer,
   8289 		 .note_key = note_key
   8290 	 };
   8291 
   8292 	 assert(write_blocks.blocks != blocks);
   8293 
   8294 	 struct ndb_writer_msg msg = { .type = NDB_WRITER_BLOCKS };
   8295 	 msg.blocks = write_blocks;
   8296 
   8297 	 ndb_writer_queue_msg(&ndb->writer, &msg);
   8298 
   8299 	 return blocks;
   8300 }
   8301 
   8302 // please call ndb_monitor_lock before calling this
   8303 static struct ndb_subscription *
   8304 ndb_monitor_find_subscription(struct ndb_monitor *monitor, uint64_t subid, int *index)
   8305 {
   8306 	struct ndb_subscription *sub, *tsub;
   8307 	int i;
   8308 
   8309 	for (i = 0, sub = NULL; i < monitor->num_subscriptions; i++) {
   8310 		tsub = &monitor->subscriptions[i];
   8311 		if (tsub->subid == subid) {
   8312 			sub = tsub;
   8313 			if (index)
   8314 				*index = i;
   8315 			break;
   8316 		}
   8317 	}
   8318 
   8319 	return sub;
   8320 }
   8321 
   8322 int ndb_poll_for_notes(struct ndb *ndb, uint64_t subid, uint64_t *note_ids,
   8323 		       int note_id_capacity)
   8324 {
   8325 	struct ndb_subscription *sub;
   8326 	int res;
   8327 
   8328 	if (subid == 0)
   8329 		return 0;
   8330 
   8331 	ndb_monitor_lock(&ndb->monitor);
   8332 
   8333 	if (!(sub = ndb_monitor_find_subscription(&ndb->monitor, subid, NULL)))
   8334 		res = 0;
   8335 	else
   8336 		res = prot_queue_try_pop_all(&sub->inbox, note_ids, note_id_capacity);
   8337 
   8338 	ndb_monitor_unlock(&ndb->monitor);
   8339 
   8340 	return res;
   8341 }
   8342 
   8343 int ndb_wait_for_notes(struct ndb *ndb, uint64_t subid, uint64_t *note_ids,
   8344                        int note_id_capacity)
   8345 {
   8346 	struct ndb_subscription *sub;
   8347 	struct prot_queue *queue_inbox;
   8348 
   8349         // this is not a valid subscription id
   8350 	if (subid == 0)
   8351 		return 0;
   8352 
   8353 	ndb_monitor_lock(&ndb->monitor);
   8354 
   8355         if (!(sub = ndb_monitor_find_subscription(&ndb->monitor, subid, NULL))) {
   8356 		ndb_monitor_unlock(&ndb->monitor);
   8357 		return 0;
   8358 	}
   8359 
   8360 	queue_inbox = &sub->inbox;
   8361 
   8362 	ndb_monitor_unlock(&ndb->monitor);
   8363 
   8364 	// there is technically a race condition if the thread yeilds at this
   8365 	// comment and a subscription is added/removed. A deadlock in the
   8366 	// writer queue would be much worse though. This function is dubious
   8367 	// anyways.
   8368 
   8369         return prot_queue_pop_all(queue_inbox, note_ids, note_id_capacity);
   8370 }
   8371 
   8372 int ndb_unsubscribe(struct ndb *ndb, uint64_t subid)
   8373 {
   8374 	struct ndb_subscription *sub;
   8375 	int index, res, elems_to_move;
   8376 
   8377 	ndb_monitor_lock(&ndb->monitor);
   8378 
   8379 	if (!(sub = ndb_monitor_find_subscription(&ndb->monitor, subid, &index))) {
   8380 		res = 0;
   8381 		goto done;
   8382 	}
   8383 
   8384 	ndb_subscription_destroy(sub);
   8385 
   8386 	elems_to_move = (--ndb->monitor.num_subscriptions) - index;
   8387 
   8388 	memmove(&ndb->monitor.subscriptions[index],
   8389 		&ndb->monitor.subscriptions[index+1],
   8390 		elems_to_move * sizeof(*sub));
   8391 
   8392 	res = 1;
   8393 
   8394 done:
   8395 	ndb_monitor_unlock(&ndb->monitor);
   8396 
   8397 	return res;
   8398 }
   8399 
   8400 int ndb_num_subscriptions(struct ndb *ndb)
   8401 {
   8402 	return ndb->monitor.num_subscriptions;
   8403 }
   8404 
   8405 uint64_t ndb_subscribe(struct ndb *ndb, struct ndb_filter *filters, int num_filters)
   8406 {
   8407 	static uint64_t subids = 0;
   8408 	struct ndb_subscription *sub;
   8409 	size_t buflen;
   8410 	uint64_t subid;
   8411 	char *buf;
   8412 
   8413 	ndb_monitor_lock(&ndb->monitor);
   8414 
   8415 	if (ndb->monitor.num_subscriptions + 1 >= MAX_SUBSCRIPTIONS) {
   8416 		fprintf(stderr, "too many subscriptions\n");
   8417 		subid = 0;
   8418 		goto done;
   8419 	}
   8420 
   8421 	sub = &ndb->monitor.subscriptions[ndb->monitor.num_subscriptions];
   8422 	subid = ++subids;
   8423 	sub->subid = subid;
   8424 
   8425 	ndb_filter_group_init(&sub->group);
   8426 	if (!ndb_filter_group_add_filters(&sub->group, filters, num_filters)) {
   8427 		subid = 0;
   8428 		goto done;
   8429 	}
   8430 
   8431 	// 500k ought to be enough for anyone
   8432 	buflen = sizeof(uint64_t) * DEFAULT_QUEUE_SIZE;
   8433 	buf = malloc(buflen);
   8434 
   8435 	if (!prot_queue_init(&sub->inbox, buf, buflen, sizeof(uint64_t))) {
   8436 		fprintf(stderr, "failed to push prot queue\n");
   8437 		subid = 0;
   8438 		goto done;
   8439 	}
   8440 
   8441 	ndb->monitor.num_subscriptions++;
   8442 done:
   8443 	ndb_monitor_unlock(&ndb->monitor);
   8444 
   8445 	return subid;
   8446 }