nostrdb

an unfairly fast embedded nostr database backed by lmdb
git clone git://jb55.com/nostrdb
Log | Files | Refs | Submodules | README | LICENSE
commit 3fe17c29d970f14a83d1d7d5722468814167a883
parent 759a77f221875121e77ce079fc63e20696a42fd4
Author: William Casarin <jb55@jb55.com>
Date:   Tue, 12 Dec 2023 13:19:58 -0800

api: don't expose many internals, like note

rust doesn't like packed structures, so hide this from bindgen

This also buttons up the API so less things are exposed which is good.

Diffstat:

Mnostrdb.c | 276+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++--


Mnostrdb.h | 538++++++++++++++++++++++++-------------------------------------------------------


Mprint_util.h | 4++--


Mtest.c | 64++++++++++++++++++++++++++++++++--------------------------------


4 files changed, 464 insertions(+), 418 deletions(-)

diff --git a/nostrdb.c b/nostrdb.c
@@ -51,6 +51,52 @@ typedef int (*ndb_migrate_fn)(struct ndb *);
 typedef int (*ndb_word_parser_fn)(void *, const char *word, int word_len,
 				  int word_index);
 
+// these must be byte-aligned, they are directly accessing the serialized data
+// representation
+#pragma pack(push, 1)
+
+
+union ndb_packed_str {
+	struct {
+		char str[3];
+		// we assume little endian everywhere. sorry not sorry.
+		unsigned char flag; // NDB_PACKED_STR, etc
+	} packed;
+
+	uint32_t offset;
+	unsigned char bytes[4];
+};
+
+struct ndb_tag {
+	uint16_t count;
+	union ndb_packed_str strs[0];
+};
+
+struct ndb_tags {
+	uint16_t padding;
+	uint16_t count;
+	struct ndb_tag tag[0];
+};
+
+// v1
+struct ndb_note {
+	unsigned char version;    // v=1
+	unsigned char padding[3]; // keep things aligned
+	unsigned char id[32];
+	unsigned char pubkey[32];
+	unsigned char sig[64];
+
+	uint64_t created_at;
+	uint32_t kind;
+	uint32_t content_length;
+	union ndb_packed_str content;
+	uint32_t strings;
+	// nothing can come after tags since it contains variadic data
+	struct ndb_tags tags;
+};
+
+#pragma pack(pop)
+
 struct ndb_migration {
 	ndb_migrate_fn fn;
 };
@@ -674,7 +720,7 @@ static int ndb_generic_filter_matches(struct ndb_filter_elements *els,
 		if (it->tag->count < 2)
 			continue;
 
-		str = ndb_note_str(note, &it->tag->strs[0]);
+		str = ndb_tag_str(note, it->tag, 0);
 
 		// we only care about packed strings (single char, etc)
 		if (str.flag != NDB_PACKED_STR)
@@ -684,7 +730,7 @@ static int ndb_generic_filter_matches(struct ndb_filter_elements *els,
 		if (str.str[0] != els->field.generic || str.str[1] != 0)
 			continue;
 
-		str = ndb_note_str(note, &it->tag->strs[1]);
+		str = ndb_tag_str(note, it->tag, 1);
 
 		switch (els->field.elem_type) {
 		case NDB_ELEMENT_ID:
@@ -1967,11 +2013,11 @@ static unsigned char *ndb_note_last_id_tag(struct ndb_note *note, char type)
 		if (iter.tag->count < 2)
 			continue;
 
-		str = ndb_note_str(note, &iter.tag->strs[0]);
+		str = ndb_tag_str(note, iter.tag, 0);
 
 		// assign liked to the last e tag
 		if (str.flag == NDB_PACKED_STR && str.str[0] == type) {
-			str = ndb_note_str(note, &iter.tag->strs[1]);
+			str = ndb_tag_str(note, iter.tag, 1);
 			if (str.flag == NDB_PACKED_ID)
 				last = str.id;
 		}
@@ -3473,7 +3519,7 @@ static int cursor_push_json_tag(struct cursor *cur, struct ndb_note *note,
                 return 0;
 
         for (i = 0; i < tag->count; i++) {
-                if (!cursor_push_json_tag_str(cur, ndb_note_str(note, &tag->strs[i])))
+                if (!cursor_push_json_tag_str(cur, ndb_tag_str(note, tag, i)))
                         return 0;
                 if (i != tag->count-1 && !cursor_push_byte(cur, ','))
 			return 0;
@@ -3644,6 +3690,16 @@ struct ndb_note * ndb_builder_note(struct ndb_builder *builder)
 	return builder->note;
 }
 
+static union ndb_packed_str ndb_offset_str(uint32_t offset)
+{
+	// ensure accidents like -1 don't corrupt our packed_str
+	union ndb_packed_str str;
+	// most significant byte is reserved for ndb_packtype
+	str.offset = offset & 0xFFFFFF;
+	return str;
+}
+
+
 /// find an existing string via str_indices. these indices only exist in the
 /// builder phase just for this purpose.
 static inline int ndb_builder_find_str(struct ndb_builder *builder,
@@ -3707,6 +3763,25 @@ static int ndb_builder_push_packed_id(struct ndb_builder *builder,
 	return 0;
 }
 
+union ndb_packed_str ndb_chars_to_packed_str(char c1, char c2)
+{
+	union ndb_packed_str str;
+	str.packed.flag = NDB_PACKED_STR;
+	str.packed.str[0] = c1;
+	str.packed.str[1] = c2;
+	str.packed.str[2] = '\0';
+	return str;
+}
+
+static union ndb_packed_str ndb_char_to_packed_str(char c)
+{
+	union ndb_packed_str str;
+	str.packed.flag = NDB_PACKED_STR;
+	str.packed.str[0] = c;
+	str.packed.str[1] = '\0';
+	return str;
+}
+
 
 /// Check for small strings to pack
 static inline int ndb_builder_try_compact_str(struct ndb_builder *builder,
@@ -4339,3 +4414,194 @@ int ndb_print_search_keys(struct ndb_txn *txn)
 
 	return 1;
 }
+
+struct ndb_tags *ndb_note_tags(struct ndb_note *note)
+{
+	return &note->tags;
+}
+
+struct ndb_str ndb_note_str(struct ndb_note *note, union ndb_packed_str *pstr)
+{
+	struct ndb_str str;
+	str.flag = pstr->packed.flag;
+
+	if (str.flag == NDB_PACKED_STR) {
+		str.str = pstr->packed.str;
+		return str;
+	}
+
+	str.str = ((const char *)note) + note->strings + (pstr->offset & 0xFFFFFF);
+	return str;
+}
+
+struct ndb_str ndb_tag_str(struct ndb_note *note, struct ndb_tag *tag, int ind)
+{
+	return ndb_note_str(note, &tag->strs[ind]);
+}
+
+struct ndb_str ndb_iter_tag_str(struct ndb_iterator *iter, int ind)
+{
+	return ndb_tag_str(iter->note, iter->tag, ind);
+}
+
+unsigned char * ndb_note_id(struct ndb_note *note)
+{
+	return note->id;
+}
+
+unsigned char * ndb_note_pubkey(struct ndb_note *note)
+{
+	return note->pubkey;
+}
+
+unsigned char * ndb_note_sig(struct ndb_note *note)
+{
+	return note->sig;
+}
+
+uint32_t ndb_note_created_at(struct ndb_note *note)
+{
+	return note->created_at;
+}
+
+uint32_t ndb_note_kind(struct ndb_note *note)
+{
+	return note->kind;
+}
+
+void _ndb_note_set_kind(struct ndb_note *note, uint32_t kind)
+{
+	note->kind = kind;
+}
+
+const char *ndb_note_content(struct ndb_note *note)
+{
+	return ndb_note_str(note, &note->content).str;
+}
+
+uint32_t ndb_note_content_length(struct ndb_note *note)
+{
+	return note->content_length;
+}
+
+struct ndb_note * ndb_note_from_bytes(unsigned char *bytes)
+{
+	struct ndb_note *note = (struct ndb_note *)bytes;
+	if (note->version != 1)
+		return 0;
+	return note;
+}
+
+void ndb_tags_iterate_start(struct ndb_note *note, struct ndb_iterator *iter)
+{
+	iter->note = note;
+	iter->tag = NULL;
+	iter->index = -1;
+}
+
+int ndb_tags_iterate_next(struct ndb_iterator *iter)
+{
+	if (iter->tag == NULL || iter->index == -1) {
+		iter->tag = iter->note->tags.tag;
+		iter->index = 0;
+		return iter->note->tags.count != 0;
+	}
+
+	struct ndb_tags *tags = &iter->note->tags;
+
+	if (++iter->index < tags->count) {
+		uint32_t tag_data_size = iter->tag->count * sizeof(iter->tag->strs[0]);
+		iter->tag = (struct ndb_tag *)(iter->tag->strs[0].bytes + tag_data_size);
+		return 1;
+	}
+
+	return 0;
+}
+
+uint16_t ndb_tags_count(struct ndb_tags *tags)
+{
+	return tags->count;
+}
+
+uint16_t ndb_tag_count(struct ndb_tag *tags)
+{
+	return tags->count;
+}
+
+enum ndb_common_kind ndb_kind_to_common_kind(int kind)
+{
+	switch (kind)
+	{
+		case 0:     return NDB_CKIND_PROFILE;
+		case 1:     return NDB_CKIND_TEXT;
+		case 3:     return NDB_CKIND_CONTACTS;
+		case 4:     return NDB_CKIND_DM;
+		case 5:     return NDB_CKIND_DELETE;
+		case 6:     return NDB_CKIND_REPOST;
+		case 7:     return NDB_CKIND_REACTION;
+		case 9735:  return NDB_CKIND_ZAP;
+		case 9734:  return NDB_CKIND_ZAP_REQUEST;
+		case 23194: return NDB_CKIND_NWC_REQUEST;
+		case 23195: return NDB_CKIND_NWC_RESPONSE;
+		case 27235: return NDB_CKIND_HTTP_AUTH;
+		case 30000: return NDB_CKIND_LIST;
+		case 30023: return NDB_CKIND_LONGFORM;
+		case 30315: return NDB_CKIND_STATUS;
+	}
+
+	return -1;
+}
+
+const char *ndb_kind_name(enum ndb_common_kind ck)
+{
+	switch (ck) {
+		case NDB_CKIND_PROFILE:      return "profile";
+		case NDB_CKIND_TEXT:         return "text";
+		case NDB_CKIND_CONTACTS:     return "contacts";
+		case NDB_CKIND_DM:           return "dm";
+		case NDB_CKIND_DELETE:       return "delete";
+		case NDB_CKIND_REPOST:       return "repost";
+		case NDB_CKIND_REACTION:     return "reaction";
+		case NDB_CKIND_ZAP:          return "zap";
+		case NDB_CKIND_ZAP_REQUEST:  return "zap_request";
+		case NDB_CKIND_NWC_REQUEST:  return "nwc_request";
+		case NDB_CKIND_NWC_RESPONSE: return "nwc_response";
+		case NDB_CKIND_HTTP_AUTH:    return "http_auth";
+		case NDB_CKIND_LIST:         return "list";
+		case NDB_CKIND_LONGFORM:     return "longform";
+		case NDB_CKIND_STATUS:       return "status";
+		case NDB_CKIND_COUNT:        return "unknown";
+	}
+
+	return "unknown";
+}
+
+const char *ndb_db_name(enum ndb_dbs db)
+{
+	switch (db) {
+		case NDB_DB_NOTE:
+			return "note";
+		case NDB_DB_META:
+			return "note_metadata";
+		case NDB_DB_PROFILE:
+			return "profile";
+		case NDB_DB_NOTE_ID:
+			return "note_index";
+		case NDB_DB_PROFILE_PK:
+			return "profile_pubkey_index";
+		case NDB_DB_NDB_META:
+			return "nostrdb_metadata";
+		case NDB_DB_PROFILE_SEARCH:
+			return "profile_search";
+		case NDB_DB_PROFILE_LAST_FETCH:
+			return "profile_last_fetch";
+		case NDB_DB_NOTE_KIND:
+			return "note_kind_index";
+		case NDB_DB_NOTE_TEXT:
+			return "note_fulltext";
+		case NDB_DBS:
+			return "count";
+	}
+
+	return "unknown";
+}
diff --git a/nostrdb.h b/nostrdb.h
@@ -20,70 +20,42 @@
 
 struct ndb_json_parser;
 struct ndb;
+struct ndb_note;
+struct ndb_tag;
+struct ndb_tags;
+struct ndb_lmdb;
+union ndb_packed_str;
 
 // sorry, swift needs help with forward declared pointers like this
 struct ndb_t {
 	struct ndb *ndb;
 };
 
-struct ndb_search_key
-{
-	char search[24];
-	unsigned char id[32];
-	uint64_t timestamp;
-};
-
-enum ndb_dbs {
-	NDB_DB_NOTE,
-	NDB_DB_META,
-	NDB_DB_PROFILE,
-	NDB_DB_NOTE_ID,
-	NDB_DB_PROFILE_PK, // profile pk index
-	NDB_DB_NDB_META,
-	NDB_DB_PROFILE_SEARCH,
-	NDB_DB_PROFILE_LAST_FETCH,
-	NDB_DB_NOTE_KIND, // note kind index
-	NDB_DB_NOTE_TEXT, // note fulltext index
-	NDB_DBS,
-};
-
-// common kinds. we collect stats on these in ndb_stat. mainly because I don't
-// want to deal with including a hashtable to the project.
-enum ndb_common_kind {
-	NDB_CKIND_PROFILE,
-	NDB_CKIND_TEXT,
-	NDB_CKIND_CONTACTS,
-	NDB_CKIND_DM,
-	NDB_CKIND_DELETE,
-	NDB_CKIND_REPOST,
-	NDB_CKIND_REACTION,
-	NDB_CKIND_ZAP,
-	NDB_CKIND_ZAP_REQUEST,
-	NDB_CKIND_NWC_REQUEST,
-	NDB_CKIND_NWC_RESPONSE,
-	NDB_CKIND_HTTP_AUTH,
-	NDB_CKIND_LIST,
-	NDB_CKIND_LONGFORM,
-	NDB_CKIND_STATUS,
-	NDB_CKIND_COUNT, // should always be last
+struct ndb_str {
+	unsigned char flag;
+	union {
+		const char *str;
+		unsigned char *id;
+	};
 };
 
-struct ndb_stat_counts {
-	size_t key_size;
-	size_t value_size;
-	size_t count;
+struct ndb_keypair {
+	unsigned char pubkey[32];
+	unsigned char secret[32];
+	
+	// this corresponds to secp256k1's keypair type. it's guaranteed to
+	// be 96 bytes according to their docs. I don't want to depend on
+	// the secp256k1 header here so we just use raw bytes.
+	unsigned char pair[96];
 };
 
-struct ndb_stat {
-	struct ndb_stat_counts dbs[NDB_DBS];
-	struct ndb_stat_counts common_kinds[NDB_CKIND_COUNT];
-	struct ndb_stat_counts other_kinds;
-};
+// function pointer for controlling what to do after we parse an id
+typedef enum ndb_idres (*ndb_id_fn)(void *, const char *);
 
-struct ndb_search {
-	struct ndb_search_key *key;
-	uint64_t profile_key;
-	void *cursor; // MDB_cursor *
+// id callback + closure data
+struct ndb_id_cb {
+	ndb_id_fn fn;
+	void *data;
 };
 
 // required to keep a read 
@@ -92,6 +64,16 @@ struct ndb_txn {
 	void *mdb_txn;
 };
 
+struct ndb_event {
+	struct ndb_note *note;
+};
+
+struct ndb_command_result {
+	int ok;
+	const char *msg;
+	int msglen;
+};
+
 // From-client event types
 enum fce_type {
 	NDB_FCE_EVENT = 0x1
@@ -111,34 +93,19 @@ enum ndb_ingest_filter_action {
 	NDB_INGEST_SKIP_VALIDATION
 };
 
-// function pointer for controlling what to do after we parse an id
-typedef enum ndb_idres (*ndb_id_fn)(void *, const char *);
-
-// id callback + closure data
-struct ndb_id_cb {
-	ndb_id_fn fn;
-	void *data;
-};
-
-struct ndb_str {
-	unsigned char flag;
-	union {
-		const char *str;
-		unsigned char *id;
-	};
-};
-
-struct ndb_event {
-	struct ndb_note *note;
+struct ndb_search_key
+{
+	char search[24];
+	unsigned char id[32];
+	uint64_t timestamp;
 };
 
-struct ndb_command_result {
-	int ok;
-	const char *msg;
-	int msglen;
+struct ndb_search {
+	struct ndb_search_key *key;
+	uint64_t profile_key;
+	void *cursor; // MDB_cursor *
 };
 
-
 // From-client event
 struct ndb_fce {
 	enum fce_type evtype;
@@ -159,88 +126,67 @@ struct ndb_tce {
 	};
 };
 
-struct ndb_keypair {
-	unsigned char pubkey[32];
-	unsigned char secret[32];
-	
-	// this corresponds to secp256k1's keypair type. it's guaranteed to
-	// be 96 bytes according to their docs. I don't want to depend on
-	// the secp256k1 header here so we just use raw bytes.
-	unsigned char pair[96];
-};
-
-#define MAX_TEXT_SEARCH_RESULTS 128
-#define MAX_TEXT_SEARCH_WORDS 8
-
-// unpacked form of the actual lmdb fulltext search key
-// see `ndb_make_text_search_key` for how the packed version is constructed
-struct ndb_text_search_key
-{
-	int str_len;
-	const char *str;
-	uint64_t timestamp;
-	uint64_t note_id;
-	int word_index;
-};
-
-struct ndb_text_search_result {
-	struct ndb_text_search_key key;
-	int prefix_chars;
-};
+typedef enum ndb_ingest_filter_action (*ndb_ingest_filter_fn)(void *, struct ndb_note *);
 
-struct ndb_text_search_results {
-	struct ndb_text_search_result results[MAX_TEXT_SEARCH_RESULTS];
-	int num_results;
+enum ndb_filter_fieldtype {
+	NDB_FILTER_IDS     = 1,
+	NDB_FILTER_AUTHORS = 2,
+	NDB_FILTER_KINDS   = 3,
+	NDB_FILTER_GENERIC = 4,
+	NDB_FILTER_SINCE   = 5,
+	NDB_FILTER_UNTIL   = 6,
+	NDB_FILTER_LIMIT   = 7,
 };
+#define NDB_NUM_FILTERS 7
 
-// these must be byte-aligned, they are directly accessing the serialized data
-// representation
-#pragma pack(push, 1)
-
-
-union ndb_packed_str {
-	struct {
-		char str[3];
-		// we assume little endian everywhere. sorry not sorry.
-		unsigned char flag; // NDB_PACKED_STR, etc
-	} packed;
-
-	uint32_t offset;
-	unsigned char bytes[4];
+// when matching generic tags, we need to know if we're dealing with
+// a pointer to a 32-byte ID or a null terminated string
+enum ndb_generic_element_type {
+	NDB_ELEMENT_UNKNOWN = 0,
+	NDB_ELEMENT_STRING  = 1,
+	NDB_ELEMENT_ID      = 2,
 };
 
-struct ndb_tag {
-	uint16_t count;
-	union ndb_packed_str strs[0];
+enum ndb_search_order {
+	NDB_ORDER_DESCENDING,
+	NDB_ORDER_ASCENDING,
 };
 
-struct ndb_tags {
-	uint16_t padding;
-	uint16_t count;
-	struct ndb_tag tag[0];
+enum ndb_dbs {
+	NDB_DB_NOTE,
+	NDB_DB_META,
+	NDB_DB_PROFILE,
+	NDB_DB_NOTE_ID,
+	NDB_DB_PROFILE_PK, // profile pk index
+	NDB_DB_NDB_META,
+	NDB_DB_PROFILE_SEARCH,
+	NDB_DB_PROFILE_LAST_FETCH,
+	NDB_DB_NOTE_KIND, // note kind index
+	NDB_DB_NOTE_TEXT, // note fulltext index
+	NDB_DBS,
 };
 
-// v1
-struct ndb_note {
-	unsigned char version;    // v=1
-	unsigned char padding[3]; // keep things aligned
-	unsigned char id[32];
-	unsigned char pubkey[32];
-	unsigned char sig[64];
-
-	uint64_t created_at;
-	uint32_t kind;
-	uint32_t content_length;
-	union ndb_packed_str content;
-	uint32_t strings;
-	// nothing can come after tags since it contains variadic data
-	struct ndb_tags tags;
+// common kinds. we collect stats on these in ndb_stat. mainly because I don't
+// want to deal with including a hashtable to the project.
+enum ndb_common_kind {
+	NDB_CKIND_PROFILE,
+	NDB_CKIND_TEXT,
+	NDB_CKIND_CONTACTS,
+	NDB_CKIND_DM,
+	NDB_CKIND_DELETE,
+	NDB_CKIND_REPOST,
+	NDB_CKIND_REACTION,
+	NDB_CKIND_ZAP,
+	NDB_CKIND_ZAP_REQUEST,
+	NDB_CKIND_NWC_REQUEST,
+	NDB_CKIND_NWC_RESPONSE,
+	NDB_CKIND_HTTP_AUTH,
+	NDB_CKIND_LIST,
+	NDB_CKIND_LONGFORM,
+	NDB_CKIND_STATUS,
+	NDB_CKIND_COUNT, // should always be last
 };
 
-#pragma pack(pop)
-
-typedef enum ndb_ingest_filter_action (*ndb_ingest_filter_fn)(void *, struct ndb_note *);
-
 struct ndb_builder {
 	struct cursor mem;
 	struct cursor note_cur;
@@ -258,30 +204,6 @@ struct ndb_iterator {
 	int index;
 };
 
-enum ndb_filter_fieldtype {
-	NDB_FILTER_IDS     = 1,
-	NDB_FILTER_AUTHORS = 2,
-	NDB_FILTER_KINDS   = 3,
-	NDB_FILTER_GENERIC = 4,
-	NDB_FILTER_SINCE   = 5,
-	NDB_FILTER_UNTIL   = 6,
-	NDB_FILTER_LIMIT   = 7,
-};
-#define NDB_NUM_FILTERS 7
-
-// when matching generic tags, we need to know if we're dealing with
-// a pointer to a 32-byte ID or a null terminated string
-enum ndb_generic_element_type {
-	NDB_ELEMENT_UNKNOWN = 0,
-	NDB_ELEMENT_STRING  = 1,
-	NDB_ELEMENT_ID      = 2,
-};
-
-enum ndb_search_order {
-	NDB_ORDER_DESCENDING,
-	NDB_ORDER_ASCENDING,
-};
-
 union ndb_filter_element {
 	const char *string;
 	const unsigned char *id;
@@ -321,6 +243,43 @@ struct ndb_text_search_config {
 	int limit;
 };
 
+struct ndb_stat_counts {
+	size_t key_size;
+	size_t value_size;
+	size_t count;
+};
+
+struct ndb_stat {
+	struct ndb_stat_counts dbs[NDB_DBS];
+	struct ndb_stat_counts common_kinds[NDB_CKIND_COUNT];
+	struct ndb_stat_counts other_kinds;
+};
+
+#define MAX_TEXT_SEARCH_RESULTS 128
+#define MAX_TEXT_SEARCH_WORDS 8
+
+// unpacked form of the actual lmdb fulltext search key
+// see `ndb_make_text_search_key` for how the packed version is constructed
+struct ndb_text_search_key
+{
+	int str_len;
+	const char *str;
+	uint64_t timestamp;
+	uint64_t note_id;
+	int word_index;
+};
+
+struct ndb_text_search_result {
+	struct ndb_text_search_key key;
+	int prefix_chars;
+};
+
+struct ndb_text_search_results {
+	struct ndb_text_search_result results[MAX_TEXT_SEARCH_RESULTS];
+	int num_results;
+};
+
+
 // CONFIG
 void ndb_default_config(struct ndb_config *);
 void ndb_config_set_ingest_threads(struct ndb_config *config, int threads);
@@ -385,7 +344,7 @@ int ndb_filter_start_generic_field(struct ndb_filter *, char tag);
 int ndb_filter_matches(struct ndb_filter *, struct ndb_note *);
 void ndb_filter_reset(struct ndb_filter *);
 void ndb_filter_end_field(struct ndb_filter *);
-void ndb_filter_free(struct ndb_filter *filter);
+void ndb_filter_free(struct ndb_filter *);
 
 // FULLTEXT SEARCH
 int ndb_text_search(struct ndb_txn *txn, const char *query, struct ndb_text_search_results *, struct ndb_text_search_config *);
@@ -393,214 +352,35 @@ void ndb_default_text_search_config(struct ndb_text_search_config *);
 void ndb_text_search_config_set_order(struct ndb_text_search_config *, enum ndb_search_order);
 void ndb_text_search_config_set_limit(struct ndb_text_search_config *, int limit);
 
-// stats
+// STATS
 int ndb_stat(struct ndb *ndb, struct ndb_stat *stat);
 void ndb_stat_counts_init(struct ndb_stat_counts *counts);
 
-static inline struct ndb_str ndb_note_str(struct ndb_note *note,
-					  union ndb_packed_str *pstr)
-{
-	struct ndb_str str;
-	str.flag = pstr->packed.flag;
-
-	if (str.flag == NDB_PACKED_STR) {
-		str.str = pstr->packed.str;
-		return str;
-	}
-
-	str.str = ((const char *)note) + note->strings + (pstr->offset & 0xFFFFFF);
-	return str;
-}
-
-static inline struct ndb_str ndb_tag_str(struct ndb_note *note,
-					 struct ndb_tag *tag, int ind)
-{
-	return ndb_note_str(note, &tag->strs[ind]);
-}
-
-static inline struct ndb_str ndb_iter_tag_str(struct ndb_iterator *iter,
-					      int ind)
-{
-	return ndb_tag_str(iter->note, iter->tag, ind);
-}
-
-static inline unsigned char * ndb_note_id(struct ndb_note *note)
-{
-	return note->id;
-}
-
-static inline unsigned char * ndb_note_pubkey(struct ndb_note *note)
-{
-	return note->pubkey;
-}
-
-static inline unsigned char * ndb_note_sig(struct ndb_note *note)
-{
-	return note->sig;
-}
-
-static inline uint32_t ndb_note_created_at(struct ndb_note *note)
-{
-	return note->created_at;
-}
-
-static inline uint32_t ndb_note_kind(struct ndb_note *note)
-{
-	return note->kind;
-}
-
-static inline const char *ndb_note_content(struct ndb_note *note)
-{
-	return ndb_note_str(note, &note->content).str;
-}
-
-static inline uint32_t ndb_note_content_length(struct ndb_note *note)
-{
-	return note->content_length;
-}
-
-static inline struct ndb_note * ndb_note_from_bytes(unsigned char *bytes)
-{
-	struct ndb_note *note = (struct ndb_note *)bytes;
-	if (note->version != 1)
-		return 0;
-	return note;
-}
-
-static inline union ndb_packed_str ndb_offset_str(uint32_t offset)
-{
-	// ensure accidents like -1 don't corrupt our packed_str
-	union ndb_packed_str str;
-	// most significant byte is reserved for ndb_packtype
-	str.offset = offset & 0xFFFFFF;
-	return str;
-}
-
-static inline union ndb_packed_str ndb_char_to_packed_str(char c)
-{
-	union ndb_packed_str str;
-	str.packed.flag = NDB_PACKED_STR;
-	str.packed.str[0] = c;
-	str.packed.str[1] = '\0';
-	return str;
-}
-
-static inline union ndb_packed_str ndb_chars_to_packed_str(char c1, char c2)
-{
-	union ndb_packed_str str;
-	str.packed.flag = NDB_PACKED_STR;
-	str.packed.str[0] = c1;
-	str.packed.str[1] = c2;
-	str.packed.str[2] = '\0';
-	return str;
-}
-
-static inline void ndb_tags_iterate_start(struct ndb_note *note,
-					  struct ndb_iterator *iter)
-{
-	iter->note = note;
-	iter->tag = NULL;
-	iter->index = -1;
-}
-
-static inline int ndb_tags_iterate_next(struct ndb_iterator *iter)
-{
-	if (iter->tag == NULL || iter->index == -1) {
-		iter->tag = iter->note->tags.tag;
-		iter->index = 0;
-		return iter->note->tags.count != 0;
-	}
-
-	struct ndb_tags *tags = &iter->note->tags;
-
-	if (++iter->index < tags->count) {
-		uint32_t tag_data_size = iter->tag->count * sizeof(iter->tag->strs[0]);
-		iter->tag = (struct ndb_tag *)(iter->tag->strs[0].bytes + tag_data_size);
-		return 1;
-	}
-
-	return 0;
-}
-
-static inline enum ndb_common_kind
-ndb_kind_to_common_kind(int kind)
-{
-	switch (kind)
-	{
-		case 0:     return NDB_CKIND_PROFILE;
-		case 1:     return NDB_CKIND_TEXT;
-		case 3:     return NDB_CKIND_CONTACTS;
-		case 4:     return NDB_CKIND_DM;
-		case 5:     return NDB_CKIND_DELETE;
-		case 6:     return NDB_CKIND_REPOST;
-		case 7:     return NDB_CKIND_REACTION;
-		case 9735:  return NDB_CKIND_ZAP;
-		case 9734:  return NDB_CKIND_ZAP_REQUEST;
-		case 23194: return NDB_CKIND_NWC_REQUEST;
-		case 23195: return NDB_CKIND_NWC_RESPONSE;
-		case 27235: return NDB_CKIND_HTTP_AUTH;
-		case 30000: return NDB_CKIND_LIST;
-		case 30023: return NDB_CKIND_LONGFORM;
-		case 30315: return NDB_CKIND_STATUS;
-	}
-
-	return -1;
-}
-
-static inline const char *
-ndb_kind_name(enum ndb_common_kind ck)
-{
-	switch (ck) {
-		case NDB_CKIND_PROFILE:      return "profile";
-		case NDB_CKIND_TEXT:         return "text";
-		case NDB_CKIND_CONTACTS:     return "contacts";
-		case NDB_CKIND_DM:           return "dm";
-		case NDB_CKIND_DELETE:       return "delete";
-		case NDB_CKIND_REPOST:       return "repost";
-		case NDB_CKIND_REACTION:     return "reaction";
-		case NDB_CKIND_ZAP:          return "zap";
-		case NDB_CKIND_ZAP_REQUEST:  return "zap_request";
-		case NDB_CKIND_NWC_REQUEST:  return "nwc_request";
-		case NDB_CKIND_NWC_RESPONSE: return "nwc_response";
-		case NDB_CKIND_HTTP_AUTH:    return "http_auth";
-		case NDB_CKIND_LIST:         return "list";
-		case NDB_CKIND_LONGFORM:     return "longform";
-		case NDB_CKIND_STATUS:       return "status";
-		case NDB_CKIND_COUNT:        return "unknown";
-	}
-
-	return "unknown";
-}
-
-static inline const char *
-ndb_db_name(enum ndb_dbs db)
-{
-	switch (db) {
-		case NDB_DB_NOTE:
-			return "note";
-		case NDB_DB_META:
-			return "note_metadata";
-		case NDB_DB_PROFILE:
-			return "profile";
-		case NDB_DB_NOTE_ID:
-			return "note_index";
-		case NDB_DB_PROFILE_PK:
-			return "profile_pubkey_index";
-		case NDB_DB_NDB_META:
-			return "nostrdb_metadata";
-		case NDB_DB_PROFILE_SEARCH:
-			return "profile_search";
-		case NDB_DB_PROFILE_LAST_FETCH:
-			return "profile_last_fetch";
-		case NDB_DB_NOTE_KIND:
-			return "note_kind_index";
-		case NDB_DB_NOTE_TEXT:
-			return "note_fulltext";
-		case NDB_DBS:
-			return "count";
-	}
-
-	return "unknown";
-}
+// NOTE
+const char *ndb_note_content(struct ndb_note *note);
+struct ndb_str ndb_note_str(struct ndb_note *note, union ndb_packed_str *pstr);
+uint32_t ndb_note_content_length(struct ndb_note *note);
+uint32_t ndb_note_created_at(struct ndb_note *note);
+uint32_t ndb_note_kind(struct ndb_note *note);
+unsigned char *ndb_note_id(struct ndb_note *note);
+unsigned char *ndb_note_pubkey(struct ndb_note *note);
+unsigned char *ndb_note_sig(struct ndb_note *note);
+void _ndb_note_set_kind(struct ndb_note *note, uint32_t kind);
+struct ndb_tags *ndb_note_tags(struct ndb_note *note);
+
+// TAGS
+void ndb_tags_iterate_start(struct ndb_note *note, struct ndb_iterator *iter);
+uint16_t ndb_tags_count(struct ndb_tags *);
+uint16_t ndb_tag_count(struct ndb_tag *);
+
+// ITER
+int ndb_tags_iterate_next(struct ndb_iterator *iter);
+struct ndb_str ndb_iter_tag_str(struct ndb_iterator *iter, int ind);
+struct ndb_str ndb_tag_str(struct ndb_note *note, struct ndb_tag *tag, int ind);
+
+// NAMES
+const char *ndb_db_name(enum ndb_dbs db);
+const char *ndb_kind_name(enum ndb_common_kind ck);
+enum ndb_common_kind ndb_kind_to_common_kind(int kind);
 
 #endif
diff --git a/print_util.h b/print_util.h
@@ -29,8 +29,8 @@ static void ndb_print_text_search_result(struct ndb_txn *txn,
 	}
 
 	printf(" ");
-	print_hex(note->id, 32);
+	print_hex(ndb_note_id(note), 32);
 
-	printf("\n%s\n\n---\n", ndb_note_str(note, &note->content).str);
+	printf("\n%s\n\n---\n", ndb_note_content(note));
 }
 
diff --git a/test.c b/test.c
@@ -49,9 +49,8 @@ static void test_filters()
 
 	const char *test_note = "{\"id\": \"160e76ca67405d7ce9ef7d2dd72f3f36401c8661a73d45498af842d40b01b736\",\"pubkey\": \"67c67870aebc327eb2a2e765e6dbb42f0f120d2c4e4e28dc16b824cf72a5acc1\",\"created_at\": 1700688516,\"kind\": 1337,\"tags\": [[\"t\",\"hashtag\"],[\"t\",\"grownostr\"],[\"p\",\"4d2e7a6a8e08007ace5a03391d21735f45caf1bf3d67b492adc28967ab46525e\"]],\"content\": \"\",\"sig\": \"20c2d070261ed269559ada40ca5ac395c389681ee3b5f7d50de19dd9b328dd70cf27d9d13875e87c968d9b49fa05f66e90f18037be4529b9e582c7e2afac3f06\"}";
 
-	assert(ndb_note_from_json(test_note, strlen(test_note), &note, buffer, sizeof(buffer)));
-
 	f = &filter;
+	assert(ndb_note_from_json(test_note, strlen(test_note), &note, buffer, sizeof(buffer)));
 
 	assert(ndb_filter_init(f));
 	assert(ndb_filter_start_field(f, NDB_FILTER_KINDS));
@@ -69,13 +68,13 @@ static void test_filters()
 	// try matching the filter
 	assert(ndb_filter_matches(f, note));
 
-	note->kind = 1;
+	_ndb_note_set_kind(note, 1);
 
 	// inverse match
 	assert(!ndb_filter_matches(f, note));
 
 	// should also match 2
-	note->kind = 2;
+	_ndb_note_set_kind(note, 2);
 	assert(ndb_filter_matches(f, note));
 
 	// don't free, just reset data pointers
@@ -92,14 +91,14 @@ static void test_filters()
 	// shouldn't match the kind filter
 	assert(!ndb_filter_matches(f, note));
 
-	note->kind = 3;
+	_ndb_note_set_kind(note, 3);
 
 	// now it should
 	assert(ndb_filter_matches(f, note));
 
 	ndb_filter_reset(f);
 	assert(ndb_filter_start_field(f, NDB_FILTER_AUTHORS));
-	assert(ndb_filter_add_id_element(f, note->pubkey));
+	assert(ndb_filter_add_id_element(f, ndb_note_pubkey(note)));
 	ndb_filter_end_field(f);
 	assert(f->current == NULL);
 	assert(ndb_filter_matches(f, note));
@@ -370,7 +369,7 @@ static void test_basic_event() {
 	assert(ok);
 	note = builder.note;
 
-	memset(note->padding, 3, sizeof(note->padding));
+	//memset(note->padding, 3, sizeof(note->padding));
 
 	ok = ndb_builder_set_content(b, hex_pk, strlen(hex_pk)); assert(ok);
 	ndb_builder_set_id(b, id); assert(ok);
@@ -390,8 +389,9 @@ static void test_basic_event() {
 	assert(ok);
 
 	// content should never be packed id
-	assert(note->content.packed.flag != NDB_PACKED_ID);
-	assert(note->tags.count == 2);
+	// TODO: figure out how to test this now that we don't expose it
+	// assert(note->content.packed.flag != NDB_PACKED_ID);
+	assert(ndb_tags_count(ndb_note_tags(note)) == 2);
 
 	// test iterator
 	struct ndb_iterator iter, *it = &iter;
@@ -400,7 +400,7 @@ static void test_basic_event() {
 	ok = ndb_tags_iterate_next(it);
 	assert(ok);
 
-	assert(it->tag->count == 2);
+	assert(ndb_tag_count(it->tag) == 2);
 	const char *p      = ndb_iter_tag_str(it, 0).str;
 	struct ndb_str hpk = ndb_iter_tag_str(it, 1);
 
@@ -412,7 +412,7 @@ static void test_basic_event() {
 
 	ok = ndb_tags_iterate_next(it);
 	assert(ok);
-	assert(it->tag->count == 3);
+	assert(ndb_tag_count(it->tag) == 3);
 	assert(!strcmp(ndb_iter_tag_str(it, 0).str, "word"));
 	assert(!strcmp(ndb_iter_tag_str(it, 1).str, "words"));
 	assert(!strcmp(ndb_iter_tag_str(it, 2).str, "w"));
@@ -434,7 +434,7 @@ static void test_empty_tags() {
 	ok = ndb_builder_finalize(b, &note, NULL);
 	assert(ok);
 
-	assert(note->tags.count == 0);
+	assert(ndb_tags_count(ndb_note_tags(note)) == 0);
 
 	ndb_tags_iterate_start(note, it);
 	ok = ndb_tags_iterate_next(it);
@@ -442,9 +442,10 @@ static void test_empty_tags() {
 }
 
 static void print_tag(struct ndb_note *note, struct ndb_tag *tag) {
-	for (int i = 0; i < tag->count; i++) {
-		union ndb_packed_str *elem = &tag->strs[i];
-		struct ndb_str str = ndb_note_str(note, elem);
+	struct ndb_str str;
+	int tag_count = ndb_tag_count(tag);
+	for (int i = 0; i < tag_count; i++) {
+		str = ndb_tag_str(note, tag, i);
 		if (str.flag == NDB_PACKED_ID) {
 			printf("<id> ");
 		} else {
@@ -470,10 +471,10 @@ static void test_parse_contact_list()
 	assert(size > 0);
 	assert(size == 34328);
 
-	memcpy(id, note->id, 32);
-	memset(note->id, 0, 32);
+	memcpy(id, ndb_note_id(note), 32);
+	memset(ndb_note_id(note), 0, 32);
 	assert(ndb_calculate_id(note, json, alloc_size));
-	assert(!memcmp(note->id, id, 32));
+	assert(!memcmp(ndb_note_id(note), id, 32));
 
 	const char* expected_content = 
 	"{\"wss://nos.lol\":{\"write\":true,\"read\":true},"
@@ -488,7 +489,7 @@ static void test_parse_contact_list()
 	assert(!strcmp(expected_content, ndb_note_content(note)));
 	assert(ndb_note_created_at(note) == 1689904312);
 	assert(ndb_note_kind(note) == 3);
-	assert(note->tags.count == 786);
+	assert(ndb_tags_count(ndb_note_tags(note)) == 786);
 	//printf("note content length %d\n", ndb_note_content_length(note));
 	printf("ndb_content_len %d, expected_len %ld\n",
 			ndb_note_content_length(note),
@@ -502,21 +503,20 @@ static void test_parse_contact_list()
 	int total_elems = 0;
 
 	while (ndb_tags_iterate_next(it)) {
-		total_elems += it->tag->count;
+		total_elems += ndb_tag_count(it->tag);
 		//printf("tag %d: ", tags);
 		if (tags == 0 || tags == 1 || tags == 2)
-			assert(it->tag->count == 3);
+			assert(ndb_tag_count(it->tag) == 3);
 
 		if (tags == 6)
-			assert(it->tag->count == 2);
+			assert(ndb_tag_count(it->tag) == 2);
 
 		if (tags == 7)
-			assert(!strcmp(ndb_note_str(note, &it->tag->strs[2]).str,
-						"wss://nostr-pub.wellorder.net"));
+			assert(!strcmp(ndb_tag_str(note, it->tag, 2).str, "wss://nostr-pub.wellorder.net"));
 
 		if (tags == 786) {
 			static unsigned char h[] = { 0x74, 0xfa, 0xe6, 0x66, 0x4c, 0x9e, 0x79, 0x98, 0x0c, 0x6a, 0xc1, 0x1c, 0x57, 0x75, 0xed, 0x30, 0x93, 0x2b, 0xe9, 0x26, 0xf5, 0xc4, 0x5b, 0xe8, 0xd6, 0x55, 0xe0, 0x0e, 0x35, 0xec, 0xa2, 0x88 };
-			assert(!memcmp(ndb_note_str(note, &it->tag->strs[1]).id, h, 32));
+			assert(!memcmp(ndb_tag_str(note, it->tag, 1).id, h, 32));
 		}
 
 		//print_tag(it->note, it->tag);
@@ -603,7 +603,7 @@ static void test_fetch_last_noteid()
 	assert(ndb_begin_query(ndb, &txn));
 	struct ndb_note *note = ndb_get_note_by_id(&txn, id, &len, NULL);
 	assert(note != NULL);
-	assert(note->created_at == 1650054135);
+	assert(ndb_note_created_at(note) == 1650054135);
 	
 	unsigned char pk[32] = { 0x32, 0xe1, 0x82, 0x76, 0x35, 0x45, 0x0e, 0xbb, 0x3c, 0x5a, 0x7d, 0x12, 0xc1, 0xf8, 0xe7, 0xb2, 0xb5, 0x14, 0x43, 0x9a, 0xc1, 0x0a, 0x67, 0xee, 0xf3, 0xd9, 0xfd, 0x9c, 0x5c, 0x68, 0xe2, 0x45 };
 
@@ -632,7 +632,7 @@ static void test_fetch_last_noteid()
 
 	struct ndb_note *n = ndb_get_note_by_key(&txn, key, NULL);
 	ndb_end_query(&txn);
-	assert(memcmp(profile_note_id, n->id, 32) == 0);
+	assert(memcmp(profile_note_id, ndb_note_id(n), 32) == 0);
 
 	//fwrite(profile, len, 1, stdout);
 
@@ -700,17 +700,17 @@ static void test_parse_json() {
 	assert(!strcmp(content, "共通語"));
 	assert(!memcmp(id, hex_id, 32));
 
-	assert(note->tags.count == 2);
+	assert(ndb_tags_count(ndb_note_tags(note)) == 2);
 
 	struct ndb_iterator iter, *it = &iter;
 	ndb_tags_iterate_start(note, it); assert(ok);
 	ok = ndb_tags_iterate_next(it); assert(ok);
-	assert(it->tag->count == 2);
+	assert(ndb_tag_count(it->tag) == 2);
 	assert(!strcmp(ndb_iter_tag_str(it, 0).str, "p"));
 	assert(!memcmp(ndb_iter_tag_str(it, 1).id, hex_id, 32));
 
 	ok = ndb_tags_iterate_next(it); assert(ok);
-	assert(it->tag->count == 3);
+	assert(ndb_tag_count(it->tag) == 3);
 	assert(!strcmp(ndb_iter_tag_str(it, 0).str, "word"));
 	assert(!strcmp(ndb_iter_tag_str(it, 1).str, "words"));
 	assert(!strcmp(ndb_iter_tag_str(it, 2).str, "w"));
@@ -725,10 +725,10 @@ static void test_strings_work_before_finalization() {
 	ok = ndb_builder_init(b, buf, sizeof(buf)); assert(ok);
 	ndb_builder_set_content(b, "hello", 5);
 
-	assert(!strcmp(ndb_note_str(b->note, &b->note->content).str, "hello"));
+	assert(!strcmp(ndb_note_content(b->note), "hello"));
 	assert(ndb_builder_finalize(b, &note, NULL));
 
-	assert(!strcmp(ndb_note_str(b->note, &b->note->content).str, "hello"));
+	assert(!strcmp(ndb_note_content(note), "hello"));
 }
 
 static void test_tce_eose() {