commit 882f6e2534ae90160b9a513961f171a803109566
parent 2f60888fb1853c480593462e4ff6e7a079ce6d94
Author: William Casarin <jb55@jb55.com>
Date: Mon, 28 Aug 2023 08:19:03 -0700
ndb: update nostrdb, fix alignment issues
Diffstat:
2 files changed, 151 insertions(+), 32 deletions(-)
diff --git a/nostrdb/nostrdb.c b/nostrdb/nostrdb.c
@@ -15,6 +15,8 @@
#include <assert.h>
#include "bindings/c/profile_json_parser.h"
+#include "bindings/c/profile_builder.h"
+#include "bindings/c/profile_verifier.h"
#include "secp256k1.h"
#include "secp256k1_ecdh.h"
#include "secp256k1_schnorrsig.h"
@@ -35,6 +37,10 @@ static const int DEFAULT_QUEUE_SIZE = 1000000;
#define NDB_PARSED_TAGS (1 << 6)
#define NDB_PARSED_ALL (NDB_PARSED_ID|NDB_PARSED_PUBKEY|NDB_PARSED_SIG|NDB_PARSED_CREATED_AT|NDB_PARSED_KIND|NDB_PARSED_CONTENT|NDB_PARSED_TAGS)
+struct ndb_profile_record_builder {
+ flatcc_builder_t *builder;
+ void *flatbuf;
+};
// controls whether to continue or stop the json parser
enum ndb_idres {
@@ -183,8 +189,7 @@ struct ndb_writer_note {
struct ndb_writer_profile {
struct ndb_writer_note note;
- void *profile_flatbuf;
- size_t profile_len;
+ struct ndb_profile_record_builder record;
};
struct ndb_ingester_msg {
@@ -275,6 +280,32 @@ cleanup:
return success;
}
+static void *ndb_lookup_by_key(struct ndb *ndb, uint64_t key,
+ enum ndb_dbs store, size_t *len)
+{
+ MDB_val k, v;
+ MDB_txn *txn;
+
+ k.mv_data = &key;
+ k.mv_size = sizeof(key);
+
+ if (mdb_txn_begin(ndb->lmdb.env, 0, 0, &txn)) {
+ ndb_debug("ndb_get_note_by_id: mdb_txn_begin failed\n");
+ return NULL;
+ }
+
+ if (mdb_get(txn, ndb->lmdb.dbs[store], &k, &v)) {
+ ndb_debug("ndb_get_profile_by_pubkey: mdb_get note failed\n");
+ mdb_txn_abort(txn);
+ return NULL;
+ }
+
+ if (len)
+ *len = v.mv_size;
+
+ return v.mv_data;
+}
+
static void *ndb_lookup_tsid(struct ndb *ndb, enum ndb_dbs ind,
enum ndb_dbs store, const unsigned char *pk,
size_t *len)
@@ -291,7 +322,7 @@ static void *ndb_lookup_tsid(struct ndb *ndb, enum ndb_dbs ind,
}
if (!ndb_get_tsid(txn, &ndb->lmdb, ind, pk, &k)) {
- ndb_debug("ndb_get_profile_by_pubkey: ndb_get_tsid failed\n");
+ //ndb_debug("ndb_get_profile_by_pubkey: ndb_get_tsid failed\n");
goto cleanup;
}
@@ -301,6 +332,7 @@ static void *ndb_lookup_tsid(struct ndb *ndb, enum ndb_dbs ind,
}
res = v.mv_data;
+ assert(((uint64_t)res % 4) == 0);
if (len)
*len = v.mv_size;
cleanup:
@@ -318,6 +350,11 @@ struct ndb_note *ndb_get_note_by_id(struct ndb *ndb, const unsigned char *id, si
return ndb_lookup_tsid(ndb, NDB_DB_NOTE_ID, NDB_DB_NOTE, id, len);
}
+struct ndb_note *ndb_get_note_by_key(struct ndb *ndb, uint64_t key, size_t *len)
+{
+ return ndb_lookup_by_key(ndb, key, NDB_DB_NOTE, len);
+}
+
static int ndb_has_note(MDB_txn *txn, struct ndb_lmdb *lmdb, const unsigned char *id)
{
MDB_val val;
@@ -343,30 +380,83 @@ static enum ndb_idres ndb_ingester_json_controller(void *data, const char *hexid
return NDB_IDRES_STOP;
}
+static int ndbprofile_parse_json(flatcc_builder_t *B,
+ const char *buf, size_t bufsiz, int flags, NdbProfile_ref_t *profile)
+{
+ flatcc_json_parser_t parser, *ctx = &parser;
+ flatcc_json_parser_init(ctx, B, buf, buf + bufsiz, flags);
+
+ if (flatcc_builder_start_buffer(B, 0, 0, 0))
+ return 0;
+
+ NdbProfile_parse_json_table(ctx, buf, buf + bufsiz, profile);
+ if (ctx->error)
+ return 0;
+
+ if (!flatcc_builder_end_buffer(B, *profile))
+ return 0;
+
+ ctx->end_loc = buf;
+
+
+ return 1;
+}
+
+void ndb_profile_record_builder_init(struct ndb_profile_record_builder *b)
+{
+ b->builder = malloc(sizeof(*b->builder));
+ b->flatbuf = NULL;
+}
-static int ndb_process_profile_note(struct ndb_note *note, void **profile,
- size_t *profile_len)
+void ndb_profile_record_builder_free(struct ndb_profile_record_builder *b)
+{
+ if (b->builder)
+ free(b->builder);
+ if (b->flatbuf)
+ free(b->flatbuf);
+
+ b->builder = NULL;
+ b->flatbuf = NULL;
+}
+
+static int ndb_process_profile_note(struct ndb_note *note,
+ struct ndb_profile_record_builder *profile)
{
int res;
- flatcc_builder_t builder;
- flatcc_json_parser_t json_parser;
+ NdbProfile_ref_t profile_table;
+ flatcc_builder_t *builder;
- flatcc_builder_init(&builder);
+ ndb_profile_record_builder_init(profile);
+ builder = profile->builder;
+ flatcc_builder_init(builder);
- //printf("parsing profile '%.*s'\n", note->content_length, ndb_note_content(note));
- res = profile_parse_json(&builder, &json_parser,
- ndb_note_content(note),
- note->content_length,
- flatcc_json_parser_f_skip_unknown);
+ NdbProfileRecord_start_as_root(builder);
- if (res != 0) {
+ //printf("parsing profile '%.*s'\n", note->content_length, ndb_note_content(note));
+ if (!(res = ndbprofile_parse_json(builder, ndb_note_content(note),
+ note->content_length,
+ flatcc_json_parser_f_skip_unknown,
+ &profile_table)))
+ {
ndb_debug("profile_parse_json failed %d '%.*s'\n", res,
- note->content_length, ndb_note_content(note));
+ note->content_length, ndb_note_content(note));
+ ndb_profile_record_builder_free(profile);
return 0;
}
- *profile = flatcc_builder_finalize_aligned_buffer(&builder, profile_len);
+ uint64_t received_at = time(NULL);
+ const char *lnurl = "fixme";
+
+ NdbProfileRecord_profile_add(builder, profile_table);
+ NdbProfileRecord_received_at_add(builder, received_at);
+
+ flatcc_builder_ref_t lnurl_off;
+ lnurl_off = flatcc_builder_create_string_str(builder, lnurl);
+
+ NdbProfileRecord_lnurl_add(builder, lnurl_off);
+
+ //*profile = flatcc_builder_finalize_aligned_buffer(builder, profile_len);
return 1;
}
@@ -382,8 +472,8 @@ static int ndb_ingester_process_event(secp256k1_context *ctx,
struct ndb_note *note;
struct ndb_ingest_controller controller;
struct ndb_id_cb cb;
- void *buf, *flatbuf;
- size_t bufsize, note_size, profile_len;
+ void *buf;
+ size_t bufsize, note_size;
// we will use this to check if we already have it in the DB during
// ID parsing
@@ -436,14 +526,17 @@ static int ndb_ingester_process_event(secp256k1_context *ctx,
// we didn't find anything. let's send it
// to the writer thread
note = realloc(note, note_size);
+ assert(((uint64_t)note % 4) == 0);
+
+ if (note->kind == 0) {
+ struct ndb_profile_record_builder *b =
+ &out->profile.record;
+
+ ndb_process_profile_note(note, b);
- if (note->kind == 0 &&
- ndb_process_profile_note(note, &flatbuf, &profile_len)) {
out->type = NDB_WRITER_PROFILE;
out->profile.note.note = note;
out->profile.note.note_len = note_size;
- out->profile.profile_flatbuf = flatbuf;
- out->profile.profile_len = profile_len;
} else {
out->type = NDB_WRITER_NOTE;
out->note.note = note;
@@ -482,11 +575,14 @@ static uint64_t ndb_get_last_key(MDB_txn *txn, MDB_dbi db)
}
static int ndb_write_profile(struct ndb_lmdb *lmdb, MDB_txn *txn,
- struct ndb_writer_profile *profile)
+ struct ndb_writer_profile *profile,
+ uint64_t note_key)
{
uint64_t profile_key;
struct ndb_tsid tsid;
struct ndb_note *note;
+ void *flatbuf;
+ size_t flatbuf_len;
int rc;
MDB_val key, val;
@@ -494,6 +590,20 @@ static int ndb_write_profile(struct ndb_lmdb *lmdb, MDB_txn *txn,
note = profile->note.note;
+ // add note_key to profile record
+ NdbProfileRecord_note_key_add(profile->record.builder, note_key);
+ NdbProfileRecord_end_as_root(profile->record.builder);
+
+ flatbuf = profile->record.flatbuf =
+ flatcc_builder_finalize_aligned_buffer(profile->record.builder, &flatbuf_len);
+
+ assert(((uint64_t)flatbuf % 8) == 0);
+
+ // TODO: this may not be safe!?
+ flatbuf_len = (flatbuf_len + 7) & ~7;
+
+ //assert(NdbProfileRecord_verify_as_root(flatbuf, flatbuf_len) == 0);
+
// get dbs
profile_db = lmdb->dbs[NDB_DB_PROFILE];
pk_db = lmdb->dbs[NDB_DB_PROFILE_PK];
@@ -504,8 +614,8 @@ static int ndb_write_profile(struct ndb_lmdb *lmdb, MDB_txn *txn,
// write profile to profile store
key.mv_data = &profile_key;
key.mv_size = sizeof(profile_key);
- val.mv_data = profile->profile_flatbuf + 4;
- val.mv_size = profile->profile_len - 4;
+ val.mv_data = flatbuf;
+ val.mv_size = flatbuf_len;
//ndb_debug("profile_len %ld\n", profile->profile_len);
if ((rc = mdb_put(txn, profile_db, &key, &val, 0))) {
@@ -579,13 +689,14 @@ static void *ndb_writer_thread(void *data)
struct ndb_writer *writer = data;
struct ndb_writer_msg msgs[THREAD_QUEUE_BATCH], *msg;
int i, popped, done, any_note;
+ uint64_t note_nkey;
MDB_txn *txn;
done = 0;
while (!done) {
txn = NULL;
popped = prot_queue_pop_all(&writer->inbox, msgs, THREAD_QUEUE_BATCH);
- ndb_debug("writer popped %d items\n", popped);
+ //ndb_debug("writer popped %d items\n", popped);
any_note = 0;
for (i = 0 ; i < popped; i++) {
@@ -614,9 +725,14 @@ static void *ndb_writer_thread(void *data)
done = 1;
continue;
case NDB_WRITER_PROFILE:
- ndb_write_note(writer->lmdb, txn, &msg->note);
- // TODO: save note_key with profile
- ndb_write_profile(writer->lmdb, txn, &msg->profile);
+ note_nkey =
+ ndb_write_note(writer->lmdb, txn, &msg->note);
+ if (msg->profile.record.builder) {
+ // only write if parsing didn't fail
+ ndb_write_profile(writer->lmdb, txn,
+ &msg->profile,
+ note_nkey);
+ }
break;
case NDB_WRITER_NOTE:
ndb_write_note(writer->lmdb, txn, &msg->note);
@@ -637,8 +753,8 @@ static void *ndb_writer_thread(void *data)
if (msg->type == NDB_WRITER_NOTE)
free(msg->note.note);
else if (msg->type == NDB_WRITER_PROFILE) {
- free(msg->profile.profile_flatbuf);
free(msg->profile.note.note);
+ ndb_profile_record_builder_free(&msg->profile.record);
}
}
}
@@ -667,7 +783,7 @@ static void *ndb_ingester_thread(void *data)
any_event = 0;
popped = prot_queue_pop_all(&thread->inbox, msgs, THREAD_QUEUE_BATCH);
- ndb_debug("ingester popped %d items\n", popped);
+ //ndb_debug("ingester popped %d items\n", popped);
for (i = 0; i < popped; i++) {
msg = &msgs[i];
@@ -1336,6 +1452,9 @@ int ndb_builder_finalize(struct ndb_builder *builder, struct ndb_note **note,
return 0;
}
+ // make sure we're aligned as a whole
+ total_size = (total_size + 7) & ~7;
+ assert((total_size % 8) == 0);
return total_size;
}
diff --git a/nostrdb/nostrdb.h b/nostrdb/nostrdb.h
@@ -120,7 +120,6 @@ struct ndb_note {
uint32_t content_length;
union ndb_packed_str content;
uint32_t strings;
- uint32_t reserved[4]; // expansion slots
// nothing can come after tags since it contains variadic data
struct ndb_tags tags;
};
@@ -157,6 +156,7 @@ int ndb_process_event(struct ndb *, const char *json, int len);
int ndb_process_events(struct ndb *, const char *ldjson, size_t len);
void *ndb_get_profile_by_pubkey(struct ndb *, const unsigned char *pubkey, size_t *len);
struct ndb_note *ndb_get_note_by_id(struct ndb *, const unsigned char *id, size_t *len);
+struct ndb_note *ndb_get_note_by_key(struct ndb *, uint64_t key, size_t *len);
void ndb_destroy(struct ndb *);
// BUILDER
