commit f0beae9326c9e197e5159a7f68c0e85e6e659e91
parent db5a509c4d134ff692bf776082a8202fa00e0081
Author: William Casarin <jb55@jb55.com>
Date: Fri, 21 Mar 2025 13:27:37 -0700
relay-index: fix a few bugs
There were a few race conditions and lmdb bugs in the
relay index implementation. Fix those!
Signed-off-by: William Casarin <jb55@jb55.com>
Diffstat:
4 files changed, 28 insertions(+), 19 deletions(-)
diff --git a/src/content_parser.c b/src/content_parser.c
@@ -24,8 +24,10 @@ struct ndb_content_parser {
static int parse_digit(struct cursor *cur, int *digit) {
int c;
- if ((c = peek_char(cur, 0)) == -1)
+ if ((c = peek_char(cur, 0)) == -1) {
+ *digit = 0;
return 0;
+ }
c -= '0';
diff --git a/src/cursor.h b/src/cursor.h
@@ -715,7 +715,6 @@ static inline int cursor_align(struct cursor *cur, int bytes) {
// pad to n-byte alignment
pad = ((size + (bytes-1)) & ~(bytes-1)) - size;
-
if (pad > 0 && !cursor_memset(cur, 0, pad))
return 0;
diff --git a/src/nostrdb.c b/src/nostrdb.c
@@ -1541,8 +1541,12 @@ static int ndb_write_note_relay(struct ndb_txn *txn, uint64_t note_key,
int rc, len;
MDB_val k, v;
- if (relay == NULL || relay_len == 0)
+ if (relay == NULL || relay_len == 0) {
+ ndb_debug("relay is NULL in ndb_write_note_relay? '%s' %d\n", relay, relay_len);
return 0;
+ }
+
+ ndb_debug("writing note_relay '%s' for notekey:%" PRIu64 "\n", relay, note_key);
if (!(len = prepare_relay_buf(relay_buf, sizeof(relay_buf), relay, relay_len))) {
fprintf(stderr, "relay url '%s' too large when writing note relay index\n", relay);
@@ -1566,6 +1570,8 @@ static int ndb_write_note_relay(struct ndb_txn *txn, uint64_t note_key,
return 0;
}
+ ndb_debug("wrote %d bytes to note relay: '%s'\n", len, relay_buf);
+
return 1;
}
@@ -1597,6 +1603,8 @@ static int ndb_write_note_relay_kind_index(struct ndb_txn *txn,
if (relay == NULL || relay_len == 0)
return 0;
+ ndb_debug("writing note_relay_kind_index '%s' for notekey:%" PRIu64 "\n", relay, note_key);
+
make_cursor(buf, buf + sizeof(buf), &cur);
if (!cursor_push(&cur, (unsigned char *)¬e_key, 8)) return 0;
@@ -1604,6 +1612,7 @@ static int ndb_write_note_relay_kind_index(struct ndb_txn *txn,
if (!cursor_push(&cur, (unsigned char *)&created_at, 8)) return 0;
if (!cursor_push_byte(&cur, (uint8_t)relay_len)) return 0;
if (!cursor_push(&cur, (unsigned char *)relay, relay_len)) return 0;
+ if (!cursor_push_byte(&cur, 0)) return 0;
if (!cursor_align(&cur, 8)) return 0;
assert(((cur.p-cur.start)%8) == 0);
@@ -2612,6 +2621,7 @@ int ndb_note_seen_on_relay(struct ndb_txn *txn, uint64_t note_key, const char *r
return 0;
rc = mdb_cursor_get(cur, &k, &v, MDB_GET_BOTH);
+ ndb_debug("seen_on_relay result: %s\n", mdb_strerror(rc));
mdb_cursor_close(cur);
return rc == MDB_SUCCESS;
@@ -4664,6 +4674,9 @@ static uint64_t ndb_write_note(struct ndb_txn *txn,
kind = note->note->kind;
+ if (note->relay != NULL)
+ relay_len = strlen(note->relay);
+
// let's quickly sanity check if we already have this note
if (ndb_get_notekey_by_id(txn, note->note->id)) {
// even if we do we still need to write relay index
@@ -4691,9 +4704,6 @@ static uint64_t ndb_write_note(struct ndb_txn *txn,
return 0;
}
- if (note->relay != NULL)
- relay_len = strlen(note->relay);
-
ndb_write_note_id_index(txn, note->note, note_key);
ndb_write_note_kind_index(txn, note->note, note_key);
ndb_write_note_tag_index(txn, note->note, note_key);
@@ -5270,7 +5280,7 @@ static int ndb_init_lmdb(const char *filename, struct ndb_lmdb *lmdb, size_t map
mdb_set_compare(txn, lmdb->dbs[NDB_DB_NOTE_RELAY_KIND], ndb_relay_kind_cmp);
// note_id -> relay index
- if ((rc = mdb_dbi_open(txn, "note_relays", MDB_CREATE | MDB_DUPSORT | MDB_DUPFIXED, &lmdb->dbs[NDB_DB_NOTE_RELAYS]))) {
+ if ((rc = mdb_dbi_open(txn, "note_relays", MDB_CREATE | MDB_DUPSORT, &lmdb->dbs[NDB_DB_NOTE_RELAYS]))) {
fprintf(stderr, "mdb_dbi_open profile last fetch, error %d\n", rc);
return 0;
}
@@ -7196,7 +7206,7 @@ int ndb_print_relay_kind_index(struct ndb_txn *txn)
printf("relay\tkind\tcreated_at\tnote_id\n");
while (mdb_cursor_get(cur, &k, &v, MDB_NEXT) == 0) {
d = (unsigned char *)k.mv_data;
- printf("%s\t", (const char *)(d + 25));
+ printf("'%s'\t", (const char *)(d + 25));
printf("%" PRIu64 "\t", *(uint64_t*)(d + 8));
printf("%" PRIu64 "\t", *(uint64_t*)(d + 16));
printf("%" PRIu64 "\n", *(uint64_t*)(d + 0));
diff --git a/test.c b/test.c
@@ -1785,7 +1785,6 @@ static void test_note_relay_index()
struct ndb *ndb;
struct ndb_txn txn;
struct ndb_config config;
- struct ndb_note *note;
struct ndb_filter filter, *f = &filter;
uint64_t note_key, subid;
struct ndb_ingest_meta meta;
@@ -1809,26 +1808,21 @@ static void test_note_relay_index()
ndb_ingest_meta_init(&meta, 1, "wss://relay.damus.io");
assert(ndb_process_event_with(ndb, json, strlen(json), &meta));
+ meta.relay = "wss://relay.mit.edu";
+ assert(ndb_process_event_with(ndb, json, strlen(json), &meta));
+ meta.relay = "wss://nostr.mom";
+ assert(ndb_process_event_with(ndb, json, strlen(json), &meta));
assert(ndb_wait_for_notes(ndb, subid, ¬e_key, 1) == 1);
assert(note_key > 0);
- assert(ndb_begin_query(ndb, &txn));
-
- assert((note = ndb_get_note_by_key(&txn, note_key, NULL)));
-
- ndb_end_query(&txn);
-
- // 3) Ingest it again from a new relay: “relay2”
- ndb_ingest_meta_init(&meta, 1, "wss://relay.mit.edu");
- assert(ndb_process_event_with(ndb, json, strlen(json), &meta));
- // TODO: subscribe to this somehow if we have a relay filter?
sleep(1);
// 4) Check that we have both relays
assert(ndb_begin_query(ndb, &txn));
assert(ndb_note_seen_on_relay(&txn, note_key, "wss://relay.damus.io"));
assert(ndb_note_seen_on_relay(&txn, note_key, "wss://relay.mit.edu"));
+ assert(ndb_note_seen_on_relay(&txn, note_key, "wss://nostr.mom"));
// walk the relays
struct ndb_note_relay_iterator iter;
@@ -1837,6 +1831,10 @@ static void test_note_relay_index()
relay = ndb_note_relay_iterate_next(&iter);
assert(relay);
+ assert(!strcmp(relay, "wss://nostr.mom"));
+
+ relay = ndb_note_relay_iterate_next(&iter);
+ assert(relay);
assert(!strcmp(relay, "wss://relay.damus.io"));
relay = ndb_note_relay_iterate_next(&iter);
