commit 64cad19042d4573d6fbfbdcded0f47129a84c438
parent 54190f661ebed53e454a12e01e56d4e65fefb574
Author: William Casarin <jb55@jb55.com>
Date: Fri, 21 Mar 2025 21:02:14 -0700
Relay queries
Add support for relay-based filtering in nostr queries.
Filters can now include a "relays" field. Optimal performance when
you include a kind as well:
{"relays":["wss://pyramid.fiatjaf.com/"], "kinds":[1]}
This corresponds to a `ndb` query like so:
$ ndb query -r wss://pyramid.fiatjaf.com/ -k 1 -l 1
using filter '{"relays":["wss://pyramid.fiatjaf.com/"],"kinds":[1],"limit":1}'
1 results in 0.094929 ms
{"id":"277dd4ed26d0b44576..}
Signed-off-by: William Casarin <jb55@jb55.com>
Diffstat:
| M | ndb.c | | | 37 | +++++++++++++++++++++++++++++++++++-- |
| M | src/nostrdb.c | | | 463 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-------------- |
| M | src/nostrdb.h | | | 2 | ++ |
3 files changed, 422 insertions(+), 80 deletions(-)
diff --git a/ndb.c b/ndb.c
@@ -15,9 +15,12 @@
static int usage()
{
printf("usage: ndb [--skip-verification] [-d db_dir] <command>\n\n");
+
printf("commands\n\n");
+
printf(" stat\n");
- printf(" query [--kind 42] [--search term] [--limit 42] [-e abcdef...] [--author abcdef... -a bcdef...]\n");
+ printf(" query [--kind 42] [--notekey key] [--search term] [--limit 42] \n");
+ printf(" [-e abcdef...] [--author abcdef... -a bcdef...] [--relay wss://relay.damus.io]\n");
printf(" profile <pubkey> print the raw profile data for a pubkey\n");
printf(" note-relays <note-id> list the relays a given note id has been seen on\n");
printf(" print-search-keys\n");
@@ -25,7 +28,9 @@ static int usage()
printf(" print-tag-keys\n");
printf(" print-relay-kind-index-keys\n");
printf(" import <line-delimited json file>\n\n");
+
printf("settings\n\n");
+
printf(" --skip-verification skip signature validation\n");
printf(" -d <db_dir> set database directory\n");
return 1;
@@ -144,6 +149,8 @@ int main(int argc, char *argv[])
struct ndb_config config;
struct timespec t1, t2;
unsigned char tmp_id[32];
+ char buf[1024];
+ buf[0] = 0;
// profiles
const char *arg_str;
@@ -207,6 +214,10 @@ int main(int argc, char *argv[])
ndb_filter_add_int_element(f, atoll(argv[1]));
argv += 2;
argc -= 2;
+ } else if (!strcmp(argv[0], "--notekey")) {
+ key = atol(argv[1]);
+ argv += 2;
+ argc -= 2;
} else if (!strcmp(argv[0], "-l") || !strcmp(argv[0], "--limit")) {
limit = atol(argv[1]);
if (current_field) {
@@ -259,6 +270,16 @@ int main(int argc, char *argv[])
ndb_filter_end_field(f);
argv += 2;
argc -= 2;
+ } else if (!strcmp(argv[0], "--relay") || !strcmp(argv[0], "-r")) {
+ if (current_field) {
+ ndb_filter_end_field(f);
+ current_field = 0;
+ }
+ ndb_filter_start_field(f, NDB_FILTER_RELAYS);
+ ndb_filter_add_str_element(f, argv[1]);
+ ndb_filter_end_field(f);
+ argv += 2;
+ argc -= 2;
} else if (!strcmp(argv[0], "-e")) {
if (current_field != 'e') {
if (!ndb_filter_start_tag_field(f, 'e')) {
@@ -316,11 +337,23 @@ int main(int argc, char *argv[])
current_field = 0;
}
+ ndb_filter_end(f);
+
+ ndb_filter_json(f, buf, sizeof(buf));
+ fprintf(stderr, "using filter '%s'\n", buf);
+
struct ndb_query_result results[10000];
ndb_begin_query(ndb, &txn);
+
clock_gettime(CLOCK_MONOTONIC, &t1);
- if (!ndb_query(&txn, f, 1, results, 10000, &count)) {
+ if (key) {
+ results[0].note = ndb_get_note_by_key(&txn, key, NULL);
+ if (results[0].note != NULL)
+ count = 1;
+ else
+ count = 0;
+ } else if (!ndb_query(&txn, f, 1, results, 10000, &count)) {
fprintf(stderr, "query error\n");
}
clock_gettime(CLOCK_MONOTONIC, &t2);
diff --git a/src/nostrdb.c b/src/nostrdb.c
@@ -238,6 +238,7 @@ enum ndb_query_plan {
NDB_PLAN_CREATED,
NDB_PLAN_TAGS,
NDB_PLAN_SEARCH,
+ NDB_PLAN_RELAY_KINDS,
};
// A id + u64 + timestamp
@@ -751,6 +752,7 @@ static const char *ndb_filter_field_name(enum ndb_filter_fieldtype field)
case NDB_FILTER_UNTIL: return "until";
case NDB_FILTER_LIMIT: return "limit";
case NDB_FILTER_SEARCH: return "search";
+ case NDB_FILTER_RELAYS: return "relays";
}
return "unknown";
@@ -862,6 +864,15 @@ static int ndb_filter_add_element(struct ndb_filter *filter, union ndb_filter_el
}
// push a pointer of the string in the databuf as an element
break;
+ case NDB_FILTER_RELAYS:
+ if (current->field.elem_type != NDB_ELEMENT_STRING) {
+ return 0;
+ }
+ if (!cursor_push(&filter->data_buf, (unsigned char *)el.string.string, el.string.len))
+ return 0;
+ if (!cursor_push_byte(&filter->data_buf, 0))
+ return 0;
+ break;
}
if (!cursor_push(&filter->elem_buf, (unsigned char *)&offset,
@@ -920,6 +931,7 @@ int ndb_filter_add_str_element_len(struct ndb_filter *filter, const char *str, i
return 0;
}
break;
+ case NDB_FILTER_RELAYS:
case NDB_FILTER_TAGS:
break;
}
@@ -950,6 +962,7 @@ int ndb_filter_add_int_element(struct ndb_filter *filter, uint64_t integer)
case NDB_FILTER_AUTHORS:
case NDB_FILTER_TAGS:
case NDB_FILTER_SEARCH:
+ case NDB_FILTER_RELAYS:
return 0;
case NDB_FILTER_KINDS:
case NDB_FILTER_SINCE:
@@ -981,6 +994,7 @@ int ndb_filter_add_id_element(struct ndb_filter *filter, const unsigned char *id
case NDB_FILTER_LIMIT:
case NDB_FILTER_KINDS:
case NDB_FILTER_SEARCH:
+ case NDB_FILTER_RELAYS:
return 0;
case NDB_FILTER_IDS:
case NDB_FILTER_AUTHORS:
@@ -1086,6 +1100,31 @@ static int compare_ids(const void *pa, const void *pb)
return memcmp(a, b, 32);
}
+static int compare_strs(const void *pa, const void *pb)
+{
+ const char *a = *(const char **)pa;
+ const char *b = *(const char **)pb;
+
+ return strcmp(a, b);
+}
+
+static int search_strs(const void *ctx, const void *mstr_ptr)
+{
+ // we reuse search_id_state here and just cast to (const char *) when
+ // needed
+ struct search_id_state *state;
+ const char *mstr_str;
+ uint32_t mstr;
+
+ state = (struct search_id_state *)ctx;
+ mstr = *(uint32_t *)mstr_ptr;
+
+ mstr_str = (const char *)ndb_filter_elements_data(state->filter, mstr);
+ assert(mstr_str);
+
+ return strcmp((const char *)state->key, mstr_str);
+}
+
static int search_ids(const void *ctx, const void *mid_ptr)
{
struct search_id_state *state;
@@ -1120,7 +1159,8 @@ static int compare_kinds(const void *pa, const void *pb)
//
// returns 1 if a filter matches a note
static int ndb_filter_matches_with(struct ndb_filter *filter,
- struct ndb_note *note, int already_matched)
+ struct ndb_note *note, int already_matched,
+ struct ndb_note_relay_iterator *relay_iter)
{
int i, j;
struct ndb_filter_elements *els;
@@ -1139,11 +1179,27 @@ static int ndb_filter_matches_with(struct ndb_filter *filter,
continue;
switch (els->field.type) {
- case NDB_FILTER_KINDS:
- for (j = 0; j < els->count; j++) {
- if ((unsigned int)els->elements[j] == note->kind)
+ case NDB_FILTER_KINDS:
+ for (j = 0; j < els->count; j++) {
+ if ((unsigned int)els->elements[j] == note->kind)
+ goto cont;
+ }
+ break;
+ case NDB_FILTER_RELAYS:
+ // for each relay the note was seen on, see if any match
+ if (!relay_iter) {
+ assert(!"expected relay iterator...");
+ break;
+ }
+ while ((state.key = (unsigned char *)ndb_note_relay_iterate_next(relay_iter))) {
+ // relays in filters are always sorted
+ if (bsearch(&state, &els->elements[0], els->count,
+ sizeof(els->elements[0]), search_strs)) {
+ ndb_note_relay_iterate_close(relay_iter);
goto cont;
+ }
}
+ ndb_note_relay_iterate_close(relay_iter);
break;
case NDB_FILTER_IDS:
state.key = ndb_note_id(note);
@@ -1201,7 +1257,14 @@ cont:
int ndb_filter_matches(struct ndb_filter *filter, struct ndb_note *note)
{
- return ndb_filter_matches_with(filter, note, 0);
+ return ndb_filter_matches_with(filter, note, 0, NULL);
+}
+
+int ndb_filter_matches_with_relay(struct ndb_filter *filter,
+ struct ndb_note *note,
+ struct ndb_note_relay_iterator *note_relay_iter)
+{
+ return ndb_filter_matches_with(filter, note, 0, note_relay_iter);
}
// because elements are stored as offsets and qsort doesn't support context,
@@ -1298,6 +1361,9 @@ void ndb_filter_end_field(struct ndb_filter *filter)
case NDB_FILTER_AUTHORS:
sort_filter_elements(filter, cur, compare_ids);
break;
+ case NDB_FILTER_RELAYS:
+ sort_filter_elements(filter, cur, compare_strs);
+ break;
case NDB_FILTER_KINDS:
qsort(&cur->elements[0], cur->count,
sizeof(cur->elements[0]), compare_kinds);
@@ -1535,7 +1601,7 @@ static int prepare_relay_buf(char *relay_buf, int bufsize, const char *relay,
// Write to the note_id -> relay_url database. This records where notes
// have been seen
static int ndb_write_note_relay(struct ndb_txn *txn, uint64_t note_key,
- const char *relay, int relay_len)
+ const char *relay, uint8_t relay_len)
{
char relay_buf[256];
int rc, len;
@@ -1575,12 +1641,90 @@ static int ndb_write_note_relay(struct ndb_txn *txn, uint64_t note_key,
return 1;
}
-static int ndb_write_note_relay_kind_index(struct ndb_txn *txn,
- uint64_t kind,
- uint64_t note_key,
- uint64_t created_at,
- const char *relay,
- int relay_len)
+struct ndb_relay_kind_key {
+ uint64_t note_key;
+ uint64_t kind;
+ uint64_t created_at;
+ uint8_t relay_len;
+ const char *relay;
+};
+
+static int ndb_relay_kind_key_init(
+ struct ndb_relay_kind_key *key,
+ uint64_t note_key,
+ uint64_t kind,
+ uint64_t created_at,
+ const char *relay)
+{
+ if (relay == NULL)
+ return 0;
+
+ key->relay = relay;
+ key->relay_len = strlen(relay);
+ if (key->relay_len > 248)
+ return 0;
+
+ key->note_key = note_key;
+ key->kind = kind;
+ key->created_at = created_at;
+ return 1;
+}
+
+
+// create a range key for a relay kind query
+static int ndb_relay_kind_key_init_high(
+ struct ndb_relay_kind_key *key,
+ const char *relay,
+ uint64_t kind,
+ uint64_t until)
+{
+ return ndb_relay_kind_key_init(key, UINT64_MAX, kind, UINT64_MAX, relay);
+}
+
+static void ndb_parse_relay_kind_key(struct ndb_relay_kind_key *key, unsigned char *buf)
+{
+ // WE ARE ASSUMING WE ARE PARSING FROM AN ALIGNED BUFFER HERE
+ assert((uint64_t)buf % 8 == 0);
+ // - note_id: 00 + 8 bytes
+ // - kind: 08 + 8 bytes
+ // - created_at: 16 + 8 bytes
+ // - relay_url_size: 24 + 1 byte
+ // - relay_url: 25 + n byte null-terminated string
+ // - pad to 8 byte alignment
+ key->note_key = *(uint64_t*) (buf + 0);
+ key->kind = *(uint64_t*) (buf + 8);
+ key->created_at = *(uint64_t*) (buf + 16);
+ key->relay_len = *(uint8_t*) (buf + 24);
+ key->relay = (const char*) (buf + 25);
+}
+
+static void ndb_debug_relay_kind_key(struct ndb_relay_kind_key *key)
+{
+ ndb_debug("note_key:%" PRIu64 " kind:%" PRIu64 " created_at:%" PRIu64 " '%s'\n",
+ key->note_key, key->kind, key->created_at, key->relay);
+}
+
+static int ndb_build_relay_kind_key(unsigned char *buf, int bufsize, struct ndb_relay_kind_key *key)
+{
+ struct cursor cur;
+ make_cursor(buf, buf + bufsize, &cur);
+
+ if (!cursor_push(&cur, (unsigned char *)&key->note_key, 8)) return 0;
+ if (!cursor_push(&cur, (unsigned char *)&key->kind, 8)) return 0;
+ if (!cursor_push(&cur, (unsigned char *)&key->created_at, 8)) return 0;
+ if (!cursor_push_byte(&cur, key->relay_len)) return 0;
+ if (!cursor_push(&cur, (unsigned char *)key->relay, key->relay_len)) return 0;
+ if (!cursor_push_byte(&cur, 0)) return 0;
+ if (!cursor_align(&cur, 8)) return 0;
+
+ assert(((cur.p-cur.start)%8) == 0);
+
+ return cur.p - cur.start;
+}
+
+static int ndb_write_note_relay_kind_index(
+ struct ndb_txn *txn,
+ struct ndb_relay_kind_key *key)
{
// The relay kind key has a layout like so
//
@@ -1592,33 +1736,20 @@ static int ndb_write_note_relay_kind_index(struct ndb_txn *txn,
// - pad to 8 byte alignment
unsigned char buf[256];
- int rc;
- struct cursor cur;
+ int rc, len;
MDB_val k, v;
// come on bro
- if (relay_len > 248)
- return 0;
-
- if (relay == NULL || relay_len == 0)
+ if (key->relay_len > 248 || key->relay == NULL || key->relay_len == 0)
return 0;
- ndb_debug("writing note_relay_kind_index '%s' for notekey:%" PRIu64 "\n", relay, note_key);
-
- make_cursor(buf, buf + sizeof(buf), &cur);
-
- if (!cursor_push(&cur, (unsigned char *)¬e_key, 8)) return 0;
- if (!cursor_push(&cur, (unsigned char *)&kind, 8)) return 0;
- if (!cursor_push(&cur, (unsigned char *)&created_at, 8)) return 0;
- if (!cursor_push_byte(&cur, (uint8_t)relay_len)) return 0;
- if (!cursor_push(&cur, (unsigned char *)relay, relay_len)) return 0;
- if (!cursor_push_byte(&cur, 0)) return 0;
- if (!cursor_align(&cur, 8)) return 0;
+ ndb_debug("writing note_relay_kind_index '%s' for notekey:%" PRIu64 "\n", key->relay, key->note_key);
- assert(((cur.p-cur.start)%8) == 0);
+ if (!(len = ndb_build_relay_kind_key(buf, sizeof(buf), key)))
+ return 0;
- k.mv_data = cur.start;
- k.mv_size = cur.p - cur.start;
+ k.mv_data = buf;
+ k.mv_size = len;
v.mv_data = NULL;
v.mv_size = 0;
@@ -1632,6 +1763,14 @@ static int ndb_write_note_relay_kind_index(struct ndb_txn *txn,
return 1;
}
+// writes the relay note kind index and the note_id -> relay db
+static int ndb_write_note_relay_indexes(struct ndb_txn *txn, struct ndb_relay_kind_key *key)
+{
+ ndb_write_note_relay_kind_index(txn, key);
+ ndb_write_note_relay(txn, key->note_key, key->relay, key->relay_len);
+ return 1;
+}
+
static int ndb_write_note_pubkey_index(struct ndb_txn *txn, struct ndb_note *note,
uint64_t note_key)
{
@@ -2212,17 +2351,23 @@ int ndb_write_last_profile_fetch(struct ndb *ndb, const unsigned char *pubkey,
// after the first element, so we have to go back one.
static int ndb_cursor_start(MDB_cursor *cur, MDB_val *k, MDB_val *v)
{
+ int rc;
// Position cursor at the next key greater than or equal to the
// specified key
- if (mdb_cursor_get(cur, k, v, MDB_SET_RANGE)) {
+
+ if ((rc = mdb_cursor_get(cur, k, v, MDB_SET_RANGE))) {
// Failed :(. It could be the last element?
- if (mdb_cursor_get(cur, k, v, MDB_LAST))
+ if ((rc = mdb_cursor_get(cur, k, v, MDB_LAST))) {
+ ndb_debug("MDB_LAST failed: '%s'\n", mdb_strerror(rc));
return 0;
+ }
} else {
// if set range worked and our key exists, it should be
// the one right before this one
- if (mdb_cursor_get(cur, k, v, MDB_PREV))
+ if ((rc = mdb_cursor_get(cur, k, v, MDB_PREV))) {
+ ndb_debug("moving back failed: '%s'\n", mdb_strerror(rc));
return 0;
+ }
}
return 1;
@@ -3415,7 +3560,7 @@ static int ndb_query_plan_execute_ids(struct ndb_txn *txn,
MDB_cursor *cur;
MDB_dbi db;
MDB_val k, v;
- int rc, i;
+ int rc, i, need_relays = 0;
struct ndb_filter_elements *ids;
struct ndb_note *note;
struct ndb_query_result res;
@@ -3423,6 +3568,7 @@ static int ndb_query_plan_execute_ids(struct ndb_txn *txn,
uint64_t note_id, until, *pint;
size_t note_size;
unsigned char *id;
+ struct ndb_note_relay_iterator note_relay_iter;
until = UINT64_MAX;
@@ -3432,6 +3578,9 @@ static int ndb_query_plan_execute_ids(struct ndb_txn *txn,
if ((pint = ndb_filter_get_int(filter, NDB_FILTER_UNTIL)))
until = *pint;
+ if (ndb_filter_find_elements(filter, NDB_FILTER_RELAYS))
+ need_relays = 1;
+
db = txn->lmdb->dbs[NDB_DB_NOTE_ID];
if ((rc = mdb_cursor_open(txn->mdb_txn, db, &cur)))
return 0;
@@ -3460,12 +3609,16 @@ static int ndb_query_plan_execute_ids(struct ndb_txn *txn,
if (!(note = ndb_get_note_by_key(txn, note_id, ¬e_size)))
continue;
+ if (need_relays)
+ ndb_note_relay_iterate_start(txn, ¬e_relay_iter, note_id);
+
// Sure this particular lookup matched the index query, but
// does it match the entire filter? Check! We also pass in
// things we've already matched via the filter so we don't have
// to check again. This can be pretty important for filters
// with a large number of entries.
- if (!ndb_filter_matches_with(filter, note, 1 << NDB_FILTER_IDS))
+ if (!ndb_filter_matches_with(filter, note, 1 << NDB_FILTER_IDS,
+ need_relays ? ¬e_relay_iter : NULL))
continue;
ndb_query_result_init(&res, note, note_size, note_id);
@@ -3521,7 +3674,7 @@ static int ndb_query_plan_execute_authors(struct ndb_txn *txn,
{
MDB_val k, v;
MDB_cursor *cur;
- int rc, i;
+ int rc, i, need_relays = 0;
uint64_t *pint, until, since, note_key;
unsigned char *author;
struct ndb_note *note;
@@ -3529,6 +3682,7 @@ static int ndb_query_plan_execute_authors(struct ndb_txn *txn,
struct ndb_filter_elements *authors;
struct ndb_query_result res;
struct ndb_tsid tsid, *ptsid;
+ struct ndb_note_relay_iterator note_relay_iter;
enum ndb_dbs db;
db = txn->lmdb->dbs[NDB_DB_NOTE_PUBKEY];
@@ -3544,6 +3698,9 @@ static int ndb_query_plan_execute_authors(struct ndb_txn *txn,
if ((pint = ndb_filter_get_int(filter, NDB_FILTER_SINCE)))
since = *pint;
+ if (ndb_filter_find_elements(filter, NDB_FILTER_RELAYS))
+ need_relays = 1;
+
if ((rc = mdb_cursor_open(txn->mdb_txn, db, &cur)))
return 0;
@@ -3576,8 +3733,15 @@ static int ndb_query_plan_execute_authors(struct ndb_txn *txn,
if (!(note = ndb_get_note_by_key(txn, note_key, ¬e_size)))
goto next;
- if (!ndb_filter_matches_with(filter, note, 1 << NDB_FILTER_AUTHORS))
+ if (need_relays)
+ ndb_note_relay_iterate_start(txn, ¬e_relay_iter, note_key);
+
+ if (!ndb_filter_matches_with(filter, note,
+ 1 << NDB_FILTER_AUTHORS,
+ need_relays ? ¬e_relay_iter : NULL))
+ {
goto next;
+ }
ndb_query_result_init(&res, note, note_size, note_key);
if (!push_query_result(results, &res))
@@ -3601,12 +3765,13 @@ static int ndb_query_plan_execute_created_at(struct ndb_txn *txn,
MDB_dbi db;
MDB_val k, v;
MDB_cursor *cur;
- int rc;
+ int rc, need_relays = 0;
struct ndb_note *note;
struct ndb_tsid key, *pkey;
uint64_t *pint, until, since, note_id;
size_t note_size;
struct ndb_query_result res;
+ struct ndb_note_relay_iterator note_relay_iter;
unsigned char high_key[32] = {0xFF};
db = txn->lmdb->dbs[NDB_DB_NOTE_ID];
@@ -3619,6 +3784,9 @@ static int ndb_query_plan_execute_created_at(struct ndb_txn *txn,
if ((pint = ndb_filter_get_int(filter, NDB_FILTER_SINCE)))
since = *pint;
+ if (ndb_filter_find_elements(filter, NDB_FILTER_RELAYS))
+ need_relays = 1;
+
if ((rc = mdb_cursor_open(txn->mdb_txn, db, &cur)))
return 0;
@@ -3642,8 +3810,11 @@ static int ndb_query_plan_execute_created_at(struct ndb_txn *txn,
if (!(note = ndb_get_note_by_key(txn, note_id, ¬e_size)))
goto next;
+ if (need_relays)
+ ndb_note_relay_iterate_start(txn, ¬e_relay_iter, note_id);
+
// does this entry match our filter?
- if (!ndb_filter_matches_with(filter, note, 0))
+ if (!ndb_filter_matches_with(filter, note, 0, need_relays ? ¬e_relay_iter : NULL))
goto next;
ndb_query_result_init(&res, note, (uint64_t)note_size, note_id);
@@ -3666,7 +3837,7 @@ static int ndb_query_plan_execute_tags(struct ndb_txn *txn,
MDB_cursor *cur;
MDB_dbi db;
MDB_val k, v;
- int len, taglen, rc, i;
+ int len, taglen, rc, i, need_relays = 0;
uint64_t *pint, until, note_id;
size_t note_size;
unsigned char key_buffer[255];
@@ -3674,12 +3845,16 @@ static int ndb_query_plan_execute_tags(struct ndb_txn *txn,
struct ndb_filter_elements *tags;
unsigned char *tag;
struct ndb_query_result res;
+ struct ndb_note_relay_iterator note_relay_iter;
db = txn->lmdb->dbs[NDB_DB_NOTE_TAGS];
if (!(tags = ndb_filter_find_elements(filter, NDB_FILTER_TAGS)))
return 0;
+ if (ndb_filter_find_elements(filter, NDB_FILTER_RELAYS))
+ need_relays = 1;
+
until = UINT64_MAX;
if ((pint = ndb_filter_get_int(filter, NDB_FILTER_UNTIL)))
until = *pint;
@@ -3722,7 +3897,12 @@ static int ndb_query_plan_execute_tags(struct ndb_txn *txn,
if (!(note = ndb_get_note_by_key(txn, note_id, ¬e_size)))
goto next;
- if (!ndb_filter_matches_with(filter, note, 1 << NDB_FILTER_TAGS))
+ if (need_relays)
+ ndb_note_relay_iterate_start(txn, ¬e_relay_iter, note_id);
+
+ if (!ndb_filter_matches_with(filter, note,
+ 1 << NDB_FILTER_TAGS,
+ need_relays ? ¬e_relay_iter : NULL))
goto next;
ndb_query_result_init(&res, note, note_size, note_id);
@@ -3739,6 +3919,120 @@ next:
return 1;
}
+static int ndb_query_plan_execute_relay_kinds(
+ struct ndb_txn *txn,
+ struct ndb_filter *filter,
+ struct ndb_query_results *results,
+ int limit)
+{
+ MDB_cursor *cur;
+ MDB_dbi db;
+ MDB_val k, v;
+ struct ndb_note *note;
+ struct ndb_filter_elements *kinds, *relays;
+ struct ndb_query_result res;
+ uint64_t kind, note_id, until, since, *pint;
+ size_t note_size;
+ const char *relay;
+ int i, j, rc, len;
+ struct ndb_relay_kind_key relay_key;
+ unsigned char keybuf[256];
+
+ // we should have kinds in a kinds filter!
+ if (!(kinds = ndb_filter_find_elements(filter, NDB_FILTER_KINDS)))
+ return 0;
+
+ if (!(relays = ndb_filter_find_elements(filter, NDB_FILTER_RELAYS)))
+ return 0;
+
+ until = UINT64_MAX;
+ if ((pint = ndb_filter_get_int(filter, NDB_FILTER_UNTIL)))
+ until = *pint;
+
+ since = 0;
+ if ((pint = ndb_filter_get_int(filter, NDB_FILTER_SINCE)))
+ since = *pint;
+
+ db = txn->lmdb->dbs[NDB_DB_NOTE_RELAY_KIND];
+
+ if ((rc = mdb_cursor_open(txn->mdb_txn, db, &cur)))
+ return 0;
+
+ for (j = 0; j < relays->count; j++) {
+ if (query_is_full(results, limit))
+ break;
+
+ if (!(relay = ndb_filter_get_string_element(filter, relays, j)))
+ continue;
+
+ for (i = 0; i < kinds->count; i++) {
+ if (query_is_full(results, limit))
+ break;
+
+ kind = kinds->elements[i];
+ ndb_debug("kind %" PRIu64 "\n", kind);
+
+ if (!ndb_relay_kind_key_init_high(&relay_key, relay, kind, until)) {
+ ndb_debug("ndb_relay_kind_key_init_high failed in relay query\n");
+ continue;
+ }
+
+ if (!(len = ndb_build_relay_kind_key(keybuf, sizeof(keybuf), &relay_key))) {
+ ndb_debug("ndb_build_relay_kind_key failed in relay query\n");
+ ndb_debug_relay_kind_key(&relay_key);
+ continue;
+ }
+
+ k.mv_data = keybuf;
+ k.mv_size = len;
+
+ ndb_debug("starting with key ");
+ ndb_debug_relay_kind_key(&relay_key);
+
+ if (!ndb_cursor_start(cur, &k, &v))
+ continue;
+
+ // scan the kind subindex
+ while (!query_is_full(results, limit)) {
+ ndb_parse_relay_kind_key(&relay_key, k.mv_data);
+
+ ndb_debug("inside kind subindex ");
+ ndb_debug_relay_kind_key(&relay_key);
+
+ if (relay_key.kind != kind)
+ break;
+
+ if (strcmp(relay_key.relay, relay))
+ break;
+
+ // don't continue the scan if we're below `since`
+ if (relay_key.created_at < since)
+ break;
+
+ note_id = relay_key.note_key;
+ if (!(note = ndb_get_note_by_key(txn, note_id, ¬e_size)))
+ goto next;
+
+ if (!ndb_filter_matches_with(filter, note,
+ (1 << NDB_FILTER_KINDS) | (1 << NDB_FILTER_RELAYS),
+ NULL))
+ goto next;
+
+ ndb_query_result_init(&res, note, note_size, note_id);
+ if (!push_query_result(results, &res))
+ break;
+
+next:
+ if (mdb_cursor_get(cur, &k, &v, MDB_PREV))
+ break;
+ }
+ }
+ }
+
+ mdb_cursor_close(cur);
+ return 1;
+}
+
static int ndb_query_plan_execute_kinds(struct ndb_txn *txn,
struct ndb_filter *filter,
struct ndb_query_results *results,
@@ -3753,12 +4047,16 @@ static int ndb_query_plan_execute_kinds(struct ndb_txn *txn,
struct ndb_query_result res;
uint64_t kind, note_id, until, since, *pint;
size_t note_size;
- int i, rc;
+ int i, rc, need_relays = 0;
+ struct ndb_note_relay_iterator note_relay_iter;
// we should have kinds in a kinds filter!
if (!(kinds = ndb_filter_find_elements(filter, NDB_FILTER_KINDS)))
return 0;
+ if (ndb_filter_find_elements(filter, NDB_FILTER_RELAYS))
+ need_relays = 1;
+
until = UINT64_MAX;
if ((pint = ndb_filter_get_int(filter, NDB_FILTER_UNTIL)))
until = *pint;
@@ -3800,7 +4098,12 @@ static int ndb_query_plan_execute_kinds(struct ndb_txn *txn,
if (!(note = ndb_get_note_by_key(txn, note_id, ¬e_size)))
goto next;
- if (!ndb_filter_matches_with(filter, note, 1 << NDB_FILTER_KINDS))
+ if (need_relays)
+ ndb_note_relay_iterate_start(txn, ¬e_relay_iter, note_id);
+
+ if (!ndb_filter_matches_with(filter, note,
+ 1 << NDB_FILTER_KINDS,
+ need_relays ? ¬e_relay_iter : NULL))
goto next;
ndb_query_result_init(&res, note, note_size, note_id);
@@ -3819,19 +4122,22 @@ next:
static enum ndb_query_plan ndb_filter_plan(struct ndb_filter *filter)
{
- struct ndb_filter_elements *ids, *kinds, *authors, *tags, *search;
+ struct ndb_filter_elements *ids, *kinds, *authors, *tags, *search, *relays;
ids = ndb_filter_find_elements(filter, NDB_FILTER_IDS);
search = ndb_filter_find_elements(filter, NDB_FILTER_SEARCH);
kinds = ndb_filter_find_elements(filter, NDB_FILTER_KINDS);
authors = ndb_filter_find_elements(filter, NDB_FILTER_AUTHORS);
tags = ndb_filter_find_elements(filter, NDB_FILTER_TAGS);
+ relays = ndb_filter_find_elements(filter, NDB_FILTER_RELAYS);
// this is rougly similar to the heuristic in strfry's dbscan
if (search) {
return NDB_PLAN_SEARCH;
} else if (ids) {
return NDB_PLAN_IDS;
+ } else if (relays && kinds && !authors) {
+ return NDB_PLAN_RELAY_KINDS;
} else if (kinds && authors && authors->count <= 10) {
return NDB_PLAN_AUTHOR_KINDS;
} else if (authors && authors->count <= 10) {
@@ -3845,7 +4151,7 @@ static enum ndb_query_plan ndb_filter_plan(struct ndb_filter *filter)
return NDB_PLAN_CREATED;
}
-static const char *ndb_query_plan_name(int plan_id)
+static const char *ndb_query_plan_name(enum ndb_query_plan plan_id)
{
switch (plan_id) {
case NDB_PLAN_IDS: return "ids";
@@ -3854,6 +4160,8 @@ static const char *ndb_query_plan_name(int plan_id)
case NDB_PLAN_TAGS: return "tags";
case NDB_PLAN_CREATED: return "created";
case NDB_PLAN_AUTHORS: return "authors";
+ case NDB_PLAN_RELAY_KINDS: return "relay_kinds";
+ case NDB_PLAN_AUTHOR_KINDS: return "author_kinds";
}
return "unknown";
@@ -3884,18 +4192,19 @@ static int ndb_query_filter(struct ndb_txn *txn, struct ndb_filter *filter,
if (!ndb_query_plan_execute_ids(txn, filter, &results, limit))
return 0;
break;
-
+ case NDB_PLAN_RELAY_KINDS:
+ if (!ndb_query_plan_execute_relay_kinds(txn, filter, &results, limit))
+ return 0;
+ break;
case NDB_PLAN_SEARCH:
if (!ndb_query_plan_execute_search(txn, filter, &results, limit))
return 0;
break;
-
// We have just kinds, just scan the kind index
case NDB_PLAN_KINDS:
if (!ndb_query_plan_execute_kinds(txn, filter, &results, limit))
return 0;
break;
-
case NDB_PLAN_TAGS:
if (!ndb_query_plan_execute_tags(txn, filter, &results, limit))
return 0;
@@ -4671,23 +4980,18 @@ static uint64_t ndb_write_note(struct ndb_txn *txn,
unsigned char *scratch, size_t scratch_size,
uint32_t ndb_flags)
{
- int rc, relay_len = 0;
+ int rc;
uint64_t note_key, kind;
+ struct ndb_relay_kind_key relay_key;
MDB_dbi note_db;
MDB_val key, val;
kind = note->note->kind;
- if (note->relay != NULL)
- relay_len = strlen(note->relay);
-
// let's quickly sanity check if we already have this note
if ((note_key = ndb_get_notekey_by_id(txn, note->note->id))) {
- // even if we do we still need to write relay index
- ndb_write_note_relay_kind_index(txn, kind, note_key,
- ndb_note_created_at(note->note),
- note->relay, relay_len);
- ndb_write_note_relay(txn, note_key, note->relay, relay_len);
+ if (ndb_relay_kind_key_init(&relay_key, note_key, kind, ndb_note_created_at(note->note), note->relay))
+ ndb_write_note_relay_indexes(txn, &relay_key);
return 0;
}
@@ -4713,10 +5017,9 @@ static uint64_t ndb_write_note(struct ndb_txn *txn,
ndb_write_note_tag_index(txn, note->note, note_key);
ndb_write_note_pubkey_index(txn, note->note, note_key);
ndb_write_note_pubkey_kind_index(txn, note->note, note_key);
- ndb_write_note_relay_kind_index(txn, kind, note_key,
- ndb_note_created_at(note->note),
- note->relay, relay_len);
- ndb_write_note_relay(txn, note_key, note->relay, relay_len);
+
+ if (ndb_relay_kind_key_init(&relay_key, note_key, kind, ndb_note_created_at(note->note), note->relay))
+ ndb_write_note_relay_indexes(txn, &relay_key);
// only parse content and do fulltext index on text and longform notes
if (kind == 1 || kind == 30023) {
@@ -4890,7 +5193,7 @@ static void *ndb_writer_thread(void *data)
struct ndb_writer *writer = data;
struct ndb_writer_msg msgs[THREAD_QUEUE_BATCH], *msg;
struct written_note written_notes[THREAD_QUEUE_BATCH];
- int i, popped, done, relay_len, needs_commit, num_notes;
+ int i, popped, done, needs_commit, num_notes;
uint64_t note_nkey;
struct ndb_txn txn;
unsigned char *scratch;
@@ -4973,18 +5276,15 @@ static void *ndb_writer_thread(void *data)
}
break;
case NDB_WRITER_NOTE_RELAY:
- relay_len = strlen(msg->note_relay.relay);
- ndb_write_note_relay(&txn,
- msg->note_relay.note_key,
- msg->note_relay.relay,
- relay_len);
- ndb_write_note_relay_kind_index(
- &txn,
- msg->note_relay.kind,
- msg->note_relay.note_key,
- msg->note_relay.created_at,
- msg->note_relay.relay,
- relay_len);
+ struct ndb_relay_kind_key relay_key;
+ if (ndb_relay_kind_key_init(&relay_key,
+ msg->note_relay.note_key,
+ msg->note_relay.kind,
+ msg->note_relay.created_at,
+ msg->note_relay.relay))
+ {
+ ndb_write_note_relay_indexes(&txn, &relay_key);
+ }
break;
case NDB_WRITER_DBMETA:
ndb_write_version(&txn, msg->ndb_meta.version);
@@ -5058,7 +5358,7 @@ static void *ndb_ingester_thread(void *data)
int rc;
ctx = secp256k1_context_create(SECP256K1_CONTEXT_VERIFY);
- ndb_debug("started ingester thread\n");
+ //ndb_debug("started ingester thread\n");
done = 0;
while (!done) {
@@ -6040,6 +6340,12 @@ int ndb_filter_json(const struct ndb_filter *filter, char *buf, int buflen)
if (!cursor_push_int_str(c, ndb_filter_get_int_element(elems, 0)))
return 0;
break;
+ case NDB_FILTER_RELAYS:
+ if (!cursor_push_str(c, "\"relays\":"))
+ return 0;
+ if (!cursor_push_json_elem_array(c, filter, elems))
+ return 0;
+ break;
}
if (i != filter->num_elements-1) {
@@ -6868,6 +7174,7 @@ static int ndb_filter_parse_json(struct ndb_json_parser *parser,
return 0;
}
break;
+ case NDB_FILTER_RELAYS:
case NDB_FILTER_TAGS:
if (!ndb_filter_parse_json_elems(parser, filter)) {
ndb_debug("failed to parse filter tags\n");
@@ -7210,7 +7517,7 @@ int ndb_print_relay_kind_index(struct ndb_txn *txn)
printf("relay\tkind\tcreated_at\tnote_id\n");
while (mdb_cursor_get(cur, &k, &v, MDB_NEXT) == 0) {
d = (unsigned char *)k.mv_data;
- printf("'%s'\t", (const char *)(d + 25));
+ printf("%s\t", (const char *)(d + 25));
printf("%" PRIu64 "\t", *(uint64_t*)(d + 8));
printf("%" PRIu64 "\t", *(uint64_t*)(d + 16));
printf("%" PRIu64 "\n", *(uint64_t*)(d + 0));
diff --git a/src/nostrdb.h b/src/nostrdb.h
@@ -162,6 +162,7 @@ enum ndb_filter_fieldtype {
NDB_FILTER_UNTIL = 6,
NDB_FILTER_LIMIT = 7,
NDB_FILTER_SEARCH = 8,
+ NDB_FILTER_RELAYS = 9,
};
#define NDB_NUM_FILTERS 7
@@ -564,6 +565,7 @@ struct ndb_filter_elements *ndb_filter_get_elements(const struct ndb_filter *, i
int ndb_filter_start_field(struct ndb_filter *, enum ndb_filter_fieldtype);
int ndb_filter_start_tag_field(struct ndb_filter *, char tag);
int ndb_filter_matches(struct ndb_filter *, struct ndb_note *);
+int ndb_filter_matches_with_relay(struct ndb_filter *, struct ndb_note *, struct ndb_note_relay_iterator *iter);
int ndb_filter_clone(struct ndb_filter *dst, struct ndb_filter *src);
int ndb_filter_end(struct ndb_filter *);
void ndb_filter_end_field(struct ndb_filter *);
