commit 034f2cc02f7966674b0171d871e783e2aae1a206
parent 6f9bd6c4f4348c5c4f485d41331db275f6fbd33b
Author: William Casarin <jb55@jb55.com>
Date: Thu, 28 Dec 2023 13:54:22 -0800
nostrdb/blocks: add note block iterator
This adds an api that walks along and pulls compact note block data out of
nostrdb.
Signed-off-by: William Casarin <jb55@jb55.com>
Diffstat:
8 files changed, 248 insertions(+), 153 deletions(-)
diff --git a/nostrdb/Makefile b/nostrdb/Makefile
@@ -2,7 +2,7 @@ CFLAGS = -Wall -Wno-misleading-indentation -Wno-unused-function -Werror -O2 -g -
HEADERS = deps/lmdb/lmdb.h deps/secp256k1/include/secp256k1.h src/sha256.h src/nostrdb.h src/cursor.h src/hex.h src/jsmn.h src/config.h src/sha256.h src/random.h src/memchr.h src/cpu.h src/nostr_bech32.h src/block.h src/str_block.h $(C_BINDINGS)
FLATCC_SRCS=deps/flatcc/src/runtime/json_parser.c deps/flatcc/src/runtime/verifier.c deps/flatcc/src/runtime/builder.c deps/flatcc/src/runtime/emitter.c deps/flatcc/src/runtime/refmap.c
BOLT11_SRCS = src/bolt11/bolt11.c src/bolt11/bech32.c src/bolt11/tal.c src/bolt11/talstr.c src/bolt11/take.c src/bolt11/list.c src/bolt11/utf8.c src/bolt11/amount.c src/bolt11/hash_u5.c
-SRCS = src/nostrdb.c src/sha256.c src/invoice.c src/nostr_bech32.c src/content_parser.c $(BOLT11_SRCS) $(FLATCC_SRCS)
+SRCS = src/nostrdb.c src/sha256.c src/invoice.c src/nostr_bech32.c src/content_parser.c src/block.c $(BOLT11_SRCS) $(FLATCC_SRCS)
LDS = $(OBJS) $(ARS)
OBJS = $(SRCS:.c=.o)
DEPS = $(OBJS) $(HEADERS) $(ARS)
diff --git a/nostrdb/src/block.c b/nostrdb/src/block.c
@@ -0,0 +1,184 @@
+
+
+#include "nostrdb.h"
+#include "block.h"
+#include <stdlib.h>
+
+struct ndb_block_iterator {
+ const char *content;
+ struct ndb_blocks *blocks;
+ struct ndb_block block;
+ struct cursor cur;
+};
+
+int push_str_block(struct cursor *buf, const char *content, struct ndb_str_block *block) {
+ return cursor_push_varint(buf, block->str - content) &&
+ cursor_push_varint(buf, block->len);
+}
+
+int pull_str_block(struct cursor *buf, const char *content, struct ndb_str_block *block) {
+ uint32_t start;
+ if (!cursor_pull_varint_u32(buf, &start))
+ return 0;
+
+ block->str = content + start;
+
+ return cursor_pull_varint_u32(buf, &block->len);
+}
+
+static int pull_nostr_bech32_type(struct cursor *cur, enum nostr_bech32_type *type)
+{
+ uint64_t inttype;
+ if (!cursor_pull_varint(cur, &inttype))
+ return 0;
+
+ if (inttype > NOSTR_BECH32_KNOWN_TYPES)
+ return 0;
+
+ *type = inttype;
+ return 1;
+}
+
+
+static int pull_bech32_mention(const char *content, struct cursor *cur, struct ndb_mention_bech32_block *block) {
+ uint16_t size;
+ unsigned char *start;
+ enum nostr_bech32_type type;
+
+ start = cur->p;
+
+ if (!pull_str_block(cur, content, &block->str))
+ return 0;
+
+ if (!cursor_pull_u16(cur, &size))
+ return 0;
+
+ if (!pull_nostr_bech32_type(cur, &type))
+ return 0;
+
+ if (!parse_nostr_bech32_buffer(cur, type, &block->bech32))
+ return 0;
+
+ cur->p = start + size;
+ return 1;
+}
+
+static int pull_invoice(const char *content, struct cursor *cur,
+ struct ndb_invoice_block *block)
+{
+ if (!pull_str_block(cur, content, &block->invstr))
+ return 0;
+
+ return ndb_decode_invoice(cur, &block->invoice);
+}
+
+static int pull_block_type(struct cursor *cur, enum ndb_block_type *type)
+{
+ uint32_t itype;
+ *type = 0;
+ if (!cursor_pull_varint_u32(cur, &itype))
+ return 0;
+
+ if (itype <= 0 || itype > NDB_NUM_BLOCK_TYPES)
+ return 0;
+
+ *type = itype;
+ return 1;
+}
+
+static int pull_block(const char *content, struct cursor *cur, struct ndb_block *block)
+{
+ unsigned char *start = cur->p;
+
+ if (!pull_block_type(cur, &block->type))
+ return 0;
+
+ switch (block->type) {
+ case BLOCK_HASHTAG:
+ case BLOCK_TEXT:
+ case BLOCK_URL:
+ if (!pull_str_block(cur, content, &block->block.str))
+ goto fail;
+ break;
+
+ case BLOCK_MENTION_INDEX:
+ if (!cursor_pull_varint_u32(cur, &block->block.mention_index))
+ goto fail;
+ break;
+
+ case BLOCK_MENTION_BECH32:
+ if (!pull_bech32_mention(content, cur, &block->block.mention_bech32))
+ goto fail;
+ break;
+
+ case BLOCK_INVOICE:
+ // we only push invoice strs here
+ if (!pull_invoice(content, cur, &block->block.invoice))
+ goto fail;
+ break;
+ }
+
+ return 1;
+fail:
+ cur->p = start;
+ return 0;
+}
+
+
+enum ndb_block_type ndb_get_block_type(struct ndb_block *block) {
+ return block->type;
+}
+
+// BLOCK ITERATORS
+struct ndb_block_iterator *ndb_blocks_iterate_start(const char *content, struct ndb_blocks *blocks) {
+ struct ndb_block_iterator *iter = malloc(sizeof(*iter));
+ if (!iter)
+ return NULL;
+
+ iter->blocks = blocks;
+ iter->content = content;
+
+ make_cursor((unsigned char *)blocks->blocks,
+ blocks->blocks + blocks->blocks_size, &iter->cur);
+
+ return iter;
+}
+
+void ndb_blocks_iterate_free(struct ndb_block_iterator *iter)
+{
+ if (iter)
+ free(iter);
+}
+
+struct ndb_block *ndb_blocks_iterate_next(struct ndb_block_iterator *iter)
+{
+ while (iter->cur.p < iter->cur.end) {
+ if (!pull_block(iter->content, &iter->cur, &iter->block))
+ return NULL;
+ else
+ return &iter->block;
+ }
+
+ return NULL;
+}
+
+// STR BLOCKS
+struct ndb_str_block *ndb_block_str(struct ndb_block *block)
+{
+ switch (block->type) {
+ case BLOCK_HASHTAG:
+ case BLOCK_TEXT:
+ case BLOCK_URL:
+ return &block->block.str;
+ case BLOCK_MENTION_INDEX:
+ return NULL;
+ case BLOCK_MENTION_BECH32:
+ return &block->block.mention_bech32.str;
+ case BLOCK_INVOICE:
+ return &block->block.invoice.invstr;
+ }
+
+ return NULL;
+}
+//const char *ndb_str_block_ptr(struct ndb_str_block *);
+//uint32_t ndb_str_block_len(struct ndb_str_block *);
diff --git a/nostrdb/src/block.h b/nostrdb/src/block.h
@@ -4,12 +4,14 @@
#include "invoice.h"
#include "str_block.h"
+#include "cursor.h"
#include "nostr_bech32.h"
+#include "nostrdb.h"
#include <inttypes.h>
#pragma pack(push, 1)
-struct ndb_note_blocks {
+struct ndb_blocks {
unsigned char version;
unsigned char padding[3];
@@ -17,41 +19,34 @@ struct ndb_note_blocks {
uint32_t num_blocks;
uint32_t blocks_size;
// future expansion
- uint32_t reserved[4];
+ uint32_t reserved[2];
unsigned char blocks[0]; // see ndb_block definition
};
#pragma pack(pop)
-enum block_type {
- BLOCK_HASHTAG = 1,
- BLOCK_TEXT = 2,
- BLOCK_MENTION_INDEX = 3,
- BLOCK_MENTION_BECH32 = 4,
- BLOCK_URL = 5,
- BLOCK_INVOICE = 6,
-};
-
-
struct ndb_mention_bech32_block {
- struct str_block str;
+ struct ndb_str_block str;
struct nostr_bech32 bech32;
};
struct ndb_invoice_block {
- struct str_block invstr;
+ struct ndb_str_block invstr;
struct ndb_invoice invoice;
};
-struct note_block {
- enum block_type type;
+struct ndb_block {
+ enum ndb_block_type type;
union {
- struct str_block str;
+ struct ndb_str_block str;
struct ndb_invoice_block invoice;
struct ndb_mention_bech32_block mention_bech32;
uint32_t mention_index;
} block;
};
+int push_str_block(struct cursor *buf, const char *content, struct ndb_str_block *block);
+int pull_str_block(struct cursor *buf, const char *content, struct ndb_str_block *block);
+
#endif // NDB_BLOCK_H
diff --git a/nostrdb/src/content_parser.c b/nostrdb/src/content_parser.c
@@ -1,6 +1,7 @@
#include "cursor.h"
#include "nostr_bech32.h"
#include "block.h"
+#include "nostrdb.h"
#include "invoice.h"
#include "bolt11/bolt11.h"
#include "bolt11/bech32.h"
@@ -8,16 +9,14 @@
#include <string.h>
#include "cursor.h"
-#include "block.h"
struct ndb_content_parser {
int bech32_strs;
struct cursor buffer;
struct cursor content;
- struct ndb_note_blocks *blocks;
+ struct ndb_blocks *blocks;
};
-
static int parse_digit(struct cursor *cur, int *digit) {
int c;
if ((c = peek_char(cur, 0)) == -1)
@@ -34,7 +33,7 @@ static int parse_digit(struct cursor *cur, int *digit) {
}
-static int parse_mention_index(struct cursor *cur, struct note_block *block) {
+static int parse_mention_index(struct cursor *cur, struct ndb_block *block) {
int d1, d2, d3, ind;
unsigned char *start = cur->p;
@@ -65,7 +64,7 @@ static int parse_mention_index(struct cursor *cur, struct note_block *block) {
return 1;
}
-static int parse_hashtag(struct cursor *cur, struct note_block *block) {
+static int parse_hashtag(struct cursor *cur, struct ndb_block *block) {
int c;
unsigned char *start = cur->p;
@@ -87,21 +86,6 @@ static int parse_hashtag(struct cursor *cur, struct note_block *block) {
return 1;
}
-static int push_str_block(struct cursor *buf, const char *content, struct str_block *block) {
- return cursor_push_varint(buf, block->str - content) &&
- cursor_push_varint(buf, block->len);
-}
-
-static int pull_str_block(struct cursor *buf, const char *content, struct str_block *block) {
- uint32_t start;
- if (!cursor_pull_varint_u32(buf, &start))
- return 0;
-
- block->str = content + start;
-
- return cursor_pull_varint_u32(buf, &block->len);
-}
-
//
// decode and push a bech32 mention into our blocks output buffer.
//
@@ -116,7 +100,7 @@ static int pull_str_block(struct cursor *buf, const char *content, struct str_bl
// This allows us to not duplicate all of the TLV encoding and decoding code
// for our on-disk nostrdb format.
//
-static int push_bech32_mention(struct ndb_content_parser *p, struct str_block *bech32)
+static int push_bech32_mention(struct ndb_content_parser *p, struct ndb_str_block *bech32)
{
// we decode the raw bech32 directly into the output buffer
struct cursor u8, u5;
@@ -178,7 +162,7 @@ fail:
return 0;
}
-static int push_invoice_str(struct ndb_content_parser *p, struct str_block *str)
+static int push_invoice_str(struct ndb_content_parser *p, struct ndb_str_block *str)
{
unsigned char *start;
struct bolt11 *bolt11;
@@ -204,97 +188,10 @@ static int push_invoice_str(struct ndb_content_parser *p, struct str_block *str)
return 1;
}
-static int pull_nostr_bech32_type(struct cursor *cur, enum nostr_bech32_type *type)
-{
- uint64_t inttype;
- if (!cursor_pull_varint(cur, &inttype))
- return 0;
-
- if (inttype > NOSTR_BECH32_KNOWN_TYPES)
- return 0;
-
- *type = inttype;
- return 1;
-}
-
-static int pull_bech32_mention(const char *content, struct cursor *cur,
- struct ndb_mention_bech32_block *block) {
- uint16_t size;
- unsigned char *start;
- enum nostr_bech32_type type;
-
- start = cur->p;
-
- if (!pull_str_block(cur, content, &block->str))
- return 0;
-
- if (!cursor_pull_u16(cur, &size))
- return 0;
-
- if (!pull_nostr_bech32_type(cur, &type))
- return 0;
-
- if (!parse_nostr_bech32_buffer(cur, type, &block->bech32))
- return 0;
-
- cur->p = start + size;
- return 1;
-}
-
-static int pull_invoice(const char *content, struct cursor *cur,
- struct ndb_invoice_block *block)
-{
- if (!pull_str_block(cur, content, &block->invstr))
- return 0;
-
- return ndb_decode_invoice(cur, &block->invoice);
-}
-
-static int pull_block(const char *content, struct cursor *cur, struct note_block *block)
-{
- unsigned char *start = cur->p;
- uint32_t type;
-
- if (!cursor_pull_varint_u32(cur, &type))
- return 0;
-
- block->type = type;
-
- switch (block->type) {
- case BLOCK_HASHTAG:
- case BLOCK_TEXT:
- case BLOCK_URL:
- if (!pull_str_block(cur, content, &block->block.str))
- goto fail;
- break;
-
- case BLOCK_MENTION_INDEX:
- if (!cursor_pull_varint_u32(cur, &block->block.mention_index))
- goto fail;
- break;
-
- case BLOCK_MENTION_BECH32:
- if (!pull_bech32_mention(content, cur, &block->block.mention_bech32))
- goto fail;
- break;
-
- case BLOCK_INVOICE:
- // we only push invoice strs here
- if (!pull_invoice(content, cur, &block->block.invoice))
- goto fail;
- break;
- }
-
- return 1;
-fail:
- cur->p = start;
- return 0;
-}
-
-int push_block(struct ndb_content_parser *p, struct note_block *block);
+int push_block(struct ndb_content_parser *p, struct ndb_block *block);
static int add_text_block(struct ndb_content_parser *p, const char *start, const char *end)
{
- struct note_block b;
+ struct ndb_block b;
if (start == end)
return 1;
@@ -307,7 +204,7 @@ static int add_text_block(struct ndb_content_parser *p, const char *start, const
}
-int push_block(struct ndb_content_parser *p, struct note_block *block)
+int push_block(struct ndb_content_parser *p, struct ndb_block *block)
{
unsigned char *start = p->buffer.p;
@@ -473,7 +370,7 @@ static int consume_url_host(struct cursor *cur)
return count != 0;
}
-static int parse_url(struct cursor *cur, struct note_block *block) {
+static int parse_url(struct cursor *cur, struct ndb_block *block) {
unsigned char *start = cur->p;
unsigned char *host;
unsigned char tmp[4096];
@@ -555,7 +452,7 @@ static int parse_url(struct cursor *cur, struct note_block *block) {
return 1;
}
-static int parse_invoice(struct cursor *cur, struct note_block *block) {
+static int parse_invoice(struct cursor *cur, struct ndb_block *block) {
unsigned char *start, *end;
// optional
@@ -584,7 +481,7 @@ static int parse_invoice(struct cursor *cur, struct note_block *block) {
}
-static int parse_mention_bech32(struct cursor *cur, struct note_block *block) {
+static int parse_mention_bech32(struct cursor *cur, struct ndb_block *block) {
unsigned char *start = cur->p;
enum nostr_bech32_type type;
@@ -605,7 +502,7 @@ static int parse_mention_bech32(struct cursor *cur, struct note_block *block) {
}
static int add_text_then_block(struct ndb_content_parser *p,
- struct note_block *block,
+ struct ndb_block *block,
unsigned char **start,
const unsigned char *pre_mention)
{
@@ -619,19 +516,19 @@ static int add_text_then_block(struct ndb_content_parser *p,
int ndb_parse_content(unsigned char *buf, int buf_size,
const char *content, int content_len,
- struct ndb_note_blocks **blocks_p)
+ struct ndb_blocks **blocks_p)
{
int cp, c;
struct ndb_content_parser parser;
- struct note_block block;
+ struct ndb_block block;
unsigned char *start, *pre_mention;
make_cursor(buf, buf + buf_size, &parser.buffer);
// allocate some space for the blocks header
- *blocks_p = parser.blocks = (struct ndb_note_blocks *)buf;
- parser.buffer.p += sizeof(struct ndb_note_blocks);
+ *blocks_p = parser.blocks = (struct ndb_blocks *)buf;
+ parser.buffer.p += sizeof(struct ndb_blocks);
make_cursor((unsigned char *)content,
(unsigned char*)content + content_len, &parser.content);
diff --git a/nostrdb/src/nostr_bech32.c b/nostrdb/src/nostr_bech32.c
@@ -8,6 +8,8 @@
#include "nostr_bech32.h"
#include <stdlib.h>
#include "cursor.h"
+#include "str_block.h"
+#include "nostrdb.h"
#include "bolt11/bech32.h"
#define MAX_TLVS 32
@@ -88,7 +90,7 @@ static int parse_nostr_bech32_nsec(struct cursor *cur, struct bech32_nsec *nsec)
static int add_relay(struct relays *relays, struct nostr_tlv *tlv)
{
- struct str_block *str;
+ struct ndb_str_block *str;
if (relays->num_relays + 1 > MAX_RELAYS)
return 0;
diff --git a/nostrdb/src/nostr_bech32.h b/nostrdb/src/nostr_bech32.h
@@ -14,7 +14,7 @@
#define MAX_RELAYS 24
struct relays {
- struct str_block relays[MAX_RELAYS];
+ struct ndb_str_block relays[MAX_RELAYS];
int num_relays;
};
@@ -54,12 +54,12 @@ struct bech32_nprofile {
struct bech32_naddr {
struct relays relays;
- struct str_block identifier;
+ struct ndb_str_block identifier;
const unsigned char *pubkey;
};
struct bech32_nrelay {
- struct str_block relay;
+ struct ndb_str_block relay;
};
struct nostr_bech32 {
diff --git a/nostrdb/src/nostrdb.h b/nostrdb/src/nostrdb.h
@@ -20,7 +20,8 @@
struct ndb_json_parser;
struct ndb;
-struct ndb_note_blocks;
+struct ndb_blocks;
+struct ndb_block;
struct ndb_note;
struct ndb_tag;
struct ndb_tags;
@@ -198,12 +199,6 @@ struct ndb_builder {
struct ndb_tag *current_tag;
};
-/*
-struct ndb_block_iterator {
- struct note_block block;
-};
-*/
-
struct ndb_iterator {
struct ndb_note *note;
struct ndb_tag *tag;
@@ -394,8 +389,30 @@ enum ndb_common_kind ndb_kind_to_common_kind(int kind);
// CONTENT PARSER
int ndb_parse_content(unsigned char *buf, int buf_size,
const char *content, int content_len,
- struct ndb_note_blocks **blocks_p);
-
-//int ndb_blocks_iterate_next(struct ndb_block_iterator *iter);
+ struct ndb_blocks **blocks_p);
+
+// BLOCKS
+enum ndb_block_type {
+ BLOCK_HASHTAG = 1,
+ BLOCK_TEXT = 2,
+ BLOCK_MENTION_INDEX = 3,
+ BLOCK_MENTION_BECH32 = 4,
+ BLOCK_URL = 5,
+ BLOCK_INVOICE = 6,
+};
+#define NDB_NUM_BLOCK_TYPES 6
+
+enum ndb_block_type ndb_block_type(struct ndb_blocks *blocks);
+
+// BLOCK ITERATORS
+struct ndb_block_iterator *ndb_blocks_iterate_start(const char *, struct ndb_blocks *);
+void ndb_blocks_iterate_free(struct ndb_block_iterator *);
+struct ndb_block *ndb_blocks_iterate_next(struct ndb_block_iterator *);
+
+// STR BLOCKS
+enum ndb_block_type ndb_get_block_type(struct ndb_block *block);
+struct ndb_str_block *ndb_block_str(struct ndb_block *);
+const char *ndb_str_block_ptr(struct ndb_str_block *);
+uint32_t ndb_str_block_len(struct ndb_str_block *);
#endif
diff --git a/nostrdb/src/str_block.h b/nostrdb/src/str_block.h
@@ -4,7 +4,7 @@
#include <inttypes.h>
-struct str_block {
+struct ndb_str_block {
const char *str;
uint32_t len;
};
