commit 8044145b1a5e7bd1d4b18c167f8c1b8081d39c50
parent 431f3822009b5a961e7e62953d5da4baa2fcaad1
Author: William Casarin <jb55@jb55.com>
Date: Sun, 17 Jan 2021 22:45:37 -0800
fetch signatures
Diffstat:
3 files changed, 228 insertions(+), 45 deletions(-)
diff --git a/Makefile b/Makefile
@@ -1,6 +1,6 @@
-CFLAGS = -Og -ggdb -Wall -Werror -Ideps -Isrc $(shell pkg-config --cflags openssl)
-LDFLAGS = $(shell pkg-config --libs openssl)
+CFLAGS = -Og -ggdb -Wall -Werror -Ideps -Isrc $(shell pkg-config --cflags openssl libcurl)
+LDFLAGS = $(shell pkg-config --libs openssl libcurl)
OBJS = src/http.o \
src/base64.o \
diff --git a/default.nix b/default.nix
@@ -3,5 +3,5 @@ with pkgs;
stdenv.mkDerivation {
name = "chibipub";
nativeBuildInputs = [ gdb pkg-config ngrok clib ];
- buildInputs = [ openssl ];
+ buildInputs = [ openssl curl ];
}
diff --git a/src/sigcheck.c b/src/sigcheck.c
@@ -1,5 +1,6 @@
-#define SCRATCH_SIZE 1048576
+#define SCRATCH_SIZE 134217728
+#define MAX_PARALLEL 10
#include <sys/mman.h>
#include <sys/stat.h>
@@ -18,6 +19,26 @@
#include "errors.h"
#include <ctype.h>
+#include <curl/curl.h>
+
+struct keyid_pubkey {
+ unsigned char *data;
+ int len;
+};
+
+enum key_writer_flags {
+ KW_FAILED = 1 << 0,
+ KW_STARTED = 1 << 1,
+};
+
+struct key_writer {
+ CURL *curl;
+ struct cursor *arena;
+ struct keyid_pubkey *pubkey;
+ const char *keyid;
+ struct cursor slice;
+ int flags;
+};
static int verify_signature(struct cursor cur, struct cursor *arena)
{
@@ -36,30 +57,26 @@ static int verify_signature(struct cursor cur, struct cursor *arena)
return 1;
}
-struct keyid {
- unsigned char *pubkey;
- int pubkey_size;
-};
-
static inline int push_keyid(struct cursor *c, const char *keyid, int keyid_len)
{
- struct keyid offset = {0};
+ struct keyid_pubkey pubkey = {0};
return push_int(c, keyid_len)
&& push_data(c, (unsigned char *)keyid, keyid_len)
- && push_data(c, (unsigned char *)&offset, sizeof(offset)); // reserved for pubkey data index
+ && push_byte(c, 0)
+ && push_data(c, (unsigned char *)&pubkey, sizeof(pubkey)); // reserved for pubkey data index
}
-static int pull_keyid(struct cursor *c, unsigned char **keyid_str,
- int *keyid_len, struct keyid **keyid)
+static int pull_keyid(struct cursor *c, const char **keyid_str, int *keyid_len,
+ struct keyid_pubkey **pubkey)
{
if (!pull_int(c, keyid_len)) {
return 0;
}
- *keyid_str = c->p;
- c->p += *keyid_len;
- *keyid = (struct keyid*)c->p;
- c->p += sizeof(struct keyid);
+ *keyid_str = (const char*)c->p;
+ c->p += (*keyid_len) + 1; /* plus zero */
+ *pubkey = (struct keyid_pubkey*)c->p;
+ c->p += sizeof(struct keyid_pubkey);
return 1;
}
@@ -68,8 +85,8 @@ static int has_keyid(struct cursor *keyids, const char *keyid, int keyid_len)
{
struct cursor scan;
int size;
- unsigned char *keyid_str;
- struct keyid *offset;
+ const char *keyid_str;
+ struct keyid_pubkey *offset;
scan.start = keyids->start;
scan.p = keyids->start;
@@ -90,20 +107,18 @@ static int has_keyid(struct cursor *keyids, const char *keyid, int keyid_len)
return 0;
}
-static int print_keyids(struct cursor *keyids)
+static int print_keyids(struct cursor keyids)
{
- struct cursor scan;
int size;
- unsigned char *keyid_str;
- struct keyid *keyid;
+ const char *keyid_str;
+ struct keyid_pubkey *pubkey;
- scan.start = keyids->start;
- scan.p = keyids->start;
- scan.end = keyids->p;
+ keyids.end = keyids.p;
+ keyids.p = keyids.start;
printf("keyids\n");
- while (scan.p < scan.end) {
- if (!pull_keyid(&scan, &keyid_str, &size, &keyid))
+ while (keyids.p < keyids.end) {
+ if (!pull_keyid(&keyids, &keyid_str, &size, &pubkey))
return 0;
printf("%.*s\n", size, keyid_str);
}
@@ -148,7 +163,11 @@ static int gather_keyids(unsigned char *json, int json_len,
}
}
- print_keyids(keyids_cur);
+ keyids_cur->end = keyids_cur->p;
+ keyids_cur->p = keyids_cur->start;
+ arena->p = keyids_cur->end;
+
+ print_keyids(*keyids_cur);
return 1;
}
@@ -184,7 +203,7 @@ static int get_cached_pubkey(unsigned char *keyid, int size,
if (0 == mkdir(".chibipub", mode)) {
return 0;
}
-
+
if (0 == mkdir(".chibipub/objects", mode)) {
return 0;
}
@@ -197,7 +216,7 @@ static int get_cached_pubkey(unsigned char *keyid, int size,
sprintf(path, ".chibipub/objects/%c%c/%.*s",
hash_str[0], hash_str[1], 64, hash_str);
- fprintf(stderr, "sig cache exists? '%s' ", path);
+ fprintf(stderr, "key cache exists? '%s' ", path);
if (access(path, F_OK)) {
fprintf(stderr, "no.\n");
return 0;
@@ -207,9 +226,144 @@ static int get_cached_pubkey(unsigned char *keyid, int size,
return read_file(path, buf, buflen, pubkey_size);
}
-static int fetch_signature(unsigned char *keyid, int keyid_len)
+static size_t write_cb(char *data, size_t n, size_t l, void *userp)
+{
+ double dcl;
+ int cl;
+ int res;
+ size_t size = n*l;
+ struct key_writer *writer = (struct key_writer *)userp;
+
+ if (!(writer->flags & KW_STARTED)) {
+ res = curl_easy_getinfo(writer->curl,
+ CURLINFO_CONTENT_LENGTH_DOWNLOAD, &dcl);
+ if (res) {
+ fprintf(stderr, "'%s' download failed, content-length unknown\n",
+ writer->keyid);
+ writer->flags |= KW_FAILED;
+ return 0;
+ }
+
+ cl = (int)dcl;
+ if (cl == -1) {
+ cl = 1024*1024;
+ }
+
+ writer->pubkey->data = cursor_alloc(writer->arena, cl);
+ if (!writer->pubkey->data) {
+ fprintf(stderr, "cursor_alloc failed, remaining (%ld) + %d (%ld total)\n",
+ writer->arena->end - writer->arena->p,
+ cl, writer->arena->end - writer->arena->start);
+ writer->flags |= KW_FAILED;
+ return 0;
+ }
+
+ writer->flags |= KW_STARTED;
+ writer->pubkey->len = 0;
+ make_cursor(writer->pubkey->data, writer->pubkey->data + cl,
+ &writer->slice);
+ }
+
+ if (!push_data(&writer->slice, (unsigned char*)data, size)) {
+ writer->flags |= KW_FAILED;
+ return 0;
+ }
+
+ writer->pubkey->len += size;
+
+ return size;
+}
+
+static int add_signature_transfer(CURLM *cm, struct key_writer *writer)
{
- printf("fetch signature %.*s\n", keyid_len, keyid);
+ CURL *eh = curl_easy_init();
+ writer->curl = eh;
+ curl_easy_setopt(eh, CURLOPT_WRITEFUNCTION, write_cb);
+ curl_easy_setopt(eh, CURLOPT_WRITEDATA, writer);
+ curl_easy_setopt(eh, CURLOPT_URL, writer->keyid);
+ curl_easy_setopt(eh, CURLOPT_PRIVATE, writer);
+ curl_multi_add_handle(cm, eh);
+
+ return 1;
+}
+
+static void prepare_transfers(CURLM **cm)
+{
+ curl_global_init(CURL_GLOBAL_ALL);
+ *cm = curl_multi_init();
+ curl_multi_setopt(*cm, CURLMOPT_MAXCONNECTS, (long)10);
+}
+
+static int handle_transfer_msg(CURLM *cm, CURLMsg *msg, int *transfers,
+ struct key_writer *to_fetch, int n_to_fetch)
+{
+ struct key_writer *writer;
+
+ if (msg->msg == CURLMSG_DONE) {
+ CURL *e = msg->easy_handle;
+
+ curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, &writer);
+
+ fprintf(stderr, "R: %d - %s <%s>\n",
+ msg->data.result,
+ curl_easy_strerror(msg->data.result),
+ writer->keyid);
+
+ curl_multi_remove_handle(cm, e);
+ curl_easy_cleanup(e);
+ } else {
+ fprintf(stderr, "E: CURLMsg (%d)\n", msg->msg);
+ }
+
+ if (*transfers < n_to_fetch) {
+ add_signature_transfer(cm, &to_fetch[*transfers]);
+ *transfers = *transfers + 1;
+ }
+
+ return 1;
+}
+
+static int perform_transfers(CURLM *cm, int transfers,
+ struct key_writer *to_fetch, int n_to_fetch)
+{
+ CURLMsg *msg;
+ int still_alive = 1;
+ int msgs_left = -1;
+
+ do {
+ curl_multi_perform(cm, &still_alive);
+
+ while ((msg = curl_multi_info_read(cm, &msgs_left))) {
+ handle_transfer_msg(
+ cm, msg, &transfers, to_fetch, n_to_fetch);
+ }
+
+ if (still_alive) {
+ curl_multi_wait(cm, NULL, 0, 1000, NULL);
+ }
+ } while (still_alive || (transfers < n_to_fetch));
+
+ curl_multi_cleanup(cm);
+ curl_global_cleanup();
+
+ return 1;
+}
+
+static int count_keyids(struct cursor scan, int *count)
+{
+ int size;
+ const char *keyid_str;
+ struct keyid_pubkey *pubkey;
+
+ *count = 0;
+
+ while (scan.p < scan.end) {
+ if (!pull_keyid(&scan, &keyid_str, &size, &pubkey)) {
+ return 0;
+ }
+ *count = *count + 1;
+ }
+
return 1;
}
@@ -218,34 +372,63 @@ static int fetch_signatures(unsigned char *json, int json_len,
{
struct cursor keyids;
struct cursor scan;
- int size;
- unsigned char *keyid_str;
- struct keyid *keyid;
+ int size, transfers;
+ int n_to_fetch, n_keyids;
+ const char *keyid_str;
+ struct key_writer *to_fetch, *kw;
+ struct keyid_pubkey *pubkey;
+ CURLM *cm;
+ n_to_fetch = 0;
if (!gather_keyids(json, json_len, arena, &keyids)) {
return 0;
}
- scan.start = keyids.start;
- scan.p = scan.start;
- scan.end = keyids.p;
+ prepare_transfers(&cm);
+
+ scan.p = scan.start = keyids.start;
+ scan.end = keyids.end;
+
+ if (!count_keyids(scan, &n_keyids)) {
+ return 0;
+ }
+
+ if (!(to_fetch = cursor_alloc(arena, n_keyids * sizeof(struct key_writer)))) {
+ return 0;
+ }
+
while (scan.p < scan.end) {
- if (!pull_keyid(&scan, &keyid_str, &size, &keyid)) {
+
+ if (!pull_keyid(&scan, &keyid_str, &size, &pubkey)) {
return 0;
}
if (get_cached_pubkey(scan.p, size, arena->p,
arena->end - arena->p,
- &keyid->pubkey_size)) {
- keyid->pubkey = arena->p;
- arena->p += keyid->pubkey_size;
+ &pubkey->len)) {
+ pubkey->data = arena->p;
+ arena->p += pubkey->len;
printf("got cached pubkey of size %d... do something\n",
- keyid->pubkey_size);
+ pubkey->len);
continue;
}
- fetch_signature(keyid_str, size);
+ pubkey->len = 0;
+
+ kw = &to_fetch[n_to_fetch++];
+ kw->keyid = keyid_str;
+ kw->arena = arena;
+ kw->pubkey = pubkey;
+ kw->flags = 0;
+ }
+
+ for (transfers = 0; transfers < MAX_PARALLEL && transfers < n_to_fetch; transfers++) {
+ add_signature_transfer(cm, &to_fetch[transfers]);
+ }
+
+ if (!perform_transfers(cm, transfers, to_fetch, n_to_fetch)) {
+ return 0;
}
return 1;
