protoverse

commit 1b2b24fa43d98378b15c09d947ddb470e7d3af04
parent 7223cbab79fd158dd605c69aebed990e8a3e5a83
Author: William Casarin <jb55@jb55.com>
Date:   Sun,  2 Aug 2020 15:05:48 -0700

Initial network packet serialization

Signed-off-by: William Casarin <jb55@jb55.com>

Diffstat:
M
.gitignore
 | 
1
+
M
Makefile
 | 
10
+++++++++-
M
client.c
 | 
9
++++++---
M
cursor.c
 | 
96
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-
M
cursor.h
 | 
10
+++++++++-
A
net.c
 | 
224
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
A
net.h
 | 
42
++++++++++++++++++++++++++++++++++++++++++
A
test.c
 | 
61
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
A
varint.c
 | 
75
+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
A
varint.h
 | 
14
++++++++++++++

10 files changed, 536 insertions(+), 6 deletions(-)
diff --git a/.gitignore b/.gitignore
@@ -2,3 +2,4 @@
 /protoverse
 /libprotoverse.a
 /TAGS
+/test
diff --git a/Makefile b/Makefile
@@ -1,7 +1,9 @@
 
 CFLAGS = -Wno-error=unused-function -O1 -g -std=c89 -Wall -Wextra -Werror -Wstrict-prototypes -Wold-style-definition -Wmissing-prototypes -Wmissing-declarations -Wdeclaration-after-statement
 
-OBJS = io.o parse.o cursor.o describe.o serve.o client.o
+OBJS = io.o parse.o cursor.o describe.o serve.o client.o net.o varint.o
+
+all: protoverse libprotoverse.a
 
 protoverse: protoverse.c $(OBJS)
 	$(CC) $(CFLAGS) $^ $(LDFLAGS) -o $@
@@ -12,6 +14,12 @@ libprotoverse.a: $(OBJS)
 clean:
 	rm -f protoverse *.o
 
+test: test.c $(OBJS)
+	$(CC) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
+check: test
+	./test
+
 TAGS: fake
 	etags *.c *.h > $@
 
diff --git a/client.c b/client.c
@@ -9,18 +9,23 @@
 #include <errno.h>
 
 #include "client.h"
+#include "cursor.h"
 
 int inet_aton(const char *cp, struct in_addr *inp);
 
 int protoverse_connect(const char *server_ip_str, int port)
 {
+	static unsigned char buf[0xFFFF];
+
 	int sockfd;
 	struct in_addr server_in_addr;
 	struct sockaddr_in server_addr;
-	static char buf[2048];
+	struct cursor cursor;
 	ssize_t sent;
 	const char msg[] = "hello, world";
 
+	make_cursor(buf, buf + sizeof(buf), &cursor);
+
 	if ((sockfd = socket(AF_INET, SOCK_DGRAM, 0)) == -1) {
 		printf("socket creation failed: %s\n", strerror(errno));
 		return 0;
@@ -35,8 +40,6 @@ int protoverse_connect(const char *server_ip_str, int port)
 	server_addr.sin_port = port == 0 || port == -1 ? 1988 : port;
 	server_addr.sin_addr = server_in_addr;
 
-	strncpy(buf, msg, sizeof(buf));
-
 	printf("sending '%s' to %s\n", msg, server_ip_str);
 	sent = sendto(sockfd, buf, sizeof(msg), MSG_CONFIRM,
 		      (struct sockaddr*)&server_addr,
diff --git a/cursor.c b/cursor.c
@@ -1,6 +1,8 @@
 
 #include "cursor.h"
 #include "typedefs.h"
+#include "varint.h"
+
 #include <stdio.h>
 #include <string.h>
 
@@ -63,7 +65,7 @@ int pull_data(struct cursor *cursor, u8 *data, int len)
 
 int push_data(struct cursor *cursor, u8 *data, int len)
 {
-	if (cursor->p + len >= cursor->end) {
+	if (cursor->p + len > cursor->end) {
 		printf("push_data oob\n");
 		return 0;
 	}
@@ -79,6 +81,61 @@ int push_int(struct cursor *cursor, int i)
 	return push_data(cursor, (u8*)&i, sizeof(i));
 }
 
+/* TODO: push_varint */
+int push_varint(struct cursor *cursor, int n)
+{
+	int ok, len;
+	unsigned char b;
+	len = 0;
+
+	while (1) {
+		b = (n & 0xFF) | 0x80;
+		n >>= 7;
+		if (n == 0) {
+			b &= 0x7F;
+			ok = push_byte(cursor, b);
+			len++;
+			if (!ok) return 0;
+			break;
+		}
+
+		ok = push_byte(cursor, b);
+		len++;
+		if (!ok) return 0;
+	}
+
+	return len;
+}
+
+/* TODO: pull_varint */
+int pull_varint(struct cursor *cursor, int *n)
+{
+	int ok, i;
+	unsigned char b;
+	*n = 0;
+
+	for (i = 0;; i++) {
+		ok = pull_byte(cursor, &b);
+		if (!ok) return 0;
+
+		*n |= ((int)b & 0x7F) << (i * 7);
+
+		/* is_last */
+		if ((b & 0x80) == 0) {
+			return 1;
+		}
+
+		if (i == 4) return 0;
+	}
+
+	return 0;
+}
+
+int pull_int(struct cursor *cursor, int *i)
+{
+	return pull_data(cursor, (u8*)i, sizeof(*i));
+}
+
 int push_u16(struct cursor *cursor, u16 i)
 {
 	return push_data(cursor, (u8*)&i, sizeof(i));
@@ -105,3 +162,40 @@ int push_str(struct cursor *cursor, const char *str)
 {
 	return push_data(cursor, (u8*)str, strlen(str));
 }
+
+/* TODO: push varint size */
+int push_prefixed_str(struct cursor *cursor, const char *str)
+{
+	int ok, len;
+	len = strlen(str);
+	ok = push_varint(cursor, len);
+	if (!ok) return 0;
+	return push_sized_str(cursor, str, len);
+}
+
+int pull_prefixed_str(struct cursor *cursor, struct cursor *dest_buf, const char **str)
+{
+	int len, ok;
+
+	ok = pull_varint(cursor, &len);
+	if (!ok) return 0;
+
+	if (dest_buf->p + len > dest_buf->end) {
+		return 0;
+	}
+
+	ok = pull_data(cursor, dest_buf->p, len);
+	if (!ok) return 0;
+
+	*str = (char*)dest_buf->p;
+	dest_buf->p += len;
+
+	ok = push_byte(dest_buf, 0);
+
+	return 1;
+}
+
+int cursor_remaining_capacity(struct cursor *cursor)
+{
+	return cursor->end - cursor->p;
+}
diff --git a/cursor.h b/cursor.h
@@ -12,16 +12,24 @@ struct cursor {
 void copy_cursor(struct cursor *src, struct cursor *dest);
 int cursor_index(struct cursor *cursor, int elem_size);
 void make_cursor(unsigned char *start, unsigned char *end, struct cursor *cursor);
+int cursor_remaining_capacity(struct cursor *cursor);
 void *index_cursor(struct cursor *cursor, unsigned short index, int elem_size);
 
 int push_u16(struct cursor *cursor, unsigned short i);
 int push_int(struct cursor *cursor, int i);
+int push_varint(struct cursor *cursor, int i);
 int push_data(struct cursor *cursor, unsigned char *data, int len);
+int push_byte(struct cursor *cursor, unsigned char c);
+
 int pull_data(struct cursor *cursor, unsigned char *data, int len);
 int pull_byte(struct cursor *cursor, unsigned char *c);
-int push_byte(struct cursor *cursor, unsigned char c);
+int pull_int(struct cursor *cursor, int *i);
+int pull_varint(struct cursor *cursor, int *i);
 
+int push_prefixed_str(struct cursor *cursor, const char *str);
 int push_str(struct cursor *cursor, const char *str);
 int push_sized_str(struct cursor *cursor, const char *str, int len);
 
+int pull_prefixed_str(struct cursor *cursor, struct cursor *dest_buf, const char **str);
+
 #endif
diff --git a/net.c b/net.c
@@ -0,0 +1,224 @@
+
+#include "net.h"
+#include "cursor.h"
+#include "varint.h"
+
+#include <sys/types.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+#include <stdio.h>
+#include <string.h>
+
+static int push_fetch_packet(struct cursor *c, struct fetch_data_packet *fetch)
+{
+	return push_prefixed_str(c, fetch->path);
+}
+
+/* TODO: CPU-independent encoding */
+static int push_chat_packet(struct cursor *c, struct chat_packet *chat)
+{
+	int ok;
+	ok = push_varint(c, chat->sender);
+	if (!ok) return 0;
+
+	return push_prefixed_str(c, chat->message);
+}
+
+static int pull_chat_packet(struct cursor *c, struct cursor *buf, struct chat_packet *chat)
+{
+	int ok;
+
+	ok = pull_varint(c, &chat->sender);
+	if (!ok) return 0;
+
+	return pull_prefixed_str(c, buf, &chat->message);
+}
+
+static int pull_fetch_packet(struct cursor *c, struct cursor *buf, struct fetch_data_packet *fetch)
+{
+	return pull_prefixed_str(c, buf, &fetch->path);
+}
+
+int send_packet(int sockfd, struct sockaddr *to_addr, int to_addr_len, struct packet *packet)
+{
+	static unsigned char buf[0xFFFF];
+	int ok, len;
+
+	len = push_packet(buf, sizeof(buf), packet);
+	if (!len) return 0;
+
+	ok = sendto(sockfd, buf, len, MSG_CONFIRM, to_addr, to_addr_len);
+
+	if (ok != len) {
+		printf("sendto: sent %d != packet_len %d\n", ok, len);
+		return 0;
+	}
+
+	return 1;
+}
+
+static int push_envelope(struct cursor *cursor, enum packet_type type, int len)
+{
+	int ok;
+	int env_len;
+
+	ok = push_varint(cursor, (int)type);
+	if (!ok) return 0;
+
+	env_len = ok;
+
+	ok = push_varint(cursor, len);
+
+	env_len += ok;
+
+	return env_len;
+}
+
+static int pull_envelope(struct cursor *cursor, enum packet_type *type, int *len)
+{
+	int ok, env_len;
+	ok = pull_varint(cursor, (int*)type);
+	if (!ok) return 0;
+
+	if (*type >= PKT_NUM_TYPES)
+		return 0;
+
+	env_len = ok;
+
+	ok = pull_varint(cursor, len);
+	if (!ok) return 0;
+
+	env_len += ok;
+
+	return env_len;
+}
+
+static int push_packet_data(struct cursor *c, struct packet *packet)
+{
+	switch (packet->type) {
+	case PKT_FETCH_DATA:
+		return push_fetch_packet(c, &packet->data.fetch);
+		break;
+	case PKT_CHAT:
+		return push_chat_packet(c, &packet->data.chat);
+		break;
+	case PKT_NUM_TYPES:
+		return 0;
+	}
+
+	return 0;
+}
+
+int push_packet(unsigned char *buf, int bufsize, struct packet *packet)
+{
+	struct cursor cursor;
+	struct cursor envelope_cursor;
+	int len, ok, envelope_size;
+	envelope_size = VARINT_MAX_LEN * 2;
+
+	make_cursor(buf, buf + envelope_size, &envelope_cursor);
+	make_cursor(buf + envelope_size, buf + bufsize, &cursor);
+
+	ok = push_packet_data(&cursor, packet);
+	if (!ok) return 0;
+
+	len = cursor.p - cursor.start;
+
+	ok = push_envelope(&envelope_cursor, packet->type, len);
+	if (!ok) return 0;
+
+	memmove(buf + ok, cursor.start, len);
+
+	return len + ok;
+}
+
+
+static int pull_packet_data(struct cursor *c, struct cursor *buf,
+			    struct packet *packet, int len)
+{
+	(void)c;
+	(void)len;
+	switch (packet->type) {
+	case PKT_FETCH_DATA:
+		return pull_fetch_packet(c, buf, &packet->data.fetch);
+	case PKT_CHAT:
+		return pull_chat_packet(c, buf, &packet->data.chat);
+	case PKT_NUM_TYPES:
+		break;
+	}
+
+	return 0;
+}
+
+int pull_packet(struct cursor *c, struct cursor *buf, struct packet *packet)
+{
+	int ok, env_size, len, capacity_left;
+
+	env_size = pull_envelope(c, &packet->type, &len);
+	if (!env_size) return 0;
+
+	capacity_left = cursor_remaining_capacity(c) - 1;
+	if (len > capacity_left) {
+		printf("sanity: packet larger (%d) than remaining buffer size: %d", len, capacity_left);
+		return 0;
+	}
+
+	ok = pull_packet_data(c, buf, packet, len);
+	if (!ok) return 0;
+
+	return len + env_size;
+}
+
+static int packet_chat_eq(struct chat_packet *a, struct chat_packet *b)
+{
+	/* fail if either is null but not both 0 or both not 0 */
+	if ((a->message == 0) ^ (b->message == 0))
+		return 0;
+
+	return a->sender == b->sender && !strcmp(a->message, b->message);
+}
+
+static int packet_fetch_eq(struct fetch_data_packet *a,
+		    struct fetch_data_packet *b)
+{
+	if ((a->path == 0) ^ (b->path == 0))
+		return 0;
+
+	return !strcmp(a->path, b->path);
+}
+
+int packet_eq(struct packet *a, struct packet *b)
+{
+	if (a->type != b->type)
+		return 0;
+
+	switch (a->type) {
+	case PKT_CHAT:
+		return packet_chat_eq(&a->data.chat, &b->data.chat);
+	case PKT_FETCH_DATA:
+		return packet_fetch_eq(&a->data.fetch, &b->data.fetch);
+	case PKT_NUM_TYPES:
+		return 0;
+	}
+
+	return 0;
+}
+
+
+void print_packet(struct packet *packet)
+{
+	switch (packet->type) {
+	case PKT_CHAT:
+		printf("(chat (sender %d) (message \"%s\"))\n",
+		       packet->data.chat.sender,
+		       packet->data.chat.message);
+		return;
+	case PKT_FETCH_DATA:
+		printf("(fetch)\n");
+		return;
+	case PKT_NUM_TYPES:
+		break;
+	}
+
+	printf("(unknown)\n");
+}
diff --git a/net.h b/net.h
@@ -0,0 +1,42 @@
+
+#ifndef PROTOVERSE_NET_H
+#define PROTOVERSE_NET_H
+
+#include <sys/socket.h>
+#include "cursor.h"
+
+enum packet_type {
+	PKT_FETCH_DATA,
+	PKT_CHAT,
+	PKT_NUM_TYPES,
+};
+
+struct fetch_data_packet {
+	const char *path;
+};
+
+struct chat_packet {
+	int sender;
+	const char *message;
+};
+
+union packet_data {
+	struct fetch_data_packet fetch;
+	struct chat_packet chat;
+};
+
+struct packet {
+	enum packet_type type;
+	union packet_data data;
+};
+
+int send_packet(int fd, struct sockaddr *to_addr, int to_addr_len, struct packet *packet);
+int recv_packet(int fd, struct packet *packet);
+
+int push_packet(unsigned char *buf, int bufsize, struct packet *packet);
+int pull_packet(struct cursor *c, struct cursor *buf, struct packet *packet);
+
+int packet_eq(struct packet *a, struct packet *b);
+void print_packet(struct packet *packet);
+
+#endif /* PROTOVERSE_NET_H */
diff --git a/test.c b/test.c
@@ -0,0 +1,61 @@
+
+#include "net.h"
+#include <assert.h>
+#include <stdio.h>
+#include <string.h>
+
+
+static void print_mem(unsigned char *a, int len)
+{
+	int i;
+	for (i = 0; i < len; i++) {
+		printf("%02x ", a[i]);
+	}
+	printf("\n");
+
+}
+
+int main(int argc, char *argv[])
+{
+	static unsigned char bufs[3][1024];
+
+	struct cursor cursors[3];
+	struct packet packet;
+	struct packet packet_out;
+
+	int pushed[2], pulled, i;
+
+	(void)argc;
+	(void)argv;
+
+	for (i = 0; i < 3; i++) {
+		make_cursor(bufs[i], bufs[i] + sizeof(bufs[i]), &cursors[i]);
+	}
+
+	packet.type = PKT_CHAT;
+	packet.data.chat.sender = 1;
+	packet.data.chat.message = "hello there";
+
+	pushed[0] = push_packet(bufs[0], sizeof(bufs[0]), &packet);
+	assert(pushed[0]);
+
+	pulled = pull_packet(&cursors[0], &cursors[1], &packet_out);
+	assert(pulled);
+
+	pushed[1] = push_packet(bufs[2], sizeof(bufs[2]), &packet_out);
+	assert(pushed[1]);
+
+	printf("chat packet\n");
+	print_mem(bufs[0], pushed[0]);
+
+	/* printf("pushed %d,%d pulled %d\n", pushed[0], pushed[1], pulled); */
+	assert(pushed[0] == pulled && pushed[1] == pushed[0]);
+
+
+	assert(!memcmp(bufs[0], bufs[2], pulled));
+	assert(packet_eq(&packet, &packet_out));
+
+	print_packet(&packet);
+
+	return 0;
+}
diff --git a/varint.c b/varint.c
@@ -0,0 +1,75 @@
+
+#include "varint.h"
+
+size_t varint_size(uint64_t v)
+{
+	if (v < 0xfd)
+		return 1;
+	if (v <= 0xffff)
+		return 3;
+	if (v <= 0xffffffff)
+		return 5;
+	return 9;
+}
+
+size_t varint_put(unsigned char buf[VARINT_MAX_LEN], uint64_t v)
+{
+	unsigned char *p = buf;
+
+	if (v < 0xfd) {
+		*(p++) = v;
+	} else if (v <= 0xffff) {
+		(*p++) = 0xfd;
+		(*p++) = v;
+		(*p++) = v >> 8;
+	} else if (v <= 0xffffffff) {
+		(*p++) = 0xfe;
+		(*p++) = v;
+		(*p++) = v >> 8;
+		(*p++) = v >> 16;
+		(*p++) = v >> 24;
+	} else {
+		(*p++) = 0xff;
+		(*p++) = v;
+		(*p++) = v >> 8;
+		(*p++) = v >> 16;
+		(*p++) = v >> 24;
+		(*p++) = v >> 32;
+		(*p++) = v >> 40;
+		(*p++) = v >> 48;
+		(*p++) = v >> 56;
+	}
+	return p - buf;
+}
+
+size_t varint_get(const unsigned char *p, size_t max, int64_t *val)
+{
+	if (max < 1)
+		return 0;
+
+	switch (*p) {
+	case 0xfd:
+		if (max < 3)
+			return 0;
+		*val = ((uint64_t)p[2] << 8) + p[1];
+		return 3;
+	case 0xfe:
+		if (max < 5)
+			return 0;
+		*val = ((uint64_t)p[4] << 24) + ((uint64_t)p[3] << 16)
+			+ ((uint64_t)p[2] << 8) + p[1];
+		return 5;
+	case 0xff:
+		if (max < 9)
+			return 0;
+		*val = ((uint64_t)p[8] << 56) + ((uint64_t)p[7] << 48)
+			+ ((uint64_t)p[6] << 40) + ((uint64_t)p[5] << 32)
+			+ ((uint64_t)p[4] << 24) + ((uint64_t)p[3] << 16)
+			+ ((uint64_t)p[2] << 8) + p[1];
+		return 9;
+	default:
+		*val = *p;
+		return 1;
+	}
+}
+
diff --git a/varint.h b/varint.h
@@ -0,0 +1,14 @@
+
+#ifndef PROTOVERSE_VARINT_H
+#define PROTOVERSE_VARINT_H
+
+#define VARINT_MAX_LEN 9
+
+#include <stddef.h>
+#include <stdint.h>
+
+size_t varint_put(unsigned char buf[VARINT_MAX_LEN], uint64_t v);
+size_t varint_size(uint64_t v);
+size_t varint_get(const unsigned char *p, size_t max, int64_t *val);
+
+#endif /* PROTOVERSE_VARINT_H */