protoverse

commit 4c8b051d35f03618b8e35a65ef253c559baadf03
parent 96883c80ad46b2c5147140831332c3e2f4d0ade7
Author: William Casarin <jb55@jb55.com>
Date:   Sun, 16 Apr 2023 13:20:20 -0700

support comments in space files

I actually need this for wast test files, just curious how much of the
token parser I can re-use for that.

Diffstat:
M
satoshis-citadel.space
 | 
6
++++++
M
src/parse.c
 | 
45
++++++++++++++++++++++++++++++++++++++++-----
M
src/protoverse.c
 | 
2
+-

3 files changed, 47 insertions(+), 6 deletions(-)
diff --git a/satoshis-citadel.space b/satoshis-citadel.space
@@ -1,4 +1,10 @@
+;; this is a comment
+;; excellent
+
 (room (shape rectangle)
+
+;; another comment
+
       (name "Satoshi's Den")
       (condition "clean")
       (condition "shiny")
diff --git a/src/parse.c b/src/parse.c
@@ -1,6 +1,7 @@
 
 #include "util.h"
 #include "parse.h"
+#include "parser.h"
 #include "io.h"
 
 #include <stdio.h>
@@ -531,6 +532,38 @@ static int read_and_push_atom(struct token_cursor *cursor, struct token_cursor *
 	return 0;
 }
 
+static void consume_whitespace(struct cursor *cur) {
+	while (cur->p < cur->end) {
+		if (!isspace(*cur->p))
+			break;
+		cur->p++;
+	}
+}
+
+static void consume_line(struct cursor *cur) {
+	int line_breaking = 0;
+	while (cur->p < cur->end) {
+		if (*cur->p == '\n' || *cur->p == '\r') {
+			line_breaking = 1;
+		} else if (line_breaking) {
+			break;
+		}
+		cur->p++;
+	}
+}
+
+static int consume_comment(struct cursor *cur) {
+	u8* start = cur->p;
+	if (!consume_byte(cur, ';'))
+		return 0;
+	if (!consume_byte(cur, ';')) {
+		cur->p = start;
+		return 0;
+	}
+	consume_line(cur);
+	return 1;
+}
+
 int tokenize_cells(u8 *buf, int buf_size, struct token_cursor *tokens)
 {
 	enum tok_state state;
@@ -538,7 +571,6 @@ int tokenize_cells(u8 *buf, int buf_size, struct token_cursor *tokens)
 	/* u8 *start = buf; */
 	u8 *token_buf = tokens->c.p;
 	u8 c;
-	int ok;
 
 	cursor.c.p = buf;
 	cursor.c.end = buf + buf_size;
@@ -546,8 +578,12 @@ int tokenize_cells(u8 *buf, int buf_size, struct token_cursor *tokens)
 	state = TS_OPEN;
 
 	while (cursor.c.p < cursor.c.end) {
-		ok = pull_byte(&cursor.c, &c);
-		if (!ok) break;
+		do {
+			consume_whitespace(&cursor.c);
+		} while (consume_comment(&cursor.c));
+
+		if (!pull_byte(&cursor.c, &c))
+			break;
 
 		if (state == TS_OPEN) {
 			if (isspace(c))
@@ -578,8 +614,7 @@ int tokenize_cells(u8 *buf, int buf_size, struct token_cursor *tokens)
 
 			cursor.c.p--;
 			/* printf("\nat %c (%ld) before reading atom\n", *cursor.p, cursor.p - start); */
-			ok = read_and_push_atom(&cursor, tokens);
-			if (!ok) {
+			if (!read_and_push_atom(&cursor, tokens)) {
 				print_token_error(&cursor);
 				return 0;
 			}
diff --git a/src/protoverse.c b/src/protoverse.c
@@ -139,7 +139,7 @@ extern char **environ;
 
 int main(int argc, const char *argv[])
 {
-	static u8 buf[4096*8];
+	static u8 buf[4096*256];
 	char **env = environ;
 	const char *space, *code_file;
 	const char *cmd;