More work on the scanner and tests for it

FossilOrigin-Name: 037c88ce86e1ba3a399c03a27d4c7252b52c2f500bc13f449e19e497b0607880

16 files changed, 317 insertions, 18 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 37e06d2..fdd5bd5 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -70,7 +70,22 @@ foreach(source ${TEST_SOURCES})
 	set(test_bin "testbin-${name}")
 	add_executable("${test_bin}" "${source}")
 	target_link_libraries("${test_bin}" PRIVATE common aplwc)
-	add_test(NAME "${name}" COMMAND "${test_bin}")
+
+	if("${name}" STREQUAL "aplwc_read")
+		file(GLOB APLWC_READ_INPUTS_GOOD CONFIGURE_DEPENDS "tests/aplwc_read/good/*.txt")
+		file(GLOB APLWC_READ_INPUTS_BAD CONFIGURE_DEPENDS "tests/aplwc_read/bad/*.txt")
+		foreach(input ${APLWC_READ_INPUTS_GOOD})
+			add_test(NAME "${name} ${input}" COMMAND "${test_bin}" "${input}")
+		endforeach()
+		foreach(input ${APLWC_READ_INPUTS_BAD})
+			add_test(NAME "${name} ${input}" COMMAND "${test_bin}" "${input}")
+			set_tests_properties("${name} ${input}" PROPERTIES
+				WILL_FAIL TRUE
+			)
+		endforeach()
+	else()
+		add_test(NAME "${name}" COMMAND "${test_bin}")
+	endif()
 endforeach()
 
 add_test(NAME "aplwc-repl-tiny" COMMAND "aplwc-repl")
diff --git a/src/bin/main.c b/src/bin/main.c
index a1c65bf..80b2ee0 100644
--- a/src/bin/main.c
+++ b/src/bin/main.c
@@ -5,7 +5,6 @@ int
 main(void)
 {
 	struct aplwc *aplwc;
-	struct aplwc_line *line;
 	uint8_t *data;
 	int ret;
 	size_t memsize;
@@ -25,13 +24,6 @@ main(void)
 		return 1;
 	}
 
-	line = aplwc_read(aplwc, "a←10+20");
-	if(line == NULL)
-		fprintf(stderr, "Could not parse line: %s\n", aplwc_errstr(aplwc));
-	else
-		aplwc_eval(line);
-	aplwc_free_line(line);
-
 	aplwc_destroy(aplwc);
 
 	free(data);
diff --git a/src/include/aplwc.h b/src/include/aplwc.h
index 75d9485..5ec3761 100644
--- a/src/include/aplwc.h
+++ b/src/include/aplwc.h
@@ -9,8 +9,9 @@ enum aplwc_rc {
 	APLWC_OUT_OF_MEMORY,
 	APLWC_NOT_YET_IMPLEMENTED,
 	APLWC_INVALID_UTF8,
+	APLWC_INVALID_SYNTAX,
 
-	APLWC_RC_LAST = APLWC_INVALID_UTF8
+	APLWC_RC_LAST = APLWC_INVALID_SYNTAX
 };
 
 struct aplwc;
diff --git a/src/lib/aplwc_internal.h b/src/lib/aplwc_internal.h
index 155508d..8661bb2 100644
--- a/src/lib/aplwc_internal.h
+++ b/src/lib/aplwc_internal.h
@@ -33,11 +33,21 @@ struct aplwc_line {
 struct string {
 	size_t bytes;
 	size_t chars;
-	uint8_t *utf8;
+	uint8_t utf8[1];
+};
+
+enum token_tag {
+	TOKEN_TAG_IDENTIFIER,
+	TOKEN_TAG_NEWLINE
+};
+
+struct token {
+	enum token_tag tag;
 };
 
 struct token_list {
 	size_t length;
+	struct token tokens[1];
 };
 
 /* eval.c */
@@ -48,12 +58,16 @@ struct memory memory_init(uint8_t *, size_t);
 struct aplwc *memory_alloc_aplwc(struct memory *);
 struct aplwc_line *memory_alloc_aplwc_line(struct memory *);
 struct string *memory_alloc_string(struct memory *, size_t);
+struct token_list *memory_alloc_token_list(struct memory *, size_t);
 void memory_free_aplwc(struct memory *, struct aplwc *);
 void memory_free_aplwc_line(struct memory *, struct aplwc_line *);
 void memory_free_string(struct memory *, struct string *);
+void memory_free_token_list(struct memory *, struct token_list *);
 
 /* string.c */
 struct string *chars_to_string(struct aplwc *, const char *);
+uint32_t string_char(struct string *, size_t);
+size_t char_length(struct string *, size_t);
 
 /* syntax.c */
 struct token_list *scan_line(struct aplwc *, struct string *);
diff --git a/src/lib/main.c b/src/lib/main.c
index cca1e68..9335bf0 100644
--- a/src/lib/main.c
+++ b/src/lib/main.c
@@ -34,6 +34,8 @@ aplwc_errstr_rc(enum aplwc_rc rc)
 		return "Not yet implemented";
 	case APLWC_INVALID_UTF8:
 		return "Invalid UTF-8";
+	case APLWC_INVALID_SYNTAX:
+		return "Invalid syntax";
 	default:
 		return "(unable to produce error string)";
 	}
@@ -77,8 +79,10 @@ aplwc_read(struct aplwc *aplwc, const char *line)
 		return NULL;
 
 	ret = memory_alloc_aplwc_line(&aplwc->memory);
-	if(ret == NULL)
+	if(ret == NULL){
+		aplwc->last_error = APLWC_OUT_OF_MEMORY;
 		return NULL;
+	}
 	
 	ret->aplwc = aplwc;
 	ret->text = text;
diff --git a/src/lib/memory.c b/src/lib/memory.c
index 136c7e6..41ac90d 100644
--- a/src/lib/memory.c
+++ b/src/lib/memory.c
@@ -18,7 +18,7 @@ alloc(struct memory *m, size_t size, size_t align)
 	size_t alloc_size, adjust;
 	void *p;
 
-	ptr = (uintptr_t)m->data;
+	ptr = ((uintptr_t)m->data) + m->offset;
 	alloc_size = size;
 
 	if(NEVER(align == 0))
@@ -30,6 +30,7 @@ alloc(struct memory *m, size_t size, size_t align)
 
 	if((m->size - m->offset) >= alloc_size){
 		p = m->data+m->offset+alloc_size-size;
+		assert((((uintptr_t)p) % align) == 0);
 		m->offset += alloc_size;
 		memset(p, 0, size);
 	}else
@@ -85,3 +86,19 @@ memory_free_string(struct memory *m, struct string *p)
 	UNUSED_PARAMETER(m);
 	UNUSED_PARAMETER(p);
 }
+
+struct token_list *
+memory_alloc_token_list(struct memory *m, size_t tokens)
+{
+	struct token_list *p;
+
+	p = alloc(m, sizeof(struct token_list) + (tokens * sizeof(struct token)), alignof(struct token_list));
+	return p;
+}
+
+void
+memory_free_token_list(struct memory *m, struct token_list *p)
+{
+	UNUSED_PARAMETER(m);
+	UNUSED_PARAMETER(p);
+}
diff --git a/src/lib/string.c b/src/lib/string.c
index f6fa49d..8499385 100644
--- a/src/lib/string.c
+++ b/src/lib/string.c
@@ -47,7 +47,7 @@ invalid:
 			charcount++;
 			switch(seqlen){
 			case 0:
-				if(NEVER(c > 0x7F))
+				if(NEVER(c == 0) || NEVER(c > 0x7F))
 					goto invalid;
 				break;
 			case 1:
@@ -73,10 +73,60 @@ invalid:
 	if(string){
 		string->bytes = bytecount;
 		string->chars = charcount;
-		string->utf8 = ((uint8_t *)string)+sizeof(*string);
 		memcpy(string->utf8, utf8, bytecount);
 	}else
 		aplwc->last_error = APLWC_OUT_OF_MEMORY;
 
 	return string;
 }
+
+uint32_t
+string_char(struct string *string, size_t offset)
+{
+	uint32_t c;
+	size_t len, i;
+
+	c = 0;
+	len = char_length(string, offset);
+	switch(char_length(string, offset)){
+	case 1:
+		c |= string->utf8[offset] & 0x7F;
+		break;
+	case 2:
+		c |= string->utf8[offset] & 0x1F;
+		break;
+	case 3:
+		c |= string->utf8[offset] & 0x0F;
+		break;
+	case 4:
+		c |= string->utf8[offset] & 0x07;
+		break;
+	}
+
+	for(i = 1; i < len; i++){
+		c <<= 6;
+		c |= string->utf8[offset] & 0x3F;
+	}
+
+	return c;
+}
+
+size_t
+char_length(struct string *string, size_t offset)
+{
+	uint8_t b;
+
+	if(offset >= string->bytes)
+		return 0;
+	b = string->utf8[offset];
+	assert(!((b & 0xC0) == 0x80));
+
+	if((b & 0x80) == 0)
+		return 1;
+	else if((b & 0xE0) == 0xC0)
+		return 2;
+	else if((b & 0xF0) == 0xE0)
+		return 3;
+	else
+		return 4;
+}
diff --git a/src/lib/syntax.c b/src/lib/syntax.c
index c1a5a85..504bb74 100644
--- a/src/lib/syntax.c
+++ b/src/lib/syntax.c
@@ -1,9 +1,156 @@
 #include "aplwc_internal.h"
 
+struct scan_state {
+	struct string *text;
+	size_t offset;
+};
+
+static void
+init_scan_state(struct scan_state *state, struct string *text)
+{
+	state->text = text;
+	state->offset = 0;
+}
+
+static int
+is_space(uint32_t c)
+{
+	return c == ' ';
+}
+
+static int
+is_letter(uint32_t c)
+{
+	return ((c >= 'a') && (c <= 'z')) || ((c >= 'A') && (c <= 'Z'));
+}
+
+static int
+is_digit(uint32_t c)
+{
+	return (c >= '0') && (c <= '9');
+}
+
+static uint32_t
+peek(struct scan_state *state)
+{
+	return string_char(state->text, state->offset);
+}
+
+static void
+advance(struct scan_state *state)
+{
+	state->offset += char_length(state->text, state->offset);
+}
+
+static void
+skip_space(struct scan_state *state)
+{
+	while(is_space(peek(state)))
+		advance(state);
+}
+
+static void
+identifier(struct scan_state *state, struct token *token)
+{
+	uint32_t c;
+	assert(is_letter(peek(state)));
+
+	do{
+		advance(state);
+		c = peek(state);
+	}while(is_letter(c) || is_digit(c) || (c == '_'));
+
+	token->tag = TOKEN_TAG_IDENTIFIER;
+}
+
+static enum aplwc_rc
+next_token(struct scan_state *state, struct token *token)
+{
+	uint32_t c;
+
+	skip_space(state);
+
+	c = peek(state);
+	if(is_letter(c)){
+		identifier(state, token);
+		return APLWC_OK;
+	}
+	if(c == '\n'){
+		advance(state);
+		token->tag = TOKEN_TAG_NEWLINE;
+		return APLWC_OK;
+	}
+
+	return APLWC_INVALID_SYNTAX;
+}
+
+static enum aplwc_rc 
+check(struct string *text, size_t *n)
+{
+	struct scan_state state;
+	struct token token;
+	enum aplwc_rc ret;
+	size_t count;
+
+	init_scan_state(&state, text);
+
+	for(count = 0; peek(&state); count++){
+		ret = next_token(&state, &token);
+		if(ret != APLWC_OK)
+			return ret;
+	}
+
+	*n = count;
+	return APLWC_OK;
+}
+
+static void
+scan(struct string *text, struct token_list *tokens)
+{
+	struct scan_state state;
+	enum aplwc_rc ret;
+	size_t count;
+
+	init_scan_state(&state, text);
+
+	for(count = 0; peek(&state); count++){
+		ret = next_token(&state, &tokens->tokens[count]);
+		assert(ret == APLWC_OK);
+	}
+}
+
 struct token_list *
 scan_line(struct aplwc *aplwc, struct string *text)
 {
-	aplwc->last_error = APLWC_NOT_YET_IMPLEMENTED;
-	UNUSED_PARAMETER(text);
-	return NULL;
+	enum aplwc_rc rc;
+	struct token_list *tokens;
+	size_t n, i;
+
+	rc = check(text, &n);
+	if(rc != APLWC_OK){
+		aplwc->last_error = rc;
+		return NULL;
+	}
+
+	tokens = memory_alloc_token_list(&aplwc->memory, n);
+	if(tokens == NULL){
+		aplwc->last_error = APLWC_OUT_OF_MEMORY;
+		return NULL;
+	}
+	tokens->length = n;
+	scan(text, tokens);
+
+	for(i = 0; i < n; i++){
+		if(tokens->tokens[i].tag != TOKEN_TAG_NEWLINE)
+			continue;
+
+		for(i++; i < n; i++){
+			if(tokens->tokens[i].tag != TOKEN_TAG_NEWLINE){
+				memory_free_token_list(&aplwc->memory, tokens);
+				return NULL;
+			}
+		}
+	}
+
+	return tokens;
 }
diff --git a/tests/aplwc_read.c b/tests/aplwc_read.c
new file mode 100644
index 0000000..b011151
--- /dev/null
+++ b/tests/aplwc_read.c
@@ -0,0 +1,48 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <aplwc.h>
+
+int
+main(int argc, char *argv[])
+{
+	uint8_t data[APLWC_MIN_MEM+100];
+	char *input;
+	struct aplwc *aplwc;
+	struct aplwc_line *line;
+	enum aplwc_rc ret;
+	FILE *f;
+	long input_size;
+
+	if(argc != 2)
+		return 1;
+
+	f = fopen(argv[1], "r");
+	if(f == NULL)
+		return 1;
+	if(fseek(f, 0, SEEK_END))
+		return 1;
+	input_size = ftell(f);
+	if(input_size < 0)
+		return 1;
+	rewind(f);
+
+	input = malloc(input_size+1);
+	if(input == NULL)
+		return 1;
+	if(fread(input, 1, input_size, f) < (size_t)input_size)
+		return 1;
+	input[input_size] = 0;
+	
+	ret = aplwc_new(&aplwc, data, sizeof(data));
+	if(ret)
+		return 1;
+
+	line = aplwc_read(aplwc, input);
+	if((line == NULL) && (aplwc_last_error(aplwc) != APLWC_OUT_OF_MEMORY))
+		return 1;
+	aplwc_free_line(line);
+	aplwc_destroy(aplwc);
+	free(input);
+
+	return 0;
+}
diff --git a/tests/aplwc_read/bad/1.txt b/tests/aplwc_read/bad/1.txt
new file mode 100644
index 0000000..422c2b7
--- /dev/null
+++ b/tests/aplwc_read/bad/1.txt
@@ -0,0 +1,2 @@
+a
+b
diff --git a/tests/aplwc_read/bad/2.txt b/tests/aplwc_read/bad/2.txt
new file mode 100644
index 0000000..94226da
--- /dev/null
+++ b/tests/aplwc_read/bad/2.txt
@@ -0,0 +1 @@
+2a
diff --git a/tests/aplwc_read/good/1.txt b/tests/aplwc_read/good/1.txt
new file mode 100644
index 0000000..7898192
--- /dev/null
+++ b/tests/aplwc_read/good/1.txt
@@ -0,0 +1 @@
+a
diff --git a/tests/aplwc_read/good/2.txt b/tests/aplwc_read/good/2.txt
new file mode 100644
index 0000000..ce63f2c
--- /dev/null
+++ b/tests/aplwc_read/good/2.txt
@@ -0,0 +1,4 @@
+a
+
+
+
diff --git a/tests/aplwc_read/good/3.txt b/tests/aplwc_read/good/3.txt
new file mode 100644
index 0000000..73c6204
--- /dev/null
+++ b/tests/aplwc_read/good/3.txt
@@ -0,0 +1 @@
+a1 b_2 cd_ef3
diff --git a/tests/aplwc_read/good/4.txt b/tests/aplwc_read/good/4.txt
new file mode 100644
index 0000000..1d7510c
--- /dev/null
+++ b/tests/aplwc_read/good/4.txt
@@ -0,0 +1 @@
+a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a a 
diff --git a/tests/aplwc_read/good/5.txt b/tests/aplwc_read/good/5.txt
new file mode 100644
index 0000000..4a873df
--- /dev/null
+++ b/tests/aplwc_read/good/5.txt
@@ -0,0 +1 @@
+aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa