Start working on the scanner.

For now, all that is implemented is some functionality which imports UTF-8 into the
memory managed by aplwc, after checking that the data is in fact valid UTF-8.

FossilOrigin-Name: c861e88777d8180cb7b37c11fc7016a900f9d5821a7e3c8e8aab54118d2fca94

11 files changed, 304 insertions, 6 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 18aaeb1..37e06d2 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -29,6 +29,7 @@ target_compile_options(common INTERFACE
 		--coverage
 		-fcondition-coverage
 		-DAPLWC_BUILD_TYPE_COVERAGE
+		-DNDEBUG
 	>
 
 	$<$<CONFIG:Release>:
@@ -47,8 +48,11 @@ target_link_options(common INTERFACE
 )
 
 add_library(aplwc
+	src/lib/eval.c
 	src/lib/main.c
 	src/lib/memory.c
+	src/lib/string.c
+	src/lib/syntax.c
 )
 target_link_libraries(aplwc PRIVATE common)
 
@@ -75,6 +79,11 @@ set_tests_properties("aplwc-repl-tiny" PROPERTIES
 	WILL_FAIL TRUE
 )
 
+add_test(NAME "aplwc-repl-large" COMMAND "aplwc-repl")
+set_tests_properties("aplwc-repl-large" PROPERTIES
+	ENVIRONMENT "APLWC_MEM=64000"
+)
+
 add_test(NAME "aplwc-repl-emptyvar" COMMAND "aplwc-repl")
 set_tests_properties("aplwc-repl-emptyvar" PROPERTIES
 	ENVIRONMENT "APLWC_MEM="
diff --git a/src/bin/main.c b/src/bin/main.c
index 80b2ee0..a1c65bf 100644
--- a/src/bin/main.c
+++ b/src/bin/main.c
@@ -5,6 +5,7 @@ int
 main(void)
 {
 	struct aplwc *aplwc;
+	struct aplwc_line *line;
 	uint8_t *data;
 	int ret;
 	size_t memsize;
@@ -24,6 +25,13 @@ main(void)
 		return 1;
 	}
 
+	line = aplwc_read(aplwc, "a←10+20");
+	if(line == NULL)
+		fprintf(stderr, "Could not parse line: %s\n", aplwc_errstr(aplwc));
+	else
+		aplwc_eval(line);
+	aplwc_free_line(line);
+
 	aplwc_destroy(aplwc);
 
 	free(data);
diff --git a/src/include/aplwc.h b/src/include/aplwc.h
index d4d101c..75d9485 100644
--- a/src/include/aplwc.h
+++ b/src/include/aplwc.h
@@ -7,17 +7,24 @@
 enum aplwc_rc {
 	APLWC_OK,
 	APLWC_OUT_OF_MEMORY,
+	APLWC_NOT_YET_IMPLEMENTED,
+	APLWC_INVALID_UTF8,
 
-	APLWC_RC_LAST = APLWC_OUT_OF_MEMORY
+	APLWC_RC_LAST = APLWC_INVALID_UTF8
 };
 
 struct aplwc;
+struct aplwc_line;
 
 enum aplwc_rc aplwc_new(struct aplwc **, uint8_t *, size_t);
 void aplwc_destroy(struct aplwc *);
 const char *aplwc_errstr_rc(enum aplwc_rc);
 const char *aplwc_errstr(struct aplwc *);
+enum aplwc_rc aplwc_last_error(struct aplwc *);
+struct aplwc_line *aplwc_read(struct aplwc *, const char *);
+void aplwc_free_line(struct aplwc_line *);
+enum aplwc_rc aplwc_eval(struct aplwc_line *);
 
-#define APLWC_MIN_MEM (128)
+#define APLWC_MIN_MEM (64)
 
 #endif /* APLWC_H */
diff --git a/src/lib/aplwc_internal.h b/src/lib/aplwc_internal.h
index 52ecd2e..155508d 100644
--- a/src/lib/aplwc_internal.h
+++ b/src/lib/aplwc_internal.h
@@ -21,16 +21,41 @@ struct memory {
 
 struct aplwc {
 	enum aplwc_rc last_error;
-	char last_error_msg[64];
 	struct memory memory;
 };
 
-/* errors.c */
-const char *rc_to_string(enum aplwc_rc);
+struct aplwc_line {
+	struct aplwc *aplwc;
+	struct string *text;
+	struct token_list *tokens;
+};
+
+struct string {
+	size_t bytes;
+	size_t chars;
+	uint8_t *utf8;
+};
+
+struct token_list {
+	size_t length;
+};
+
+/* eval.c */
+void eval_line(struct aplwc_line *);
 
 /* memory.c */
 struct memory memory_init(uint8_t *, size_t);
 struct aplwc *memory_alloc_aplwc(struct memory *);
+struct aplwc_line *memory_alloc_aplwc_line(struct memory *);
+struct string *memory_alloc_string(struct memory *, size_t);
 void memory_free_aplwc(struct memory *, struct aplwc *);
+void memory_free_aplwc_line(struct memory *, struct aplwc_line *);
+void memory_free_string(struct memory *, struct string *);
+
+/* string.c */
+struct string *chars_to_string(struct aplwc *, const char *);
+
+/* syntax.c */
+struct token_list *scan_line(struct aplwc *, struct string *);
 
 #endif /* APLWC_INTERNAL_H */
diff --git a/src/lib/eval.c b/src/lib/eval.c
new file mode 100644
index 0000000..89bdf59
--- /dev/null
+++ b/src/lib/eval.c
@@ -0,0 +1,7 @@
+#include "aplwc_internal.h"
+
+void
+eval_line(struct aplwc_line *line)
+{
+	line->aplwc->last_error = APLWC_NOT_YET_IMPLEMENTED;
+}
diff --git a/src/lib/main.c b/src/lib/main.c
index f7cbe7d..cca1e68 100644
--- a/src/lib/main.c
+++ b/src/lib/main.c
@@ -30,6 +30,10 @@ aplwc_errstr_rc(enum aplwc_rc rc)
 		return "";
 	case APLWC_OUT_OF_MEMORY:
 		return "Out of memory";
+	case APLWC_NOT_YET_IMPLEMENTED:
+		return "Not yet implemented";
+	case APLWC_INVALID_UTF8:
+		return "Invalid UTF-8";
 	default:
 		return "(unable to produce error string)";
 	}
@@ -50,3 +54,51 @@ aplwc_errstr(struct aplwc *aplwc)
 
 	return err;
 }
+
+enum aplwc_rc
+aplwc_last_error(struct aplwc *aplwc)
+{
+	return aplwc->last_error;
+}
+
+struct aplwc_line *
+aplwc_read(struct aplwc *aplwc, const char *line)
+{
+	struct string *text;
+	struct token_list *tokens;
+	struct aplwc_line *ret;
+
+	text = chars_to_string(aplwc, line);
+	if(text == NULL)
+		return NULL;
+
+	tokens = scan_line(aplwc, text);
+	if(tokens == NULL)
+		return NULL;
+
+	ret = memory_alloc_aplwc_line(&aplwc->memory);
+	if(ret == NULL)
+		return NULL;
+	
+	ret->aplwc = aplwc;
+	ret->text = text;
+	ret->tokens = tokens;
+
+	return ret;
+}
+
+void
+aplwc_free_line(struct aplwc_line *line)
+{
+	if(line == NULL)
+		return;
+	memory_free_aplwc_line(&line->aplwc->memory, line);
+}
+
+enum aplwc_rc
+aplwc_eval(struct aplwc_line *line)
+{
+	line->aplwc->last_error = APLWC_OK;
+	eval_line(line);
+	return line->aplwc->last_error;
+}
diff --git a/src/lib/memory.c b/src/lib/memory.c
index 187137e..136c7e6 100644
--- a/src/lib/memory.c
+++ b/src/lib/memory.c
@@ -53,3 +53,35 @@ memory_free_aplwc(struct memory *m, struct aplwc *p)
 	UNUSED_PARAMETER(m);
 	UNUSED_PARAMETER(p);
 }
+
+struct aplwc_line *
+memory_alloc_aplwc_line(struct memory *m)
+{
+	struct aplwc_line *p;
+
+	p = alloc(m, sizeof(struct aplwc_line), alignof(struct aplwc_line));
+	return p;
+}
+
+void
+memory_free_aplwc_line(struct memory *m, struct aplwc_line *p)
+{
+	UNUSED_PARAMETER(m);
+	UNUSED_PARAMETER(p);
+}
+
+struct string *
+memory_alloc_string(struct memory *m, size_t bytes)
+{
+	struct string *p;
+
+	p = alloc(m, sizeof(struct string) + bytes, alignof(struct string));
+	return p;
+}
+
+void
+memory_free_string(struct memory *m, struct string *p)
+{
+	UNUSED_PARAMETER(m);
+	UNUSED_PARAMETER(p);
+}
diff --git a/src/lib/string.c b/src/lib/string.c
new file mode 100644
index 0000000..f6fa49d
--- /dev/null
+++ b/src/lib/string.c
@@ -0,0 +1,82 @@
+#include "aplwc_internal.h"
+
+struct string *
+chars_to_string(struct aplwc *aplwc, const char *chars)
+{
+	const uint8_t *utf8;
+	size_t bytecount, charcount;
+	struct string *string;
+	uint32_t c;
+	uint8_t b;
+	int seqlen, extra;
+
+	utf8 = (const uint8_t *)chars;
+	seqlen = extra = 0;
+	for(bytecount = 0, charcount = 0; utf8[bytecount]; bytecount++){
+		b = utf8[bytecount];
+		if(extra == 0){
+			c = 0;
+			if((b & 0x80) == 0){
+				seqlen = 0;
+				c |= (b & 0x7F);
+			}else if((b & 0xE0) == 0xC0){
+				seqlen = 1;
+				c |= (b & 0x1F);
+			}else if((b & 0xF0) == 0xE0){
+				seqlen = 2;
+				c |= (b & 0x0F);
+			}else if((b & 0xF8) == 0xF0){
+				seqlen = 3;
+				c |= (b & 0x7);
+			}else{
+invalid:
+				aplwc->last_error = APLWC_INVALID_UTF8;
+				return NULL;
+			}
+			extra = seqlen;
+		}else{
+			extra--;
+			c <<= 6;
+			if((b & 0xC0) == 0x80)
+				c |= (b & 0x3F);
+			else
+				goto invalid;	
+		}
+
+		if(extra == 0){
+			charcount++;
+			switch(seqlen){
+			case 0:
+				if(NEVER(c > 0x7F))
+					goto invalid;
+				break;
+			case 1:
+				if((c < 0x80) || NEVER(c > 0x07FF))
+					goto invalid;
+				break;
+			case 2:
+				if((c < 0x0800) || NEVER(c > 0xFFFF))
+					goto invalid;
+				break;
+			case 3:
+				if((c < 0x010000) || NEVER(c > 0x10FFFF))
+					goto invalid;
+				break;
+			}
+		}
+	}
+	
+	if(extra != 0)
+		goto invalid;
+
+	string = memory_alloc_string(&aplwc->memory, bytecount);
+	if(string){
+		string->bytes = bytecount;
+		string->chars = charcount;
+		string->utf8 = ((uint8_t *)string)+sizeof(*string);
+		memcpy(string->utf8, utf8, bytecount);
+	}else
+		aplwc->last_error = APLWC_OUT_OF_MEMORY;
+
+	return string;
+}
diff --git a/src/lib/syntax.c b/src/lib/syntax.c
new file mode 100644
index 0000000..c1a5a85
--- /dev/null
+++ b/src/lib/syntax.c
@@ -0,0 +1,9 @@
+#include "aplwc_internal.h"
+
+struct token_list *
+scan_line(struct aplwc *aplwc, struct string *text)
+{
+	aplwc->last_error = APLWC_NOT_YET_IMPLEMENTED;
+	UNUSED_PARAMETER(text);
+	return NULL;
+}
diff --git a/tests/aplwc_errstr.c b/tests/aplwc_errstr.c
index 0efb777..8014c1e 100644
--- a/tests/aplwc_errstr.c
+++ b/tests/aplwc_errstr.c
@@ -35,7 +35,7 @@ main(void)
 	if(err == NULL)
 		return 1;
 
-	for(ret = 0; ret < APLWC_RC_LAST; ret++){
+	for(ret = 0; ret <= APLWC_RC_LAST; ret++){
 		err = aplwc_errstr_rc(ret);
 		if(err == NULL)
 			return 1;
diff --git a/tests/aplwc_utf8.c b/tests/aplwc_utf8.c
new file mode 100644
index 0000000..b97b978
--- /dev/null
+++ b/tests/aplwc_utf8.c
@@ -0,0 +1,67 @@
+#include <aplwc.h>
+
+uint8_t bad1[] = {0xff,0};
+uint8_t bad2[] = {0x80,0};
+uint8_t bad3[] = {0xc0,0x80,0};
+uint8_t bad4[] = {0xc0,0x40,0};
+uint8_t bad5[] = {0xe0,0x80,0x80,0};
+uint8_t bad6[] = {0xc0,0};
+uint8_t bad7[] = {0xF0,0x80,0x80,0x80,0};
+
+uint8_t good1[] = {0x61,0x70,0x6c,0x77,0x63,0};
+uint8_t good2[] = {0x41,0xe2,0x89,0xa2,0xce,0x91,0x2e,0};
+uint8_t good3[] = {0xed,0x95,0x9c,0xea,0xb5,0xad,0xec,0x96,0xb4,0};
+uint8_t good4[] = {0xe6,0x97,0xa5,0xe6,0x9c,0xac,0xe8,0xaa,0x9e,0};
+uint8_t good5[] = {0xef,0xbb,0xbf,0xf0,0xa3,0x8e,0xb4,0};
+
+int
+main(void)
+{
+	uint8_t data[APLWC_MIN_MEM+64000];
+	struct aplwc *aplwc;
+	struct aplwc_line *line;
+	enum aplwc_rc ret;
+	size_t i;
+
+	uint8_t *bad_tests[] = {
+		bad1,
+		bad2,
+		bad3,
+		bad4,
+		bad5,
+		bad6,
+		bad7
+	};
+
+	uint8_t *good_tests[] = {
+		good1,
+		good2,
+		good3,
+		good4,
+		good5
+	};
+
+	ret = aplwc_new(&aplwc, data, sizeof(data));
+	if(ret)
+		return 1;
+
+	for(i = 0; i < (sizeof(bad_tests)/sizeof(*bad_tests)); i++){
+		line = aplwc_read(aplwc, (char *)bad_tests[i]);
+		if(line != NULL)
+			return 1;
+		if(aplwc_last_error(aplwc) != APLWC_INVALID_UTF8)
+			return 1;
+		aplwc_free_line(line);
+	}
+
+	for(i = 0; i < (sizeof(good_tests)/sizeof(*good_tests)); i++){
+		line = aplwc_read(aplwc, (char *)good_tests[i]);
+		if((line == NULL) && (aplwc_last_error(aplwc) == APLWC_INVALID_UTF8))
+			return 1;
+		aplwc_free_line(line);
+	}
+
+	aplwc_destroy(aplwc);
+
+	return 0;
+}