More work on the scanner and tests for it

FossilOrigin-Name: 037c88ce86e1ba3a399c03a27d4c7252b52c2f500bc13f449e19e497b0607880

5 files changed, 240 insertions, 8 deletions

diff --git a/src/lib/aplwc_internal.h b/src/lib/aplwc_internal.h
index 155508d..8661bb2 100644
--- a/src/lib/aplwc_internal.h
+++ b/src/lib/aplwc_internal.h
@@ -33,11 +33,21 @@ struct aplwc_line {
 struct string {
 	size_t bytes;
 	size_t chars;
-	uint8_t *utf8;
+	uint8_t utf8[1];
+};
+
+enum token_tag {
+	TOKEN_TAG_IDENTIFIER,
+	TOKEN_TAG_NEWLINE
+};
+
+struct token {
+	enum token_tag tag;
 };
 
 struct token_list {
 	size_t length;
+	struct token tokens[1];
 };
 
 /* eval.c */
@@ -48,12 +58,16 @@ struct memory memory_init(uint8_t *, size_t);
 struct aplwc *memory_alloc_aplwc(struct memory *);
 struct aplwc_line *memory_alloc_aplwc_line(struct memory *);
 struct string *memory_alloc_string(struct memory *, size_t);
+struct token_list *memory_alloc_token_list(struct memory *, size_t);
 void memory_free_aplwc(struct memory *, struct aplwc *);
 void memory_free_aplwc_line(struct memory *, struct aplwc_line *);
 void memory_free_string(struct memory *, struct string *);
+void memory_free_token_list(struct memory *, struct token_list *);
 
 /* string.c */
 struct string *chars_to_string(struct aplwc *, const char *);
+uint32_t string_char(struct string *, size_t);
+size_t char_length(struct string *, size_t);
 
 /* syntax.c */
 struct token_list *scan_line(struct aplwc *, struct string *);
diff --git a/src/lib/main.c b/src/lib/main.c
index cca1e68..9335bf0 100644
--- a/src/lib/main.c
+++ b/src/lib/main.c
@@ -34,6 +34,8 @@ aplwc_errstr_rc(enum aplwc_rc rc)
 		return "Not yet implemented";
 	case APLWC_INVALID_UTF8:
 		return "Invalid UTF-8";
+	case APLWC_INVALID_SYNTAX:
+		return "Invalid syntax";
 	default:
 		return "(unable to produce error string)";
 	}
@@ -77,8 +79,10 @@ aplwc_read(struct aplwc *aplwc, const char *line)
 		return NULL;
 
 	ret = memory_alloc_aplwc_line(&aplwc->memory);
-	if(ret == NULL)
+	if(ret == NULL){
+		aplwc->last_error = APLWC_OUT_OF_MEMORY;
 		return NULL;
+	}
 	
 	ret->aplwc = aplwc;
 	ret->text = text;
diff --git a/src/lib/memory.c b/src/lib/memory.c
index 136c7e6..41ac90d 100644
--- a/src/lib/memory.c
+++ b/src/lib/memory.c
@@ -18,7 +18,7 @@ alloc(struct memory *m, size_t size, size_t align)
 	size_t alloc_size, adjust;
 	void *p;
 
-	ptr = (uintptr_t)m->data;
+	ptr = ((uintptr_t)m->data) + m->offset;
 	alloc_size = size;
 
 	if(NEVER(align == 0))
@@ -30,6 +30,7 @@ alloc(struct memory *m, size_t size, size_t align)
 
 	if((m->size - m->offset) >= alloc_size){
 		p = m->data+m->offset+alloc_size-size;
+		assert((((uintptr_t)p) % align) == 0);
 		m->offset += alloc_size;
 		memset(p, 0, size);
 	}else
@@ -85,3 +86,19 @@ memory_free_string(struct memory *m, struct string *p)
 	UNUSED_PARAMETER(m);
 	UNUSED_PARAMETER(p);
 }
+
+struct token_list *
+memory_alloc_token_list(struct memory *m, size_t tokens)
+{
+	struct token_list *p;
+
+	p = alloc(m, sizeof(struct token_list) + (tokens * sizeof(struct token)), alignof(struct token_list));
+	return p;
+}
+
+void
+memory_free_token_list(struct memory *m, struct token_list *p)
+{
+	UNUSED_PARAMETER(m);
+	UNUSED_PARAMETER(p);
+}
diff --git a/src/lib/string.c b/src/lib/string.c
index f6fa49d..8499385 100644
--- a/src/lib/string.c
+++ b/src/lib/string.c
@@ -47,7 +47,7 @@ invalid:
 			charcount++;
 			switch(seqlen){
 			case 0:
-				if(NEVER(c > 0x7F))
+				if(NEVER(c == 0) || NEVER(c > 0x7F))
 					goto invalid;
 				break;
 			case 1:
@@ -73,10 +73,60 @@ invalid:
 	if(string){
 		string->bytes = bytecount;
 		string->chars = charcount;
-		string->utf8 = ((uint8_t *)string)+sizeof(*string);
 		memcpy(string->utf8, utf8, bytecount);
 	}else
 		aplwc->last_error = APLWC_OUT_OF_MEMORY;
 
 	return string;
 }
+
+uint32_t
+string_char(struct string *string, size_t offset)
+{
+	uint32_t c;
+	size_t len, i;
+
+	c = 0;
+	len = char_length(string, offset);
+	switch(char_length(string, offset)){
+	case 1:
+		c |= string->utf8[offset] & 0x7F;
+		break;
+	case 2:
+		c |= string->utf8[offset] & 0x1F;
+		break;
+	case 3:
+		c |= string->utf8[offset] & 0x0F;
+		break;
+	case 4:
+		c |= string->utf8[offset] & 0x07;
+		break;
+	}
+
+	for(i = 1; i < len; i++){
+		c <<= 6;
+		c |= string->utf8[offset] & 0x3F;
+	}
+
+	return c;
+}
+
+size_t
+char_length(struct string *string, size_t offset)
+{
+	uint8_t b;
+
+	if(offset >= string->bytes)
+		return 0;
+	b = string->utf8[offset];
+	assert(!((b & 0xC0) == 0x80));
+
+	if((b & 0x80) == 0)
+		return 1;
+	else if((b & 0xE0) == 0xC0)
+		return 2;
+	else if((b & 0xF0) == 0xE0)
+		return 3;
+	else
+		return 4;
+}
diff --git a/src/lib/syntax.c b/src/lib/syntax.c
index c1a5a85..504bb74 100644
--- a/src/lib/syntax.c
+++ b/src/lib/syntax.c
@@ -1,9 +1,156 @@
 #include "aplwc_internal.h"
 
+struct scan_state {
+	struct string *text;
+	size_t offset;
+};
+
+static void
+init_scan_state(struct scan_state *state, struct string *text)
+{
+	state->text = text;
+	state->offset = 0;
+}
+
+static int
+is_space(uint32_t c)
+{
+	return c == ' ';
+}
+
+static int
+is_letter(uint32_t c)
+{
+	return ((c >= 'a') && (c <= 'z')) || ((c >= 'A') && (c <= 'Z'));
+}
+
+static int
+is_digit(uint32_t c)
+{
+	return (c >= '0') && (c <= '9');
+}
+
+static uint32_t
+peek(struct scan_state *state)
+{
+	return string_char(state->text, state->offset);
+}
+
+static void
+advance(struct scan_state *state)
+{
+	state->offset += char_length(state->text, state->offset);
+}
+
+static void
+skip_space(struct scan_state *state)
+{
+	while(is_space(peek(state)))
+		advance(state);
+}
+
+static void
+identifier(struct scan_state *state, struct token *token)
+{
+	uint32_t c;
+	assert(is_letter(peek(state)));
+
+	do{
+		advance(state);
+		c = peek(state);
+	}while(is_letter(c) || is_digit(c) || (c == '_'));
+
+	token->tag = TOKEN_TAG_IDENTIFIER;
+}
+
+static enum aplwc_rc
+next_token(struct scan_state *state, struct token *token)
+{
+	uint32_t c;
+
+	skip_space(state);
+
+	c = peek(state);
+	if(is_letter(c)){
+		identifier(state, token);
+		return APLWC_OK;
+	}
+	if(c == '\n'){
+		advance(state);
+		token->tag = TOKEN_TAG_NEWLINE;
+		return APLWC_OK;
+	}
+
+	return APLWC_INVALID_SYNTAX;
+}
+
+static enum aplwc_rc 
+check(struct string *text, size_t *n)
+{
+	struct scan_state state;
+	struct token token;
+	enum aplwc_rc ret;
+	size_t count;
+
+	init_scan_state(&state, text);
+
+	for(count = 0; peek(&state); count++){
+		ret = next_token(&state, &token);
+		if(ret != APLWC_OK)
+			return ret;
+	}
+
+	*n = count;
+	return APLWC_OK;
+}
+
+static void
+scan(struct string *text, struct token_list *tokens)
+{
+	struct scan_state state;
+	enum aplwc_rc ret;
+	size_t count;
+
+	init_scan_state(&state, text);
+
+	for(count = 0; peek(&state); count++){
+		ret = next_token(&state, &tokens->tokens[count]);
+		assert(ret == APLWC_OK);
+	}
+}
+
 struct token_list *
 scan_line(struct aplwc *aplwc, struct string *text)
 {
-	aplwc->last_error = APLWC_NOT_YET_IMPLEMENTED;
-	UNUSED_PARAMETER(text);
-	return NULL;
+	enum aplwc_rc rc;
+	struct token_list *tokens;
+	size_t n, i;
+
+	rc = check(text, &n);
+	if(rc != APLWC_OK){
+		aplwc->last_error = rc;
+		return NULL;
+	}
+
+	tokens = memory_alloc_token_list(&aplwc->memory, n);
+	if(tokens == NULL){
+		aplwc->last_error = APLWC_OUT_OF_MEMORY;
+		return NULL;
+	}
+	tokens->length = n;
+	scan(text, tokens);
+
+	for(i = 0; i < n; i++){
+		if(tokens->tokens[i].tag != TOKEN_TAG_NEWLINE)
+			continue;
+
+		for(i++; i < n; i++){
+			if(tokens->tokens[i].tag != TOKEN_TAG_NEWLINE){
+				memory_free_token_list(&aplwc->memory, tokens);
+				return NULL;
+			}
+		}
+	}
+
+	return tokens;
 }