More work on the scanner: numbers

FossilOrigin-Name: b19f7075ec69fcf519de1ed8c33e417f48e74a2d2452f5c59b93bf9101cb2d7e

15 files changed, 99 insertions, 22 deletions

diff --git a/src/lib/aplwc_internal.h b/src/lib/aplwc_internal.h
index 8661bb2..833a351 100644
--- a/src/lib/aplwc_internal.h
+++ b/src/lib/aplwc_internal.h
@@ -38,11 +38,14 @@ struct string {
 
 enum token_tag {
 	TOKEN_TAG_IDENTIFIER,
-	TOKEN_TAG_NEWLINE
+	TOKEN_TAG_NEWLINE,
+	TOKEN_TAG_NUMBER
 };
 
 struct token {
 	enum token_tag tag;
+	size_t start;
+	size_t end;
 };
 
 struct token_list {
diff --git a/src/lib/string.c b/src/lib/string.c
index 8499385..123e81b 100644
--- a/src/lib/string.c
+++ b/src/lib/string.c
@@ -105,7 +105,7 @@ string_char(struct string *string, size_t offset)
 
 	for(i = 1; i < len; i++){
 		c <<= 6;
-		c |= string->utf8[offset] & 0x3F;
+		c |= string->utf8[offset+i] & 0x3F;
 	}
 
 	return c;
diff --git a/src/lib/syntax.c b/src/lib/syntax.c
index 504bb74..6b1d0cd 100644
--- a/src/lib/syntax.c
+++ b/src/lib/syntax.c
@@ -3,6 +3,7 @@
 struct scan_state {
 	struct string *text;
 	size_t offset;
+	int skip_number_and_identifier;
 };
 
 static void
@@ -10,6 +11,7 @@ init_scan_state(struct scan_state *state, struct string *text)
 {
 	state->text = text;
 	state->offset = 0;
+	state->skip_number_and_identifier = 0;
 }
 
 static int
@@ -36,10 +38,12 @@ peek(struct scan_state *state)
 	return string_char(state->text, state->offset);
 }
 
-static void
+static uint32_t 
 advance(struct scan_state *state)
 {
 	state->offset += char_length(state->text, state->offset);
+	state->skip_number_and_identifier = 0;
+	return peek(state);
 }
 
 static void
@@ -49,39 +53,98 @@ skip_space(struct scan_state *state)
 		advance(state);
 }
 
-static void
+static int 
 identifier(struct scan_state *state, struct token *token)
 {
 	uint32_t c;
-	assert(is_letter(peek(state)));
+	if(state->skip_number_and_identifier || (!is_letter(peek(state))))
+		return 0;
 
-	do{
-		advance(state);
-		c = peek(state);
-	}while(is_letter(c) || is_digit(c) || (c == '_'));
+	do
+		c = advance(state);
+	while(is_letter(c) || is_digit(c) || (c == '_'));
 
 	token->tag = TOKEN_TAG_IDENTIFIER;
+	state->skip_number_and_identifier = 1;
+	return 1;
 }
 
-static enum aplwc_rc
-next_token(struct scan_state *state, struct token *token)
+static int 
+number(struct scan_state *state, struct token *token)
 {
 	uint32_t c;
-
-	skip_space(state);
+	int has_digits;
 
 	c = peek(state);
-	if(is_letter(c)){
-		identifier(state, token);
-		return APLWC_OK;
+
+	if(state->skip_number_and_identifier)
+		return 0;
+
+	has_digits = 0;
+
+	if(c == 0xAF) /* High minus */
+		c = advance(state);
+	
+	if(is_digit(c)){ /* integer part */
+		has_digits = 1;	
+		do
+			c = advance(state);
+		while(is_digit(c));
 	}
-	if(c == '\n'){
-		advance(state);
-		token->tag = TOKEN_TAG_NEWLINE;
-		return APLWC_OK;
+
+	if(c == '.'){
+		c = advance(state);
+
+		if(is_digit(c)){ /* fractional part */
+			has_digits = 1;
+			do
+				c = advance(state);
+			while(is_digit(c));
+		}
 	}
 
-	return APLWC_INVALID_SYNTAX;
+	if(has_digits == 0)
+		return 0;
+
+	token->tag = TOKEN_TAG_NUMBER;
+	state->skip_number_and_identifier = 1;
+	return 1;
+}
+
+static int 
+newline(struct scan_state *state, struct token *token)
+{
+	if(peek(state) != '\n')
+		return 0;
+
+	advance(state);
+	token->tag = TOKEN_TAG_NEWLINE;
+	return 1;
+}
+
+static enum aplwc_rc
+next_token(struct scan_state *state, struct token *token)
+{
+	size_t i;
+	int (*scans[])(struct scan_state *, struct token *token) = {
+		identifier,
+		number,
+		newline
+	};
+
+	skip_space(state);
+	token->start = state->offset;
+	for(i = 0; i < NELEM(scans); i++){
+		if(scans[i](state, token))
+			break;
+		state->offset = token->start;
+	}
+	token->end = state->offset;
+	skip_space(state);
+	if(token->start == token->end)
+		return APLWC_INVALID_SYNTAX;
+	else
+		return APLWC_OK;
 }
 
 static enum aplwc_rc 
diff --git a/tests/aplwc_read.c b/tests/aplwc_read.c
index b011151..c54f598 100644
--- a/tests/aplwc_read.c
+++ b/tests/aplwc_read.c
@@ -5,7 +5,7 @@
 int
 main(int argc, char *argv[])
 {
-	uint8_t data[APLWC_MIN_MEM+100];
+	uint8_t data[APLWC_MIN_MEM+1000];
 	char *input;
 	struct aplwc *aplwc;
 	struct aplwc_line *line;
diff --git a/tests/aplwc_read/bad/3.txt b/tests/aplwc_read/bad/3.txt
new file mode 100644
index 0000000..730edf3
--- /dev/null
+++ b/tests/aplwc_read/bad/3.txt
@@ -0,0 +1 @@
+¯abc
diff --git a/tests/aplwc_read/bad/4.txt b/tests/aplwc_read/bad/4.txt
new file mode 100644
index 0000000..dddd5d5
--- /dev/null
+++ b/tests/aplwc_read/bad/4.txt
@@ -0,0 +1 @@
+¯
diff --git a/tests/aplwc_read/bad/5.txt b/tests/aplwc_read/bad/5.txt
new file mode 100644
index 0000000..91043bd
--- /dev/null
+++ b/tests/aplwc_read/bad/5.txt
@@ -0,0 +1 @@
+¯.
diff --git a/tests/aplwc_read/bad/6.txt b/tests/aplwc_read/bad/6.txt
new file mode 100644
index 0000000..9c558e3
--- /dev/null
+++ b/tests/aplwc_read/bad/6.txt
@@ -0,0 +1 @@
+.
diff --git a/tests/aplwc_read/good/10.txt b/tests/aplwc_read/good/10.txt
new file mode 100644
index 0000000..ea0adda
--- /dev/null
+++ b/tests/aplwc_read/good/10.txt
@@ -0,0 +1 @@
+¯10
diff --git a/tests/aplwc_read/good/11.txt b/tests/aplwc_read/good/11.txt
new file mode 100644
index 0000000..64774b3
--- /dev/null
+++ b/tests/aplwc_read/good/11.txt
@@ -0,0 +1 @@
+¯10.20
diff --git a/tests/aplwc_read/good/12.txt b/tests/aplwc_read/good/12.txt
new file mode 100644
index 0000000..a5cc92f
--- /dev/null
+++ b/tests/aplwc_read/good/12.txt
@@ -0,0 +1 @@
+¯.20
diff --git a/tests/aplwc_read/good/6.txt b/tests/aplwc_read/good/6.txt
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/tests/aplwc_read/good/6.txt
@@ -0,0 +1 @@
+1
diff --git a/tests/aplwc_read/good/7.txt b/tests/aplwc_read/good/7.txt
new file mode 100644
index 0000000..190a180
--- /dev/null
+++ b/tests/aplwc_read/good/7.txt
@@ -0,0 +1 @@
+123
diff --git a/tests/aplwc_read/good/8.txt b/tests/aplwc_read/good/8.txt
new file mode 100644
index 0000000..0d9a755
--- /dev/null
+++ b/tests/aplwc_read/good/8.txt
@@ -0,0 +1 @@
+22 aa
diff --git a/tests/aplwc_read/good/9.txt b/tests/aplwc_read/good/9.txt
new file mode 100644
index 0000000..2301365
--- /dev/null
+++ b/tests/aplwc_read/good/9.txt
@@ -0,0 +1 @@
+.555