More work on the scanner: numbers

FossilOrigin-Name: b19f7075ec69fcf519de1ed8c33e417f48e74a2d2452f5c59b93bf9101cb2d7e

1 files changed, 82 insertions, 19 deletions

diff --git a/src/lib/syntax.c b/src/lib/syntax.c
index 504bb74..6b1d0cd 100644
--- a/src/lib/syntax.c
+++ b/src/lib/syntax.c
@@ -3,6 +3,7 @@
 struct scan_state {
 	struct string *text;
 	size_t offset;
+	int skip_number_and_identifier;
 };
 
 static void
@@ -10,6 +11,7 @@ init_scan_state(struct scan_state *state, struct string *text)
 {
 	state->text = text;
 	state->offset = 0;
+	state->skip_number_and_identifier = 0;
 }
 
 static int
@@ -36,10 +38,12 @@ peek(struct scan_state *state)
 	return string_char(state->text, state->offset);
 }
 
-static void
+static uint32_t 
 advance(struct scan_state *state)
 {
 	state->offset += char_length(state->text, state->offset);
+	state->skip_number_and_identifier = 0;
+	return peek(state);
 }
 
 static void
@@ -49,39 +53,98 @@ skip_space(struct scan_state *state)
 		advance(state);
 }
 
-static void
+static int 
 identifier(struct scan_state *state, struct token *token)
 {
 	uint32_t c;
-	assert(is_letter(peek(state)));
+	if(state->skip_number_and_identifier || (!is_letter(peek(state))))
+		return 0;
 
-	do{
-		advance(state);
-		c = peek(state);
-	}while(is_letter(c) || is_digit(c) || (c == '_'));
+	do
+		c = advance(state);
+	while(is_letter(c) || is_digit(c) || (c == '_'));
 
 	token->tag = TOKEN_TAG_IDENTIFIER;
+	state->skip_number_and_identifier = 1;
+	return 1;
 }
 
-static enum aplwc_rc
-next_token(struct scan_state *state, struct token *token)
+static int 
+number(struct scan_state *state, struct token *token)
 {
 	uint32_t c;
-
-	skip_space(state);
+	int has_digits;
 
 	c = peek(state);
-	if(is_letter(c)){
-		identifier(state, token);
-		return APLWC_OK;
+
+	if(state->skip_number_and_identifier)
+		return 0;
+
+	has_digits = 0;
+
+	if(c == 0xAF) /* High minus */
+		c = advance(state);
+	
+	if(is_digit(c)){ /* integer part */
+		has_digits = 1;	
+		do
+			c = advance(state);
+		while(is_digit(c));
 	}
-	if(c == '\n'){
-		advance(state);
-		token->tag = TOKEN_TAG_NEWLINE;
-		return APLWC_OK;
+
+	if(c == '.'){
+		c = advance(state);
+
+		if(is_digit(c)){ /* fractional part */
+			has_digits = 1;
+			do
+				c = advance(state);
+			while(is_digit(c));
+		}
 	}
 
-	return APLWC_INVALID_SYNTAX;
+	if(has_digits == 0)
+		return 0;
+
+	token->tag = TOKEN_TAG_NUMBER;
+	state->skip_number_and_identifier = 1;
+	return 1;
+}
+
+static int 
+newline(struct scan_state *state, struct token *token)
+{
+	if(peek(state) != '\n')
+		return 0;
+
+	advance(state);
+	token->tag = TOKEN_TAG_NEWLINE;
+	return 1;
+}
+
+static enum aplwc_rc
+next_token(struct scan_state *state, struct token *token)
+{
+	size_t i;
+	int (*scans[])(struct scan_state *, struct token *token) = {
+		identifier,
+		number,
+		newline
+	};
+
+	skip_space(state);
+	token->start = state->offset;
+	for(i = 0; i < NELEM(scans); i++){
+		if(scans[i](state, token))
+			break;
+		state->offset = token->start;
+	}
+	token->end = state->offset;
+	skip_space(state);
+	if(token->start == token->end)
+		return APLWC_INVALID_SYNTAX;
+	else
+		return APLWC_OK;
 }
 
 static enum aplwc_rc