Use the parsed unicode stuff to decide what is valid identifier characters, and symbol characters.unicode-props

FossilOrigin-Name: afa25da5a44e8cb032d74269a6d4b7ec1e8842f18cefbd54d5f84a0b6d780f70

1 files changed, 7 insertions, 9 deletions

diff --git a/src/lib/syntax.c b/src/lib/syntax.c
index 1740a4e..d864bf4 100644
--- a/src/lib/syntax.c
+++ b/src/lib/syntax.c
@@ -42,12 +42,6 @@ is_space(uint32_t c)
 }
 
 static int
-is_letter(uint32_t c)
-{
-	return ((c >= 'a') && (c <= 'z')) || ((c >= 'A') && (c <= 'Z'));
-}
-
-static int
 is_digit(uint32_t c)
 {
 	return (c >= '0') && (c <= '9');
@@ -111,6 +105,10 @@ single_sym(struct scan_state *state, struct token_list *tokens, size_t token_ind
 	case '(':    T.tag = TOKEN_TAG_LEFT_PAREN; break;
 	case ')':    T.tag = TOKEN_TAG_RIGHT_PAREN; break;
 	default:
+		if(!is_pattern_syntax(c)){
+			rc = APLWC_INVALID_SYNTAX;
+			goto end;
+		}
 		T.tag = TOKEN_TAG_UNKNOWN_SYM;
 		if(!check_only){
 			if((rc = newsym(state, start, 1, &sym)) != APLWC_OK)
@@ -139,7 +137,7 @@ identifier(struct scan_state *state, struct token_list *tokens, size_t token_ind
 
 	rc = APLWC_INVALID_SYNTAX;
 
-	if(state->skip_number_and_identifier || (!is_letter(peek(state))))
+	if(state->skip_number_and_identifier || (!is_id_start(peek(state))))
 		goto end;
 
 	start = state->offset;
@@ -147,7 +145,7 @@ identifier(struct scan_state *state, struct token_list *tokens, size_t token_ind
 	do{
 		c = advance(state);
 		len++;
-	}while(is_letter(c) || is_digit(c) || (c == '_'));
+	}while(is_id_continue(c));
 
 	T.tag = TOKEN_TAG_IDENTIFIER;
 	if(!check_only){
@@ -186,7 +184,7 @@ keyword(struct scan_state *state, struct token_list *tokens, size_t token_index,
 	do{
 		c = advance(state);
 		len++;
-	}while(is_letter(c));
+	}while(is_id_continue(c));
 
 	rc = APLWC_OK;
 	T.tag = TOKEN_TAG_KEYWORD;