4 files changed, 28 insertions, 15 deletions

diff --git a/src/lib/aplwc_internal.h b/src/lib/aplwc_internal.h
index 8c13dfe..c123d63 100644
--- a/src/lib/aplwc_internal.h
+++ b/src/lib/aplwc_internal.h
@@ -180,4 +180,9 @@ enum aplwc_rc getsym(struct aplwc *, struct string *, symbol_t *);
 /* module.c */
 enum aplwc_rc load_module(struct aplwc *, struct string *, struct token_list *);
 
+/* unicode.c */
+int is_id_start(uint32_t);
+int is_id_continue(uint32_t);
+int is_pattern_syntax(uint32_t);
+
 #endif /* APLWC_INTERNAL_H */
diff --git a/src/lib/string.c b/src/lib/string.c
index db08b28..4101b80 100644
--- a/src/lib/string.c
+++ b/src/lib/string.c
@@ -180,7 +180,7 @@ substring(struct aplwc *aplwc, struct string *s, size_t start, size_t length)
 
 	bytes = 0;
 	for(i = 0; i < length; i++)
-		bytes += char_length(s, start+i);
+		bytes += char_length(s, start+bytes);
 
 	r = memory_alloc_string(aplwc, bytes);
 	if(r){
diff --git a/src/lib/syntax.c b/src/lib/syntax.c
index 1740a4e..d864bf4 100644
--- a/src/lib/syntax.c
+++ b/src/lib/syntax.c
@@ -42,12 +42,6 @@ is_space(uint32_t c)
 }
 
 static int
-is_letter(uint32_t c)
-{
-	return ((c >= 'a') && (c <= 'z')) || ((c >= 'A') && (c <= 'Z'));
-}
-
-static int
 is_digit(uint32_t c)
 {
 	return (c >= '0') && (c <= '9');
@@ -111,6 +105,10 @@ single_sym(struct scan_state *state, struct token_list *tokens, size_t token_ind
 	case '(':    T.tag = TOKEN_TAG_LEFT_PAREN; break;
 	case ')':    T.tag = TOKEN_TAG_RIGHT_PAREN; break;
 	default:
+		if(!is_pattern_syntax(c)){
+			rc = APLWC_INVALID_SYNTAX;
+			goto end;
+		}
 		T.tag = TOKEN_TAG_UNKNOWN_SYM;
 		if(!check_only){
 			if((rc = newsym(state, start, 1, &sym)) != APLWC_OK)
@@ -139,7 +137,7 @@ identifier(struct scan_state *state, struct token_list *tokens, size_t token_ind
 
 	rc = APLWC_INVALID_SYNTAX;
 
-	if(state->skip_number_and_identifier || (!is_letter(peek(state))))
+	if(state->skip_number_and_identifier || (!is_id_start(peek(state))))
 		goto end;
 
 	start = state->offset;
@@ -147,7 +145,7 @@ identifier(struct scan_state *state, struct token_list *tokens, size_t token_ind
 	do{
 		c = advance(state);
 		len++;
-	}while(is_letter(c) || is_digit(c) || (c == '_'));
+	}while(is_id_continue(c));
 
 	T.tag = TOKEN_TAG_IDENTIFIER;
 	if(!check_only){
@@ -186,7 +184,7 @@ keyword(struct scan_state *state, struct token_list *tokens, size_t token_index,
 	do{
 		c = advance(state);
 		len++;
-	}while(is_letter(c));
+	}while(is_id_continue(c));
 
 	rc = APLWC_OK;
 	T.tag = TOKEN_TAG_KEYWORD;
diff --git a/src/lib/unicode.c b/src/lib/unicode.c
index b5c6471..48cb91e 100644
--- a/src/lib/unicode.c
+++ b/src/lib/unicode.c
@@ -18,18 +18,28 @@
  */
 
 #include "aplwc_internal.h"
-#include "unicode.h"
+#include "unicode_properties.c"
 
 static int
 is(uint32_t c, enum unicode_property p)
 {
-	int n;
+	int n, a, b, i;
 	struct unicode_range *r;
 
-	UNUSED_PARAMETER(c);
-
 	unicode_get(p, &n, &r);
-
+	a = 0;
+	b = n-1;
+	while(a <= b){
+		i = a + (b-a) / 2;
+		if(c >= r[i].from && c <= r[i].to)
+			return 1;
+		else if(c < r[i].from)
+			b = i-1;
+		else{
+			assert(c > r[i].to);
+			a = i+1;
+		}
+	}
 	return 0;
 }