Use the parsed unicode stuff to decide what is valid identifier characters, and symbol characters.unicode-props

FossilOrigin-Name: afa25da5a44e8cb032d74269a6d4b7ec1e8842f18cefbd54d5f84a0b6d780f70

8 files changed, 55 insertions, 24 deletions

diff --git a/Makefile.in b/Makefile.in
index 6beeab9..83c4626 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -541,7 +541,7 @@ lib_LIBRARIES = libaplwc.a
 include_HEADERS = src/include/aplwc.h
 noinst_HEADERS = src/include/aplwc_macros.h
 BUILT_SOURCES = $(top_builddir)/generated/boot.h \
-	$(top_builddir)/generated/unicode.h
+	$(top_builddir)/generated/unicode_properties.c
 CLEANFILES = $(BUILT_SOURCES)
 EXTRA_DIST = src/lib/boot.aplwc $(LOG_COMPILER) \
 	tests/scripts/simfail.sh $(TXT_LOG_COMPILER) $(TEST_INPUTS) \
@@ -1731,7 +1731,7 @@ $(top_builddir)/generated/boot.h: genboot src/lib/boot.aplwc
 	$(MKDIR_P) $(top_builddir)/generated
 	./genboot $(top_srcdir)/src/lib/boot.aplwc > $@
 
-$(top_builddir)/generated/unicode.h: genunicode src/unicode/DerivedCoreProperties.txt src/unicode/PropList.txt
+$(top_builddir)/generated/unicode_properties.c: genunicode src/unicode/DerivedCoreProperties.txt src/unicode/PropList.txt
 	$(MKDIR_P) $(top_builddir)/generated
 	./genunicode $(top_srcdir)/src/unicode > $@
 
diff --git a/src/lib/aplwc_internal.h b/src/lib/aplwc_internal.h
index 8c13dfe..c123d63 100644
--- a/src/lib/aplwc_internal.h
+++ b/src/lib/aplwc_internal.h
@@ -180,4 +180,9 @@ enum aplwc_rc getsym(struct aplwc *, struct string *, symbol_t *);
 /* module.c */
 enum aplwc_rc load_module(struct aplwc *, struct string *, struct token_list *);
 
+/* unicode.c */
+int is_id_start(uint32_t);
+int is_id_continue(uint32_t);
+int is_pattern_syntax(uint32_t);
+
 #endif /* APLWC_INTERNAL_H */
diff --git a/src/lib/string.c b/src/lib/string.c
index db08b28..4101b80 100644
--- a/src/lib/string.c
+++ b/src/lib/string.c
@@ -180,7 +180,7 @@ substring(struct aplwc *aplwc, struct string *s, size_t start, size_t length)
 
 	bytes = 0;
 	for(i = 0; i < length; i++)
-		bytes += char_length(s, start+i);
+		bytes += char_length(s, start+bytes);
 
 	r = memory_alloc_string(aplwc, bytes);
 	if(r){
diff --git a/src/lib/syntax.c b/src/lib/syntax.c
index 1740a4e..d864bf4 100644
--- a/src/lib/syntax.c
+++ b/src/lib/syntax.c
@@ -42,12 +42,6 @@ is_space(uint32_t c)
 }
 
 static int
-is_letter(uint32_t c)
-{
-	return ((c >= 'a') && (c <= 'z')) || ((c >= 'A') && (c <= 'Z'));
-}
-
-static int
 is_digit(uint32_t c)
 {
 	return (c >= '0') && (c <= '9');
@@ -111,6 +105,10 @@ single_sym(struct scan_state *state, struct token_list *tokens, size_t token_ind
 	case '(':    T.tag = TOKEN_TAG_LEFT_PAREN; break;
 	case ')':    T.tag = TOKEN_TAG_RIGHT_PAREN; break;
 	default:
+		if(!is_pattern_syntax(c)){
+			rc = APLWC_INVALID_SYNTAX;
+			goto end;
+		}
 		T.tag = TOKEN_TAG_UNKNOWN_SYM;
 		if(!check_only){
 			if((rc = newsym(state, start, 1, &sym)) != APLWC_OK)
@@ -139,7 +137,7 @@ identifier(struct scan_state *state, struct token_list *tokens, size_t token_ind
 
 	rc = APLWC_INVALID_SYNTAX;
 
-	if(state->skip_number_and_identifier || (!is_letter(peek(state))))
+	if(state->skip_number_and_identifier || (!is_id_start(peek(state))))
 		goto end;
 
 	start = state->offset;
@@ -147,7 +145,7 @@ identifier(struct scan_state *state, struct token_list *tokens, size_t token_ind
 	do{
 		c = advance(state);
 		len++;
-	}while(is_letter(c) || is_digit(c) || (c == '_'));
+	}while(is_id_continue(c));
 
 	T.tag = TOKEN_TAG_IDENTIFIER;
 	if(!check_only){
@@ -186,7 +184,7 @@ keyword(struct scan_state *state, struct token_list *tokens, size_t token_index,
 	do{
 		c = advance(state);
 		len++;
-	}while(is_letter(c));
+	}while(is_id_continue(c));
 
 	rc = APLWC_OK;
 	T.tag = TOKEN_TAG_KEYWORD;
diff --git a/src/lib/unicode.c b/src/lib/unicode.c
index b5c6471..48cb91e 100644
--- a/src/lib/unicode.c
+++ b/src/lib/unicode.c
@@ -18,18 +18,28 @@
  */
 
 #include "aplwc_internal.h"
-#include "unicode.h"
+#include "unicode_properties.c"
 
 static int
 is(uint32_t c, enum unicode_property p)
 {
-	int n;
+	int n, a, b, i;
 	struct unicode_range *r;
 
-	UNUSED_PARAMETER(c);
-
 	unicode_get(p, &n, &r);
-
+	a = 0;
+	b = n-1;
+	while(a <= b){
+		i = a + (b-a) / 2;
+		if(c >= r[i].from && c <= r[i].to)
+			return 1;
+		else if(c < r[i].from)
+			b = i-1;
+		else{
+			assert(c > r[i].to);
+			a = i+1;
+		}
+	}
 	return 0;
 }
 
diff --git a/src/tools/Makefile.am b/src/tools/Makefile.am
index 1930970..b05ab8a 100644
--- a/src/tools/Makefile.am
+++ b/src/tools/Makefile.am
@@ -4,13 +4,13 @@ genboot_SOURCES = src/tools/genboot.c
 genunicode_SOURCES = src/tools/genunicode.c
 
 BUILT_SOURCES += $(top_builddir)/generated/boot.h
-BUILT_SOURCES += $(top_builddir)/generated/unicode.h
+BUILT_SOURCES += $(top_builddir)/generated/unicode_properties.c
 
 $(top_builddir)/generated/boot.h: genboot src/lib/boot.aplwc
 	$(MKDIR_P) $(top_builddir)/generated
 	./genboot $(top_srcdir)/src/lib/boot.aplwc > $@
 
-$(top_builddir)/generated/unicode.h: genunicode src/unicode/DerivedCoreProperties.txt src/unicode/PropList.txt
+$(top_builddir)/generated/unicode_properties.c: genunicode src/unicode/DerivedCoreProperties.txt src/unicode/PropList.txt
 	$(MKDIR_P) $(top_builddir)/generated
 	./genunicode $(top_srcdir)/src/unicode > $@
 
diff --git a/src/tools/genunicode.c b/src/tools/genunicode.c
index 0d5d428..76f2c60 100644
--- a/src/tools/genunicode.c
+++ b/src/tools/genunicode.c
@@ -41,6 +41,7 @@ struct line {
 	char property[64];
 };
 
+static void emit_line(struct line *, int *);
 static int peek(FILE *);
 static int parse_line(FILE *, struct line *);
 static int parse_int(FILE *);
@@ -55,7 +56,7 @@ main(int argc, char *argv[])
 	FILE *f;
 	char *dir, *buf;
 	size_t dir_size, tmp_size, max_size, i, j, nfiles;
-	struct line line;
+	struct line line, prev_line;
 	int current, n;
 
 	assert(argc == 2);
@@ -86,23 +87,28 @@ main(int argc, char *argv[])
 		printf("%s%s", (i == 0) ? "\n\t" : ",\n\t", properties[i]);
 	printf("\n};\n\n");
 
-	printf("struct unicode_range {\n\tint from;\n\tint to;\n};\n\n");
+	printf("struct unicode_range {\n\tuint32_t from;\n\tuint32_t to;\n};\n\n");
 
 	for(i = 0; i < sizeof(files)/sizeof(files[0]); i++){
 		memcpy(buf+dir_size+1, files[i], strlen(files[i])+1);
 		f = fopen(buf, "r");
 		assert(f != NULL);
 
+		printf("/* %s */\n\n", files[i]);
+
+		memset(&prev_line, 0, sizeof(prev_line));
 		while(parse_line(f, &line)){
 			if(!line.valid)
 				continue;
 
 			if((current == -1) || (strcmp(line.property, properties[current]) != 0)){
 				if(current != -1){
+					emit_line(&prev_line, &n);
 					printf("\n};\n");
 					printf("#define %s_count %d\n\n", properties[current], n);
 				}
 
+				prev_line.valid = 0;
 				current = -1;
 				for(j = 0; j < sizeof(properties)/sizeof(properties[0]); j++){
 					if(strcmp(line.property, properties[j]) == 0){
@@ -115,8 +121,13 @@ main(int argc, char *argv[])
 			}
 
 			if(current != -1){
-				printf("%s{%d, %d}", (n == 0) ? "\n\t" : ",\n\t", line.from, line.to);
-				n++;
+				if(prev_line.valid){
+					if(prev_line.to+1 == line.from)
+						line.from = prev_line.from;
+					else
+						emit_line(&prev_line, &n);
+				}
+				prev_line = line;
 			}
 		}
 		fclose(f);
@@ -137,6 +148,13 @@ main(int argc, char *argv[])
 	return 0;
 }
 
+static void
+emit_line(struct line *line, int *n)
+{
+	printf("%s{0x%X, 0x%X}", (*n == 0) ? "\n\t" : ",\n\t", line->from, line->to);
+	(*n)++;
+}
+
 static int
 peek(FILE *f)
 {
diff --git a/tests/scripts/good/3.aplwc b/tests/scripts/good/3.aplwc
index 50f8ca5..b24682c 100755
--- a/tests/scripts/good/3.aplwc
+++ b/tests/scripts/good/3.aplwc
@@ -1,3 +1,3 @@
 #!/usr/bin/env aplwc
 
-:export plus minus foo
+:export plus minus foo æøå_