Merge unicode-props into trunk

FossilOrigin-Name: 95c406d482112373a41982bf36f3a9159fceff5ff95162cab34e97fd0cfaa73c

5 files changed, 77 insertions, 11 deletions

diff --git a/src/lib/Makefile.am b/src/lib/Makefile.am
index c95fe1c..9cbbfae 100644
--- a/src/lib/Makefile.am
+++ b/src/lib/Makefile.am
@@ -8,6 +8,7 @@ libaplwc_a_SOURCES = \
 	src/lib/module.c \
 	src/lib/string.c \
 	src/lib/symtab.c \
-	src/lib/syntax.c
+	src/lib/syntax.c \
+	src/lib/unicode.c
 
 EXTRA_DIST += src/lib/boot.aplwc
diff --git a/src/lib/aplwc_internal.h b/src/lib/aplwc_internal.h
index 8c13dfe..c123d63 100644
--- a/src/lib/aplwc_internal.h
+++ b/src/lib/aplwc_internal.h
@@ -180,4 +180,9 @@ enum aplwc_rc getsym(struct aplwc *, struct string *, symbol_t *);
 /* module.c */
 enum aplwc_rc load_module(struct aplwc *, struct string *, struct token_list *);
 
+/* unicode.c */
+int is_id_start(uint32_t);
+int is_id_continue(uint32_t);
+int is_pattern_syntax(uint32_t);
+
 #endif /* APLWC_INTERNAL_H */
diff --git a/src/lib/string.c b/src/lib/string.c
index db08b28..4101b80 100644
--- a/src/lib/string.c
+++ b/src/lib/string.c
@@ -180,7 +180,7 @@ substring(struct aplwc *aplwc, struct string *s, size_t start, size_t length)
 
 	bytes = 0;
 	for(i = 0; i < length; i++)
-		bytes += char_length(s, start+i);
+		bytes += char_length(s, start+bytes);
 
 	r = memory_alloc_string(aplwc, bytes);
 	if(r){
diff --git a/src/lib/syntax.c b/src/lib/syntax.c
index 1740a4e..d864bf4 100644
--- a/src/lib/syntax.c
+++ b/src/lib/syntax.c
@@ -42,12 +42,6 @@ is_space(uint32_t c)
 }
 
 static int
-is_letter(uint32_t c)
-{
-	return ((c >= 'a') && (c <= 'z')) || ((c >= 'A') && (c <= 'Z'));
-}
-
-static int
 is_digit(uint32_t c)
 {
 	return (c >= '0') && (c <= '9');
@@ -111,6 +105,10 @@ single_sym(struct scan_state *state, struct token_list *tokens, size_t token_ind
 	case '(':    T.tag = TOKEN_TAG_LEFT_PAREN; break;
 	case ')':    T.tag = TOKEN_TAG_RIGHT_PAREN; break;
 	default:
+		if(!is_pattern_syntax(c)){
+			rc = APLWC_INVALID_SYNTAX;
+			goto end;
+		}
 		T.tag = TOKEN_TAG_UNKNOWN_SYM;
 		if(!check_only){
 			if((rc = newsym(state, start, 1, &sym)) != APLWC_OK)
@@ -139,7 +137,7 @@ identifier(struct scan_state *state, struct token_list *tokens, size_t token_ind
 
 	rc = APLWC_INVALID_SYNTAX;
 
-	if(state->skip_number_and_identifier || (!is_letter(peek(state))))
+	if(state->skip_number_and_identifier || (!is_id_start(peek(state))))
 		goto end;
 
 	start = state->offset;
@@ -147,7 +145,7 @@ identifier(struct scan_state *state, struct token_list *tokens, size_t token_ind
 	do{
 		c = advance(state);
 		len++;
-	}while(is_letter(c) || is_digit(c) || (c == '_'));
+	}while(is_id_continue(c));
 
 	T.tag = TOKEN_TAG_IDENTIFIER;
 	if(!check_only){
@@ -186,7 +184,7 @@ keyword(struct scan_state *state, struct token_list *tokens, size_t token_index,
 	do{
 		c = advance(state);
 		len++;
-	}while(is_letter(c));
+	}while(is_id_continue(c));
 
 	rc = APLWC_OK;
 	T.tag = TOKEN_TAG_KEYWORD;
diff --git a/src/lib/unicode.c b/src/lib/unicode.c
new file mode 100644
index 0000000..48cb91e
--- /dev/null
+++ b/src/lib/unicode.c
@@ -0,0 +1,62 @@
+/* 
+ * Copyright (C) 2026 Peter Mikkelsen <petermikkelsen10@gmail.com>
+ *
+ * This file is part of aplwc.
+ *
+ * Aplwc is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Aplwc is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with aplwc.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#include "aplwc_internal.h"
+#include "unicode_properties.c"
+
+static int
+is(uint32_t c, enum unicode_property p)
+{
+	int n, a, b, i;
+	struct unicode_range *r;
+
+	unicode_get(p, &n, &r);
+	a = 0;
+	b = n-1;
+	while(a <= b){
+		i = a + (b-a) / 2;
+		if(c >= r[i].from && c <= r[i].to)
+			return 1;
+		else if(c < r[i].from)
+			b = i-1;
+		else{
+			assert(c > r[i].to);
+			a = i+1;
+		}
+	}
+	return 0;
+}
+
+int
+is_id_start(uint32_t c)
+{
+	return is(c, ID_Start);
+}
+
+int
+is_id_continue(uint32_t c)
+{
+	return is(c, ID_Continue);
+}
+
+int
+is_pattern_syntax(uint32_t c)
+{
+	return is(c, Pattern_Syntax);
+}