From 0b6ea0024a5a51db015797e952d55bfac49b4b6d Mon Sep 17 00:00:00 2001 From: peter Date: Sun, 13 Sep 2026 15:40:54 +0000 Subject: Start work on using the real unicode properties as the base for what is allowed in an identifier and a symbol and so on. FossilOrigin-Name: c8b81a4236423008e26c8457e90bcadf82fd00594d981ee99de3908a77c153d4 --- src/lib/unicode.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 src/lib/unicode.c (limited to 'src/lib') diff --git a/src/lib/unicode.c b/src/lib/unicode.c new file mode 100644 index 0000000..862dd5f --- /dev/null +++ b/src/lib/unicode.c @@ -0,0 +1,20 @@ +/* + * Copyright (C) 2026 Peter Mikkelsen + * + * This file is part of aplwc. + * + * Aplwc is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Aplwc is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with aplwc. If not, see . + */ + +#include "aplwc_internal.h" -- cgit v1.2.3 From a4c65d3fa539b8e07a617977d11eb781ebc7dce5 Mon Sep 17 00:00:00 2001 From: peter Date: Tue, 22 Sep 2026 20:05:28 +0000 Subject: Start writing a genunicode program to process the unicode data files. While we are here, add some missing copyright/license notices. FossilOrigin-Name: 72aa02406da41a413284af19d52bcc1a091cac210b64f9887b123a95f20128e0 --- src/lib/Makefile.am | 3 ++- src/lib/unicode.c | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'src/lib') diff --git a/src/lib/Makefile.am b/src/lib/Makefile.am index c95fe1c..9cbbfae 100644 --- a/src/lib/Makefile.am +++ b/src/lib/Makefile.am @@ -8,6 +8,7 @@ libaplwc_a_SOURCES = \ src/lib/module.c \ src/lib/string.c \ src/lib/symtab.c \ - src/lib/syntax.c + src/lib/syntax.c \ + src/lib/unicode.c EXTRA_DIST += src/lib/boot.aplwc diff --git a/src/lib/unicode.c b/src/lib/unicode.c index 862dd5f..a913b6d 100644 --- a/src/lib/unicode.c +++ b/src/lib/unicode.c @@ -18,3 +18,4 @@ */ #include "aplwc_internal.h" +#include "unicode.h" -- cgit v1.2.3 From a5b212017eaa73d30a1cd7d22635990a5b8a52ad Mon Sep 17 00:00:00 2001 From: peter Date: Thu, 24 Sep 2026 20:48:52 +0000 Subject: More work on the genunicode program FossilOrigin-Name: edfc2aebb5a66067934b9a05bb88b9d7e3aab4d2070d710e34c4339a0d303a5c --- src/lib/unicode.c | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) (limited to 'src/lib') diff --git a/src/lib/unicode.c b/src/lib/unicode.c index a913b6d..b5c6471 100644 --- a/src/lib/unicode.c +++ b/src/lib/unicode.c @@ -19,3 +19,34 @@ #include "aplwc_internal.h" #include "unicode.h" + +static int +is(uint32_t c, enum unicode_property p) +{ + int n; + struct unicode_range *r; + + UNUSED_PARAMETER(c); + + unicode_get(p, &n, &r); + + return 0; +} + +int +is_id_start(uint32_t c) +{ + return is(c, ID_Start); +} + +int +is_id_continue(uint32_t c) +{ + return is(c, ID_Continue); +} + +int +is_pattern_syntax(uint32_t c) +{ + return is(c, Pattern_Syntax); +} -- cgit v1.2.3 From bdfbb6fd4bb9be35c9c338e18dc8e9b447cfe8ca Mon Sep 17 00:00:00 2001 From: peter Date: Fri, 25 Sep 2026 21:05:02 +0000 Subject: Use the parsed unicode stuff to decide what is valid identifier characters, and symbol characters. FossilOrigin-Name: afa25da5a44e8cb032d74269a6d4b7ec1e8842f18cefbd54d5f84a0b6d780f70 --- src/lib/aplwc_internal.h | 5 +++++ src/lib/string.c | 2 +- src/lib/syntax.c | 16 +++++++--------- src/lib/unicode.c | 20 +++++++++++++++----- 4 files changed, 28 insertions(+), 15 deletions(-) (limited to 'src/lib') diff --git a/src/lib/aplwc_internal.h b/src/lib/aplwc_internal.h index 8c13dfe..c123d63 100644 --- a/src/lib/aplwc_internal.h +++ b/src/lib/aplwc_internal.h @@ -180,4 +180,9 @@ enum aplwc_rc getsym(struct aplwc *, struct string *, symbol_t *); /* module.c */ enum aplwc_rc load_module(struct aplwc *, struct string *, struct token_list *); +/* unicode.c */ +int is_id_start(uint32_t); +int is_id_continue(uint32_t); +int is_pattern_syntax(uint32_t); + #endif /* APLWC_INTERNAL_H */ diff --git a/src/lib/string.c b/src/lib/string.c index db08b28..4101b80 100644 --- a/src/lib/string.c +++ b/src/lib/string.c @@ -180,7 +180,7 @@ substring(struct aplwc *aplwc, struct string *s, size_t start, size_t length) bytes = 0; for(i = 0; i < length; i++) - bytes += char_length(s, start+i); + bytes += char_length(s, start+bytes); r = memory_alloc_string(aplwc, bytes); if(r){ diff --git a/src/lib/syntax.c b/src/lib/syntax.c index 1740a4e..d864bf4 100644 --- a/src/lib/syntax.c +++ b/src/lib/syntax.c @@ -41,12 +41,6 @@ is_space(uint32_t c) return c == ' '; } -static int -is_letter(uint32_t c) -{ - return ((c >= 'a') && (c <= 'z')) || ((c >= 'A') && (c <= 'Z')); -} - static int is_digit(uint32_t c) { @@ -111,6 +105,10 @@ single_sym(struct scan_state *state, struct token_list *tokens, size_t token_ind case '(': T.tag = TOKEN_TAG_LEFT_PAREN; break; case ')': T.tag = TOKEN_TAG_RIGHT_PAREN; break; default: + if(!is_pattern_syntax(c)){ + rc = APLWC_INVALID_SYNTAX; + goto end; + } T.tag = TOKEN_TAG_UNKNOWN_SYM; if(!check_only){ if((rc = newsym(state, start, 1, &sym)) != APLWC_OK) @@ -139,7 +137,7 @@ identifier(struct scan_state *state, struct token_list *tokens, size_t token_ind rc = APLWC_INVALID_SYNTAX; - if(state->skip_number_and_identifier || (!is_letter(peek(state)))) + if(state->skip_number_and_identifier || (!is_id_start(peek(state)))) goto end; start = state->offset; @@ -147,7 +145,7 @@ identifier(struct scan_state *state, struct token_list *tokens, size_t token_ind do{ c = advance(state); len++; - }while(is_letter(c) || is_digit(c) || (c == '_')); + }while(is_id_continue(c)); T.tag = TOKEN_TAG_IDENTIFIER; if(!check_only){ @@ -186,7 +184,7 @@ keyword(struct scan_state *state, struct token_list *tokens, size_t token_index, do{ c = advance(state); len++; - }while(is_letter(c)); + }while(is_id_continue(c)); rc = APLWC_OK; T.tag = TOKEN_TAG_KEYWORD; diff --git a/src/lib/unicode.c b/src/lib/unicode.c index b5c6471..48cb91e 100644 --- a/src/lib/unicode.c +++ b/src/lib/unicode.c @@ -18,18 +18,28 @@ */ #include "aplwc_internal.h" -#include "unicode.h" +#include "unicode_properties.c" static int is(uint32_t c, enum unicode_property p) { - int n; + int n, a, b, i; struct unicode_range *r; - UNUSED_PARAMETER(c); - unicode_get(p, &n, &r); - + a = 0; + b = n-1; + while(a <= b){ + i = a + (b-a) / 2; + if(c >= r[i].from && c <= r[i].to) + return 1; + else if(c < r[i].from) + b = i-1; + else{ + assert(c > r[i].to); + a = i+1; + } + } return 0; } -- cgit v1.2.3