From a4c65d3fa539b8e07a617977d11eb781ebc7dce5 Mon Sep 17 00:00:00 2001 From: peter Date: Tue, 22 Sep 2026 20:05:28 +0000 Subject: Start writing a genunicode program to process the unicode data files. While we are here, add some missing copyright/license notices. FossilOrigin-Name: 72aa02406da41a413284af19d52bcc1a091cac210b64f9887b123a95f20128e0 --- src/tools/genunicode.c | 68 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 src/tools/genunicode.c (limited to 'src/tools/genunicode.c') diff --git a/src/tools/genunicode.c b/src/tools/genunicode.c new file mode 100644 index 0000000..0329ace --- /dev/null +++ b/src/tools/genunicode.c @@ -0,0 +1,68 @@ +/* + * Copyright (C) 2026 Peter Mikkelsen + * + * This file is part of aplwc. + * + * Aplwc is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Aplwc is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with aplwc. If not, see . + */ + +#include +#include +#include +#include + +static char *files[] = { + "DerivedCoreProperties.txt", + "PropList.txt" +}; + +int +main(int argc, char *argv[]) +{ + FILE *f; + char *dir, *buf; + size_t dir_size, tmp_size, max_size, i, nfiles; + + assert(argc == 2); + dir = argv[1]; + dir_size = strlen(dir); + + if(dir[dir_size-1] == '/') + dir_size--; + + nfiles = sizeof(files)/sizeof(files[0]); + + max_size = 0; + for(i = 0; i < nfiles; i++){ + tmp_size = strlen(files[i]); + if(tmp_size > max_size) + max_size = tmp_size; + } + max_size += dir_size + 2; + buf = malloc(max_size); + assert(buf != NULL); + memcpy(buf, dir, dir_size); + buf[dir_size] = '/'; + + for(i = 0; i < sizeof(files)/sizeof(files[0]); i++){ + memcpy(buf+dir_size+1, files[i], strlen(files[i])+1); + f = fopen(buf, "r"); + assert(f != NULL); + fclose(f); + + printf("/* Unicode file %s read from %s */\n", files[i], buf); + } + + return 0; +} -- cgit v1.2.3 From a5b212017eaa73d30a1cd7d22635990a5b8a52ad Mon Sep 17 00:00:00 2001 From: peter Date: Thu, 24 Sep 2026 20:48:52 +0000 Subject: More work on the genunicode program FossilOrigin-Name: edfc2aebb5a66067934b9a05bb88b9d7e3aab4d2070d710e34c4339a0d303a5c --- src/tools/genunicode.c | 177 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 175 insertions(+), 2 deletions(-) (limited to 'src/tools/genunicode.c') diff --git a/src/tools/genunicode.c b/src/tools/genunicode.c index 0329ace..0d5d428 100644 --- a/src/tools/genunicode.c +++ b/src/tools/genunicode.c @@ -21,18 +21,42 @@ #include #include #include +#include static char *files[] = { "DerivedCoreProperties.txt", "PropList.txt" }; +static char *properties[] = { + "ID_Start", + "ID_Continue", + "Pattern_Syntax" +}; + +struct line { + int from; + int to; + int valid; + char property[64]; +}; + +static int peek(FILE *); +static int parse_line(FILE *, struct line *); +static int parse_int(FILE *); +static void parse_dots(FILE *); +static void parse_sep(FILE *); + +static jmp_buf env; + int main(int argc, char *argv[]) { FILE *f; char *dir, *buf; - size_t dir_size, tmp_size, max_size, i, nfiles; + size_t dir_size, tmp_size, max_size, i, j, nfiles; + struct line line; + int current, n; assert(argc == 2); dir = argv[1]; @@ -54,15 +78,164 @@ main(int argc, char *argv[]) assert(buf != NULL); memcpy(buf, dir, dir_size); buf[dir_size] = '/'; + current = -1; + n = 0; + + printf("enum unicode_property {"); + for(i = 0; i < sizeof(properties)/sizeof(properties[0]); i++) + printf("%s%s", (i == 0) ? "\n\t" : ",\n\t", properties[i]); + printf("\n};\n\n"); + + printf("struct unicode_range {\n\tint from;\n\tint to;\n};\n\n"); for(i = 0; i < sizeof(files)/sizeof(files[0]); i++){ memcpy(buf+dir_size+1, files[i], strlen(files[i])+1); f = fopen(buf, "r"); assert(f != NULL); + + while(parse_line(f, &line)){ + if(!line.valid) + continue; + + if((current == -1) || (strcmp(line.property, properties[current]) != 0)){ + if(current != -1){ + printf("\n};\n"); + printf("#define %s_count %d\n\n", properties[current], n); + } + + current = -1; + for(j = 0; j < sizeof(properties)/sizeof(properties[0]); j++){ + if(strcmp(line.property, properties[j]) == 0){ + current = j; + n = 0; + printf("static struct unicode_range %s_ranges[] = {", properties[j]); + break; + } + } + } + + if(current != -1){ + printf("%s{%d, %d}", (n == 0) ? "\n\t" : ",\n\t", line.from, line.to); + n++; + } + } fclose(f); + } - printf("/* Unicode file %s read from %s */\n", files[i], buf); + printf("static void\n"); + printf("unicode_get(enum unicode_property p, int *n, struct unicode_range **r)\n{\n"); + printf("\tswitch(p){\n"); + for(i = 0; i < sizeof(properties)/sizeof(properties[0]); i++){ + printf("\tcase %s:\n", properties[i]); + printf("\t\t*n = %s_count;\n", properties[i]); + printf("\t\t*r = %s_ranges;\n", properties[i]); + printf("\t\tbreak;\n"); } + printf("\t}\n}\n\n"); + free(buf); return 0; } + +static int +peek(FILE *f) +{ + int c; + + c = fgetc(f); + ungetc(c, f); + return c; +} + +static int +parse_line(FILE *f, struct line *l) +{ + int c, i; + + if(setjmp(env)){ + l->valid = 0; + do + c = fgetc(f); + while(c != EOF && c != '\n'); + return c != EOF; + }else{ + l->valid = 1; + } + + l->from = parse_int(f); + if(peek(f) == '.'){ + parse_dots(f); + l->to = parse_int(f); + }else{ + l->to = l->from; + } + parse_sep(f); + + for(i = 0; ; i++){ + c = fgetc(f); + l->property[i] = c; + if(c == ' ' || c == EOF){ + if(i == 0) + longjmp(env, 1); + break; + } + } + l->property[i] = 0; + + for(;;){ + c = fgetc(f); + if(c == '\n' || c == EOF) + break; + } + return 1; +} + +static int +parse_int(FILE *f) +{ + int n, m, c; + + n = 0; + + for(;;){ + c = peek(f); + if((c >= '0') && (c <= '9')) + m = c - '0'; + else if((c >= 'A') && (c <= 'F')) + m = 10 + (c - 'A'); + else if(c == '.' || c == ' ') + return n; + else + longjmp(env, 1); + fgetc(f); + n <<= 4; + n += m; + } +} + +static void +parse_dots(FILE *f) +{ + int i; + + for(i = 0; i < 2; i++){ + if(peek(f) != '.') + longjmp(env, 1); + fgetc(f); + } +} + +static void +parse_sep(FILE *f) +{ + while(peek(f) == ' ') + fgetc(f); + + if(peek(f) == ';') + fgetc(f); + else + longjmp(env, 1); + + while(peek(f) == ' ') + fgetc(f); +} -- cgit v1.2.3 From bdfbb6fd4bb9be35c9c338e18dc8e9b447cfe8ca Mon Sep 17 00:00:00 2001 From: peter Date: Fri, 25 Sep 2026 21:05:02 +0000 Subject: Use the parsed unicode stuff to decide what is valid identifier characters, and symbol characters. FossilOrigin-Name: afa25da5a44e8cb032d74269a6d4b7ec1e8842f18cefbd54d5f84a0b6d780f70 --- src/tools/genunicode.c | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) (limited to 'src/tools/genunicode.c') diff --git a/src/tools/genunicode.c b/src/tools/genunicode.c index 0d5d428..76f2c60 100644 --- a/src/tools/genunicode.c +++ b/src/tools/genunicode.c @@ -41,6 +41,7 @@ struct line { char property[64]; }; +static void emit_line(struct line *, int *); static int peek(FILE *); static int parse_line(FILE *, struct line *); static int parse_int(FILE *); @@ -55,7 +56,7 @@ main(int argc, char *argv[]) FILE *f; char *dir, *buf; size_t dir_size, tmp_size, max_size, i, j, nfiles; - struct line line; + struct line line, prev_line; int current, n; assert(argc == 2); @@ -86,23 +87,28 @@ main(int argc, char *argv[]) printf("%s%s", (i == 0) ? "\n\t" : ",\n\t", properties[i]); printf("\n};\n\n"); - printf("struct unicode_range {\n\tint from;\n\tint to;\n};\n\n"); + printf("struct unicode_range {\n\tuint32_t from;\n\tuint32_t to;\n};\n\n"); for(i = 0; i < sizeof(files)/sizeof(files[0]); i++){ memcpy(buf+dir_size+1, files[i], strlen(files[i])+1); f = fopen(buf, "r"); assert(f != NULL); + printf("/* %s */\n\n", files[i]); + + memset(&prev_line, 0, sizeof(prev_line)); while(parse_line(f, &line)){ if(!line.valid) continue; if((current == -1) || (strcmp(line.property, properties[current]) != 0)){ if(current != -1){ + emit_line(&prev_line, &n); printf("\n};\n"); printf("#define %s_count %d\n\n", properties[current], n); } + prev_line.valid = 0; current = -1; for(j = 0; j < sizeof(properties)/sizeof(properties[0]); j++){ if(strcmp(line.property, properties[j]) == 0){ @@ -115,8 +121,13 @@ main(int argc, char *argv[]) } if(current != -1){ - printf("%s{%d, %d}", (n == 0) ? "\n\t" : ",\n\t", line.from, line.to); - n++; + if(prev_line.valid){ + if(prev_line.to+1 == line.from) + line.from = prev_line.from; + else + emit_line(&prev_line, &n); + } + prev_line = line; } } fclose(f); @@ -137,6 +148,13 @@ main(int argc, char *argv[]) return 0; } +static void +emit_line(struct line *line, int *n) +{ + printf("%s{0x%X, 0x%X}", (*n == 0) ? "\n\t" : ",\n\t", line->from, line->to); + (*n)++; +} + static int peek(FILE *f) { -- cgit v1.2.3