Start writing a genunicode program to process the unicode data files. While we are here, add some missing copyright/license notices.

FossilOrigin-Name: 72aa02406da41a413284af19d52bcc1a091cac210b64f9887b123a95f20128e0

11 files changed, 213 insertions, 15 deletions

diff --git a/Makefile.am b/Makefile.am
index faeeead..602d0df 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -21,3 +21,4 @@ include tests/Makefile.am
 
 TESTS += $(TEST_INPUTS) $(TEST_SCRIPTS_SH) $(TEST_SCRIPTS_APLWC)
 EXTRA_DIST += $(TEST_INPUTS) $(TEST_SCRIPTS_SH) $(TEST_SCRIPTS_APLWC)
+CLEANFILES += $(BUILT_SOURCES)
diff --git a/Makefile.in b/Makefile.in
index 1644fb0..6beeab9 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -79,7 +79,7 @@ NORMAL_UNINSTALL = :
 PRE_UNINSTALL = :
 POST_UNINSTALL = :
 bin_PROGRAMS = aplwc$(EXEEXT)
-noinst_PROGRAMS = genboot$(EXEEXT)
+noinst_PROGRAMS = genboot$(EXEEXT) genunicode$(EXEEXT)
 check_PROGRAMS = testbin-aplwc_errstr$(EXEEXT) \
 	testbin-aplwc_utf8$(EXEEXT) testbin-aplwc_eval$(EXEEXT) \
 	testbin-aplwc_read$(EXEEXT)
@@ -141,7 +141,7 @@ am__dirstamp = $(am__leading_dot)dirstamp
 am_libaplwc_a_OBJECTS = src/lib/eval.$(OBJEXT) src/lib/main.$(OBJEXT) \
 	src/lib/memory.$(OBJEXT) src/lib/module.$(OBJEXT) \
 	src/lib/string.$(OBJEXT) src/lib/symtab.$(OBJEXT) \
-	src/lib/syntax.$(OBJEXT)
+	src/lib/syntax.$(OBJEXT) src/lib/unicode.$(OBJEXT)
 libaplwc_a_OBJECTS = $(am_libaplwc_a_OBJECTS)
 am_aplwc_OBJECTS = src/bin/main.$(OBJEXT) src/bin/util.$(OBJEXT)
 aplwc_OBJECTS = $(am_aplwc_OBJECTS)
@@ -149,6 +149,9 @@ aplwc_DEPENDENCIES = libaplwc.a
 am_genboot_OBJECTS = src/tools/genboot.$(OBJEXT)
 genboot_OBJECTS = $(am_genboot_OBJECTS)
 genboot_LDADD = $(LDADD)
+am_genunicode_OBJECTS = src/tools/genunicode.$(OBJEXT)
+genunicode_OBJECTS = $(am_genunicode_OBJECTS)
+genunicode_LDADD = $(LDADD)
 am_testbin_aplwc_errstr_OBJECTS = tests/aplwc_errstr.$(OBJEXT)
 testbin_aplwc_errstr_OBJECTS = $(am_testbin_aplwc_errstr_OBJECTS)
 testbin_aplwc_errstr_DEPENDENCIES = libaplwc.a
@@ -181,9 +184,10 @@ am__depfiles_remade = src/bin/$(DEPDIR)/main.Po \
 	src/lib/$(DEPDIR)/main.Po src/lib/$(DEPDIR)/memory.Po \
 	src/lib/$(DEPDIR)/module.Po src/lib/$(DEPDIR)/string.Po \
 	src/lib/$(DEPDIR)/symtab.Po src/lib/$(DEPDIR)/syntax.Po \
-	src/tools/$(DEPDIR)/genboot.Po tests/$(DEPDIR)/aplwc_errstr.Po \
-	tests/$(DEPDIR)/aplwc_eval.Po tests/$(DEPDIR)/aplwc_read.Po \
-	tests/$(DEPDIR)/aplwc_utf8.Po
+	src/lib/$(DEPDIR)/unicode.Po src/tools/$(DEPDIR)/genboot.Po \
+	src/tools/$(DEPDIR)/genunicode.Po \
+	tests/$(DEPDIR)/aplwc_errstr.Po tests/$(DEPDIR)/aplwc_eval.Po \
+	tests/$(DEPDIR)/aplwc_read.Po tests/$(DEPDIR)/aplwc_utf8.Po
 am__mv = mv -f
 COMPILE = $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) \
 	$(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS)
@@ -198,12 +202,13 @@ am__v_CCLD_ = $(am__v_CCLD_@AM_DEFAULT_V@)
 am__v_CCLD_0 = @echo "  CCLD    " $@;
 am__v_CCLD_1 = 
 SOURCES = $(libaplwc_a_SOURCES) $(aplwc_SOURCES) $(genboot_SOURCES) \
-	$(testbin_aplwc_errstr_SOURCES) $(testbin_aplwc_eval_SOURCES) \
-	$(testbin_aplwc_read_SOURCES) $(testbin_aplwc_utf8_SOURCES)
-DIST_SOURCES = $(libaplwc_a_SOURCES) $(aplwc_SOURCES) \
-	$(genboot_SOURCES) $(testbin_aplwc_errstr_SOURCES) \
+	$(genunicode_SOURCES) $(testbin_aplwc_errstr_SOURCES) \
 	$(testbin_aplwc_eval_SOURCES) $(testbin_aplwc_read_SOURCES) \
 	$(testbin_aplwc_utf8_SOURCES)
+DIST_SOURCES = $(libaplwc_a_SOURCES) $(aplwc_SOURCES) \
+	$(genboot_SOURCES) $(genunicode_SOURCES) \
+	$(testbin_aplwc_errstr_SOURCES) $(testbin_aplwc_eval_SOURCES) \
+	$(testbin_aplwc_read_SOURCES) $(testbin_aplwc_utf8_SOURCES)
 am__can_run_installinfo = \
   case $$AM_UPDATE_INFO_DIR in \
     n|no|NO) false;; \
@@ -535,8 +540,9 @@ top_srcdir = @top_srcdir@
 lib_LIBRARIES = libaplwc.a
 include_HEADERS = src/include/aplwc.h
 noinst_HEADERS = src/include/aplwc_macros.h
-BUILT_SOURCES = $(top_builddir)/generated/boot.h
-CLEANFILES = $(top_builddir)/generated/boot.h
+BUILT_SOURCES = $(top_builddir)/generated/boot.h \
+	$(top_builddir)/generated/unicode.h
+CLEANFILES = $(BUILT_SOURCES)
 EXTRA_DIST = src/lib/boot.aplwc $(LOG_COMPILER) \
 	tests/scripts/simfail.sh $(TXT_LOG_COMPILER) $(TEST_INPUTS) \
 	$(TEST_SCRIPTS_SH) $(TEST_SCRIPTS_APLWC)
@@ -585,9 +591,11 @@ libaplwc_a_SOURCES = \
 	src/lib/module.c \
 	src/lib/string.c \
 	src/lib/symtab.c \
-	src/lib/syntax.c
+	src/lib/syntax.c \
+	src/lib/unicode.c
 
 genboot_SOURCES = src/tools/genboot.c
+genunicode_SOURCES = src/tools/genunicode.c
 LOG_COMPILER = $(top_srcdir)/tests/run_script_test.sh
 testbin_aplwc_errstr_SOURCES = tests/aplwc_errstr.c
 testbin_aplwc_errstr_LDADD = libaplwc.a
@@ -751,6 +759,8 @@ src/lib/symtab.$(OBJEXT): src/lib/$(am__dirstamp) \
 	src/lib/$(DEPDIR)/$(am__dirstamp)
 src/lib/syntax.$(OBJEXT): src/lib/$(am__dirstamp) \
 	src/lib/$(DEPDIR)/$(am__dirstamp)
+src/lib/unicode.$(OBJEXT): src/lib/$(am__dirstamp) \
+	src/lib/$(DEPDIR)/$(am__dirstamp)
 
 libaplwc.a: $(libaplwc_a_OBJECTS) $(libaplwc_a_DEPENDENCIES) $(EXTRA_libaplwc_a_DEPENDENCIES) 
 	$(AM_V_at)-rm -f libaplwc.a
@@ -782,6 +792,12 @@ src/tools/genboot.$(OBJEXT): src/tools/$(am__dirstamp) \
 genboot$(EXEEXT): $(genboot_OBJECTS) $(genboot_DEPENDENCIES) $(EXTRA_genboot_DEPENDENCIES) 
 	@rm -f genboot$(EXEEXT)
 	$(AM_V_CCLD)$(LINK) $(genboot_OBJECTS) $(genboot_LDADD) $(LIBS)
+src/tools/genunicode.$(OBJEXT): src/tools/$(am__dirstamp) \
+	src/tools/$(DEPDIR)/$(am__dirstamp)
+
+genunicode$(EXEEXT): $(genunicode_OBJECTS) $(genunicode_DEPENDENCIES) $(EXTRA_genunicode_DEPENDENCIES) 
+	@rm -f genunicode$(EXEEXT)
+	$(AM_V_CCLD)$(LINK) $(genunicode_OBJECTS) $(genunicode_LDADD) $(LIBS)
 tests/$(am__dirstamp):
 	@$(MKDIR_P) tests
 	@: >>tests/$(am__dirstamp)
@@ -832,7 +848,9 @@ distclean-compile:
 @AMDEP_TRUE@@am__include@ @am__quote@src/lib/$(DEPDIR)/string.Po@am__quote@ # am--include-marker
 @AMDEP_TRUE@@am__include@ @am__quote@src/lib/$(DEPDIR)/symtab.Po@am__quote@ # am--include-marker
 @AMDEP_TRUE@@am__include@ @am__quote@src/lib/$(DEPDIR)/syntax.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@src/lib/$(DEPDIR)/unicode.Po@am__quote@ # am--include-marker
 @AMDEP_TRUE@@am__include@ @am__quote@src/tools/$(DEPDIR)/genboot.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@src/tools/$(DEPDIR)/genunicode.Po@am__quote@ # am--include-marker
 @AMDEP_TRUE@@am__include@ @am__quote@tests/$(DEPDIR)/aplwc_errstr.Po@am__quote@ # am--include-marker
 @AMDEP_TRUE@@am__include@ @am__quote@tests/$(DEPDIR)/aplwc_eval.Po@am__quote@ # am--include-marker
 @AMDEP_TRUE@@am__include@ @am__quote@tests/$(DEPDIR)/aplwc_read.Po@am__quote@ # am--include-marker
@@ -1594,7 +1612,9 @@ distclean: distclean-am
 	-rm -f src/lib/$(DEPDIR)/string.Po
 	-rm -f src/lib/$(DEPDIR)/symtab.Po
 	-rm -f src/lib/$(DEPDIR)/syntax.Po
+	-rm -f src/lib/$(DEPDIR)/unicode.Po
 	-rm -f src/tools/$(DEPDIR)/genboot.Po
+	-rm -f src/tools/$(DEPDIR)/genunicode.Po
 	-rm -f tests/$(DEPDIR)/aplwc_errstr.Po
 	-rm -f tests/$(DEPDIR)/aplwc_eval.Po
 	-rm -f tests/$(DEPDIR)/aplwc_read.Po
@@ -1655,7 +1675,9 @@ maintainer-clean: maintainer-clean-am
 	-rm -f src/lib/$(DEPDIR)/string.Po
 	-rm -f src/lib/$(DEPDIR)/symtab.Po
 	-rm -f src/lib/$(DEPDIR)/syntax.Po
+	-rm -f src/lib/$(DEPDIR)/unicode.Po
 	-rm -f src/tools/$(DEPDIR)/genboot.Po
+	-rm -f src/tools/$(DEPDIR)/genunicode.Po
 	-rm -f tests/$(DEPDIR)/aplwc_errstr.Po
 	-rm -f tests/$(DEPDIR)/aplwc_eval.Po
 	-rm -f tests/$(DEPDIR)/aplwc_read.Po
@@ -1709,6 +1731,10 @@ $(top_builddir)/generated/boot.h: genboot src/lib/boot.aplwc
 	$(MKDIR_P) $(top_builddir)/generated
 	./genboot $(top_srcdir)/src/lib/boot.aplwc > $@
 
+$(top_builddir)/generated/unicode.h: genunicode src/unicode/DerivedCoreProperties.txt src/unicode/PropList.txt
+	$(MKDIR_P) $(top_builddir)/generated
+	./genunicode $(top_srcdir)/src/unicode > $@
+
 # Tell versions [3.59,3.63) of GNU make to not export all variables.
 # Otherwise a system limit (for SysV at least) may be exceeded.
 .NOEXPORT:
diff --git a/src/lib/Makefile.am b/src/lib/Makefile.am
index c95fe1c..9cbbfae 100644
--- a/src/lib/Makefile.am
+++ b/src/lib/Makefile.am
@@ -8,6 +8,7 @@ libaplwc_a_SOURCES = \
 	src/lib/module.c \
 	src/lib/string.c \
 	src/lib/symtab.c \
-	src/lib/syntax.c
+	src/lib/syntax.c \
+	src/lib/unicode.c
 
 EXTRA_DIST += src/lib/boot.aplwc
diff --git a/src/lib/unicode.c b/src/lib/unicode.c
index 862dd5f..a913b6d 100644
--- a/src/lib/unicode.c
+++ b/src/lib/unicode.c
@@ -18,3 +18,4 @@
  */
 
 #include "aplwc_internal.h"
+#include "unicode.h"
diff --git a/src/tools/Makefile.am b/src/tools/Makefile.am
index b1079e7..1930970 100644
--- a/src/tools/Makefile.am
+++ b/src/tools/Makefile.am
@@ -1,10 +1,16 @@
-noinst_PROGRAMS += genboot
+noinst_PROGRAMS += genboot genunicode
 
 genboot_SOURCES = src/tools/genboot.c
+genunicode_SOURCES = src/tools/genunicode.c
 
 BUILT_SOURCES += $(top_builddir)/generated/boot.h
-CLEANFILES += $(top_builddir)/generated/boot.h
+BUILT_SOURCES += $(top_builddir)/generated/unicode.h
 
 $(top_builddir)/generated/boot.h: genboot src/lib/boot.aplwc
 	$(MKDIR_P) $(top_builddir)/generated
 	./genboot $(top_srcdir)/src/lib/boot.aplwc > $@
+
+$(top_builddir)/generated/unicode.h: genunicode src/unicode/DerivedCoreProperties.txt src/unicode/PropList.txt
+	$(MKDIR_P) $(top_builddir)/generated
+	./genunicode $(top_srcdir)/src/unicode > $@
+
diff --git a/src/tools/genboot.c b/src/tools/genboot.c
index e89b3aa..9d0a7e4 100644
--- a/src/tools/genboot.c
+++ b/src/tools/genboot.c
@@ -1,3 +1,22 @@
+/* 
+ * Copyright (C) 2026 Peter Mikkelsen <petermikkelsen10@gmail.com>
+ *
+ * This file is part of aplwc.
+ *
+ * Aplwc is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Aplwc is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with aplwc.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 #include <stdio.h>
 #include <assert.h>
 
diff --git a/src/tools/genunicode.c b/src/tools/genunicode.c
new file mode 100644
index 0000000..0329ace
--- /dev/null
+++ b/src/tools/genunicode.c
@@ -0,0 +1,68 @@
+/* 
+ * Copyright (C) 2026 Peter Mikkelsen <petermikkelsen10@gmail.com>
+ *
+ * This file is part of aplwc.
+ *
+ * Aplwc is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Aplwc is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with aplwc.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <assert.h>
+
+static char *files[] = {
+	"DerivedCoreProperties.txt",
+	"PropList.txt"
+};
+
+int
+main(int argc, char *argv[])
+{
+	FILE *f;
+	char *dir, *buf;
+	size_t dir_size, tmp_size, max_size, i, nfiles;
+
+	assert(argc == 2);
+	dir = argv[1];
+	dir_size = strlen(dir);
+
+	if(dir[dir_size-1] == '/')
+		dir_size--;
+
+	nfiles = sizeof(files)/sizeof(files[0]);
+
+	max_size = 0;
+	for(i = 0; i < nfiles; i++){
+		tmp_size = strlen(files[i]);
+		if(tmp_size > max_size)
+			max_size = tmp_size;
+	}
+	max_size += dir_size + 2;
+	buf = malloc(max_size);
+	assert(buf != NULL);
+	memcpy(buf, dir, dir_size);
+	buf[dir_size] = '/';
+
+	for(i = 0; i < sizeof(files)/sizeof(files[0]); i++){
+		memcpy(buf+dir_size+1, files[i], strlen(files[i])+1);
+		f = fopen(buf, "r");
+		assert(f != NULL);
+		fclose(f);
+
+		printf("/* Unicode file %s read from %s */\n", files[i], buf);
+	}
+
+	return 0;
+}
diff --git a/tests/aplwc_errstr.c b/tests/aplwc_errstr.c
index cee1ea1..d1e6a20 100644
--- a/tests/aplwc_errstr.c
+++ b/tests/aplwc_errstr.c
@@ -1,3 +1,22 @@
+/* 
+ * Copyright (C) 2026 Peter Mikkelsen <petermikkelsen10@gmail.com>
+ *
+ * This file is part of aplwc.
+ *
+ * Aplwc is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Aplwc is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with aplwc.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 #include <stdlib.h>
 #include <string.h>
 #include <aplwc.h>
diff --git a/tests/aplwc_eval.c b/tests/aplwc_eval.c
index 3d314e7..deb1610 100644
--- a/tests/aplwc_eval.c
+++ b/tests/aplwc_eval.c
@@ -1,3 +1,22 @@
+/* 
+ * Copyright (C) 2026 Peter Mikkelsen <petermikkelsen10@gmail.com>
+ *
+ * This file is part of aplwc.
+ *
+ * Aplwc is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Aplwc is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with aplwc.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
diff --git a/tests/aplwc_read.c b/tests/aplwc_read.c
index a5a79a9..902c9d2 100644
--- a/tests/aplwc_read.c
+++ b/tests/aplwc_read.c
@@ -1,3 +1,22 @@
+/* 
+ * Copyright (C) 2026 Peter Mikkelsen <petermikkelsen10@gmail.com>
+ *
+ * This file is part of aplwc.
+ *
+ * Aplwc is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Aplwc is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with aplwc.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
diff --git a/tests/aplwc_utf8.c b/tests/aplwc_utf8.c
index 316845d..44dd85a 100644
--- a/tests/aplwc_utf8.c
+++ b/tests/aplwc_utf8.c
@@ -1,3 +1,22 @@
+/* 
+ * Copyright (C) 2026 Peter Mikkelsen <petermikkelsen10@gmail.com>
+ *
+ * This file is part of aplwc.
+ *
+ * Aplwc is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * Aplwc is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with aplwc.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
 #include <stdlib.h>
 #include <string.h>
 #include <aplwc.h>