More work on the scanner and tests for it

FossilOrigin-Name: 037c88ce86e1ba3a399c03a27d4c7252b52c2f500bc13f449e19e497b0607880

1 files changed, 15 insertions, 1 deletions

diff --git a/src/lib/aplwc_internal.h b/src/lib/aplwc_internal.h
index 155508d..8661bb2 100644
--- a/src/lib/aplwc_internal.h
+++ b/src/lib/aplwc_internal.h
@@ -33,11 +33,21 @@ struct aplwc_line {
 struct string {
 	size_t bytes;
 	size_t chars;
-	uint8_t *utf8;
+	uint8_t utf8[1];
+};
+
+enum token_tag {
+	TOKEN_TAG_IDENTIFIER,
+	TOKEN_TAG_NEWLINE
+};
+
+struct token {
+	enum token_tag tag;
 };
 
 struct token_list {
 	size_t length;
+	struct token tokens[1];
 };
 
 /* eval.c */
@@ -48,12 +58,16 @@ struct memory memory_init(uint8_t *, size_t);
 struct aplwc *memory_alloc_aplwc(struct memory *);
 struct aplwc_line *memory_alloc_aplwc_line(struct memory *);
 struct string *memory_alloc_string(struct memory *, size_t);
+struct token_list *memory_alloc_token_list(struct memory *, size_t);
 void memory_free_aplwc(struct memory *, struct aplwc *);
 void memory_free_aplwc_line(struct memory *, struct aplwc_line *);
 void memory_free_string(struct memory *, struct string *);
+void memory_free_token_list(struct memory *, struct token_list *);
 
 /* string.c */
 struct string *chars_to_string(struct aplwc *, const char *);
+uint32_t string_char(struct string *, size_t);
+size_t char_length(struct string *, size_t);
 
 /* syntax.c */
 struct token_list *scan_line(struct aplwc *, struct string *);