diff options
| -rw-r--r-- | CMakeLists.txt | 1 | ||||
| -rw-r--r-- | src/lib/aplwc_internal.h | 42 | ||||
| -rw-r--r-- | src/lib/boot.aplwc | 6 | ||||
| -rw-r--r-- | src/lib/main.c | 4 | ||||
| -rw-r--r-- | src/lib/memory.c | 50 | ||||
| -rw-r--r-- | src/lib/module.c | 195 | ||||
| -rw-r--r-- | src/lib/string.c | 24 | ||||
| -rw-r--r-- | src/lib/symtab.c | 51 | ||||
| -rw-r--r-- | src/lib/syntax.c | 322 | ||||
| -rwxr-xr-x | tests/scripts/bad/5.aplwc (renamed from tests/scripts/good/3.aplwc) | 6 |
10 files changed, 602 insertions, 99 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 3e2c034..2fb0813 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -55,6 +55,7 @@ add_library(aplwc src/lib/memory.c src/lib/module.c src/lib/string.c + src/lib/symtab.c src/lib/syntax.c ) target_link_libraries(aplwc PRIVATE common) diff --git a/src/lib/aplwc_internal.h b/src/lib/aplwc_internal.h index d97cbf7..0bdd5b5 100644 --- a/src/lib/aplwc_internal.h +++ b/src/lib/aplwc_internal.h @@ -27,10 +27,22 @@ #include <stdbool.h> #include <string.h> +typedef int symbol_t; + +enum keyword { + KW_MODULE, + KW_SYMBOL, + KW_EXPORT, + + N_KWDS +}; + struct aplwc { struct aplwc_fns fns; enum aplwc_rc last_error; struct aplwc_module *modules; + struct symbol *symtab; + symbol_t keyword_symbols[N_KWDS]; /* Memory management */ uint8_t *data; @@ -79,6 +91,10 @@ struct token { enum token_tag tag; size_t start; size_t end; + union { + symbol_t symbol; + enum keyword keyword; + } data; }; struct token_list { @@ -99,9 +115,25 @@ struct aplwc_value { struct aplwc_module { struct string *name; struct token_list *tokens; + struct module_member *imports; + struct module_member *members; struct aplwc_module *link; }; +struct module_member { + int exported; + struct aplwc_module *module; + symbol_t name; + struct module_member *real_member; /* a symbol points to its identifier member. An imported symbol points to the defining member */ + struct module_member *link; +}; + +struct symbol { + symbol_t id; + struct string *text; + struct symbol *link; +}; + /* eval.c */ struct aplwc_value *eval_line(struct aplwc *, struct aplwc_line *); @@ -113,7 +145,8 @@ struct string *memory_alloc_string(struct aplwc *, size_t); struct token_list *memory_alloc_token_list(struct aplwc *, size_t); struct aplwc_value *memory_alloc_aplwc_value(struct aplwc *); struct aplwc_module *memory_alloc_aplwc_module(struct aplwc *); -struct module_imports *memory_alloc_module_imports(struct aplwc *, size_t); +struct module_member *memory_alloc_module_member(struct aplwc *); +struct symbol *memory_alloc_symbol(struct aplwc *); void memory_root_new(struct aplwc *, void *, struct aplwc_ref *); void *memory_root_get(struct aplwc_ref *); void memory_root_free(struct aplwc_ref *); @@ -134,10 +167,15 @@ struct string *chars_to_string(struct aplwc *, const char *, int len); uint32_t string_char(struct string *, size_t); size_t char_length(struct string *, size_t); int string_compare(struct string *, struct string *); +struct string *substring(struct aplwc *, struct string *, size_t, size_t); /* syntax.c */ struct token_list *scan_string(struct aplwc *, struct string *, int); -enum aplwc_rc check_tokens(struct token_list *); +enum aplwc_rc check_tokens(struct aplwc_module *, struct token_list *); +enum aplwc_rc keyword_init(struct aplwc *); + +/* symtab.c */ +enum aplwc_rc getsym(struct aplwc *, struct string *, symbol_t *); /* module.c */ enum aplwc_rc load_module(struct aplwc *, struct string *, struct token_list *); diff --git a/src/lib/boot.aplwc b/src/lib/boot.aplwc index ef6b479..32c8cbf 100644 --- a/src/lib/boot.aplwc +++ b/src/lib/boot.aplwc @@ -18,9 +18,9 @@ ⍝ :module Boot -:symbol plus '+' -:symbol minus '-' -:export '+' '-' +:symbol plus + +:symbol minus - +:export + - ∇ r←a plus b r←a (1⌶) b diff --git a/src/lib/main.c b/src/lib/main.c index afb99e9..d25ad99 100644 --- a/src/lib/main.c +++ b/src/lib/main.c @@ -39,7 +39,9 @@ aplwc_new(struct aplwc_fns fns, enum aplwc_rc *rcp) aplwc->modules = NULL; aplwc->fns = fns; + aplwc->symtab = NULL; memory_init(aplwc); + keyword_init(aplwc); rc = aplwc_load_module(aplwc, "Boot", (const char *)boot_src, boot_src_len); end: @@ -123,7 +125,7 @@ aplwc_read(struct aplwc *aplwc, const char *linestr, AplwcLine *lineptr) goto end; MEMPTR(1, tokens); - if((aplwc->last_error = check_tokens(tokens)) != APLWC_OK) + if((aplwc->last_error = check_tokens(NULL, tokens)) != APLWC_OK) goto end; if((line = memory_alloc_aplwc_line(aplwc)) == NULL) diff --git a/src/lib/memory.c b/src/lib/memory.c index 03f2056..fb143ec 100644 --- a/src/lib/memory.c +++ b/src/lib/memory.c @@ -28,7 +28,9 @@ enum chunk_tag { CHUNK_TAG_STRING, CHUNK_TAG_TOKEN_LIST, CHUNK_TAG_APLWC_VALUE, - CHUNK_TAG_APLWC_MODULE + CHUNK_TAG_APLWC_MODULE, + CHUNK_TAG_MODULE_MEMBER, + CHUNK_TAG_SYMBOL }; struct chunk { @@ -39,6 +41,8 @@ struct chunk { unsigned int last:1; }; +static void debug_check_mem(struct aplwc *); + void memory_init(struct aplwc *aplwc) { @@ -69,14 +73,21 @@ data(struct chunk *c) struct chunk * next(struct chunk *c) { + static struct chunk *prev; uint8_t *p; if(c->last) return NULL; p = (uint8_t*)c; + assert((((uintptr_t)p) % ALIGNMENT) == 0); + p += c->size; + assert((((uintptr_t)p) % ALIGNMENT) == 0); + + prev = c; + UNUSED_PARAMETER(prev); - return (struct chunk *)(p + c->size); + return (struct chunk *)p; } #define VISIT_FIELD(X, tag, ptr) X(((struct tag *)data(c))->ptr) @@ -95,13 +106,25 @@ next(struct chunk *c) case CHUNK_TAG_APLWC_MODULE: \ VISIT_FIELD(X, aplwc_module, name); \ VISIT_FIELD(X, aplwc_module, tokens); \ + VISIT_FIELD(X, aplwc_module, imports); \ + VISIT_FIELD(X, aplwc_module, members); \ VISIT_FIELD(X, aplwc_module, link); \ break; \ + case CHUNK_TAG_MODULE_MEMBER: \ + VISIT_FIELD(X, module_member, module); \ + VISIT_FIELD(X, module_member, real_member); \ + VISIT_FIELD(X, module_member, link); \ + break; \ + case CHUNK_TAG_SYMBOL: \ + VISIT_FIELD(X, symbol, text); \ + VISIT_FIELD(X, symbol, link); \ + break; \ case CHUNK_TAG_FREE: \ break; \ } #define VISIT_ROOT_PTRS(X) do{ \ X(aplwc->modules); \ + X(aplwc->symtab); \ for(frame = aplwc->memptrs; frame; frame = frame->link) \ for(i = 0; i < frame->count; i++) \ if(frame->ptrs[i]) \ @@ -325,6 +348,7 @@ compact(struct aplwc *aplwc) if(c->tag == CHUNK_TAG_FREE){ prev->size += c->size; prev->last = c->last; + c = prev; }else{ /* swap prev and c */ oldptr = data(c); @@ -357,6 +381,8 @@ debug_check_mem(struct aplwc *aplwc) prev = NULL; for(c = (struct chunk *)aplwc->data; c; prev = c, c = next(c)){ assert(((uint8_t*)c) < (aplwc->data+aplwc->size)); + assert((c->size % ALIGNMENT) == 0); + assert(c->stack == NULL); size += c->size; assert((c->last == 0) != (size == aplwc->size)); if(c->tag == CHUNK_TAG_FREE) @@ -414,6 +440,8 @@ retry: if(new->last){ new->last = 0; free->last = 1; + }else{ + free->last = 0; } new->size = size+HEADER_SIZE; @@ -529,6 +557,24 @@ memory_alloc_aplwc_module(struct aplwc *aplwc) return p; } +struct module_member * +memory_alloc_module_member(struct aplwc *aplwc) +{ + struct module_member *p; + + p = alloc(aplwc, sizeof(struct module_member), CHUNK_TAG_MODULE_MEMBER); + return p; +} + +struct symbol * +memory_alloc_symbol(struct aplwc *aplwc) +{ + struct symbol *p; + + p = alloc(aplwc, sizeof(struct symbol), CHUNK_TAG_SYMBOL); + return p; +} + void clear_ref(struct aplwc_ref *ref) { diff --git a/src/lib/module.c b/src/lib/module.c index 51bbfdd..4243b77 100644 --- a/src/lib/module.c +++ b/src/lib/module.c @@ -18,16 +18,103 @@ */ #include "aplwc_internal.h" +static struct module_member * +add_member(struct aplwc *aplwc, struct aplwc_module *module, symbol_t name, struct module_member *real_member, int export) +{ + struct module_member *m; + MEMPTRS(aplwc, 2); + MEMPTR(0, module); + MEMPTR(1, real_member); + + for(m = module->members; m; m = m->link){ + if(m->name == name){ + if(real_member) + assert(m->real_member == NULL || m->real_member == real_member); + m->real_member = real_member; + m->exported |= export; + goto end; + } + } + + if((m = memory_alloc_module_member(aplwc)) == NULL) + goto end; + + m->exported = export; + m->name = name; + m->real_member = real_member; + m->link = module->members; + module->members = m; +end: + MEMPTRS_EXIT(aplwc); + return m; +} static int -read_keyword_decl(struct aplwc_module *module, size_t offset) +read_keyword_decl(struct aplwc *aplwc, struct aplwc_module *module, size_t offset) { int len; + size_t max; + struct module_member *m; - for(len = 1; (offset + len) < module->tokens->length; len++) - if(module->tokens->tokens[offset+len].tag == TOKEN_TAG_NEWLINE) - break; + MEMPTRS(aplwc, 2); + MEMPTR(0, module); + + m = NULL; + MEMPTR(1, m); + +#define t(n) module->tokens->tokens[offset+(n)] + + max = module->tokens->length - offset; + + assert(t(0).tag == TOKEN_TAG_KEYWORD); + switch(t(0).data.keyword){ + case KW_MODULE: + if((max < 1) || (t(1).tag != TOKEN_TAG_IDENTIFIER)) + goto fail; + len = 1; + break; + case KW_SYMBOL: + if((max < 1) || (t(1).tag != TOKEN_TAG_IDENTIFIER)) + goto fail; + if((max < 2) || (t(2).tag != TOKEN_TAG_UNKNOWN_SYM)) + goto fail; + len = 2; + + if((m = add_member(aplwc, module, t(1).data.symbol, NULL, 0)) == NULL) + goto fail; + if(add_member(aplwc, module, t(2).data.symbol, m, 0) == NULL) + goto fail; + break; + case KW_EXPORT: + for(len = 1; (size_t)len < max; len++){ + switch(t(len).tag){ + case TOKEN_TAG_UNKNOWN_SYM: + case TOKEN_TAG_IDENTIFIER: + if(add_member(aplwc, module, t(len).data.symbol, NULL, 1) == NULL) + goto fail; + break; + case TOKEN_TAG_NEWLINE: + goto end; + default: +fail: + len = -1; + goto end; + } + } + break; + default: + assert(0); + return -1; + } + + if((len >= 0) && (max > (size_t)len)) + if(t(len+1).tag != TOKEN_TAG_NEWLINE) + goto fail; + len++; /* For the keyword itself */ +end: + MEMPTRS_EXIT(aplwc); return len; +#undef t } static int @@ -46,14 +133,14 @@ read_function_decl(struct aplwc_module *module, size_t offset) } static int -read_decl(struct aplwc_module *module, size_t offset) +read_decl(struct aplwc *aplwc, struct aplwc_module *module, size_t offset) { switch(module->tokens->tokens[offset].tag){ case TOKEN_TAG_COMMENT: case TOKEN_TAG_NEWLINE: return 0; case TOKEN_TAG_KEYWORD: - return read_keyword_decl(module, offset); + return read_keyword_decl(aplwc, module, offset); case TOKEN_TAG_DEL: return read_function_decl(module, offset); default: @@ -62,49 +149,109 @@ read_decl(struct aplwc_module *module, size_t offset) } static enum aplwc_rc -read_decls(struct aplwc_module *module) +read_decls(struct aplwc *aplwc, struct aplwc_module *module) { size_t offset; int skip; + enum aplwc_rc rc; + MEMPTRS(aplwc, 1); + MEMPTR(0, module); + + rc = APLWC_OK; for(offset = 0; offset < module->tokens->length; offset++){ - skip = read_decl(module, offset); - if(skip == -1) - return APLWC_INVALID_SYNTAX; + skip = read_decl(aplwc, module, offset); + if(skip == -1){ + rc = APLWC_INVALID_SYNTAX; + goto end; + } offset += skip; } - return APLWC_OK; + +end: + MEMPTRS_EXIT(aplwc); + return rc; +} + +static struct aplwc_module * +module_lookup(struct aplwc *aplwc, struct string *name) +{ + struct aplwc_module *m; + + for(m = aplwc->modules; m; m = m->link){ + if(string_compare(m->name, name) == 0) + break; + } + + return m; +} + +static enum aplwc_rc +resolve_imports(struct aplwc *aplwc, struct aplwc_module *module) +{ + struct aplwc_module *imported; + struct string *name; + struct module_member *m; + enum aplwc_rc rc; + + MEMPTRS(aplwc, 1); + MEMPTR(0, module); + + rc = APLWC_OK; + if((name = chars_to_string(aplwc, "Boot", -1)) == NULL){ + rc = aplwc->last_error; + goto end; + } + + if((imported = module_lookup(aplwc, name)) == NULL) + goto end; /* only because it is Boot for now... */ + + for(m = imported->members; m; m = m->link){ + if(m->exported == 0) + continue; + if(add_member(aplwc, module, m->name, m, 0) == NULL){ + rc = aplwc->last_error; + goto end; + } + } +end: + MEMPTRS_EXIT(aplwc); + return rc; } enum aplwc_rc load_module(struct aplwc *aplwc, struct string *name, struct token_list *tokens) { - enum aplwc_rc rc; struct aplwc_module *module; - MEMPTRS(aplwc, 2); + MEMPTRS(aplwc, 3); MEMPTR(0, name); MEMPTR(1, tokens); - module = memory_alloc_aplwc_module(aplwc); - if(module == NULL){ - rc = aplwc_last_error(aplwc); + if((module = memory_alloc_aplwc_module(aplwc)) == NULL) goto end; - } + MEMPTR(2, module); module->name = name; module->tokens = tokens; + module->imports = NULL; + module->members = NULL; module->link = NULL; - rc = read_decls(module); - if(rc == APLWC_OK){ - module->link = aplwc->modules; - aplwc->modules = module; - }else - aplwc->last_error = rc; + if((aplwc->last_error = read_decls(aplwc, module)) != APLWC_OK) + goto end; + + if((aplwc->last_error = resolve_imports(aplwc, module)) != APLWC_OK) + goto end; + + if((aplwc->last_error = check_tokens(module, tokens)) != APLWC_OK) + goto end; + + module->link = aplwc->modules; + aplwc->modules = module; end: MEMPTRS_EXIT(aplwc); - return rc; + return aplwc->last_error; } diff --git a/src/lib/string.c b/src/lib/string.c index 05549e0..db08b28 100644 --- a/src/lib/string.c +++ b/src/lib/string.c @@ -168,3 +168,27 @@ string_compare(struct string *a, struct string *b) else return 0; } + +struct string * +substring(struct aplwc *aplwc, struct string *s, size_t start, size_t length) +{ + size_t bytes, i; + struct string *r; + + MEMPTRS(aplwc, 1); + MEMPTR(0, s); + + bytes = 0; + for(i = 0; i < length; i++) + bytes += char_length(s, start+i); + + r = memory_alloc_string(aplwc, bytes); + if(r){ + r->bytes = bytes; + r->chars = length; + memcpy(r->utf8, s->utf8+start, bytes); + } + + MEMPTRS_EXIT(aplwc); + return r; +} diff --git a/src/lib/symtab.c b/src/lib/symtab.c new file mode 100644 index 0000000..d81414d --- /dev/null +++ b/src/lib/symtab.c @@ -0,0 +1,51 @@ +/* + * Copyright (C) 2026 Peter Mikkelsen <petermikkelsen10@gmail.com> + * + * This file is part of aplwc. + * + * Aplwc is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Aplwc is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with aplwc. If not, see <https://www.gnu.org/licenses/>. + */ + +#include "aplwc_internal.h" + +enum aplwc_rc +getsym(struct aplwc *aplwc, struct string *string, symbol_t *id) +{ + static symbol_t next_id; + struct symbol *sym; + enum aplwc_rc rc; + + MEMPTRS(aplwc, 1); + + rc = APLWC_OK; + for(sym = aplwc->symtab; sym; sym = sym->link){ + if(string_compare(sym->text, string) == 0) + goto end; + } + + MEMPTR(0, string); + if((sym = memory_alloc_symbol(aplwc)) == NULL){ + rc = aplwc->last_error; + goto end; + } + sym->id = next_id++; + sym->text = string; + sym->link = aplwc->symtab; + aplwc->symtab = sym; + +end: + *id = sym->id; + MEMPTRS_EXIT(aplwc); + return rc; +} diff --git a/src/lib/syntax.c b/src/lib/syntax.c index 23138ed..2a613c8 100644 --- a/src/lib/syntax.c +++ b/src/lib/syntax.c @@ -20,14 +20,16 @@ #include "aplwc_internal.h" struct scan_state { - struct string *text; + struct aplwc *aplwc; + struct string **text; size_t offset; int skip_number_and_identifier; }; static void -init_scan_state(struct scan_state *state, struct string *text) +init_scan_state(struct scan_state *state, struct aplwc *aplwc, struct string **text) { + state->aplwc = aplwc; state->text = text; state->offset = 0; state->skip_number_and_identifier = 0; @@ -54,17 +56,33 @@ is_digit(uint32_t c) static uint32_t peek(struct scan_state *state) { - return string_char(state->text, state->offset); + return string_char(*state->text, state->offset); } static uint32_t advance(struct scan_state *state) { - state->offset += char_length(state->text, state->offset); + state->offset += char_length(*state->text, state->offset); state->skip_number_and_identifier = 0; return peek(state); } +static symbol_t +newsym(struct scan_state *state, size_t from, size_t length) +{ + struct string *s; + symbol_t id; + enum aplwc_rc rc; + + s = substring(state->aplwc, *state->text, from, length); + assert(s); /* I know this is wrong */ + + rc = getsym(state->aplwc, s, &id); + assert(rc == APLWC_OK); /* I know this is wrong */ + + return id; +} + static void skip_space(struct scan_state *state) { @@ -72,70 +90,142 @@ skip_space(struct scan_state *state) advance(state); } +#define T tokens->tokens[token_index] + static int -single_sym(struct scan_state *state, struct token *token) +single_sym(struct scan_state *state, struct token_list *tokens, size_t token_index, int check_only) { uint32_t c; + size_t start; + symbol_t sym; + + MEMPTRS(state->aplwc, 1); + MEMPTR(0, tokens); c = peek(state); + start = state->offset; switch(c){ - case 0x2190: token->tag = TOKEN_TAG_LEFT_ARROW; break; - case 0x22C4: token->tag = TOKEN_TAG_DIAMOND; break; - case 0x2207: token->tag = TOKEN_TAG_DEL; break; - case 0x2336: token->tag = TOKEN_TAG_IBEAM; break; - case 0x2395: token->tag = TOKEN_TAG_QUAD; break; - case '\n': token->tag = TOKEN_TAG_NEWLINE; break; - case '(': token->tag = TOKEN_TAG_LEFT_PAREN; break; - case ')': token->tag = TOKEN_TAG_RIGHT_PAREN; break; - default: token->tag = TOKEN_TAG_UNKNOWN_SYM; break; + case 0x2190: T.tag = TOKEN_TAG_LEFT_ARROW; break; + case 0x22C4: T.tag = TOKEN_TAG_DIAMOND; break; + case 0x2207: T.tag = TOKEN_TAG_DEL; break; + case 0x2336: T.tag = TOKEN_TAG_IBEAM; break; + case 0x2395: T.tag = TOKEN_TAG_QUAD; break; + case '\n': T.tag = TOKEN_TAG_NEWLINE; break; + case '(': T.tag = TOKEN_TAG_LEFT_PAREN; break; + case ')': T.tag = TOKEN_TAG_RIGHT_PAREN; break; + default: + T.tag = TOKEN_TAG_UNKNOWN_SYM; + if(!check_only){ + sym = newsym(state, start, 1); + T.data.symbol = sym; + } + break; } advance(state); + MEMPTRS_EXIT(state->aplwc); return 1; } static int -identifier(struct scan_state *state, struct token *token) +identifier(struct scan_state *state, struct token_list *tokens, size_t token_index, int check_only) { uint32_t c; + size_t start, len; + int ok; + symbol_t sym; + + MEMPTRS(state->aplwc, 1); + MEMPTR(0, tokens); + + ok = 0; + if(state->skip_number_and_identifier || (!is_letter(peek(state)))) - return 0; + goto end; - do + start = state->offset; + len = 0; + do{ c = advance(state); - while(is_letter(c) || is_digit(c) || (c == '_')); + len++; + }while(is_letter(c) || is_digit(c) || (c == '_')); + + T.tag = TOKEN_TAG_IDENTIFIER; + if(!check_only){ + sym = newsym(state, start, len); + T.data.symbol = sym; + } - token->tag = TOKEN_TAG_IDENTIFIER; state->skip_number_and_identifier = 1; - return 1; + ok = 1; +end: + MEMPTRS_EXIT(state->aplwc); + return ok; } static int -keyword(struct scan_state *state, struct token *token) +keyword(struct scan_state *state, struct token_list *tokens, size_t token_index, int check_only) { uint32_t c; + size_t start, len; + int ok; + symbol_t sym; + enum keyword kw; + + MEMPTRS(state->aplwc, 1); + MEMPTR(0, tokens); + + ok = 0; + if(peek(state) != ':') - return 0; + goto end; + c = advance(state); - do + start = state->offset; + len = 0; + do{ c = advance(state); - while(is_letter(c)); + len++; + }while(is_letter(c)); - token->tag = TOKEN_TAG_KEYWORD; - return 1; + T.tag = TOKEN_TAG_KEYWORD; + if(!check_only){ + sym = newsym(state, start, len); + + for(kw = 0; kw < N_KWDS; kw++){ + if(sym == state->aplwc->keyword_symbols[kw]){ + T.data.keyword = kw; + break; + } + } + if(kw == N_KWDS) + ok = 0; + + } + + ok = 1; +end: + MEMPTRS_EXIT(state->aplwc); + return ok; } static int -number(struct scan_state *state, struct token *token) +number(struct scan_state *state, struct token_list *tokens, size_t token_index, int check_only) { uint32_t c; - int has_digits; + int has_digits, ok; + + MEMPTRS(state->aplwc, 1); + MEMPTR(0, tokens); + UNUSED_PARAMETER(check_only); + + ok = 0; c = peek(state); if(state->skip_number_and_identifier) - return 0; + goto end; has_digits = 0; @@ -161,21 +251,31 @@ number(struct scan_state *state, struct token *token) } if(has_digits == 0) - return 0; + goto end; - token->tag = TOKEN_TAG_NUMBER; + T.tag = TOKEN_TAG_NUMBER; state->skip_number_and_identifier = 1; - return 1; + ok = 1; +end: + MEMPTRS_EXIT(state->aplwc); + return ok; } static int -text(struct scan_state *state, struct token *token) +text(struct scan_state *state, struct token_list *tokens, size_t token_index, int check_only) { uint32_t c; + int ok; + + MEMPTRS(state->aplwc, 1); + MEMPTR(0, tokens); + UNUSED_PARAMETER(check_only); + + ok = 0; c = peek(state); if(c != '\'') - return 0; + goto end; do c = advance(state); @@ -184,34 +284,48 @@ text(struct scan_state *state, struct token *token) if(c == '\'') advance(state); else - return 0; + goto end; - token->tag = TOKEN_TAG_TEXT; - return 1; + T.tag = TOKEN_TAG_TEXT; + ok = 1; +end: + MEMPTRS_EXIT(state->aplwc); + return ok; } static int -comment(struct scan_state *state, struct token *token) +comment(struct scan_state *state, struct token_list *tokens, size_t token_index, int check_only) { uint32_t c; + int ok; + MEMPTRS(state->aplwc, 1); + MEMPTR(0, tokens); + + UNUSED_PARAMETER(check_only); + + ok = 0; c = peek(state); if(c != 0x235D) - return 0; + goto end; do c = advance(state); while(c != '\n'); - token->tag = TOKEN_TAG_COMMENT; - return 1; + T.tag = TOKEN_TAG_COMMENT; + ok = 1; +end: + MEMPTRS_EXIT(state->aplwc); + return ok; } static enum aplwc_rc -next_token(struct scan_state *state, struct token *token) +next_token(struct scan_state *state, struct token_list *tokens, size_t token_index, int check_only) { + enum aplwc_rc rc; size_t i; - int (*scans[])(struct scan_state *, struct token *token) = { + int (*scans[])(struct scan_state *, struct token_list *, size_t, int) = { identifier, keyword, number, @@ -219,55 +333,76 @@ next_token(struct scan_state *state, struct token *token) comment, single_sym /* Must be last */ }; + MEMPTRS(state->aplwc, 1); + MEMPTR(0, tokens); skip_space(state); - token->start = state->offset; + T.start = state->offset; for(i = 0; i < NELEM(scans); i++){ - if(scans[i](state, token)) + if(scans[i](state, tokens, token_index, check_only)) break; - state->offset = token->start; + state->offset = tokens->tokens[token_index].start; } - token->end = state->offset; + T.end = state->offset; skip_space(state); - if(token->start == token->end) - return APLWC_INVALID_SYNTAX; - else - return APLWC_OK; + rc = (T.start == T.end) ? APLWC_INVALID_SYNTAX : APLWC_OK; + MEMPTRS_EXIT(state->aplwc); + return rc; } +#undef T + static enum aplwc_rc -check(struct string *text, size_t *n) +check(struct aplwc *aplwc, struct string *text, size_t *n) { struct scan_state state; - struct token token; + struct token_list *tokens; enum aplwc_rc ret; size_t count; - init_scan_state(&state, text); + MEMPTRS(aplwc, 2); + MEMPTR(0, text); + + if((tokens = memory_alloc_token_list(aplwc, 1)) == NULL){ + ret = aplwc->last_error; + goto end; + } + MEMPTR(1, tokens); + + init_scan_state(&state, aplwc, &text); for(count = 0; peek(&state); count++){ - ret = next_token(&state, &token); + ret = next_token(&state, tokens, 0, 1); if(ret != APLWC_OK) - return ret; + goto end; } *n = count; - return APLWC_OK; + ret = APLWC_OK; +end: + MEMPTRS_EXIT(aplwc); + return ret; } static void -scan(struct string *text, struct token_list *tokens) +scan(struct aplwc *aplwc, struct string *text, struct token_list *tokens) { struct scan_state state; enum aplwc_rc ret; size_t count; - init_scan_state(&state, text); + MEMPTRS(aplwc, 2); + MEMPTR(0, text); + MEMPTR(1, tokens); + + init_scan_state(&state, aplwc, &text); for(count = 0; peek(&state); count++){ - ret = next_token(&state, &tokens->tokens[count]); + ret = next_token(&state, tokens, count, 0); assert(ret == APLWC_OK); } + + MEMPTRS_EXIT(aplwc); } struct token_list * @@ -277,11 +412,11 @@ scan_string(struct aplwc *aplwc, struct string *text, int oneline) struct token_list *tokens; size_t n, i; - MEMPTRS(aplwc, 1); + MEMPTRS(aplwc, 2); MEMPTR(0, text); tokens = NULL; - rc = check(text, &n); + rc = check(aplwc, text, &n); if(rc != APLWC_OK) goto end; @@ -290,9 +425,11 @@ scan_string(struct aplwc *aplwc, struct string *text, int oneline) rc = aplwc->last_error; goto end; } + MEMPTR(1, tokens); + tokens->text = text; tokens->length = n; - scan(text, tokens); + scan(aplwc, text, tokens); for(i = 0; oneline && (i < n); i++){ if(tokens->tokens[i].tag != TOKEN_TAG_NEWLINE) @@ -313,8 +450,34 @@ end: return tokens; } +static void +resolve_unknown(struct aplwc_module *module, struct token *t) +{ + symbol_t s; + struct module_member *m; + + assert(t && (t->tag == TOKEN_TAG_UNKNOWN_SYM)); + if(module == NULL) + return; + + s = t->data.symbol; + + for(m = module->members; m; m = m->link){ + if(m->name == s){ +found: + t->tag = TOKEN_TAG_IDENTIFIER; + return; + } + } + + for(m = module->imports; m; m = m->link){ + if(m->name == s) + goto found; + } +} + enum aplwc_rc -check_tokens(struct token_list *tokens) +check_tokens(struct aplwc_module *module, struct token_list *tokens) { size_t i; enum aplwc_rc rc; @@ -322,7 +485,9 @@ check_tokens(struct token_list *tokens) for(i = 0, rc = APLWC_OK; (rc == APLWC_OK) && (i < tokens->length); i++){ switch(tokens->tokens[i].tag){ case TOKEN_TAG_UNKNOWN_SYM: - rc = APLWC_INVALID_SYNTAX; + resolve_unknown(module, &tokens->tokens[i]); + if(tokens->tokens[i].tag == TOKEN_TAG_UNKNOWN_SYM) + return APLWC_INVALID_SYNTAX; break; default: break; @@ -331,3 +496,30 @@ check_tokens(struct token_list *tokens) return rc; } + +enum aplwc_rc +keyword_init(struct aplwc *aplwc) +{ + enum keyword kw; + const char *cstr; + struct string *str; + symbol_t sym; + + aplwc->last_error = APLWC_OK; + for(kw = 0; kw < N_KWDS; kw++){ + switch(kw){ + case KW_MODULE: cstr = "module"; break; + case KW_SYMBOL: cstr = "symbol"; break; + case KW_EXPORT: cstr = "export"; break; + case N_KWDS: assert(0); break; + } + + if((str = chars_to_string(aplwc, cstr, -1)) == NULL) + break; + if(getsym(aplwc, str, &sym) != APLWC_OK) + break; + aplwc->keyword_symbols[kw] = sym; + } + + return aplwc->last_error; +} diff --git a/tests/scripts/good/3.aplwc b/tests/scripts/bad/5.aplwc index f94a4c6..26b3140 100755 --- a/tests/scripts/good/3.aplwc +++ b/tests/scripts/bad/5.aplwc @@ -1,4 +1,6 @@ #!/usr/bin/env aplwc-repl - -:keyword 1 2
\ No newline at end of file +∇ foo + ' + ' +∇ |