1 files changed, 257 insertions, 65 deletions

diff --git a/src/lib/syntax.c b/src/lib/syntax.c
index 23138ed..2a613c8 100644
--- a/src/lib/syntax.c
+++ b/src/lib/syntax.c
@@ -20,14 +20,16 @@
 #include "aplwc_internal.h"
 
 struct scan_state {
-	struct string *text;
+	struct aplwc *aplwc;
+	struct string **text;
 	size_t offset;
 	int skip_number_and_identifier;
 };
 
 static void
-init_scan_state(struct scan_state *state, struct string *text)
+init_scan_state(struct scan_state *state, struct aplwc *aplwc, struct string **text)
 {
+	state->aplwc = aplwc;
 	state->text = text;
 	state->offset = 0;
 	state->skip_number_and_identifier = 0;
@@ -54,17 +56,33 @@ is_digit(uint32_t c)
 static uint32_t
 peek(struct scan_state *state)
 {
-	return string_char(state->text, state->offset);
+	return string_char(*state->text, state->offset);
 }
 
 static uint32_t 
 advance(struct scan_state *state)
 {
-	state->offset += char_length(state->text, state->offset);
+	state->offset += char_length(*state->text, state->offset);
 	state->skip_number_and_identifier = 0;
 	return peek(state);
 }
 
+static symbol_t
+newsym(struct scan_state *state, size_t from, size_t length)
+{
+	struct string *s;
+	symbol_t id;
+	enum aplwc_rc rc;
+
+	s = substring(state->aplwc, *state->text, from, length);
+	assert(s); /* I know this is wrong */
+
+	rc = getsym(state->aplwc, s, &id);
+	assert(rc == APLWC_OK); /* I know this is wrong */
+
+	return id;
+}
+
 static void
 skip_space(struct scan_state *state)
 {
@@ -72,70 +90,142 @@ skip_space(struct scan_state *state)
 		advance(state);
 }
 
+#define T tokens->tokens[token_index]
+
 static int
-single_sym(struct scan_state *state, struct token *token)
+single_sym(struct scan_state *state, struct token_list *tokens, size_t token_index, int check_only)
 {
 	uint32_t c;
+	size_t start;
+	symbol_t sym;
+
+	MEMPTRS(state->aplwc, 1);
+	MEMPTR(0, tokens);
 
 	c = peek(state);
+	start = state->offset;
 
 	switch(c){
-	case 0x2190: token->tag = TOKEN_TAG_LEFT_ARROW; break;
-	case 0x22C4: token->tag = TOKEN_TAG_DIAMOND; break;
-	case 0x2207: token->tag = TOKEN_TAG_DEL; break;
-	case 0x2336: token->tag = TOKEN_TAG_IBEAM; break;
-	case 0x2395: token->tag = TOKEN_TAG_QUAD; break;
-	case '\n':   token->tag = TOKEN_TAG_NEWLINE; break;
-	case '(':    token->tag = TOKEN_TAG_LEFT_PAREN; break;
-	case ')':    token->tag = TOKEN_TAG_RIGHT_PAREN; break;
-	default:     token->tag = TOKEN_TAG_UNKNOWN_SYM; break;
+	case 0x2190: T.tag = TOKEN_TAG_LEFT_ARROW; break;
+	case 0x22C4: T.tag = TOKEN_TAG_DIAMOND; break;
+	case 0x2207: T.tag = TOKEN_TAG_DEL; break;
+	case 0x2336: T.tag = TOKEN_TAG_IBEAM; break;
+	case 0x2395: T.tag = TOKEN_TAG_QUAD; break;
+	case '\n':   T.tag = TOKEN_TAG_NEWLINE; break;
+	case '(':    T.tag = TOKEN_TAG_LEFT_PAREN; break;
+	case ')':    T.tag = TOKEN_TAG_RIGHT_PAREN; break;
+	default:
+		T.tag = TOKEN_TAG_UNKNOWN_SYM;
+		if(!check_only){
+			sym = newsym(state, start, 1);
+			T.data.symbol = sym;
+		}
+		break;
 	}
 
 	advance(state);
+	MEMPTRS_EXIT(state->aplwc);
 	return 1;
 }
 
 static int 
-identifier(struct scan_state *state, struct token *token)
+identifier(struct scan_state *state, struct token_list *tokens, size_t token_index, int check_only)
 {
 	uint32_t c;
+	size_t start, len;
+	int ok;
+	symbol_t sym;
+
+	MEMPTRS(state->aplwc, 1);
+	MEMPTR(0, tokens);
+
+	ok = 0;
+
 	if(state->skip_number_and_identifier || (!is_letter(peek(state))))
-		return 0;
+		goto end;
 
-	do
+	start = state->offset;
+	len = 0;
+	do{
 		c = advance(state);
-	while(is_letter(c) || is_digit(c) || (c == '_'));
+		len++;
+	}while(is_letter(c) || is_digit(c) || (c == '_'));
+
+	T.tag = TOKEN_TAG_IDENTIFIER;
+	if(!check_only){
+		sym = newsym(state, start, len);
+		T.data.symbol = sym;
+	}
 
-	token->tag = TOKEN_TAG_IDENTIFIER;
 	state->skip_number_and_identifier = 1;
-	return 1;
+	ok = 1;
+end:
+	MEMPTRS_EXIT(state->aplwc);
+	return ok;
 }
 
 static int
-keyword(struct scan_state *state, struct token *token)
+keyword(struct scan_state *state, struct token_list *tokens, size_t token_index, int check_only)
 {
 	uint32_t c;
+	size_t start, len;
+	int ok;
+	symbol_t sym;
+	enum keyword kw;
+
+	MEMPTRS(state->aplwc, 1);
+	MEMPTR(0, tokens);
+
+	ok = 0;
+
 	if(peek(state) != ':')
-		return 0;
+		goto end;
+	c = advance(state);
 
-	do
+	start = state->offset;
+	len = 0;
+	do{
 		c = advance(state);
-	while(is_letter(c));
+		len++;
+	}while(is_letter(c));
 
-	token->tag = TOKEN_TAG_KEYWORD;
-	return 1;
+	T.tag = TOKEN_TAG_KEYWORD;
+	if(!check_only){
+		sym = newsym(state, start, len);
+
+		for(kw = 0; kw < N_KWDS; kw++){
+			if(sym == state->aplwc->keyword_symbols[kw]){
+				T.data.keyword = kw;
+				break;
+			}
+		}
+		if(kw == N_KWDS)
+			ok = 0;
+
+	}
+
+	ok = 1;
+end:
+	MEMPTRS_EXIT(state->aplwc);
+	return ok;
 }
 
 static int 
-number(struct scan_state *state, struct token *token)
+number(struct scan_state *state, struct token_list *tokens, size_t token_index, int check_only)
 {
 	uint32_t c;
-	int has_digits;
+	int has_digits, ok;
+
+	MEMPTRS(state->aplwc, 1);
+	MEMPTR(0, tokens);
 
+	UNUSED_PARAMETER(check_only);
+
+	ok = 0;
 	c = peek(state);
 
 	if(state->skip_number_and_identifier)
-		return 0;
+		goto end;
 
 	has_digits = 0;
 
@@ -161,21 +251,31 @@ number(struct scan_state *state, struct token *token)
 	}
 
 	if(has_digits == 0)
-		return 0;
+		goto end;
 
-	token->tag = TOKEN_TAG_NUMBER;
+	T.tag = TOKEN_TAG_NUMBER;
 	state->skip_number_and_identifier = 1;
-	return 1;
+	ok = 1;
+end:
+	MEMPTRS_EXIT(state->aplwc);
+	return ok;
 }
 
 static int
-text(struct scan_state *state, struct token *token)
+text(struct scan_state *state, struct token_list *tokens, size_t token_index, int check_only)
 {
 	uint32_t c;
+	int ok;
+
+	MEMPTRS(state->aplwc, 1);
+	MEMPTR(0, tokens);
 
+	UNUSED_PARAMETER(check_only);
+
+	ok = 0;
 	c = peek(state);
 	if(c != '\'')
-		return 0;
+		goto end;
 
 	do
 		c = advance(state);
@@ -184,34 +284,48 @@ text(struct scan_state *state, struct token *token)
 	if(c == '\'')
 		advance(state);
 	else
-		return 0;
+		goto end;
 
-	token->tag = TOKEN_TAG_TEXT;
-	return 1;
+	T.tag = TOKEN_TAG_TEXT;
+	ok = 1;
+end:
+	MEMPTRS_EXIT(state->aplwc);
+	return ok;
 }
 
 static int
-comment(struct scan_state *state, struct token *token)
+comment(struct scan_state *state, struct token_list *tokens, size_t token_index, int check_only)
 {
 	uint32_t c;
+	int ok;
 
+	MEMPTRS(state->aplwc, 1);
+	MEMPTR(0, tokens);
+
+	UNUSED_PARAMETER(check_only);
+
+	ok = 0;
 	c = peek(state);
 	if(c != 0x235D)
-		return 0;
+		goto end;
 
 	do
 		c = advance(state);
 	while(c != '\n');
 
-	token->tag = TOKEN_TAG_COMMENT;
-	return 1;
+	T.tag = TOKEN_TAG_COMMENT;
+	ok = 1;
+end:
+	MEMPTRS_EXIT(state->aplwc);
+	return ok;
 }
 
 static enum aplwc_rc
-next_token(struct scan_state *state, struct token *token)
+next_token(struct scan_state *state, struct token_list *tokens, size_t token_index, int check_only)
 {
+	enum aplwc_rc rc;
 	size_t i;
-	int (*scans[])(struct scan_state *, struct token *token) = {
+	int (*scans[])(struct scan_state *, struct token_list *, size_t, int) = {
 		identifier,
 		keyword,
 		number,
@@ -219,55 +333,76 @@ next_token(struct scan_state *state, struct token *token)
 		comment,
 		single_sym /* Must be last */
 	};
+	MEMPTRS(state->aplwc, 1);
+	MEMPTR(0, tokens);
 
 	skip_space(state);
-	token->start = state->offset;
+	T.start = state->offset;
 	for(i = 0; i < NELEM(scans); i++){
-		if(scans[i](state, token))
+		if(scans[i](state, tokens, token_index, check_only))
 			break;
-		state->offset = token->start;
+		state->offset = tokens->tokens[token_index].start;
 	}
-	token->end = state->offset;
+	T.end = state->offset;
 	skip_space(state);
-	if(token->start == token->end)
-		return APLWC_INVALID_SYNTAX;
-	else
-		return APLWC_OK;
+	rc = (T.start == T.end) ? APLWC_INVALID_SYNTAX : APLWC_OK;
+	MEMPTRS_EXIT(state->aplwc);
+	return rc;
 }
 
+#undef T
+
 static enum aplwc_rc 
-check(struct string *text, size_t *n)
+check(struct aplwc *aplwc, struct string *text, size_t *n)
 {
 	struct scan_state state;
-	struct token token;
+	struct token_list *tokens;
 	enum aplwc_rc ret;
 	size_t count;
 
-	init_scan_state(&state, text);
+	MEMPTRS(aplwc, 2);
+	MEMPTR(0, text);
+
+	if((tokens = memory_alloc_token_list(aplwc, 1)) == NULL){
+		ret = aplwc->last_error;
+		goto end;
+	}
+	MEMPTR(1, tokens);
+
+	init_scan_state(&state, aplwc, &text);
 
 	for(count = 0; peek(&state); count++){
-		ret = next_token(&state, &token);
+		ret = next_token(&state, tokens, 0, 1);
 		if(ret != APLWC_OK)
-			return ret;
+			goto end;
 	}
 
 	*n = count;
-	return APLWC_OK;
+	ret = APLWC_OK;
+end:
+	MEMPTRS_EXIT(aplwc);
+	return ret; 
 }
 
 static void
-scan(struct string *text, struct token_list *tokens)
+scan(struct aplwc *aplwc, struct string *text, struct token_list *tokens)
 {
 	struct scan_state state;
 	enum aplwc_rc ret;
 	size_t count;
 
-	init_scan_state(&state, text);
+	MEMPTRS(aplwc, 2);
+	MEMPTR(0, text);
+	MEMPTR(1, tokens);
+
+	init_scan_state(&state, aplwc, &text);
 
 	for(count = 0; peek(&state); count++){
-		ret = next_token(&state, &tokens->tokens[count]);
+		ret = next_token(&state, tokens, count, 0);
 		assert(ret == APLWC_OK);
 	}
+
+	MEMPTRS_EXIT(aplwc);
 }
 
 struct token_list *
@@ -277,11 +412,11 @@ scan_string(struct aplwc *aplwc, struct string *text, int oneline)
 	struct token_list *tokens;
 	size_t n, i;
 
-	MEMPTRS(aplwc, 1);
+	MEMPTRS(aplwc, 2);
 	MEMPTR(0, text);
 
 	tokens = NULL;
-	rc = check(text, &n);
+	rc = check(aplwc, text, &n);
 	if(rc != APLWC_OK)
 		goto end;
 
@@ -290,9 +425,11 @@ scan_string(struct aplwc *aplwc, struct string *text, int oneline)
 		rc = aplwc->last_error;
 		goto end;
 	}
+	MEMPTR(1, tokens);
+
 	tokens->text = text;
 	tokens->length = n;
-	scan(text, tokens);
+	scan(aplwc, text, tokens);
 
 	for(i = 0; oneline && (i < n); i++){
 		if(tokens->tokens[i].tag != TOKEN_TAG_NEWLINE)
@@ -313,8 +450,34 @@ end:
 	return tokens;
 }
 
+static void
+resolve_unknown(struct aplwc_module *module, struct token *t)
+{
+	symbol_t s;
+	struct module_member *m;
+
+	assert(t && (t->tag == TOKEN_TAG_UNKNOWN_SYM));
+	if(module == NULL)
+		return;
+
+	s = t->data.symbol;
+
+	for(m = module->members; m; m = m->link){
+		if(m->name == s){
+found:
+			t->tag = TOKEN_TAG_IDENTIFIER;
+			return;
+		}
+	}
+
+	for(m = module->imports; m; m = m->link){
+		if(m->name == s)
+			goto found;
+	}
+}
+
 enum aplwc_rc
-check_tokens(struct token_list *tokens)
+check_tokens(struct aplwc_module *module, struct token_list *tokens)
 {
 	size_t i;
 	enum aplwc_rc rc;
@@ -322,7 +485,9 @@ check_tokens(struct token_list *tokens)
 	for(i = 0, rc = APLWC_OK; (rc == APLWC_OK) && (i < tokens->length); i++){
 		switch(tokens->tokens[i].tag){
 		case TOKEN_TAG_UNKNOWN_SYM:
-			rc = APLWC_INVALID_SYNTAX;
+			resolve_unknown(module, &tokens->tokens[i]);
+			if(tokens->tokens[i].tag == TOKEN_TAG_UNKNOWN_SYM)
+				return APLWC_INVALID_SYNTAX;
 			break;
 		default:
 			break;
@@ -331,3 +496,30 @@ check_tokens(struct token_list *tokens)
 
 	return rc;
 }
+
+enum aplwc_rc
+keyword_init(struct aplwc *aplwc)
+{
+	enum keyword kw;
+	const char *cstr;
+	struct string *str;
+	symbol_t sym;
+
+	aplwc->last_error = APLWC_OK;
+	for(kw = 0; kw < N_KWDS; kw++){
+		switch(kw){
+		case KW_MODULE: cstr = "module"; break;
+		case KW_SYMBOL: cstr = "symbol"; break;
+		case KW_EXPORT: cstr = "export"; break;
+		case N_KWDS: assert(0); break;
+		}
+
+		if((str = chars_to_string(aplwc, cstr, -1)) == NULL)
+			break;
+		if(getsym(aplwc, str, &sym) != APLWC_OK)
+			break;
+		aplwc->keyword_symbols[kw] = sym;
+	}
+
+	return aplwc->last_error;
+}