1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
|
#include "aplwc_internal.h"
struct scan_state {
struct string *text;
size_t offset;
};
static void
init_scan_state(struct scan_state *state, struct string *text)
{
state->text = text;
state->offset = 0;
}
static int
is_space(uint32_t c)
{
return c == ' ';
}
static int
is_letter(uint32_t c)
{
return ((c >= 'a') && (c <= 'z')) || ((c >= 'A') && (c <= 'Z'));
}
static int
is_digit(uint32_t c)
{
return (c >= '0') && (c <= '9');
}
static uint32_t
peek(struct scan_state *state)
{
return string_char(state->text, state->offset);
}
static void
advance(struct scan_state *state)
{
state->offset += char_length(state->text, state->offset);
}
static void
skip_space(struct scan_state *state)
{
while(is_space(peek(state)))
advance(state);
}
static void
identifier(struct scan_state *state, struct token *token)
{
uint32_t c;
assert(is_letter(peek(state)));
do{
advance(state);
c = peek(state);
}while(is_letter(c) || is_digit(c) || (c == '_'));
token->tag = TOKEN_TAG_IDENTIFIER;
}
static enum aplwc_rc
next_token(struct scan_state *state, struct token *token)
{
uint32_t c;
skip_space(state);
c = peek(state);
if(is_letter(c)){
identifier(state, token);
return APLWC_OK;
}
if(c == '\n'){
advance(state);
token->tag = TOKEN_TAG_NEWLINE;
return APLWC_OK;
}
return APLWC_INVALID_SYNTAX;
}
static enum aplwc_rc
check(struct string *text, size_t *n)
{
struct scan_state state;
struct token token;
enum aplwc_rc ret;
size_t count;
init_scan_state(&state, text);
for(count = 0; peek(&state); count++){
ret = next_token(&state, &token);
if(ret != APLWC_OK)
return ret;
}
*n = count;
return APLWC_OK;
}
static void
scan(struct string *text, struct token_list *tokens)
{
struct scan_state state;
enum aplwc_rc ret;
size_t count;
init_scan_state(&state, text);
for(count = 0; peek(&state); count++){
ret = next_token(&state, &tokens->tokens[count]);
assert(ret == APLWC_OK);
}
}
struct token_list *
scan_line(struct aplwc *aplwc, struct string *text)
{
enum aplwc_rc rc;
struct token_list *tokens;
size_t n, i;
rc = check(text, &n);
if(rc != APLWC_OK){
aplwc->last_error = rc;
return NULL;
}
tokens = memory_alloc_token_list(&aplwc->memory, n);
if(tokens == NULL){
aplwc->last_error = APLWC_OUT_OF_MEMORY;
return NULL;
}
tokens->length = n;
scan(text, tokens);
for(i = 0; i < n; i++){
if(tokens->tokens[i].tag != TOKEN_TAG_NEWLINE)
continue;
for(i++; i < n; i++){
if(tokens->tokens[i].tag != TOKEN_TAG_NEWLINE){
memory_free_token_list(&aplwc->memory, tokens);
return NULL;
}
}
}
return tokens;
}
|