77 lines
1.3 KiB
C
77 lines
1.3 KiB
C
#include <stddef.h>
|
|
|
|
#include "error.h"
|
|
#include "vector.h"
|
|
#include "tokenizer.h"
|
|
|
|
DECL_SPAN(token);
|
|
|
|
static bool identifier_start(char);
|
|
static bool identifier(char);
|
|
static bool digit(char);
|
|
|
|
VEC(token) tokenize(SPAN(char) source) {
|
|
VEC(token) tokens = {};
|
|
|
|
size_t offset = 0;
|
|
size_t line = 1;
|
|
size_t line_start_offset = 0;
|
|
|
|
while (offset < LENGTH(source)) {
|
|
struct source_location at = {
|
|
.line = line,
|
|
.column = offset - line_start_offset + 1,
|
|
.offset = offset,
|
|
};
|
|
|
|
auto ch = AT(source, offset);
|
|
|
|
if (ch == '\n') {
|
|
line++;
|
|
offset++;
|
|
line_start_offset = offset;
|
|
|
|
} else if (ch == ' ' || ch == '\t') {
|
|
offset++;
|
|
|
|
} else if (identifier_start(ch)) {
|
|
auto start = offset;
|
|
while (identifier(AT(source, offset))) {
|
|
offset++;
|
|
}
|
|
auto length = offset - start;
|
|
|
|
VEC_PUSH(tokens, ((token) {
|
|
.at = at,
|
|
.length = length,
|
|
.id = TOK_IDENTIFIER,
|
|
}));
|
|
|
|
} else {
|
|
VEC_PUSH(tokens, ((token) {
|
|
.at = at,
|
|
.length = 1,
|
|
.id = ch,
|
|
}));
|
|
|
|
offset++;
|
|
}
|
|
}
|
|
|
|
return tokens;
|
|
}
|
|
|
|
static bool identifier_start(char c) {
|
|
return identifier(c) && !digit(c);
|
|
}
|
|
|
|
static bool identifier(char c) {
|
|
return ('0' <= c && c <= '9')
|
|
|| ('A' <= c && c <= 'Z')
|
|
|| ('a' <= c && c <= 'z')
|
|
|| c == '_';
|
|
}
|
|
|
|
static bool digit(char c) {
|
|
return '0' <= c && c <= '9';
|
|
}
|