cusel/compiler/tokenizer.c
2026-11-21 00:46:50 +02:00

77 lines
1.3 KiB
C

#include <stddef.h>
#include "error.h"
#include "vector.h"
#include "tokenizer.h"
DECL_SPAN(token);
static bool identifier_start(char);
static bool identifier(char);
static bool digit(char);
VEC(token) tokenize(SPAN(char) source) {
VEC(token) tokens = {};
size_t offset = 0;
size_t line = 1;
size_t line_start_offset = 0;
while (offset < LENGTH(source)) {
struct source_location at = {
.line = line,
.column = offset - line_start_offset + 1,
.offset = offset,
};
auto ch = AT(source, offset);
if (ch == '\n') {
line++;
offset++;
line_start_offset = offset;
} else if (ch == ' ' || ch == '\t') {
offset++;
} else if (identifier_start(ch)) {
auto start = offset;
while (identifier(AT(source, offset))) {
offset++;
}
auto length = offset - start;
VEC_PUSH(tokens, ((token) {
.at = at,
.length = length,
.id = TOK_IDENTIFIER,
}));
} else {
VEC_PUSH(tokens, ((token) {
.at = at,
.length = 1,
.id = ch,
}));
offset++;
}
}
return tokens;
}
static bool identifier_start(char c) {
return identifier(c) && !digit(c);
}
static bool identifier(char c) {
return ('0' <= c && c <= '9')
|| ('A' <= c && c <= 'Z')
|| ('a' <= c && c <= 'z')
|| c == '_';
}
static bool digit(char c) {
return '0' <= c && c <= '9';
}