cusel/compiler/tokenizer.h
2026-11-21 00:46:50 +02:00

20 lines
289 B
C

struct source_location {
size_t offset;
size_t line;
size_t column;
};
enum token_id : int {
TOK_IDENTIFIER = 256,
};
typedef struct {
struct source_location at;
size_t length;
enum token_id id;
} token;
DECL_VEC(token);
DECL_SPAN(char);
VEC(token) tokenize(SPAN(char) source);