sic

commit 54a02e87ccad26778eb18531bc8969407b699ad9
parent 618272bc406d90e052965e24a5727d7a6fee7559
Author: citbl <citbl@citbl.org>
Date:   Tue, 12 May 2026 22:56:48 +1000

lexing numbers and fixes

Diffstat:
M
src/lexer.c
 | 
34
+++++++++++++++++++++++++++-------
M
test.sic
 | 
4
+++-

2 files changed, 30 insertions(+), 8 deletions(-)
diff --git a/src/lexer.c b/src/lexer.c
@@ -27,7 +27,7 @@ void print_tokens(Lexer* lex)
     Token_Type typ;
     size_t i;
 
-    printf("------- print tokens --------\n");
+    printf("------- print tokens (%zu) -------\n", lex->len);
 
     for (i = 0; i < lex->len; i++) {
         t = lex->tokens[i];
@@ -37,6 +37,12 @@ void print_tokens(Lexer* lex)
         case LIT_STRING:
             printf("STRING LITERAL: %s\n", lex->tokens[i].lexeme.value);
             break;
+        case LIT_DECIMAL:
+            printf("DECIMAL LITERAL: %s\n", lex->tokens[i].lexeme.value);
+            break;
+        case LIT_INT:
+            printf("DECIMAL LITERAL: %s\n", lex->tokens[i].lexeme.value);
+            break;
         default:
             printf("print_tokens: unhandled token %i", typ);
             break;
@@ -92,27 +98,40 @@ static void err(Lexer* lex, const char* message)
 
 static void lex_number(Lexer* lex, Token* tok)
 {
-    char c;
+    char c = lex->code[lex->state.pos];
     Str* str = &tok->lexeme;
+    str_append(str, c);
     tok->type = LIT_INT;
-    while (1) {
+    while (lex->state.pos < lex->code_len) {
         advance(lex);
         c = peek(lex);
+        // TODO lexing numbers: commas and underscores
         if (c == '.' && tok->type == LIT_DECIMAL) {
             err(lex, "parsing number failed with more than one decimal point '.'\n");
         }
         if (c == '.' && tok->type == LIT_INT) tok->type = LIT_DECIMAL;
+        if (c != '.' && !isdigit((unsigned char)c)) break;
         str_append(str, c);
-        if (c != '.' && !isdigit(c)) break;
     }
-    str->value[str->len] = '\0';
+    add_token(lex, *tok);
+}
+
+static Token new_token(Lexer* lex)
+{
+    return (Token){
+        .filename = lex->filename,
+        .path = lex->path,
+        .col = lex->state.col,
+        .line = lex->state.line,
+        .type = NOTYETSET,
+        .lexeme = {0},
+    };
 }
 
 Lexer* lexer_lex(Lexer* lex)
 {
+    Token t;
     char c = '\0';
-    Token t = {
-        .filename = lex->filename, .path = lex->path, .col = -1, .line = -1, .type = NOTYETSET, .lexeme = {0}};
 
     lex->tokens = calloc(250, sizeof(Token));
     lex->state.pos = 0;
@@ -122,6 +141,7 @@ Lexer* lexer_lex(Lexer* lex)
     // longest valid token first
     while (lex->state.pos < lex->code_len) {
         c = lex->code[lex->state.pos];
+        t = new_token(lex);
 
         if (c == '/' && peek(lex) == '/') {
             run_until_char(lex, '\n');
diff --git a/test.sic b/test.sic
@@ -1,5 +1,7 @@
 // this is a comment
 
 str name = "Johnny Mnemonic";
-int age = 45.4.2;
+int age = 45;
+dec height = 188.3;
+