nightshade

commit 32aeaade738feadea0de8ac1484e3f79dd38cc53
parent 7c72e44ab79f728eb9c77b34df8eb788c15df3fd
Author: citbl <citbl@citbl.org>
Date:   Mon,  1 Jun 2026 21:17:18 +1000

handle tabs, bug fixes

Diffstat:
M
src/lexer.adb
 | 
70
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++----------
M
src/lexer.ads
 | 
23
++++++++++++++++++-----

2 files changed, 78 insertions(+), 15 deletions(-)
diff --git a/src/lexer.adb b/src/lexer.adb
@@ -54,13 +54,26 @@ package body Lexer is
 
    -- skip spaces
 
-   procedure Skip_Spaces_Comments (L : in out Lexer) is
+   procedure Handle_Space_Comments (L : in out Lexer) is
       C : Character;
+      T : Token;
    begin
       while true loop
          while true loop
             C := Peek (L);
-            if Is_Space (C) or Is_Line_Terminator (C) then
+            if Is_Line_Terminator (C) then
+               -- we care about line return in this impl
+               T :=
+                 (Kind   => Separator,
+                  Lexeme => To_Unbounded_String (""),
+                  Line   => L.Line,
+                  Col    => L.Col);
+               Add_Token (L, T);
+            end if;
+            if Is_Space (C)
+              or Is_Line_Terminator (C)
+              or C = Character'Val (9) -- \t
+            then
                Nudge (L);
             else
                exit;
@@ -76,7 +89,7 @@ package body Lexer is
          <<Continue>>
          null;
       end loop;
-   end Skip_Spaces_Comments;
+   end Handle_Space_Comments;
 
    -- make ident
 
@@ -105,8 +118,6 @@ package body Lexer is
 
       Word := To_Unbounded_String (L.Source (Start .. L.Pos - 1));
 
-      -- TODO upgrade to keywords
-
       declare
          W : constant String := To_String (Word);
       begin
@@ -220,15 +231,21 @@ package body Lexer is
       Start : Natural;
       C     : Character;
       C2    : Character;
+      Line  : Positive;
+      Col   : Positive;
    begin
-      Skip_Spaces_Comments (L);
+      Handle_Space_Comments (L);
+
+      Start := L.Pos;
+      Line := L.Line;
+      Col := L.Col;
 
       if L.Pos < L.Source'Length then
          TK := Unknown;
       else
          TK := EOF;
       end if;
-      Start := L.Pos;
+
       C := Peek (L);
       C2 := Peek2 (L);
 
@@ -252,8 +269,40 @@ package body Lexer is
          return T;
       end if;
 
+      if C = '(' then
+         TK := L_Paren;
+      elsif C = ')' then
+         TK := R_Paren;
+      elsif C = '[' then
+         TK := L_Bracket;
+      elsif C = ']' then
+         TK := R_Bracket;
+      elsif C = '{' then
+         TK := L_Brace;
+      elsif C = '}' then
+         TK := R_Brace;
+      elsif C = ':' then
+         if C2 = ':' then
+            TK := ColonColon;
+            Nudge (L);
+         else
+            TK := Colon;
+         end if;
+      --  else
+      --     Put_Line
+      --       (Integer'Image (Line)
+      --        & " :"
+      --        & Integer'Image (Col)
+      --        & ": "
+      --        & "Unhandled: "
+      --        & Integer'Image (Character'Pos (C)));
+      end if;
+
       T :=
-        (Kind => TK, Lexeme => To_Unbounded_String (""), Line => 1, Col => 1);
+        (Kind   => TK,
+         Lexeme => To_Unbounded_String (""),
+         Line   => Line,
+         Col    => Col);
       Nudge (L);
       return T;
    end Next_Token;
@@ -281,12 +330,13 @@ package body Lexer is
       I     : Natural := 0;
       Total : Natural := Natural (L.Tokens.Length);
    begin
-      Put_Line ("total: " & (L.Tokens.Length'Image));
+      Put_Line ("total: " & (L.Tokens.Length'Image) & " tokens");
 
       while I < Total loop
+
          Put_Line
            (Integer'Image (L.Tokens (I).Line)
-            & ":"
+            & " :"
             & Integer'Image (L.Tokens (I).Col)
             & ": "
             & Token_Kind'Image (L.Tokens (I).Kind)
diff --git a/src/lexer.ads b/src/lexer.ads
@@ -8,7 +8,16 @@ package Lexer is
       Int_Literal,
       Str_Literal,
       Bool_Literal,
-	  Keyword,
+      L_Paren,
+      R_Paren,
+      L_Bracket,
+      R_Bracket,
+      L_Brace,
+      R_Brace,
+      ColonColon,
+      Colon,
+      Separator,
+      Keyword,
       Unknown,
       EOF);
 
@@ -21,9 +30,12 @@ package Lexer is
    package Token_Vectors is new
      Ada.Containers.Vectors (Index_Type => Natural, Element_Type => Token);
 
-   type Lexer(Src_Len: Positive; FN_Len: Positive) is record
-      Source    :  String(1 .. Src_Len);
-      File_Name :  String(1 .. FN_Len);
+   type Lexer
+     (Src_Len : Positive;
+      FN_Len  : Positive)
+   is record
+      Source    : String (1 .. Src_Len);
+      File_Name : String (1 .. FN_Len);
       Tokens    : Token_Vectors.Vector;
       Pos       : Positive;
       Line      : Positive;
@@ -32,7 +44,8 @@ package Lexer is
 
    function Init (File_Name : String; File_Contents : String) return Lexer;
 
-   procedure Print_Tokens(L : in out Lexer);
+   procedure Add_Token (L : in out Lexer; T : Token);
+   procedure Print_Tokens (L : in out Lexer);
 
    procedure Lex (L : in out Lexer);
 end Lexer;