fix: lexer

flake.lock

@@ -61,16 +61,16 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1739206421,
-        "narHash": "sha256-PwQASeL2cGVmrtQYlrBur0U20Xy07uSWVnFup2PHnDs=",
+        "lastModified": 1740019556,
+        "narHash": "sha256-vn285HxnnlHLWnv59Og7muqECNMS33mWLM14soFIv2g=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "44534bc021b85c8d78e465021e21f33b856e2540",
+        "rev": "dad564433178067be1fbdfcce23b546254b6d641",
         "type": "github"
       },
       "original": {
         "owner": "nixos",
-        "ref": "nixos-24.11",
+        "ref": "nixpkgs-unstable",
         "repo": "nixpkgs",
         "type": "github"
       }

flake.nix

@@ -1,14 +1,17 @@
 {
-  inputs.nixpkgs.url = "github:nixos/nixpkgs/nixos-24.11";
+  inputs.nixpkgs.url = "github:nixos/nixpkgs/nixpkgs-unstable";
   inputs.git-hooks.url = "github:cachix/git-hooks.nix";
   inputs.git-hooks.inputs.nixpkgs.follows = "nixpkgs";
-  outputs = inputs: let
-    inherit (import nix/boilerplate.nix inputs) call genPkgs;
-  in {
-    # overlays = import nix/overlays.nix;
-    packages = call (import nix/packages.nix);
-    checks = call (import nix/checks.nix);
-    devShells = call (import nix/shells.nix);
-    formatter = genPkgs (p: p.alejandra);
-  };
+  outputs =
+    inputs:
+    let
+      inherit (import nix/boilerplate.nix inputs) call genPkgs;
+    in
+    {
+      # overlays = import nix/overlays.nix;
+      packages = call (import nix/packages.nix);
+      checks = call (import nix/checks.nix);
+      devShells = call (import nix/shells.nix);
+      formatter = genPkgs (p: p.nixfmt-rfc-style);
+    };
 }

lyt/src/lexer.rs

@@ -1,10 +1,4 @@
-use std::{
-    iter::{self, from_fn},
-    num::ParseIntError,
-    path::Path,
-    str::Chars,
-    sync::Arc,
-};
+use std::{iter::Peekable, num::ParseIntError, path::Path, str::Chars, sync::Arc};
 
 // TODO: tree_sitter ?
 
@@ -24,7 +18,7 @@ enum BareToken {
 }
 
 #[derive(Debug, PartialEq)]
-struct Token {
+pub struct Token {
     location: Option<Location>, // Not all tokens are associated with a location, such as EndOfFile
     token: BareToken,
 }
@@ -52,6 +46,12 @@ enum LexerError {
     Unexpected(char),
 }
 
+impl From<ParseIntError> for LexerError {
+    fn from(value: ParseIntError) -> Self {
+        LexerError::ParseIntError(value)
+    }
+}
+
 #[derive(Debug, PartialEq)]
 pub struct Error {
     pub location: Location,
@@ -62,7 +62,7 @@ pub struct Lexer<'a> {
     line: usize,
     col: usize,
     source: Source,
-    chars: Chars<'a>,
+    chars: Peekable<Chars<'a>>,
     collected: Vec<char>,
 
     done: bool,
@@ -70,21 +70,23 @@ pub struct Lexer<'a> {
 }
 
 impl<'a> Lexer<'a> {
-    fn new(code: &'a str, source: Source) -> Self {
-        let mut lexer = Lexer {
+    pub fn new(code: &'a str, source: Source) -> Self {
+        let lexer = Lexer {
             done: false,
             sent_eof: false,
             line: 1,
             col: 0,
             source: source.clone(),
-            chars: code.chars(),
+            chars: code.chars().peekable(),
             collected: vec![],
         };
-        lexer.advance();
         lexer
     }
 
     fn advance(&mut self) {
+        if self.done {
+            return;
+        }
         match self.chars.next() {
             Some('\n') => {
                 self.line += 1;
@@ -117,7 +119,7 @@ impl<'a> Lexer<'a> {
         t
     }
 
-    fn produce_error(&mut self, error: LexerError) -> Error {
+    fn produce_error(&self, error: LexerError) -> Error {
         Error {
             location: self.current_location(),
             error,
@@ -131,12 +133,51 @@ impl<'a> Lexer<'a> {
             Some(c) => c,
         };
         Ok(Some(match current {
-            c if c.is_ascii_digit() => self.number()?,
+            c if c.is_digit(10) => self.number()?,
+            '\n' => self.produce(BareToken::NewLine),
+            '+' => self.produce(BareToken::Plus),
+
+            c if c.is_whitespace() => {
+                self.whitespace();
+                return self.token();
+            }
+
             c => return Err(self.produce_error(LexerError::Unexpected(*c))),
         }))
     }
 
-    fn number(&mut self) -> Result<Token, Error> {}
+    fn whitespace(&mut self) {
+        while let Some(c) = self.chars.peek() {
+            if c.is_whitespace() {
+                self.advance()
+            } else {
+                break;
+            }
+        }
+        self.collected.clear();
+    }
+
+    fn number(&mut self) -> Result<Token, Error> {
+        while let Some(c) = self.chars.peek() {
+            if c.is_ascii_digit() {
+                self.advance()
+            } else {
+                break;
+            }
+        }
+
+        dbg!(&self.collected);
+        Ok(self.produce(BareToken::Integer(
+            self.collected
+                .iter()
+                .collect::<String>()
+                .parse()
+                .map_err(|e: ParseIntError| {
+                    dbg!(&e);
+                    self.produce_error(e.into())
+                })?,
+        )))
+    }
 }
 
 impl<'a> Iterator for Lexer<'a> {
@@ -153,9 +194,9 @@ impl<'a> Iterator for Lexer<'a> {
             }));
         }
         match self.token() {
-            Ok(Some(t)) => Some(Ok(t)),
-            Ok(None) => None,
-            Err(e) => Some(Err(e)),
+            Ok(Some(t)) => return Some(Ok(t)),
+            Ok(None) => return self.next(),
+            Err(e) => return Some(Err(e)),
         }
     }
 }

lyt/src/parser.rs

@@ -28,9 +28,9 @@ impl<'a> From<lexer::Error> for Error {
 
 impl Parser {
     pub fn parse_str(&mut self, code: &'static str) -> Result<Expression, Error> {
-        let mut lexer = lexer::Lexer::default();
-        let tokens = lexer.lex_str(code);
-        Ok(self.parse(tokens?)?)
+        let lexer = lexer::Lexer::new(code, lexer::Source::Unknown);
+        let tokens = lexer.collect::<Result<Vec<lexer::Token>, lexer::Error>>()?;
+        Ok(self.parse(tokens)?)
     }
 
     pub fn parse(&mut self, tokens: Vec<lexer::Token>) -> Result<Expression, Error> {

nix/boilerplate.nix

@@ -1,17 +1,25 @@
-inputs @ {
+inputs@{
   nixpkgs,
   self,
   ...
-}: let
+}:
+let
   forSelfOverlay =
-    if builtins.hasAttr "overlays" self && builtins.hasAttr "forSelf" self.overlays
-    then self.overlays.forSelf
-    else (_: p: p);
-in rec {
+    if builtins.hasAttr "overlays" self && builtins.hasAttr "forSelf" self.overlays then
+      self.overlays.forSelf
+    else
+      (_: p: p);
+in
+rec {
   # TODO: Iterate all Nix's supported systems?
-  systems = ["aarch64-linux" "x86_64-linux" "x86_64-darwin" "aarch64-darwin"];
+  systems = [
+    "aarch64-linux"
+    "x86_64-linux"
+    "x86_64-darwin"
+    "aarch64-darwin"
+  ];
   forSystems = nixpkgs.lib.genAttrs systems;
-  pkgsFor = system: ((import nixpkgs {inherit system;}).extend forSelfOverlay);
+  pkgsFor = system: ((import nixpkgs { inherit system; }).extend forSelfOverlay);
   genPkgs = func: (forSystems (system: func (pkgsFor system)));
-  call = imported: genPkgs (pkgs: imported (inputs // {inherit pkgs;}));
+  call = imported: genPkgs (pkgs: imported (inputs // { inherit pkgs; }));
 }

nix/checks.nix

@@ -2,22 +2,24 @@
   pkgs,
   git-hooks,
   ...
-}: {
+}:
+let
+  check-command = command: {
+    enable = true;
+    name = command;
+    entry = command;
+    pass_filenames = false;
+    stages = [ "pre-commit" ];
+  };
+in
+{
   git-hooks = git-hooks.lib.${pkgs.system}.run {
     src = ./..;
     hooks = {
-      alejandra.enable = true;
+      nixfmt-rfc-style.enable = true;
       cargo-check.enable = true;
       convco.enable = true;
-      cargo-test = {
-        enable = true;
-        name = "cargo-test";
-        entry = "cargo test";
-        # types = ["rust"];
-        # language = "rust";
-        pass_filenames = false;
-        stages = ["pre-commit"];
-      };
+      cargo-test = check-command "cargo test";
       clippy.enable = true;
       rustfmt.enable = true;
     };

nix/packages.nix

@@ -1,4 +1,5 @@
-{pkgs, ...}: let
+{ pkgs, ... }:
+let
   inherit (builtins) fromTOML readFile;
   pname = "lyt";
   src = ./..;
@@ -8,7 +9,8 @@
     cargoHash = "sha256-ZrT35F8EoawLgJVcp15kBLGkQZkhi1dxkYEAsV1ZlaU=";
     useFetchCargoVendor = true;
   };
-in {
+in
+{
   ${pname} = main-package;
   default = main-package;
 }

nix/shells.nix

@@ -2,20 +2,20 @@
   self,
   pkgs,
   ...
-}: let
+}:
+let
   inherit (pkgs) system;
-in {
+in
+{
   default = pkgs.mkShell {
     inherit (self.checks.${system}.git-hooks) shellHook;
-    inputsFrom = [self.packages.${system}.default];
+    inputsFrom = [ self.packages.${system}.default ];
     packages = with pkgs; [
-      convco
-      rustPackages.clippy
-      typescript-language-server
-      rust-analyzer
-      rustfmt
+      rustup
+      gdb
+      gdbgui
+
       nixd
-      lldb
     ];
   };
 }