fix: lexer

This commit is contained in:
Daniel Flanagan 2025-02-21 07:51:02 -06:00
parent 56e99a3083
commit 13835d9511
8 changed files with 123 additions and 67 deletions

8
flake.lock generated
View file

@ -61,16 +61,16 @@
},
"nixpkgs": {
"locked": {
"lastModified": 1739206421,
"narHash": "sha256-PwQASeL2cGVmrtQYlrBur0U20Xy07uSWVnFup2PHnDs=",
"lastModified": 1740019556,
"narHash": "sha256-vn285HxnnlHLWnv59Og7muqECNMS33mWLM14soFIv2g=",
"owner": "nixos",
"repo": "nixpkgs",
"rev": "44534bc021b85c8d78e465021e21f33b856e2540",
"rev": "dad564433178067be1fbdfcce23b546254b6d641",
"type": "github"
},
"original": {
"owner": "nixos",
"ref": "nixos-24.11",
"ref": "nixpkgs-unstable",
"repo": "nixpkgs",
"type": "github"
}

View file

@ -1,14 +1,17 @@
{
inputs.nixpkgs.url = "github:nixos/nixpkgs/nixos-24.11";
inputs.nixpkgs.url = "github:nixos/nixpkgs/nixpkgs-unstable";
inputs.git-hooks.url = "github:cachix/git-hooks.nix";
inputs.git-hooks.inputs.nixpkgs.follows = "nixpkgs";
outputs = inputs: let
inherit (import nix/boilerplate.nix inputs) call genPkgs;
in {
# overlays = import nix/overlays.nix;
packages = call (import nix/packages.nix);
checks = call (import nix/checks.nix);
devShells = call (import nix/shells.nix);
formatter = genPkgs (p: p.alejandra);
};
outputs =
inputs:
let
inherit (import nix/boilerplate.nix inputs) call genPkgs;
in
{
# overlays = import nix/overlays.nix;
packages = call (import nix/packages.nix);
checks = call (import nix/checks.nix);
devShells = call (import nix/shells.nix);
formatter = genPkgs (p: p.nixfmt-rfc-style);
};
}

View file

@ -1,10 +1,4 @@
use std::{
iter::{self, from_fn},
num::ParseIntError,
path::Path,
str::Chars,
sync::Arc,
};
use std::{iter::Peekable, num::ParseIntError, path::Path, str::Chars, sync::Arc};
// TODO: tree_sitter ?
@ -24,7 +18,7 @@ enum BareToken {
}
#[derive(Debug, PartialEq)]
struct Token {
pub struct Token {
location: Option<Location>, // Not all tokens are associated with a location, such as EndOfFile
token: BareToken,
}
@ -52,6 +46,12 @@ enum LexerError {
Unexpected(char),
}
impl From<ParseIntError> for LexerError {
fn from(value: ParseIntError) -> Self {
LexerError::ParseIntError(value)
}
}
#[derive(Debug, PartialEq)]
pub struct Error {
pub location: Location,
@ -62,7 +62,7 @@ pub struct Lexer<'a> {
line: usize,
col: usize,
source: Source,
chars: Chars<'a>,
chars: Peekable<Chars<'a>>,
collected: Vec<char>,
done: bool,
@ -70,21 +70,23 @@ pub struct Lexer<'a> {
}
impl<'a> Lexer<'a> {
fn new(code: &'a str, source: Source) -> Self {
let mut lexer = Lexer {
pub fn new(code: &'a str, source: Source) -> Self {
let lexer = Lexer {
done: false,
sent_eof: false,
line: 1,
col: 0,
source: source.clone(),
chars: code.chars(),
chars: code.chars().peekable(),
collected: vec![],
};
lexer.advance();
lexer
}
fn advance(&mut self) {
if self.done {
return;
}
match self.chars.next() {
Some('\n') => {
self.line += 1;
@ -117,7 +119,7 @@ impl<'a> Lexer<'a> {
t
}
fn produce_error(&mut self, error: LexerError) -> Error {
fn produce_error(&self, error: LexerError) -> Error {
Error {
location: self.current_location(),
error,
@ -131,12 +133,51 @@ impl<'a> Lexer<'a> {
Some(c) => c,
};
Ok(Some(match current {
c if c.is_ascii_digit() => self.number()?,
c if c.is_digit(10) => self.number()?,
'\n' => self.produce(BareToken::NewLine),
'+' => self.produce(BareToken::Plus),
c if c.is_whitespace() => {
self.whitespace();
return self.token();
}
c => return Err(self.produce_error(LexerError::Unexpected(*c))),
}))
}
fn number(&mut self) -> Result<Token, Error> {}
fn whitespace(&mut self) {
while let Some(c) = self.chars.peek() {
if c.is_whitespace() {
self.advance()
} else {
break;
}
}
self.collected.clear();
}
fn number(&mut self) -> Result<Token, Error> {
while let Some(c) = self.chars.peek() {
if c.is_ascii_digit() {
self.advance()
} else {
break;
}
}
dbg!(&self.collected);
Ok(self.produce(BareToken::Integer(
self.collected
.iter()
.collect::<String>()
.parse()
.map_err(|e: ParseIntError| {
dbg!(&e);
self.produce_error(e.into())
})?,
)))
}
}
impl<'a> Iterator for Lexer<'a> {
@ -153,9 +194,9 @@ impl<'a> Iterator for Lexer<'a> {
}));
}
match self.token() {
Ok(Some(t)) => Some(Ok(t)),
Ok(None) => None,
Err(e) => Some(Err(e)),
Ok(Some(t)) => return Some(Ok(t)),
Ok(None) => return self.next(),
Err(e) => return Some(Err(e)),
}
}
}

View file

@ -28,9 +28,9 @@ impl<'a> From<lexer::Error> for Error {
impl Parser {
pub fn parse_str(&mut self, code: &'static str) -> Result<Expression, Error> {
let mut lexer = lexer::Lexer::default();
let tokens = lexer.lex_str(code);
Ok(self.parse(tokens?)?)
let lexer = lexer::Lexer::new(code, lexer::Source::Unknown);
let tokens = lexer.collect::<Result<Vec<lexer::Token>, lexer::Error>>()?;
Ok(self.parse(tokens)?)
}
pub fn parse(&mut self, tokens: Vec<lexer::Token>) -> Result<Expression, Error> {

View file

@ -1,17 +1,25 @@
inputs @ {
inputs@{
nixpkgs,
self,
...
}: let
}:
let
forSelfOverlay =
if builtins.hasAttr "overlays" self && builtins.hasAttr "forSelf" self.overlays
then self.overlays.forSelf
else (_: p: p);
in rec {
if builtins.hasAttr "overlays" self && builtins.hasAttr "forSelf" self.overlays then
self.overlays.forSelf
else
(_: p: p);
in
rec {
# TODO: Iterate all Nix's supported systems?
systems = ["aarch64-linux" "x86_64-linux" "x86_64-darwin" "aarch64-darwin"];
systems = [
"aarch64-linux"
"x86_64-linux"
"x86_64-darwin"
"aarch64-darwin"
];
forSystems = nixpkgs.lib.genAttrs systems;
pkgsFor = system: ((import nixpkgs {inherit system;}).extend forSelfOverlay);
pkgsFor = system: ((import nixpkgs { inherit system; }).extend forSelfOverlay);
genPkgs = func: (forSystems (system: func (pkgsFor system)));
call = imported: genPkgs (pkgs: imported (inputs // {inherit pkgs;}));
call = imported: genPkgs (pkgs: imported (inputs // { inherit pkgs; }));
}

View file

@ -2,22 +2,24 @@
pkgs,
git-hooks,
...
}: {
}:
let
check-command = command: {
enable = true;
name = command;
entry = command;
pass_filenames = false;
stages = [ "pre-commit" ];
};
in
{
git-hooks = git-hooks.lib.${pkgs.system}.run {
src = ./..;
hooks = {
alejandra.enable = true;
nixfmt-rfc-style.enable = true;
cargo-check.enable = true;
convco.enable = true;
cargo-test = {
enable = true;
name = "cargo-test";
entry = "cargo test";
# types = ["rust"];
# language = "rust";
pass_filenames = false;
stages = ["pre-commit"];
};
cargo-test = check-command "cargo test";
clippy.enable = true;
rustfmt.enable = true;
};

View file

@ -1,4 +1,5 @@
{pkgs, ...}: let
{ pkgs, ... }:
let
inherit (builtins) fromTOML readFile;
pname = "lyt";
src = ./..;
@ -8,7 +9,8 @@
cargoHash = "sha256-ZrT35F8EoawLgJVcp15kBLGkQZkhi1dxkYEAsV1ZlaU=";
useFetchCargoVendor = true;
};
in {
in
{
${pname} = main-package;
default = main-package;
}

View file

@ -2,20 +2,20 @@
self,
pkgs,
...
}: let
}:
let
inherit (pkgs) system;
in {
in
{
default = pkgs.mkShell {
inherit (self.checks.${system}.git-hooks) shellHook;
inputsFrom = [self.packages.${system}.default];
inputsFrom = [ self.packages.${system}.default ];
packages = with pkgs; [
convco
rustPackages.clippy
typescript-language-server
rust-analyzer
rustfmt
rustup
gdb
gdbgui
nixd
lldb
];
};
}