fix: lexer
This commit is contained in:
parent
56e99a3083
commit
13835d9511
8 changed files with 123 additions and 67 deletions
8
flake.lock
generated
8
flake.lock
generated
|
@ -61,16 +61,16 @@
|
|||
},
|
||||
"nixpkgs": {
|
||||
"locked": {
|
||||
"lastModified": 1739206421,
|
||||
"narHash": "sha256-PwQASeL2cGVmrtQYlrBur0U20Xy07uSWVnFup2PHnDs=",
|
||||
"lastModified": 1740019556,
|
||||
"narHash": "sha256-vn285HxnnlHLWnv59Og7muqECNMS33mWLM14soFIv2g=",
|
||||
"owner": "nixos",
|
||||
"repo": "nixpkgs",
|
||||
"rev": "44534bc021b85c8d78e465021e21f33b856e2540",
|
||||
"rev": "dad564433178067be1fbdfcce23b546254b6d641",
|
||||
"type": "github"
|
||||
},
|
||||
"original": {
|
||||
"owner": "nixos",
|
||||
"ref": "nixos-24.11",
|
||||
"ref": "nixpkgs-unstable",
|
||||
"repo": "nixpkgs",
|
||||
"type": "github"
|
||||
}
|
||||
|
|
23
flake.nix
23
flake.nix
|
@ -1,14 +1,17 @@
|
|||
{
|
||||
inputs.nixpkgs.url = "github:nixos/nixpkgs/nixos-24.11";
|
||||
inputs.nixpkgs.url = "github:nixos/nixpkgs/nixpkgs-unstable";
|
||||
inputs.git-hooks.url = "github:cachix/git-hooks.nix";
|
||||
inputs.git-hooks.inputs.nixpkgs.follows = "nixpkgs";
|
||||
outputs = inputs: let
|
||||
inherit (import nix/boilerplate.nix inputs) call genPkgs;
|
||||
in {
|
||||
# overlays = import nix/overlays.nix;
|
||||
packages = call (import nix/packages.nix);
|
||||
checks = call (import nix/checks.nix);
|
||||
devShells = call (import nix/shells.nix);
|
||||
formatter = genPkgs (p: p.alejandra);
|
||||
};
|
||||
outputs =
|
||||
inputs:
|
||||
let
|
||||
inherit (import nix/boilerplate.nix inputs) call genPkgs;
|
||||
in
|
||||
{
|
||||
# overlays = import nix/overlays.nix;
|
||||
packages = call (import nix/packages.nix);
|
||||
checks = call (import nix/checks.nix);
|
||||
devShells = call (import nix/shells.nix);
|
||||
formatter = genPkgs (p: p.nixfmt-rfc-style);
|
||||
};
|
||||
}
|
||||
|
|
|
@ -1,10 +1,4 @@
|
|||
use std::{
|
||||
iter::{self, from_fn},
|
||||
num::ParseIntError,
|
||||
path::Path,
|
||||
str::Chars,
|
||||
sync::Arc,
|
||||
};
|
||||
use std::{iter::Peekable, num::ParseIntError, path::Path, str::Chars, sync::Arc};
|
||||
|
||||
// TODO: tree_sitter ?
|
||||
|
||||
|
@ -24,7 +18,7 @@ enum BareToken {
|
|||
}
|
||||
|
||||
#[derive(Debug, PartialEq)]
|
||||
struct Token {
|
||||
pub struct Token {
|
||||
location: Option<Location>, // Not all tokens are associated with a location, such as EndOfFile
|
||||
token: BareToken,
|
||||
}
|
||||
|
@ -52,6 +46,12 @@ enum LexerError {
|
|||
Unexpected(char),
|
||||
}
|
||||
|
||||
impl From<ParseIntError> for LexerError {
|
||||
fn from(value: ParseIntError) -> Self {
|
||||
LexerError::ParseIntError(value)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq)]
|
||||
pub struct Error {
|
||||
pub location: Location,
|
||||
|
@ -62,7 +62,7 @@ pub struct Lexer<'a> {
|
|||
line: usize,
|
||||
col: usize,
|
||||
source: Source,
|
||||
chars: Chars<'a>,
|
||||
chars: Peekable<Chars<'a>>,
|
||||
collected: Vec<char>,
|
||||
|
||||
done: bool,
|
||||
|
@ -70,21 +70,23 @@ pub struct Lexer<'a> {
|
|||
}
|
||||
|
||||
impl<'a> Lexer<'a> {
|
||||
fn new(code: &'a str, source: Source) -> Self {
|
||||
let mut lexer = Lexer {
|
||||
pub fn new(code: &'a str, source: Source) -> Self {
|
||||
let lexer = Lexer {
|
||||
done: false,
|
||||
sent_eof: false,
|
||||
line: 1,
|
||||
col: 0,
|
||||
source: source.clone(),
|
||||
chars: code.chars(),
|
||||
chars: code.chars().peekable(),
|
||||
collected: vec![],
|
||||
};
|
||||
lexer.advance();
|
||||
lexer
|
||||
}
|
||||
|
||||
fn advance(&mut self) {
|
||||
if self.done {
|
||||
return;
|
||||
}
|
||||
match self.chars.next() {
|
||||
Some('\n') => {
|
||||
self.line += 1;
|
||||
|
@ -117,7 +119,7 @@ impl<'a> Lexer<'a> {
|
|||
t
|
||||
}
|
||||
|
||||
fn produce_error(&mut self, error: LexerError) -> Error {
|
||||
fn produce_error(&self, error: LexerError) -> Error {
|
||||
Error {
|
||||
location: self.current_location(),
|
||||
error,
|
||||
|
@ -131,12 +133,51 @@ impl<'a> Lexer<'a> {
|
|||
Some(c) => c,
|
||||
};
|
||||
Ok(Some(match current {
|
||||
c if c.is_ascii_digit() => self.number()?,
|
||||
c if c.is_digit(10) => self.number()?,
|
||||
'\n' => self.produce(BareToken::NewLine),
|
||||
'+' => self.produce(BareToken::Plus),
|
||||
|
||||
c if c.is_whitespace() => {
|
||||
self.whitespace();
|
||||
return self.token();
|
||||
}
|
||||
|
||||
c => return Err(self.produce_error(LexerError::Unexpected(*c))),
|
||||
}))
|
||||
}
|
||||
|
||||
fn number(&mut self) -> Result<Token, Error> {}
|
||||
fn whitespace(&mut self) {
|
||||
while let Some(c) = self.chars.peek() {
|
||||
if c.is_whitespace() {
|
||||
self.advance()
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
self.collected.clear();
|
||||
}
|
||||
|
||||
fn number(&mut self) -> Result<Token, Error> {
|
||||
while let Some(c) = self.chars.peek() {
|
||||
if c.is_ascii_digit() {
|
||||
self.advance()
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
dbg!(&self.collected);
|
||||
Ok(self.produce(BareToken::Integer(
|
||||
self.collected
|
||||
.iter()
|
||||
.collect::<String>()
|
||||
.parse()
|
||||
.map_err(|e: ParseIntError| {
|
||||
dbg!(&e);
|
||||
self.produce_error(e.into())
|
||||
})?,
|
||||
)))
|
||||
}
|
||||
}
|
||||
|
||||
impl<'a> Iterator for Lexer<'a> {
|
||||
|
@ -153,9 +194,9 @@ impl<'a> Iterator for Lexer<'a> {
|
|||
}));
|
||||
}
|
||||
match self.token() {
|
||||
Ok(Some(t)) => Some(Ok(t)),
|
||||
Ok(None) => None,
|
||||
Err(e) => Some(Err(e)),
|
||||
Ok(Some(t)) => return Some(Ok(t)),
|
||||
Ok(None) => return self.next(),
|
||||
Err(e) => return Some(Err(e)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -28,9 +28,9 @@ impl<'a> From<lexer::Error> for Error {
|
|||
|
||||
impl Parser {
|
||||
pub fn parse_str(&mut self, code: &'static str) -> Result<Expression, Error> {
|
||||
let mut lexer = lexer::Lexer::default();
|
||||
let tokens = lexer.lex_str(code);
|
||||
Ok(self.parse(tokens?)?)
|
||||
let lexer = lexer::Lexer::new(code, lexer::Source::Unknown);
|
||||
let tokens = lexer.collect::<Result<Vec<lexer::Token>, lexer::Error>>()?;
|
||||
Ok(self.parse(tokens)?)
|
||||
}
|
||||
|
||||
pub fn parse(&mut self, tokens: Vec<lexer::Token>) -> Result<Expression, Error> {
|
||||
|
|
|
@ -1,17 +1,25 @@
|
|||
inputs @ {
|
||||
inputs@{
|
||||
nixpkgs,
|
||||
self,
|
||||
...
|
||||
}: let
|
||||
}:
|
||||
let
|
||||
forSelfOverlay =
|
||||
if builtins.hasAttr "overlays" self && builtins.hasAttr "forSelf" self.overlays
|
||||
then self.overlays.forSelf
|
||||
else (_: p: p);
|
||||
in rec {
|
||||
if builtins.hasAttr "overlays" self && builtins.hasAttr "forSelf" self.overlays then
|
||||
self.overlays.forSelf
|
||||
else
|
||||
(_: p: p);
|
||||
in
|
||||
rec {
|
||||
# TODO: Iterate all Nix's supported systems?
|
||||
systems = ["aarch64-linux" "x86_64-linux" "x86_64-darwin" "aarch64-darwin"];
|
||||
systems = [
|
||||
"aarch64-linux"
|
||||
"x86_64-linux"
|
||||
"x86_64-darwin"
|
||||
"aarch64-darwin"
|
||||
];
|
||||
forSystems = nixpkgs.lib.genAttrs systems;
|
||||
pkgsFor = system: ((import nixpkgs {inherit system;}).extend forSelfOverlay);
|
||||
pkgsFor = system: ((import nixpkgs { inherit system; }).extend forSelfOverlay);
|
||||
genPkgs = func: (forSystems (system: func (pkgsFor system)));
|
||||
call = imported: genPkgs (pkgs: imported (inputs // {inherit pkgs;}));
|
||||
call = imported: genPkgs (pkgs: imported (inputs // { inherit pkgs; }));
|
||||
}
|
||||
|
|
|
@ -2,22 +2,24 @@
|
|||
pkgs,
|
||||
git-hooks,
|
||||
...
|
||||
}: {
|
||||
}:
|
||||
let
|
||||
check-command = command: {
|
||||
enable = true;
|
||||
name = command;
|
||||
entry = command;
|
||||
pass_filenames = false;
|
||||
stages = [ "pre-commit" ];
|
||||
};
|
||||
in
|
||||
{
|
||||
git-hooks = git-hooks.lib.${pkgs.system}.run {
|
||||
src = ./..;
|
||||
hooks = {
|
||||
alejandra.enable = true;
|
||||
nixfmt-rfc-style.enable = true;
|
||||
cargo-check.enable = true;
|
||||
convco.enable = true;
|
||||
cargo-test = {
|
||||
enable = true;
|
||||
name = "cargo-test";
|
||||
entry = "cargo test";
|
||||
# types = ["rust"];
|
||||
# language = "rust";
|
||||
pass_filenames = false;
|
||||
stages = ["pre-commit"];
|
||||
};
|
||||
cargo-test = check-command "cargo test";
|
||||
clippy.enable = true;
|
||||
rustfmt.enable = true;
|
||||
};
|
||||
|
|
|
@ -1,4 +1,5 @@
|
|||
{pkgs, ...}: let
|
||||
{ pkgs, ... }:
|
||||
let
|
||||
inherit (builtins) fromTOML readFile;
|
||||
pname = "lyt";
|
||||
src = ./..;
|
||||
|
@ -8,7 +9,8 @@
|
|||
cargoHash = "sha256-ZrT35F8EoawLgJVcp15kBLGkQZkhi1dxkYEAsV1ZlaU=";
|
||||
useFetchCargoVendor = true;
|
||||
};
|
||||
in {
|
||||
in
|
||||
{
|
||||
${pname} = main-package;
|
||||
default = main-package;
|
||||
}
|
||||
|
|
|
@ -2,20 +2,20 @@
|
|||
self,
|
||||
pkgs,
|
||||
...
|
||||
}: let
|
||||
}:
|
||||
let
|
||||
inherit (pkgs) system;
|
||||
in {
|
||||
in
|
||||
{
|
||||
default = pkgs.mkShell {
|
||||
inherit (self.checks.${system}.git-hooks) shellHook;
|
||||
inputsFrom = [self.packages.${system}.default];
|
||||
inputsFrom = [ self.packages.${system}.default ];
|
||||
packages = with pkgs; [
|
||||
convco
|
||||
rustPackages.clippy
|
||||
typescript-language-server
|
||||
rust-analyzer
|
||||
rustfmt
|
||||
rustup
|
||||
gdb
|
||||
gdbgui
|
||||
|
||||
nixd
|
||||
lldb
|
||||
];
|
||||
};
|
||||
}
|
||||
|
|
Loading…
Add table
Reference in a new issue