From 12c7b851d866401ac4a32c746ed237b051aa3dd3 Mon Sep 17 00:00:00 2001 From: Mike Buland Date: Thu, 20 Aug 2026 08:16:58 -0700 Subject: Making progress cleaning up. --- deepcrow/src/lexer.rs | 105 ++++++++++++++++++-------------------------------- 1 file changed, 37 insertions(+), 68 deletions(-) (limited to 'deepcrow/src/lexer.rs') diff --git a/deepcrow/src/lexer.rs b/deepcrow/src/lexer.rs index 2528cb6..98fc1c7 100644 --- a/deepcrow/src/lexer.rs +++ b/deepcrow/src/lexer.rs @@ -3,7 +3,7 @@ use std::iter::Iterator; use std::str::CharIndices; use std::fmt; -use crate::Position; +use crate::{Position,Range}; #[derive(Debug,Copy,Clone,PartialEq,Eq)] pub enum ErrorType { @@ -17,60 +17,45 @@ pub enum TokenType<'a> { EndFlat, EndPoint, - View, - Output, - Show, - Loop, - If, - ElIf, - Else, - Sharp, Equals, Period, Identifier(&'a str), - Literal(&'a str), + LiteralStr(&'a str), Text(&'a str), Error{ what: ErrorType }, } #[derive(Copy,Clone)] -pub struct Symbol<'a> { - symbol: TokenType<'a>, - start: Position, - end: Position, +pub struct Token<'a> { + token: TokenType<'a>, + pos: Range, } -impl<'a> fmt::Debug for Symbol<'a> { +impl<'a> fmt::Debug for Token<'a> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{:?} @ {:?}-{:?}", self.symbol, self.start, self.end ) + write!(f, "{:?} @ {:?}", self.token, self.pos ) } } -impl<'a> Symbol<'a> { - pub fn new( symbol: TokenType<'a>, start: Position, end: Position ) -> Self { +impl<'a> Token<'a> { + pub fn new( token: TokenType<'a>, start: Position, end: Position ) -> Self { Self { - symbol, start, end, + token, Range::new( start, end ), } } pub fn check_type bool>(&self, f: T ) -> bool { - f( &self.symbol ) - } - - pub fn symbol(&self) -> &TokenType<'a> { - &self.symbol + f( &self.token ) } - pub fn start(&self) -> &Position { - &self.start + pub fn token(&self) -> &TokenType<'a> { + &self.token } - /* - pub fn end(&self) -> &Position { - &self.end + pub fn pos(&self) -> &Range { + &self.pos } - */ } #[derive(Debug)] @@ -81,9 +66,7 @@ enum Mode { pub struct Lexer<'a> { data: &'a str, - chars: CharIndices<'a>, - cur: [Option<(usize, char)>;2], - icur: usize, + src: LookAhead>, mode: Mode, pos: Position, } @@ -107,15 +90,10 @@ fn is_valid_token_char( c: char, first: bool ) -> bool { impl<'a> Lexer<'a> { pub fn new( data: &'a str ) -> Lexer<'a> { - let mut chars = data.char_indices(); - let cur = chars.next(); - let cur2 = chars.next(); - //println!(" - cur: {:?}, peek: {:?}", cur, cur2 ); + let mut chars = ; Lexer { data, - chars, - cur: [cur, cur2], - icur: 0, + src: LookAhead::new(data.char_indices()), mode: Mode::Text, pos: Position::new(1,1), } @@ -170,8 +148,8 @@ impl<'a> Lexer<'a> { } } - fn error( &self, what: ErrorType ) -> Option> { - Some(Symbol::new( TokenType::Error { + fn error( &self, what: ErrorType ) -> Option> { + Some(Token::new( TokenType::Error { what: what }, self.pos, self.pos )) } @@ -182,7 +160,7 @@ impl<'a> Lexer<'a> { } } - fn next_symbol(&mut self) -> Option> { + fn next_symbol(&mut self) -> Option> { // If we hit the end then we're already done. if self.cur().is_none() { return None; @@ -206,7 +184,7 @@ impl<'a> Lexer<'a> { } } - fn parse_text(&mut self) -> Option> { + fn parse_text(&mut self) -> Option> { let start = self.cur_index(); let start_pos = self.pos.clone(); while self.next().is_some() && !self.is_start_tag() { } @@ -217,11 +195,11 @@ impl<'a> Lexer<'a> { if start == end { None } else { - Some(Symbol::new( TokenType::Text(s), start_pos, end_pos )) + Some(Token::new( TokenType::Text(s), start_pos, end_pos )) } } - fn parse_token(&mut self) -> Option> { + fn parse_token(&mut self) -> Option> { self.skip_ws(); match self.cur() { Some('"') => { @@ -229,15 +207,15 @@ impl<'a> Lexer<'a> { } Some('#') => { self.next(); - return Some(Symbol::new(TokenType::Sharp, self.pos, self.pos)); + return Some(Token::new(TokenType::Sharp, self.pos, self.pos)); } Some('=') => { self.next(); - return Some(Symbol::new(TokenType::Equals, self.pos, self.pos)); + return Some(Token::new(TokenType::Equals, self.pos, self.pos)); } Some('.') => { self.next(); - return Some(Symbol::new(TokenType::Period, self.pos, self.pos)); + return Some(Token::new(TokenType::Period, self.pos, self.pos)); } _ => {} } @@ -253,20 +231,11 @@ impl<'a> Lexer<'a> { if start == end { None } else { - Some(Symbol::new( match s { - "view" => TokenType::View, - "show" => TokenType::Show, - "output" => TokenType::Output, - "loop" => TokenType::Loop, - "if" => TokenType::If, - "elif" => TokenType::ElIf, - "else" => TokenType::Else, - _ => TokenType::Identifier(s) - }, start_pos, end_pos )) + Some(Token::new( TokenType::Identifier(s), start_pos, end_pos )) } } - fn parse_literal_str(&mut self) -> Option> { + fn parse_literal_str(&mut self) -> Option> { if let Some(chr) = self.cur() && chr != '"' { return self.error( ErrorType::UnexpectedChar(chr) ); } @@ -277,7 +246,7 @@ impl<'a> Lexer<'a> { let end_pos = self.pos.clone(); self.next(); let s = &self.data[start..end]; - Some(Symbol::new(TokenType::Literal(s), start_pos, end_pos )) + Some(Token::new(TokenType::LiteralStr(s), start_pos, end_pos )) } fn is_start_tag(&mut self) -> bool { @@ -289,7 +258,7 @@ impl<'a> Lexer<'a> { } } - fn parse_start_tag(&mut self) -> Option> { + fn parse_start_tag(&mut self) -> Option> { let start_pos = self.pos.clone(); match self.cur() { Some('[') => { @@ -298,7 +267,7 @@ impl<'a> Lexer<'a> { let end_pos = self.pos.clone(); self.next(); self.mode = Mode::InTag; - Some(Symbol::new( TokenType::StartFlat, start_pos, end_pos ) ) + Some(Token::new( TokenType::StartFlat, start_pos, end_pos ) ) } else { None } @@ -309,7 +278,7 @@ impl<'a> Lexer<'a> { let end_pos = self.pos.clone(); self.next(); self.mode = Mode::InTag; - Some(Symbol::new(TokenType::StartPoint, start_pos, end_pos ) ) + Some(Token::new(TokenType::StartPoint, start_pos, end_pos ) ) } else { None } @@ -329,7 +298,7 @@ impl<'a> Lexer<'a> { } } - fn parse_end_tag(&mut self) -> Option> { + fn parse_end_tag(&mut self) -> Option> { self.skip_ws(); let start_pos = self.pos.clone(); if let Some(chr) = self.cur() && chr == '|' { @@ -339,14 +308,14 @@ impl<'a> Lexer<'a> { let end_pos = self.pos.clone(); self.next(); self.mode = Mode::Text; - Some(Symbol::new( TokenType::EndFlat, start_pos, end_pos)) + Some(Token::new( TokenType::EndFlat, start_pos, end_pos)) } Some('>') => { self.next(); let end_pos = self.pos.clone(); self.next(); self.mode = Mode::Text; - Some(Symbol::new( TokenType::EndPoint, start_pos, end_pos)) + Some(Token::new( TokenType::EndPoint, start_pos, end_pos)) } _ => { None @@ -359,7 +328,7 @@ impl<'a> Lexer<'a> { } impl<'a> Iterator for Lexer<'a> { - type Item = Symbol<'a>; + type Item = Token<'a>; fn next(&mut self) -> Option { self.next_symbol() -- cgit v1.2.3