From 3dd9ca69340512e42867b43db689e494bada2000 Mon Sep 17 00:00:00 2001 From: Mike Buland Date: Fri, 5 Jun 2026 14:46:35 -0700 Subject: Symbols have a wrapper with positional data now. It required some weird changes, I'm not sure I'm happy with them all yet, but I"m getting closer. It parses all basics, now we need to build an AST so it's actually useful :-P --- src/lexer.rs | 105 ++++++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 78 insertions(+), 27 deletions(-) (limited to 'src/lexer.rs') diff --git a/src/lexer.rs b/src/lexer.rs index 94825b2..04be350 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -2,13 +2,15 @@ use std::iter::Iterator; //use core::error::Error; use std::str::CharIndices; +use crate::Position; + #[derive(Debug,Copy,Clone)] pub enum ErrorType { UnexpectedChar(char), } #[derive(Debug,Copy,Clone)] -pub enum Symbol<'a> { +pub enum SymbolType<'a> { StartFlat, StartPoint, EndFlat, @@ -20,10 +22,41 @@ pub enum Symbol<'a> { Token(&'a str), Literal(&'a str), Text(&'a str), - Error{ line: u32, row: u32, what: ErrorType }, + Error{ what: ErrorType }, EOS, } +#[derive(Debug,Copy,Clone)] +pub struct Symbol<'a> { + symbol: SymbolType<'a>, + start: Position, + end: Position, +} + +impl<'a> Symbol<'a> { + pub fn new( symbol: SymbolType<'a>, start: Position, end: Position ) -> Self { + Self { + symbol, start, end, + } + } + + pub fn check_type bool>(&self, f: T ) -> bool { + f( &self.symbol ) + } + + pub fn symbol(&self) -> &SymbolType<'a> { + &self.symbol + } + + pub fn start(&self) -> &Position { + &self.start + } + + pub fn end(&self) -> &Position { + &self.end + } +} + enum Mode { Text, InTag, @@ -35,8 +68,7 @@ pub struct Lexer<'a> { cur: [Option<(usize, char)>;2], icur: usize, mode: Mode, - line: u32, - row: u32, + pos: Position, } fn is_valid_token_char( c: char ) -> bool { @@ -61,8 +93,7 @@ impl<'a> Lexer<'a> { cur: [cur, cur2], icur: 0, mode: Mode::Text, - line: 0, - row: 0, + pos: Position::new(1,1), } } @@ -71,6 +102,12 @@ impl<'a> Lexer<'a> { self.icur = (self.icur+1)%2; //println!(" - cur: {:?}, peek: {:?}", self.cur(), self.peek() ); if let Some((_,chr)) = self.cur[self.icur] { + if chr == '\n' { + self.pos.column = 1; + self.pos.line += 1; + } else { + self.pos.column += 1; + } Some(chr) } else { None @@ -110,11 +147,9 @@ impl<'a> Lexer<'a> { } fn error( &self, what: ErrorType ) -> Option> { - Some(Symbol::Error { - line: self.line, - row: self.row, + Some(Symbol::new( SymbolType::Error { what: what - }) + }, self.pos, self.pos )) } fn skip_ws( &mut self ) { @@ -149,14 +184,16 @@ impl<'a> Lexer<'a> { fn parse_text(&mut self) -> Option> { let start = self.cur_index(); + let start_pos = self.pos.clone(); while self.next().is_some() && !self.is_start_tag() { } let end = self.cur_index(); + let end_pos = self.pos.clone(); let s = &self.data[start..end]; //println!(" text: >>>{}<<<", s); if start == end { None } else { - Some(Symbol::Text(s)) + Some(Symbol::new( SymbolType::Text(s), start_pos, end_pos )) } } @@ -168,25 +205,27 @@ impl<'a> Lexer<'a> { } Some('=') => { self.next(); - return Some(Symbol::Equals); + return Some(Symbol::new(SymbolType::Equals, self.pos, self.pos)); } _ => {} } let start = self.cur_index(); + let start_pos = self.pos.clone(); while self.next().is_some_and(|ch| is_valid_token_char(ch) ) && !self.is_end_tag() {} let end = self.cur_index(); + let end_pos = self.pos.clone(); let s = &self.data[start..end]; if start == end { None } else { - Some(match s { - "fragment" => Symbol::Fragment, - "section" => Symbol::Section, - "output" => Symbol::Output, - _ => Symbol::Token(s) - }) + Some(Symbol::new( match s { + "fragment" => SymbolType::Fragment, + "section" => SymbolType::Section, + "output" => SymbolType::Output, + _ => SymbolType::Token(s) + }, start_pos, end_pos )) } } @@ -195,11 +234,13 @@ impl<'a> Lexer<'a> { return self.error( ErrorType::UnexpectedChar(chr) ); } let start = self.peek_index(); + let start_pos = self.pos.clone(); while self.next().is_some_and(|chr| chr != '"') { } let end = self.cur_index(); + let end_pos = self.pos.clone(); self.next(); let s = &self.data[start..end]; - Some(Symbol::Literal(s)) + Some(Symbol::new(SymbolType::Literal(s), start_pos, end_pos )) } fn is_start_tag(&mut self) -> bool { @@ -212,21 +253,26 @@ impl<'a> Lexer<'a> { } fn parse_start_tag(&mut self) -> Option> { + let start_pos = self.pos.clone(); match self.cur() { Some('[') => { if let Some(p) = self.peek() && p == '|' { - self.next(); self.next(); + self.next(); + let end_pos = self.pos.clone(); + self.next(); self.mode = Mode::InTag; - Some(Symbol::StartFlat) + Some(Symbol::new( SymbolType::StartFlat, start_pos, end_pos ) ) } else { None } } Some('<') => { if let Some(p) = self.peek() && p == '|' { - self.next(); self.next(); + self.next(); + let end_pos = self.pos.clone(); + self.next(); self.mode = Mode::InTag; - Some(Symbol::StartPoint) + Some(Symbol::new(SymbolType::StartPoint, start_pos, end_pos ) ) } else { None } @@ -247,17 +293,22 @@ impl<'a> Lexer<'a> { } fn parse_end_tag(&mut self) -> Option> { + let start_pos = self.pos.clone(); if let Some(chr) = self.cur() && chr == '|' { match self.peek() { Some(']') => { - self.next(); self.next(); + self.next(); + let end_pos = self.pos.clone(); + self.next(); self.mode = Mode::Text; - Some(Symbol::EndFlat) + Some(Symbol::new( SymbolType::EndFlat, start_pos, end_pos)) } Some('>') => { - self.next(); self.next(); + self.next(); + let end_pos = self.pos.clone(); + self.next(); self.mode = Mode::Text; - Some(Symbol::EndPoint) + Some(Symbol::new( SymbolType::EndPoint, start_pos, end_pos)) } _ => { None -- cgit v1.2.3