From 062048ef6e3e060bcf841ea2ec92e026f09d8da0 Mon Sep 17 00:00:00 2001 From: eichlan Date: Tue, 23 Jun 2026 15:31:51 -0700 Subject: Roorganized into workspace and packages. This is to accomidate a new proc_macro package, which can't have anything else in it. --- src/lexer.rs | 368 ----------------------------------------------------------- 1 file changed, 368 deletions(-) delete mode 100644 src/lexer.rs (limited to 'src/lexer.rs') diff --git a/src/lexer.rs b/src/lexer.rs deleted file mode 100644 index d4d39e9..0000000 --- a/src/lexer.rs +++ /dev/null @@ -1,368 +0,0 @@ -use std::iter::Iterator; -//use core::error::Error; -use std::str::CharIndices; -use std::fmt; - -use crate::Position; - -#[derive(Debug,Copy,Clone,PartialEq,Eq)] -pub enum ErrorType { - UnexpectedChar(char), -} - -#[derive(Debug,Copy,Clone,PartialEq,Eq)] -pub enum SymbolType<'a> { - StartFlat, - StartPoint, - EndFlat, - EndPoint, - - View, - Output, - Show, - Loop, - If, - ElIf, - Else, - - Sharp, - Equals, - Period, - Token(&'a str), - Literal(&'a str), - Text(&'a str), - Error{ what: ErrorType }, -} - -#[derive(Copy,Clone)] -pub struct Symbol<'a> { - symbol: SymbolType<'a>, - start: Position, - end: Position, -} - -impl<'a> fmt::Debug for Symbol<'a> { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{:?} @ {:?}-{:?}", self.symbol, self.start, self.end ) - } -} - -impl<'a> Symbol<'a> { - pub fn new( symbol: SymbolType<'a>, start: Position, end: Position ) -> Self { - Self { - symbol, start, end, - } - } - - pub fn check_type bool>(&self, f: T ) -> bool { - f( &self.symbol ) - } - - pub fn symbol(&self) -> &SymbolType<'a> { - &self.symbol - } - - pub fn start(&self) -> &Position { - &self.start - } - - /* - pub fn end(&self) -> &Position { - &self.end - } - */ -} - -#[derive(Debug)] -enum Mode { - Text, - InTag, -} - -pub struct Lexer<'a> { - data: &'a str, - chars: CharIndices<'a>, - cur: [Option<(usize, char)>;2], - icur: usize, - mode: Mode, - pos: Position, -} - -fn is_valid_token_char( c: char, first: bool ) -> bool { - if c.is_whitespace() { - return false; - } - if first { - match c { - 'a'..'z' | 'A'..'Z' | '_' => true, - _ => false - } - } else { - match c { - 'a'..'z' | 'A'..'Z' | '0'..'9' | '_' => true, - _ => false - } - } -} - -impl<'a> Lexer<'a> { - pub fn new( data: &'a str ) -> Lexer<'a> { - let mut chars = data.char_indices(); - let cur = chars.next(); - let cur2 = chars.next(); - //println!(" - cur: {:?}, peek: {:?}", cur, cur2 ); - Lexer { - data, - chars, - cur: [cur, cur2], - icur: 0, - mode: Mode::Text, - pos: Position::new(1,1), - } - } - - fn next(&mut self) -> Option { - self.cur[self.icur] = self.chars.next(); - self.icur = (self.icur+1)%2; - //println!(" - cur: {:?}, peek: {:?}", self.cur(), self.peek() ); - if let Some((_,chr)) = self.cur[self.icur] { - if chr == '\n' { - self.pos.column = 1; - self.pos.line += 1; - } else { - self.pos.column += 1; - } - Some(chr) - } else { - None - } - } - - fn cur(&self) -> Option { - if let Some((_, chr)) = self.cur[self.icur] { - Some(chr) - } else { - None - } - } - - fn cur_index(&self) -> usize { - if let Some((idx, _)) = self.cur[self.icur] { - idx - } else { - self.chars.offset() - } - } - - fn peek(&self) -> Option { - if let Some((_,chr)) = self.cur[(self.icur+1)%2] { - Some(chr) - } else { - None - } - } - - fn peek_index(&self) -> usize { - if let Some((idx, _)) = self.cur[(self.icur+1)%2] { - idx - } else { - self.chars.offset() - } - } - - fn error( &self, what: ErrorType ) -> Option> { - Some(Symbol::new( SymbolType::Error { - what: what - }, self.pos, self.pos )) - } - - fn skip_ws( &mut self ) { - while self.cur().is_some_and(|x|x.is_whitespace()) { - self.next(); - } - } - - fn next_symbol(&mut self) -> Option> { - // If we hit the end then we're already done. - if self.cur().is_none() { - return None; - } - - match self.mode { - Mode::Text => { - if let Some(sym) = self.parse_start_tag() { - Some(sym) - } else { - self.parse_text() - } - } - Mode::InTag => { - if let Some(sym) = self.parse_end_tag() { - Some(sym) - } else { - self.parse_token() - } - } - } - } - - fn parse_text(&mut self) -> Option> { - let start = self.cur_index(); - let start_pos = self.pos.clone(); - while self.next().is_some() && !self.is_start_tag() { } - let end = self.cur_index(); - let end_pos = self.pos.clone(); - let s = &self.data[start..end]; - //println!(" text: >>>{}<<<", s); - if start == end { - None - } else { - Some(Symbol::new( SymbolType::Text(s), start_pos, end_pos )) - } - } - - fn parse_token(&mut self) -> Option> { - self.skip_ws(); - match self.cur() { - Some('"') => { - return self.parse_literal_str(); - } - Some('#') => { - self.next(); - return Some(Symbol::new(SymbolType::Sharp, self.pos, self.pos)); - } - Some('=') => { - self.next(); - return Some(Symbol::new(SymbolType::Equals, self.pos, self.pos)); - } - Some('.') => { - self.next(); - return Some(Symbol::new(SymbolType::Period, self.pos, self.pos)); - } - _ => {} - } - let start = self.cur_index(); - let start_pos = self.pos.clone(); - - let mut first = true; - while self.next().is_some_and(|ch| is_valid_token_char(ch, first) ) && - !self.is_end_tag() { first = false; } - let end = self.cur_index(); - let end_pos = self.pos.clone(); - let s = &self.data[start..end]; - if start == end { - None - } else { - Some(Symbol::new( match s { - "view" => SymbolType::View, - "show" => SymbolType::Show, - "output" => SymbolType::Output, - "loop" => SymbolType::Loop, - "if" => SymbolType::If, - "elif" => SymbolType::ElIf, - "else" => SymbolType::Else, - _ => SymbolType::Token(s) - }, start_pos, end_pos )) - } - } - - fn parse_literal_str(&mut self) -> Option> { - if let Some(chr) = self.cur() && chr != '"' { - return self.error( ErrorType::UnexpectedChar(chr) ); - } - let start = self.peek_index(); - let start_pos = self.pos.clone(); - while self.next().is_some_and(|chr| chr != '"') { } - let end = self.cur_index(); - let end_pos = self.pos.clone(); - self.next(); - let s = &self.data[start..end]; - Some(Symbol::new(SymbolType::Literal(s), start_pos, end_pos )) - } - - fn is_start_tag(&mut self) -> bool { - if let Some(cur) = self.cur() && (cur == '[' || cur == '<') && - let Some(peek) = self.peek() && peek == '|' { - true - } else { - false - } - } - - fn parse_start_tag(&mut self) -> Option> { - let start_pos = self.pos.clone(); - match self.cur() { - Some('[') => { - if let Some(p) = self.peek() && p == '|' { - self.next(); - let end_pos = self.pos.clone(); - self.next(); - self.mode = Mode::InTag; - Some(Symbol::new( SymbolType::StartFlat, start_pos, end_pos ) ) - } else { - None - } - } - Some('<') => { - if let Some(p) = self.peek() && p == '|' { - self.next(); - let end_pos = self.pos.clone(); - self.next(); - self.mode = Mode::InTag; - Some(Symbol::new(SymbolType::StartPoint, start_pos, end_pos ) ) - } else { - None - } - } - _ => { - None - } - } - } - - fn is_end_tag(&mut self) -> bool { - if let Some(cur) = self.cur() && cur == '|' && - let Some(peek) = self.peek() && (peek == '>' || peek == ']') { - true - } else { - false - } - } - - fn parse_end_tag(&mut self) -> Option> { - self.skip_ws(); - let start_pos = self.pos.clone(); - if let Some(chr) = self.cur() && chr == '|' { - match self.peek() { - Some(']') => { - self.next(); - let end_pos = self.pos.clone(); - self.next(); - self.mode = Mode::Text; - Some(Symbol::new( SymbolType::EndFlat, start_pos, end_pos)) - } - Some('>') => { - self.next(); - let end_pos = self.pos.clone(); - self.next(); - self.mode = Mode::Text; - Some(Symbol::new( SymbolType::EndPoint, start_pos, end_pos)) - } - _ => { - None - } - } - } else { - None - } - } -} - -impl<'a> Iterator for Lexer<'a> { - type Item = Symbol<'a>; - - fn next(&mut self) -> Option { - self.next_symbol() - } -} - -- cgit v1.2.3