From 062048ef6e3e060bcf841ea2ec92e026f09d8da0 Mon Sep 17 00:00:00 2001 From: eichlan Date: Tue, 23 Jun 2026 15:31:51 -0700 Subject: Roorganized into workspace and packages. This is to accomidate a new proc_macro package, which can't have anything else in it. --- crimtag/src/lexer.rs | 368 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 368 insertions(+) create mode 100644 crimtag/src/lexer.rs (limited to 'crimtag/src/lexer.rs') diff --git a/crimtag/src/lexer.rs b/crimtag/src/lexer.rs new file mode 100644 index 0000000..d4d39e9 --- /dev/null +++ b/crimtag/src/lexer.rs @@ -0,0 +1,368 @@ +use std::iter::Iterator; +//use core::error::Error; +use std::str::CharIndices; +use std::fmt; + +use crate::Position; + +#[derive(Debug,Copy,Clone,PartialEq,Eq)] +pub enum ErrorType { + UnexpectedChar(char), +} + +#[derive(Debug,Copy,Clone,PartialEq,Eq)] +pub enum SymbolType<'a> { + StartFlat, + StartPoint, + EndFlat, + EndPoint, + + View, + Output, + Show, + Loop, + If, + ElIf, + Else, + + Sharp, + Equals, + Period, + Token(&'a str), + Literal(&'a str), + Text(&'a str), + Error{ what: ErrorType }, +} + +#[derive(Copy,Clone)] +pub struct Symbol<'a> { + symbol: SymbolType<'a>, + start: Position, + end: Position, +} + +impl<'a> fmt::Debug for Symbol<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?} @ {:?}-{:?}", self.symbol, self.start, self.end ) + } +} + +impl<'a> Symbol<'a> { + pub fn new( symbol: SymbolType<'a>, start: Position, end: Position ) -> Self { + Self { + symbol, start, end, + } + } + + pub fn check_type bool>(&self, f: T ) -> bool { + f( &self.symbol ) + } + + pub fn symbol(&self) -> &SymbolType<'a> { + &self.symbol + } + + pub fn start(&self) -> &Position { + &self.start + } + + /* + pub fn end(&self) -> &Position { + &self.end + } + */ +} + +#[derive(Debug)] +enum Mode { + Text, + InTag, +} + +pub struct Lexer<'a> { + data: &'a str, + chars: CharIndices<'a>, + cur: [Option<(usize, char)>;2], + icur: usize, + mode: Mode, + pos: Position, +} + +fn is_valid_token_char( c: char, first: bool ) -> bool { + if c.is_whitespace() { + return false; + } + if first { + match c { + 'a'..'z' | 'A'..'Z' | '_' => true, + _ => false + } + } else { + match c { + 'a'..'z' | 'A'..'Z' | '0'..'9' | '_' => true, + _ => false + } + } +} + +impl<'a> Lexer<'a> { + pub fn new( data: &'a str ) -> Lexer<'a> { + let mut chars = data.char_indices(); + let cur = chars.next(); + let cur2 = chars.next(); + //println!(" - cur: {:?}, peek: {:?}", cur, cur2 ); + Lexer { + data, + chars, + cur: [cur, cur2], + icur: 0, + mode: Mode::Text, + pos: Position::new(1,1), + } + } + + fn next(&mut self) -> Option { + self.cur[self.icur] = self.chars.next(); + self.icur = (self.icur+1)%2; + //println!(" - cur: {:?}, peek: {:?}", self.cur(), self.peek() ); + if let Some((_,chr)) = self.cur[self.icur] { + if chr == '\n' { + self.pos.column = 1; + self.pos.line += 1; + } else { + self.pos.column += 1; + } + Some(chr) + } else { + None + } + } + + fn cur(&self) -> Option { + if let Some((_, chr)) = self.cur[self.icur] { + Some(chr) + } else { + None + } + } + + fn cur_index(&self) -> usize { + if let Some((idx, _)) = self.cur[self.icur] { + idx + } else { + self.chars.offset() + } + } + + fn peek(&self) -> Option { + if let Some((_,chr)) = self.cur[(self.icur+1)%2] { + Some(chr) + } else { + None + } + } + + fn peek_index(&self) -> usize { + if let Some((idx, _)) = self.cur[(self.icur+1)%2] { + idx + } else { + self.chars.offset() + } + } + + fn error( &self, what: ErrorType ) -> Option> { + Some(Symbol::new( SymbolType::Error { + what: what + }, self.pos, self.pos )) + } + + fn skip_ws( &mut self ) { + while self.cur().is_some_and(|x|x.is_whitespace()) { + self.next(); + } + } + + fn next_symbol(&mut self) -> Option> { + // If we hit the end then we're already done. + if self.cur().is_none() { + return None; + } + + match self.mode { + Mode::Text => { + if let Some(sym) = self.parse_start_tag() { + Some(sym) + } else { + self.parse_text() + } + } + Mode::InTag => { + if let Some(sym) = self.parse_end_tag() { + Some(sym) + } else { + self.parse_token() + } + } + } + } + + fn parse_text(&mut self) -> Option> { + let start = self.cur_index(); + let start_pos = self.pos.clone(); + while self.next().is_some() && !self.is_start_tag() { } + let end = self.cur_index(); + let end_pos = self.pos.clone(); + let s = &self.data[start..end]; + //println!(" text: >>>{}<<<", s); + if start == end { + None + } else { + Some(Symbol::new( SymbolType::Text(s), start_pos, end_pos )) + } + } + + fn parse_token(&mut self) -> Option> { + self.skip_ws(); + match self.cur() { + Some('"') => { + return self.parse_literal_str(); + } + Some('#') => { + self.next(); + return Some(Symbol::new(SymbolType::Sharp, self.pos, self.pos)); + } + Some('=') => { + self.next(); + return Some(Symbol::new(SymbolType::Equals, self.pos, self.pos)); + } + Some('.') => { + self.next(); + return Some(Symbol::new(SymbolType::Period, self.pos, self.pos)); + } + _ => {} + } + let start = self.cur_index(); + let start_pos = self.pos.clone(); + + let mut first = true; + while self.next().is_some_and(|ch| is_valid_token_char(ch, first) ) && + !self.is_end_tag() { first = false; } + let end = self.cur_index(); + let end_pos = self.pos.clone(); + let s = &self.data[start..end]; + if start == end { + None + } else { + Some(Symbol::new( match s { + "view" => SymbolType::View, + "show" => SymbolType::Show, + "output" => SymbolType::Output, + "loop" => SymbolType::Loop, + "if" => SymbolType::If, + "elif" => SymbolType::ElIf, + "else" => SymbolType::Else, + _ => SymbolType::Token(s) + }, start_pos, end_pos )) + } + } + + fn parse_literal_str(&mut self) -> Option> { + if let Some(chr) = self.cur() && chr != '"' { + return self.error( ErrorType::UnexpectedChar(chr) ); + } + let start = self.peek_index(); + let start_pos = self.pos.clone(); + while self.next().is_some_and(|chr| chr != '"') { } + let end = self.cur_index(); + let end_pos = self.pos.clone(); + self.next(); + let s = &self.data[start..end]; + Some(Symbol::new(SymbolType::Literal(s), start_pos, end_pos )) + } + + fn is_start_tag(&mut self) -> bool { + if let Some(cur) = self.cur() && (cur == '[' || cur == '<') && + let Some(peek) = self.peek() && peek == '|' { + true + } else { + false + } + } + + fn parse_start_tag(&mut self) -> Option> { + let start_pos = self.pos.clone(); + match self.cur() { + Some('[') => { + if let Some(p) = self.peek() && p == '|' { + self.next(); + let end_pos = self.pos.clone(); + self.next(); + self.mode = Mode::InTag; + Some(Symbol::new( SymbolType::StartFlat, start_pos, end_pos ) ) + } else { + None + } + } + Some('<') => { + if let Some(p) = self.peek() && p == '|' { + self.next(); + let end_pos = self.pos.clone(); + self.next(); + self.mode = Mode::InTag; + Some(Symbol::new(SymbolType::StartPoint, start_pos, end_pos ) ) + } else { + None + } + } + _ => { + None + } + } + } + + fn is_end_tag(&mut self) -> bool { + if let Some(cur) = self.cur() && cur == '|' && + let Some(peek) = self.peek() && (peek == '>' || peek == ']') { + true + } else { + false + } + } + + fn parse_end_tag(&mut self) -> Option> { + self.skip_ws(); + let start_pos = self.pos.clone(); + if let Some(chr) = self.cur() && chr == '|' { + match self.peek() { + Some(']') => { + self.next(); + let end_pos = self.pos.clone(); + self.next(); + self.mode = Mode::Text; + Some(Symbol::new( SymbolType::EndFlat, start_pos, end_pos)) + } + Some('>') => { + self.next(); + let end_pos = self.pos.clone(); + self.next(); + self.mode = Mode::Text; + Some(Symbol::new( SymbolType::EndPoint, start_pos, end_pos)) + } + _ => { + None + } + } + } else { + None + } + } +} + +impl<'a> Iterator for Lexer<'a> { + type Item = Symbol<'a>; + + fn next(&mut self) -> Option { + self.next_symbol() + } +} + -- cgit v1.2.3