use std::iter::Iterator; //use core::error::Error; use std::str::CharIndices; #[derive(Debug)] pub enum Symbol<'a> { StartFlat, StartPoint, EndFlat, EndPoint, Fragment, Section, Output, Equals, Token(&'a str), Literal(&'a str), Text(&'a str), Error{ line: u32, row: u32, what: String }, EOS, } enum Mode { Text, InTag, } pub struct Lexer<'a> { data: &'a str, chars: CharIndices<'a>, cur: [Option<(usize, char)>;2], icur: usize, mode: Mode, line: u32, row: u32, } fn is_valid_token_char( c: char ) -> bool { if c.is_whitespace() { return false; } match c { 'a'..'z' | 'A'..'Z' | '0'..'9' | '_' | '-' => true, _ => false } } impl<'a> Lexer<'a> { pub fn new( data: &'a str ) -> Lexer<'a> { let mut chars = data.char_indices(); let cur = chars.next(); let cur2 = chars.next(); //println!(" - cur: {:?}, peek: {:?}", cur, cur2 ); Lexer { data, chars, cur: [cur, cur2], icur: 0, mode: Mode::Text, line: 0, row: 0, } } fn next(&mut self) -> Option { self.cur[self.icur] = self.chars.next(); self.icur = (self.icur+1)%2; //println!(" - cur: {:?}, peek: {:?}", self.cur(), self.peek() ); if let Some((_,chr)) = self.cur[self.icur] { Some(chr) } else { None } } fn cur(&self) -> Option { if let Some((_, chr)) = self.cur[self.icur] { Some(chr) } else { None } } fn cur_index(&self) -> usize { if let Some((idx, _)) = self.cur[self.icur] { idx } else { self.chars.offset() } } fn peek(&self) -> Option { if let Some((_,chr)) = self.cur[(self.icur+1)%2] { Some(chr) } else { None } } fn peek_index(&self) -> usize { if let Some((idx, _)) = self.cur[(self.icur+1)%2] { idx } else { self.chars.offset() } } fn error( &self, what: String ) -> Option> { Some(Symbol::Error { line: self.line, row: self.row, what: what }) } fn skip_ws( &mut self ) { while self.cur().is_some_and(|x|x.is_whitespace()) { self.next(); } } fn next_symbol(&mut self) -> Option> { // If we hit the end then we're already done. if self.cur().is_none() { return None; } match self.mode { Mode::Text => { if let Some(sym) = self.parse_start_tag() { Some(sym) } else { self.parse_text() } } Mode::InTag => { if let Some(sym) = self.parse_end_tag() { Some(sym) } else { self.parse_token() } } } } fn parse_text(&mut self) -> Option> { let start = self.cur_index(); while self.next().is_some() && !self.is_start_tag() { } let end = self.cur_index(); let s = &self.data[start..end]; //println!(" text: >>>{}<<<", s); if start == end { None } else { Some(Symbol::Text(s)) } } fn parse_token(&mut self) -> Option> { self.skip_ws(); match self.cur() { Some('"') => { return self.parse_literal_str(); } Some('=') => { return Some(Symbol::Equals); } _ => {} } let start = self.cur_index(); while self.next().is_some_and(|ch| is_valid_token_char(ch) ) && !self.is_end_tag() {} let end = self.cur_index(); let s = &self.data[start..end]; if start == end { None } else { Some(match s { "fragment" => Symbol::Fragment, "section" => Symbol::Section, "output" => Symbol::Output, _ => Symbol::Token(s) }) } } fn parse_literal_str(&mut self) -> Option> { if let Some(chr) = self.cur() && chr != '"' { return self.error(format!("Expected '\"' but found '{}'", chr)); } let start = self.peek_index(); while self.next().is_some_and(|chr| chr != '"') { } let end = self.cur_index(); self.next(); let s = &self.data[start..end]; Some(Symbol::Literal(s)) } fn is_start_tag(&mut self) -> bool { if let Some(cur) = self.cur() && (cur == '[' || cur == '<') && let Some(peek) = self.peek() && peek == '|' { true } else { false } } fn parse_start_tag(&mut self) -> Option> { match self.cur() { Some('[') => { if let Some(p) = self.peek() && p == '|' { self.next(); self.next(); self.mode = Mode::InTag; Some(Symbol::StartFlat) } else { None } } Some('<') => { if let Some(p) = self.peek() && p == '|' { self.next(); self.next(); self.mode = Mode::InTag; Some(Symbol::StartPoint) } else { None } } _ => { None } } } fn is_end_tag(&mut self) -> bool { if let Some(cur) = self.cur() && cur == '|' && let Some(peek) = self.peek() && (peek == '>' || peek == ']') { true } else { false } } fn parse_end_tag(&mut self) -> Option> { if let Some(chr) = self.cur() && chr == '|' { match self.peek() { Some(']') => { self.next(); self.next(); self.mode = Mode::Text; Some(Symbol::EndFlat) } Some('>') => { self.next(); self.next(); self.mode = Mode::Text; Some(Symbol::EndPoint) } _ => { None } } } else { None } } } impl<'a> Iterator for Lexer<'a> { type Item = Symbol<'a>; fn next(&mut self) -> Option { self.next_symbol() } }