use std::iter::Iterator; //use core::error::Error; use std::str::CharIndices; use std::fmt; use crate::{Position,Range}; #[derive(Debug,Copy,Clone,PartialEq,Eq)] pub enum ErrorType { UnexpectedChar(char), } #[derive(Debug,Copy,Clone,PartialEq,Eq)] pub enum TokenType<'a> { StartFlat, StartPoint, EndFlat, EndPoint, Sharp, Equals, Period, Identifier(&'a str), LiteralStr(&'a str), Text(&'a str), Error{ what: ErrorType }, } #[derive(Copy,Clone)] pub struct Token<'a> { token: TokenType<'a>, pos: Range, } impl<'a> fmt::Debug for Token<'a> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{:?} @ {:?}", self.token, self.pos ) } } impl<'a> Token<'a> { pub fn new( token: TokenType<'a>, start: Position, end: Position ) -> Self { Self { token, Range::new( start, end ), } } pub fn check_type bool>(&self, f: T ) -> bool { f( &self.token ) } pub fn token(&self) -> &TokenType<'a> { &self.token } pub fn pos(&self) -> &Range { &self.pos } } #[derive(Debug)] enum Mode { Text, InTag, } pub struct Lexer<'a> { data: &'a str, src: LookAhead>, mode: Mode, pos: Position, } fn is_valid_token_char( c: char, first: bool ) -> bool { if c.is_whitespace() { return false; } if first { match c { 'a'..'z' | 'A'..'Z' | '_' => true, _ => false } } else { match c { 'a'..'z' | 'A'..'Z' | '0'..'9' | '_' => true, _ => false } } } impl<'a> Lexer<'a> { pub fn new( data: &'a str ) -> Lexer<'a> { let mut chars = ; Lexer { data, src: LookAhead::new(data.char_indices()), mode: Mode::Text, pos: Position::new(1,1), } } fn next(&mut self) -> Option { self.cur[self.icur] = self.chars.next(); self.icur = (self.icur+1)%2; //println!(" - cur: {:?}, peek: {:?}", self.cur(), self.peek() ); if let Some((_,chr)) = self.cur[self.icur] { if chr == '\n' { self.pos.column = 1; self.pos.line += 1; } else { self.pos.column += 1; } Some(chr) } else { None } } fn cur(&self) -> Option { if let Some((_, chr)) = self.cur[self.icur] { Some(chr) } else { None } } fn cur_index(&self) -> usize { if let Some((idx, _)) = self.cur[self.icur] { idx } else { self.chars.offset() } } fn peek(&self) -> Option { if let Some((_,chr)) = self.cur[(self.icur+1)%2] { Some(chr) } else { None } } fn peek_index(&self) -> usize { if let Some((idx, _)) = self.cur[(self.icur+1)%2] { idx } else { self.chars.offset() } } fn error( &self, what: ErrorType ) -> Option> { Some(Token::new( TokenType::Error { what: what }, self.pos, self.pos )) } fn skip_ws( &mut self ) { while self.cur().is_some_and(|x|x.is_whitespace()) { self.next(); } } fn next_symbol(&mut self) -> Option> { // If we hit the end then we're already done. if self.cur().is_none() { return None; } match self.mode { Mode::Text => { if let Some(sym) = self.parse_start_tag() { Some(sym) } else { self.parse_text() } } Mode::InTag => { if let Some(sym) = self.parse_end_tag() { Some(sym) } else { self.parse_token() } } } } fn parse_text(&mut self) -> Option> { let start = self.cur_index(); let start_pos = self.pos.clone(); while self.next().is_some() && !self.is_start_tag() { } let end = self.cur_index(); let end_pos = self.pos.clone(); let s = &self.data[start..end]; //println!(" text: >>>{}<<<", s); if start == end { None } else { Some(Token::new( TokenType::Text(s), start_pos, end_pos )) } } fn parse_token(&mut self) -> Option> { self.skip_ws(); match self.cur() { Some('"') => { return self.parse_literal_str(); } Some('#') => { self.next(); return Some(Token::new(TokenType::Sharp, self.pos, self.pos)); } Some('=') => { self.next(); return Some(Token::new(TokenType::Equals, self.pos, self.pos)); } Some('.') => { self.next(); return Some(Token::new(TokenType::Period, self.pos, self.pos)); } _ => {} } let start = self.cur_index(); let start_pos = self.pos.clone(); let mut first = true; while self.next().is_some_and(|ch| is_valid_token_char(ch, first) ) && !self.is_end_tag() { first = false; } let end = self.cur_index(); let end_pos = self.pos.clone(); let s = &self.data[start..end]; if start == end { None } else { Some(Token::new( TokenType::Identifier(s), start_pos, end_pos )) } } fn parse_literal_str(&mut self) -> Option> { if let Some(chr) = self.cur() && chr != '"' { return self.error( ErrorType::UnexpectedChar(chr) ); } let start = self.peek_index(); let start_pos = self.pos.clone(); while self.next().is_some_and(|chr| chr != '"') { } let end = self.cur_index(); let end_pos = self.pos.clone(); self.next(); let s = &self.data[start..end]; Some(Token::new(TokenType::LiteralStr(s), start_pos, end_pos )) } fn is_start_tag(&mut self) -> bool { if let Some(cur) = self.cur() && (cur == '[' || cur == '<') && let Some(peek) = self.peek() && peek == '|' { true } else { false } } fn parse_start_tag(&mut self) -> Option> { let start_pos = self.pos.clone(); match self.cur() { Some('[') => { if let Some(p) = self.peek() && p == '|' { self.next(); let end_pos = self.pos.clone(); self.next(); self.mode = Mode::InTag; Some(Token::new( TokenType::StartFlat, start_pos, end_pos ) ) } else { None } } Some('<') => { if let Some(p) = self.peek() && p == '|' { self.next(); let end_pos = self.pos.clone(); self.next(); self.mode = Mode::InTag; Some(Token::new(TokenType::StartPoint, start_pos, end_pos ) ) } else { None } } _ => { None } } } fn is_end_tag(&mut self) -> bool { if let Some(cur) = self.cur() && cur == '|' && let Some(peek) = self.peek() && (peek == '>' || peek == ']') { true } else { false } } fn parse_end_tag(&mut self) -> Option> { self.skip_ws(); let start_pos = self.pos.clone(); if let Some(chr) = self.cur() && chr == '|' { match self.peek() { Some(']') => { self.next(); let end_pos = self.pos.clone(); self.next(); self.mode = Mode::Text; Some(Token::new( TokenType::EndFlat, start_pos, end_pos)) } Some('>') => { self.next(); let end_pos = self.pos.clone(); self.next(); self.mode = Mode::Text; Some(Token::new( TokenType::EndPoint, start_pos, end_pos)) } _ => { None } } } else { None } } } impl<'a> Iterator for Lexer<'a> { type Item = Token<'a>; fn next(&mut self) -> Option { self.next_symbol() } }