Roorganized into workspace and packages.

This is to accomidate a new proc_macro package, which can't have
anything else in it.

1 files changed, 368 insertions, 0 deletions

diff --git a/crimtag/src/lexer.rs b/crimtag/src/lexer.rs
new file mode 100644
index 0000000..d4d39e9
--- /dev/null
+++ b/crimtag/src/lexer.rs
@@ -0,0 +1,368 @@
+use std::iter::Iterator;
+//use core::error::Error;
+use std::str::CharIndices;
+use std::fmt;
+
+use crate::Position;
+
+#[derive(Debug,Copy,Clone,PartialEq,Eq)]
+pub enum ErrorType {
+    UnexpectedChar(char),
+}
+
+#[derive(Debug,Copy,Clone,PartialEq,Eq)]
+pub enum SymbolType<'a> {
+    StartFlat,
+    StartPoint,
+    EndFlat,
+    EndPoint,
+
+    View,
+    Output,
+    Show,
+    Loop,
+    If,
+    ElIf,
+    Else,
+
+    Sharp,
+    Equals,
+    Period,
+    Token(&'a str),
+    Literal(&'a str),
+    Text(&'a str),
+    Error{ what: ErrorType },
+}
+
+#[derive(Copy,Clone)]
+pub struct Symbol<'a> {
+    symbol: SymbolType<'a>,
+    start: Position,
+    end: Position,
+}
+
+impl<'a> fmt::Debug for Symbol<'a> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "{:?} @ {:?}-{:?}", self.symbol, self.start, self.end )
+    }
+}
+
+impl<'a> Symbol<'a> {
+    pub fn new( symbol: SymbolType<'a>, start: Position, end: Position ) -> Self {
+        Self {
+            symbol, start, end,
+        }
+    }
+
+    pub fn check_type<T: Fn( &SymbolType ) -> bool>(&self, f: T ) -> bool {
+        f( &self.symbol )
+    }
+
+    pub fn symbol(&self) -> &SymbolType<'a> {
+        &self.symbol
+    }
+
+    pub fn start(&self) -> &Position {
+        &self.start
+    }
+
+    /*
+    pub fn end(&self) -> &Position {
+        &self.end
+    }
+    */
+}
+
+#[derive(Debug)]
+enum Mode {
+    Text,
+    InTag,
+}
+
+pub struct Lexer<'a> {
+    data: &'a str,
+    chars: CharIndices<'a>,
+    cur: [Option<(usize, char)>;2],
+    icur: usize,
+    mode: Mode,
+    pos: Position,
+}
+
+fn is_valid_token_char( c: char, first: bool ) -> bool {
+    if c.is_whitespace() {
+        return false;
+    }
+    if first {
+        match c {
+            'a'..'z' | 'A'..'Z' | '_' => true,
+            _ => false
+        }
+    } else {
+        match c {
+            'a'..'z' | 'A'..'Z' | '0'..'9' | '_' => true,
+            _ => false
+        }
+    }
+}
+
+impl<'a> Lexer<'a> {
+    pub fn new( data: &'a str ) -> Lexer<'a> {
+        let mut chars = data.char_indices();
+        let cur = chars.next();
+        let cur2 = chars.next();
+        //println!(" - cur: {:?}, peek: {:?}", cur, cur2 );
+        Lexer {
+            data,
+            chars,
+            cur: [cur, cur2],
+            icur: 0,
+            mode: Mode::Text,
+            pos: Position::new(1,1),
+        }
+    }
+
+    fn next(&mut self) -> Option<char> {
+        self.cur[self.icur] = self.chars.next();
+        self.icur = (self.icur+1)%2;
+        //println!(" - cur: {:?}, peek: {:?}", self.cur(), self.peek() );
+        if let Some((_,chr)) = self.cur[self.icur] {
+            if chr == '\n' {
+                self.pos.column = 1;
+                self.pos.line += 1;
+            } else {
+                self.pos.column += 1;
+            }
+            Some(chr)
+        } else {
+            None
+        }
+    }
+
+    fn cur(&self) -> Option<char> {
+        if let Some((_, chr)) = self.cur[self.icur] {
+            Some(chr)
+        } else {
+            None
+        }
+    }
+
+    fn cur_index(&self) -> usize {
+        if let Some((idx, _)) = self.cur[self.icur] {
+            idx
+        } else {
+            self.chars.offset()
+        }
+    }
+
+    fn peek(&self) -> Option<char> {
+        if let Some((_,chr)) = self.cur[(self.icur+1)%2] {
+            Some(chr)
+        } else {
+            None
+        }
+    }
+
+    fn peek_index(&self) -> usize {
+        if let Some((idx, _)) = self.cur[(self.icur+1)%2] {
+            idx
+        } else {
+            self.chars.offset()
+        }
+    }
+
+    fn error( &self, what: ErrorType ) -> Option<Symbol<'a>> {
+        Some(Symbol::new( SymbolType::Error {
+            what: what
+        }, self.pos, self.pos ))
+    }
+
+    fn skip_ws( &mut self ) {
+        while self.cur().is_some_and(|x|x.is_whitespace()) {
+            self.next();
+        }
+    }
+
+    fn next_symbol(&mut self) -> Option<Symbol<'a>> {
+        // If we hit the end then we're already done.
+        if self.cur().is_none() {
+            return None;
+        }
+
+        match self.mode {
+            Mode::Text => {
+                if let Some(sym) = self.parse_start_tag() {
+                    Some(sym)
+                } else {
+                    self.parse_text()
+                }
+            }
+            Mode::InTag => {
+                if let Some(sym) = self.parse_end_tag() {
+                    Some(sym)
+                } else {
+                    self.parse_token()
+                }
+            }
+        }
+    }
+
+    fn parse_text(&mut self) -> Option<Symbol<'a>> {
+        let start = self.cur_index();
+        let start_pos = self.pos.clone();
+        while self.next().is_some() && !self.is_start_tag() { }
+        let end = self.cur_index();
+        let end_pos = self.pos.clone();
+        let s = &self.data[start..end];
+        //println!(" text: >>>{}<<<", s);
+        if start == end {
+            None
+        } else {
+            Some(Symbol::new( SymbolType::Text(s), start_pos, end_pos ))
+        }
+    }
+
+    fn parse_token(&mut self) -> Option<Symbol<'a>> {
+        self.skip_ws();
+        match self.cur() {
+            Some('"') => {
+                return self.parse_literal_str();
+            }
+            Some('#') => {
+                self.next();
+                return Some(Symbol::new(SymbolType::Sharp, self.pos, self.pos));
+            }
+            Some('=') => {
+                self.next();
+                return Some(Symbol::new(SymbolType::Equals, self.pos, self.pos));
+            }
+            Some('.') => {
+                self.next();
+                return Some(Symbol::new(SymbolType::Period, self.pos, self.pos));
+            }
+            _ => {}
+        }
+        let start = self.cur_index();
+        let start_pos = self.pos.clone();
+       
+        let mut first = true;
+        while self.next().is_some_and(|ch| is_valid_token_char(ch, first) ) &&
+            !self.is_end_tag() { first = false; }
+        let end = self.cur_index();
+        let end_pos = self.pos.clone();
+        let s = &self.data[start..end];
+        if start == end {
+            None
+        } else {
+            Some(Symbol::new( match s {
+                "view" => SymbolType::View,
+                "show" => SymbolType::Show,
+                "output" => SymbolType::Output,
+                "loop" => SymbolType::Loop,
+                "if" => SymbolType::If,
+                "elif" => SymbolType::ElIf,
+                "else" => SymbolType::Else,
+                _ => SymbolType::Token(s)
+            }, start_pos, end_pos ))
+        }
+    }
+
+    fn parse_literal_str(&mut self) -> Option<Symbol<'a>> {
+        if let Some(chr) = self.cur() && chr != '"' {
+            return self.error( ErrorType::UnexpectedChar(chr) );
+        }
+        let start = self.peek_index();
+        let start_pos = self.pos.clone();
+        while self.next().is_some_and(|chr| chr != '"') { }
+        let end = self.cur_index();
+        let end_pos = self.pos.clone();
+        self.next();
+        let s = &self.data[start..end];
+        Some(Symbol::new(SymbolType::Literal(s), start_pos, end_pos ))
+    }
+
+    fn is_start_tag(&mut self) -> bool {
+        if let Some(cur) = self.cur() && (cur == '[' || cur == '<') &&
+            let Some(peek) = self.peek() && peek == '|' {
+            true
+        } else {
+            false
+        }
+    }
+
+    fn parse_start_tag(&mut self) -> Option<Symbol<'a>> {
+        let start_pos = self.pos.clone();
+        match self.cur() {
+            Some('[') => {
+                if let Some(p) = self.peek() && p == '|' {
+                    self.next();
+                    let end_pos = self.pos.clone();
+                    self.next();
+                    self.mode = Mode::InTag;
+                    Some(Symbol::new( SymbolType::StartFlat, start_pos, end_pos ) )
+                } else {
+                    None
+                }
+            }
+            Some('<') => {
+                if let Some(p) = self.peek() && p == '|' {
+                    self.next();
+                    let end_pos = self.pos.clone();
+                    self.next();
+                    self.mode = Mode::InTag;
+                    Some(Symbol::new(SymbolType::StartPoint, start_pos, end_pos ) )
+                } else {
+                    None
+                }
+            }
+            _ => {
+                None
+            }
+        }
+    }
+    
+    fn is_end_tag(&mut self) -> bool {
+        if let Some(cur) = self.cur() && cur == '|' &&
+            let Some(peek) = self.peek() && (peek == '>' || peek == ']') {
+            true
+        } else {
+            false
+        }
+    }
+    
+    fn parse_end_tag(&mut self) -> Option<Symbol<'a>> {
+        self.skip_ws();
+        let start_pos = self.pos.clone();
+        if let Some(chr) = self.cur() && chr == '|' {
+            match self.peek() {
+                Some(']') => {
+                    self.next();
+                    let end_pos = self.pos.clone();
+                    self.next();
+                    self.mode = Mode::Text;
+                    Some(Symbol::new( SymbolType::EndFlat, start_pos, end_pos))
+                }
+                Some('>') => {
+                    self.next();
+                    let end_pos = self.pos.clone();
+                    self.next();
+                    self.mode = Mode::Text;
+                    Some(Symbol::new( SymbolType::EndPoint, start_pos, end_pos))
+                }
+                _ => {
+                    None
+                }
+            }
+        } else {
+            None
+        }
+    }
+}
+
+impl<'a> Iterator for Lexer<'a> {
+    type Item = Symbol<'a>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        self.next_symbol()
+    }
+}
+