Making progress cleaning up.HEADmain

1 files changed, 37 insertions, 68 deletions

diff --git a/deepcrow/src/lexer.rs b/deepcrow/src/lexer.rs
index 2528cb6..98fc1c7 100644
--- a/deepcrow/src/lexer.rs
+++ b/deepcrow/src/lexer.rs
@@ -3,7 +3,7 @@ use std::iter::Iterator;
 use std::str::CharIndices;
 use std::fmt;
 
-use crate::Position;
+use crate::{Position,Range};
 
 #[derive(Debug,Copy,Clone,PartialEq,Eq)]
 pub enum ErrorType {
@@ -17,60 +17,45 @@ pub enum TokenType<'a> {
     EndFlat,
     EndPoint,
 
-    View,
-    Output,
-    Show,
-    Loop,
-    If,
-    ElIf,
-    Else,
-
     Sharp,
     Equals,
     Period,
     Identifier(&'a str),
-    Literal(&'a str),
+    LiteralStr(&'a str),
     Text(&'a str),
     Error{ what: ErrorType },
 }
 
 #[derive(Copy,Clone)]
-pub struct Symbol<'a> {
-    symbol: TokenType<'a>,
-    start: Position,
-    end: Position,
+pub struct Token<'a> {
+    token: TokenType<'a>,
+    pos: Range,
 }
 
-impl<'a> fmt::Debug for Symbol<'a> {
+impl<'a> fmt::Debug for Token<'a> {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "{:?} @ {:?}-{:?}", self.symbol, self.start, self.end )
+        write!(f, "{:?} @ {:?}", self.token, self.pos )
     }
 }
 
-impl<'a> Symbol<'a> {
-    pub fn new( symbol: TokenType<'a>, start: Position, end: Position ) -> Self {
+impl<'a> Token<'a> {
+    pub fn new( token: TokenType<'a>, start: Position, end: Position ) -> Self {
         Self {
-            symbol, start, end,
+            token, Range::new( start, end ),
         }
     }
 
     pub fn check_type<T: Fn( &TokenType ) -> bool>(&self, f: T ) -> bool {
-        f( &self.symbol )
-    }
-
-    pub fn symbol(&self) -> &TokenType<'a> {
-        &self.symbol
+        f( &self.token )
     }
 
-    pub fn start(&self) -> &Position {
-        &self.start
+    pub fn token(&self) -> &TokenType<'a> {
+        &self.token
     }
 
-    /*
-    pub fn end(&self) -> &Position {
-        &self.end
+    pub fn pos(&self) -> &Range {
+        &self.pos
     }
-    */
 }
 
 #[derive(Debug)]
@@ -81,9 +66,7 @@ enum Mode {
 
 pub struct Lexer<'a> {
     data: &'a str,
-    chars: CharIndices<'a>,
-    cur: [Option<(usize, char)>;2],
-    icur: usize,
+    src: LookAhead<CharIndices<'a>>,
     mode: Mode,
     pos: Position,
 }
@@ -107,15 +90,10 @@ fn is_valid_token_char( c: char, first: bool ) -> bool {
 
 impl<'a> Lexer<'a> {
     pub fn new( data: &'a str ) -> Lexer<'a> {
-        let mut chars = data.char_indices();
-        let cur = chars.next();
-        let cur2 = chars.next();
-        //println!(" - cur: {:?}, peek: {:?}", cur, cur2 );
+        let mut chars = ;
         Lexer {
             data,
-            chars,
-            cur: [cur, cur2],
-            icur: 0,
+            src: LookAhead::new(data.char_indices()),
             mode: Mode::Text,
             pos: Position::new(1,1),
         }
@@ -170,8 +148,8 @@ impl<'a> Lexer<'a> {
         }
     }
 
-    fn error( &self, what: ErrorType ) -> Option<Symbol<'a>> {
-        Some(Symbol::new( TokenType::Error {
+    fn error( &self, what: ErrorType ) -> Option<Token<'a>> {
+        Some(Token::new( TokenType::Error {
             what: what
         }, self.pos, self.pos ))
     }
@@ -182,7 +160,7 @@ impl<'a> Lexer<'a> {
         }
     }
 
-    fn next_symbol(&mut self) -> Option<Symbol<'a>> {
+    fn next_symbol(&mut self) -> Option<Token<'a>> {
         // If we hit the end then we're already done.
         if self.cur().is_none() {
             return None;
@@ -206,7 +184,7 @@ impl<'a> Lexer<'a> {
         }
     }
 
-    fn parse_text(&mut self) -> Option<Symbol<'a>> {
+    fn parse_text(&mut self) -> Option<Token<'a>> {
         let start = self.cur_index();
         let start_pos = self.pos.clone();
         while self.next().is_some() && !self.is_start_tag() { }
@@ -217,11 +195,11 @@ impl<'a> Lexer<'a> {
         if start == end {
             None
         } else {
-            Some(Symbol::new( TokenType::Text(s), start_pos, end_pos ))
+            Some(Token::new( TokenType::Text(s), start_pos, end_pos ))
         }
     }
 
-    fn parse_token(&mut self) -> Option<Symbol<'a>> {
+    fn parse_token(&mut self) -> Option<Token<'a>> {
         self.skip_ws();
         match self.cur() {
             Some('"') => {
@@ -229,15 +207,15 @@ impl<'a> Lexer<'a> {
             }
             Some('#') => {
                 self.next();
-                return Some(Symbol::new(TokenType::Sharp, self.pos, self.pos));
+                return Some(Token::new(TokenType::Sharp, self.pos, self.pos));
             }
             Some('=') => {
                 self.next();
-                return Some(Symbol::new(TokenType::Equals, self.pos, self.pos));
+                return Some(Token::new(TokenType::Equals, self.pos, self.pos));
             }
             Some('.') => {
                 self.next();
-                return Some(Symbol::new(TokenType::Period, self.pos, self.pos));
+                return Some(Token::new(TokenType::Period, self.pos, self.pos));
             }
             _ => {}
         }
@@ -253,20 +231,11 @@ impl<'a> Lexer<'a> {
         if start == end {
             None
         } else {
-            Some(Symbol::new( match s {
-                "view" => TokenType::View,
-                "show" => TokenType::Show,
-                "output" => TokenType::Output,
-                "loop" => TokenType::Loop,
-                "if" => TokenType::If,
-                "elif" => TokenType::ElIf,
-                "else" => TokenType::Else,
-                _ => TokenType::Identifier(s)
-            }, start_pos, end_pos ))
+            Some(Token::new( TokenType::Identifier(s), start_pos, end_pos ))
         }
     }
 
-    fn parse_literal_str(&mut self) -> Option<Symbol<'a>> {
+    fn parse_literal_str(&mut self) -> Option<Token<'a>> {
         if let Some(chr) = self.cur() && chr != '"' {
             return self.error( ErrorType::UnexpectedChar(chr) );
         }
@@ -277,7 +246,7 @@ impl<'a> Lexer<'a> {
         let end_pos = self.pos.clone();
         self.next();
         let s = &self.data[start..end];
-        Some(Symbol::new(TokenType::Literal(s), start_pos, end_pos ))
+        Some(Token::new(TokenType::LiteralStr(s), start_pos, end_pos ))
     }
 
     fn is_start_tag(&mut self) -> bool {
@@ -289,7 +258,7 @@ impl<'a> Lexer<'a> {
         }
     }
 
-    fn parse_start_tag(&mut self) -> Option<Symbol<'a>> {
+    fn parse_start_tag(&mut self) -> Option<Token<'a>> {
         let start_pos = self.pos.clone();
         match self.cur() {
             Some('[') => {
@@ -298,7 +267,7 @@ impl<'a> Lexer<'a> {
                     let end_pos = self.pos.clone();
                     self.next();
                     self.mode = Mode::InTag;
-                    Some(Symbol::new( TokenType::StartFlat, start_pos, end_pos ) )
+                    Some(Token::new( TokenType::StartFlat, start_pos, end_pos ) )
                 } else {
                     None
                 }
@@ -309,7 +278,7 @@ impl<'a> Lexer<'a> {
                     let end_pos = self.pos.clone();
                     self.next();
                     self.mode = Mode::InTag;
-                    Some(Symbol::new(TokenType::StartPoint, start_pos, end_pos ) )
+                    Some(Token::new(TokenType::StartPoint, start_pos, end_pos ) )
                 } else {
                     None
                 }
@@ -329,7 +298,7 @@ impl<'a> Lexer<'a> {
         }
     }
     
-    fn parse_end_tag(&mut self) -> Option<Symbol<'a>> {
+    fn parse_end_tag(&mut self) -> Option<Token<'a>> {
         self.skip_ws();
         let start_pos = self.pos.clone();
         if let Some(chr) = self.cur() && chr == '|' {
@@ -339,14 +308,14 @@ impl<'a> Lexer<'a> {
                     let end_pos = self.pos.clone();
                     self.next();
                     self.mode = Mode::Text;
-                    Some(Symbol::new( TokenType::EndFlat, start_pos, end_pos))
+                    Some(Token::new( TokenType::EndFlat, start_pos, end_pos))
                 }
                 Some('>') => {
                     self.next();
                     let end_pos = self.pos.clone();
                     self.next();
                     self.mode = Mode::Text;
-                    Some(Symbol::new( TokenType::EndPoint, start_pos, end_pos))
+                    Some(Token::new( TokenType::EndPoint, start_pos, end_pos))
                 }
                 _ => {
                     None
@@ -359,7 +328,7 @@ impl<'a> Lexer<'a> {
 }
 
 impl<'a> Iterator for Lexer<'a> {
-    type Item = Symbol<'a>;
+    type Item = Token<'a>;
 
     fn next(&mut self) -> Option<Self::Item> {
         self.next_symbol()