Symbols have a wrapper with positional data now.

It required some weird changes, I'm not sure I'm happy with them all
yet, but I"m getting closer. It parses all basics, now we need to build
an AST so it's actually useful :-P

1 files changed, 78 insertions, 27 deletions

diff --git a/src/lexer.rs b/src/lexer.rs
index 94825b2..04be350 100644
--- a/src/lexer.rs
+++ b/src/lexer.rs
@@ -2,13 +2,15 @@ use std::iter::Iterator;
 //use core::error::Error;
 use std::str::CharIndices;
 
+use crate::Position;
+
 #[derive(Debug,Copy,Clone)]
 pub enum ErrorType {
     UnexpectedChar(char),
 }
 
 #[derive(Debug,Copy,Clone)]
-pub enum Symbol<'a> {
+pub enum SymbolType<'a> {
     StartFlat,
     StartPoint,
     EndFlat,
@@ -20,10 +22,41 @@ pub enum Symbol<'a> {
     Token(&'a str),
     Literal(&'a str),
     Text(&'a str),
-    Error{ line: u32, row: u32, what: ErrorType },
+    Error{ what: ErrorType },
     EOS,
 }
 
+#[derive(Debug,Copy,Clone)]
+pub struct Symbol<'a> {
+    symbol: SymbolType<'a>,
+    start: Position,
+    end: Position,
+}
+
+impl<'a> Symbol<'a> {
+    pub fn new( symbol: SymbolType<'a>, start: Position, end: Position ) -> Self {
+        Self {
+            symbol, start, end,
+        }
+    }
+
+    pub fn check_type<T: Fn( &SymbolType ) -> bool>(&self, f: T ) -> bool {
+        f( &self.symbol )
+    }
+
+    pub fn symbol(&self) -> &SymbolType<'a> {
+        &self.symbol
+    }
+
+    pub fn start(&self) -> &Position {
+        &self.start
+    }
+
+    pub fn end(&self) -> &Position {
+        &self.end
+    }
+}
+
 enum Mode {
     Text,
     InTag,
@@ -35,8 +68,7 @@ pub struct Lexer<'a> {
     cur: [Option<(usize, char)>;2],
     icur: usize,
     mode: Mode,
-    line: u32,
-    row: u32,
+    pos: Position,
 }
 
 fn is_valid_token_char( c: char ) -> bool {
@@ -61,8 +93,7 @@ impl<'a> Lexer<'a> {
             cur: [cur, cur2],
             icur: 0,
             mode: Mode::Text,
-            line: 0,
-            row: 0,
+            pos: Position::new(1,1),
         }
     }
 
@@ -71,6 +102,12 @@ impl<'a> Lexer<'a> {
         self.icur = (self.icur+1)%2;
         //println!(" - cur: {:?}, peek: {:?}", self.cur(), self.peek() );
         if let Some((_,chr)) = self.cur[self.icur] {
+            if chr == '\n' {
+                self.pos.column = 1;
+                self.pos.line += 1;
+            } else {
+                self.pos.column += 1;
+            }
             Some(chr)
         } else {
             None
@@ -110,11 +147,9 @@ impl<'a> Lexer<'a> {
     }
 
     fn error( &self, what: ErrorType ) -> Option<Symbol<'a>> {
-        Some(Symbol::Error {
-            line: self.line,
-            row: self.row,
+        Some(Symbol::new( SymbolType::Error {
             what: what
-        })
+        }, self.pos, self.pos ))
     }
 
     fn skip_ws( &mut self ) {
@@ -149,14 +184,16 @@ impl<'a> Lexer<'a> {
 
     fn parse_text(&mut self) -> Option<Symbol<'a>> {
         let start = self.cur_index();
+        let start_pos = self.pos.clone();
         while self.next().is_some() && !self.is_start_tag() { }
         let end = self.cur_index();
+        let end_pos = self.pos.clone();
         let s = &self.data[start..end];
         //println!(" text: >>>{}<<<", s);
         if start == end {
             None
         } else {
-            Some(Symbol::Text(s))
+            Some(Symbol::new( SymbolType::Text(s), start_pos, end_pos ))
         }
     }
 
@@ -168,25 +205,27 @@ impl<'a> Lexer<'a> {
             }
             Some('=') => {
                 self.next();
-                return Some(Symbol::Equals);
+                return Some(Symbol::new(SymbolType::Equals, self.pos, self.pos));
             }
             _ => {}
         }
         let start = self.cur_index();
+        let start_pos = self.pos.clone();
         
         while self.next().is_some_and(|ch| is_valid_token_char(ch) ) &&
             !self.is_end_tag() {}
         let end = self.cur_index();
+        let end_pos = self.pos.clone();
         let s = &self.data[start..end];
         if start == end {
             None
         } else {
-            Some(match s {
-                "fragment" => Symbol::Fragment,
-                "section" => Symbol::Section,
-                "output" => Symbol::Output,
-                _ => Symbol::Token(s)
-            })
+            Some(Symbol::new( match s {
+                "fragment" => SymbolType::Fragment,
+                "section" => SymbolType::Section,
+                "output" => SymbolType::Output,
+                _ => SymbolType::Token(s)
+            }, start_pos, end_pos ))
         }
     }
 
@@ -195,11 +234,13 @@ impl<'a> Lexer<'a> {
             return self.error( ErrorType::UnexpectedChar(chr) );
         }
         let start = self.peek_index();
+        let start_pos = self.pos.clone();
         while self.next().is_some_and(|chr| chr != '"') { }
         let end = self.cur_index();
+        let end_pos = self.pos.clone();
         self.next();
         let s = &self.data[start..end];
-        Some(Symbol::Literal(s))
+        Some(Symbol::new(SymbolType::Literal(s), start_pos, end_pos ))
     }
 
     fn is_start_tag(&mut self) -> bool {
@@ -212,21 +253,26 @@ impl<'a> Lexer<'a> {
     }
 
     fn parse_start_tag(&mut self) -> Option<Symbol<'a>> {
+        let start_pos = self.pos.clone();
         match self.cur() {
             Some('[') => {
                 if let Some(p) = self.peek() && p == '|' {
-                    self.next(); self.next();
+                    self.next();
+                    let end_pos = self.pos.clone();
+                    self.next();
                     self.mode = Mode::InTag;
-                    Some(Symbol::StartFlat)
+                    Some(Symbol::new( SymbolType::StartFlat, start_pos, end_pos ) )
                 } else {
                     None
                 }
             }
             Some('<') => {
                 if let Some(p) = self.peek() && p == '|' {
-                    self.next(); self.next();
+                    self.next();
+                    let end_pos = self.pos.clone();
+                    self.next();
                     self.mode = Mode::InTag;
-                    Some(Symbol::StartPoint)
+                    Some(Symbol::new(SymbolType::StartPoint, start_pos, end_pos ) )
                 } else {
                     None
                 }
@@ -247,17 +293,22 @@ impl<'a> Lexer<'a> {
     }
     
     fn parse_end_tag(&mut self) -> Option<Symbol<'a>> {
+        let start_pos = self.pos.clone();
         if let Some(chr) = self.cur() && chr == '|' {
             match self.peek() {
                 Some(']') => {
-                    self.next(); self.next();
+                    self.next();
+                    let end_pos = self.pos.clone();
+                    self.next();
                     self.mode = Mode::Text;
-                    Some(Symbol::EndFlat)
+                    Some(Symbol::new( SymbolType::EndFlat, start_pos, end_pos))
                 }
                 Some('>') => {
-                    self.next(); self.next();
+                    self.next();
+                    let end_pos = self.pos.clone();
+                    self.next();
                     self.mode = Mode::Text;
-                    Some(Symbol::EndPoint)
+                    Some(Symbol::new( SymbolType::EndPoint, start_pos, end_pos))
                 }
                 _ => {
                     None