Symbols have a wrapper with positional data now.

It required some weird changes, I'm not sure I'm happy with them all
yet, but I"m getting closer. It parses all basics, now we need to build
an AST so it's actually useful :-P

3 files changed, 133 insertions, 52 deletions

diff --git a/src/lexer.rs b/src/lexer.rs
index 94825b2..04be350 100644
--- a/src/lexer.rs
+++ b/src/lexer.rs
@@ -2,13 +2,15 @@ use std::iter::Iterator;
 //use core::error::Error;
 use std::str::CharIndices;
 
+use crate::Position;
+
 #[derive(Debug,Copy,Clone)]
 pub enum ErrorType {
     UnexpectedChar(char),
 }
 
 #[derive(Debug,Copy,Clone)]
-pub enum Symbol<'a> {
+pub enum SymbolType<'a> {
     StartFlat,
     StartPoint,
     EndFlat,
@@ -20,10 +22,41 @@ pub enum Symbol<'a> {
     Token(&'a str),
     Literal(&'a str),
     Text(&'a str),
-    Error{ line: u32, row: u32, what: ErrorType },
+    Error{ what: ErrorType },
     EOS,
 }
 
+#[derive(Debug,Copy,Clone)]
+pub struct Symbol<'a> {
+    symbol: SymbolType<'a>,
+    start: Position,
+    end: Position,
+}
+
+impl<'a> Symbol<'a> {
+    pub fn new( symbol: SymbolType<'a>, start: Position, end: Position ) -> Self {
+        Self {
+            symbol, start, end,
+        }
+    }
+
+    pub fn check_type<T: Fn( &SymbolType ) -> bool>(&self, f: T ) -> bool {
+        f( &self.symbol )
+    }
+
+    pub fn symbol(&self) -> &SymbolType<'a> {
+        &self.symbol
+    }
+
+    pub fn start(&self) -> &Position {
+        &self.start
+    }
+
+    pub fn end(&self) -> &Position {
+        &self.end
+    }
+}
+
 enum Mode {
     Text,
     InTag,
@@ -35,8 +68,7 @@ pub struct Lexer<'a> {
     cur: [Option<(usize, char)>;2],
     icur: usize,
     mode: Mode,
-    line: u32,
-    row: u32,
+    pos: Position,
 }
 
 fn is_valid_token_char( c: char ) -> bool {
@@ -61,8 +93,7 @@ impl<'a> Lexer<'a> {
             cur: [cur, cur2],
             icur: 0,
             mode: Mode::Text,
-            line: 0,
-            row: 0,
+            pos: Position::new(1,1),
         }
     }
 
@@ -71,6 +102,12 @@ impl<'a> Lexer<'a> {
         self.icur = (self.icur+1)%2;
         //println!(" - cur: {:?}, peek: {:?}", self.cur(), self.peek() );
         if let Some((_,chr)) = self.cur[self.icur] {
+            if chr == '\n' {
+                self.pos.column = 1;
+                self.pos.line += 1;
+            } else {
+                self.pos.column += 1;
+            }
             Some(chr)
         } else {
             None
@@ -110,11 +147,9 @@ impl<'a> Lexer<'a> {
     }
 
     fn error( &self, what: ErrorType ) -> Option<Symbol<'a>> {
-        Some(Symbol::Error {
-            line: self.line,
-            row: self.row,
+        Some(Symbol::new( SymbolType::Error {
             what: what
-        })
+        }, self.pos, self.pos ))
     }
 
     fn skip_ws( &mut self ) {
@@ -149,14 +184,16 @@ impl<'a> Lexer<'a> {
 
     fn parse_text(&mut self) -> Option<Symbol<'a>> {
         let start = self.cur_index();
+        let start_pos = self.pos.clone();
         while self.next().is_some() && !self.is_start_tag() { }
         let end = self.cur_index();
+        let end_pos = self.pos.clone();
         let s = &self.data[start..end];
         //println!(" text: >>>{}<<<", s);
         if start == end {
             None
         } else {
-            Some(Symbol::Text(s))
+            Some(Symbol::new( SymbolType::Text(s), start_pos, end_pos ))
         }
     }
 
@@ -168,25 +205,27 @@ impl<'a> Lexer<'a> {
             }
             Some('=') => {
                 self.next();
-                return Some(Symbol::Equals);
+                return Some(Symbol::new(SymbolType::Equals, self.pos, self.pos));
             }
             _ => {}
         }
         let start = self.cur_index();
+        let start_pos = self.pos.clone();
         
         while self.next().is_some_and(|ch| is_valid_token_char(ch) ) &&
             !self.is_end_tag() {}
         let end = self.cur_index();
+        let end_pos = self.pos.clone();
         let s = &self.data[start..end];
         if start == end {
             None
         } else {
-            Some(match s {
-                "fragment" => Symbol::Fragment,
-                "section" => Symbol::Section,
-                "output" => Symbol::Output,
-                _ => Symbol::Token(s)
-            })
+            Some(Symbol::new( match s {
+                "fragment" => SymbolType::Fragment,
+                "section" => SymbolType::Section,
+                "output" => SymbolType::Output,
+                _ => SymbolType::Token(s)
+            }, start_pos, end_pos ))
         }
     }
 
@@ -195,11 +234,13 @@ impl<'a> Lexer<'a> {
             return self.error( ErrorType::UnexpectedChar(chr) );
         }
         let start = self.peek_index();
+        let start_pos = self.pos.clone();
         while self.next().is_some_and(|chr| chr != '"') { }
         let end = self.cur_index();
+        let end_pos = self.pos.clone();
         self.next();
         let s = &self.data[start..end];
-        Some(Symbol::Literal(s))
+        Some(Symbol::new(SymbolType::Literal(s), start_pos, end_pos ))
     }
 
     fn is_start_tag(&mut self) -> bool {
@@ -212,21 +253,26 @@ impl<'a> Lexer<'a> {
     }
 
     fn parse_start_tag(&mut self) -> Option<Symbol<'a>> {
+        let start_pos = self.pos.clone();
         match self.cur() {
             Some('[') => {
                 if let Some(p) = self.peek() && p == '|' {
-                    self.next(); self.next();
+                    self.next();
+                    let end_pos = self.pos.clone();
+                    self.next();
                     self.mode = Mode::InTag;
-                    Some(Symbol::StartFlat)
+                    Some(Symbol::new( SymbolType::StartFlat, start_pos, end_pos ) )
                 } else {
                     None
                 }
             }
             Some('<') => {
                 if let Some(p) = self.peek() && p == '|' {
-                    self.next(); self.next();
+                    self.next();
+                    let end_pos = self.pos.clone();
+                    self.next();
                     self.mode = Mode::InTag;
-                    Some(Symbol::StartPoint)
+                    Some(Symbol::new(SymbolType::StartPoint, start_pos, end_pos ) )
                 } else {
                     None
                 }
@@ -247,17 +293,22 @@ impl<'a> Lexer<'a> {
     }
     
     fn parse_end_tag(&mut self) -> Option<Symbol<'a>> {
+        let start_pos = self.pos.clone();
         if let Some(chr) = self.cur() && chr == '|' {
             match self.peek() {
                 Some(']') => {
-                    self.next(); self.next();
+                    self.next();
+                    let end_pos = self.pos.clone();
+                    self.next();
                     self.mode = Mode::Text;
-                    Some(Symbol::EndFlat)
+                    Some(Symbol::new( SymbolType::EndFlat, start_pos, end_pos))
                 }
                 Some('>') => {
-                    self.next(); self.next();
+                    self.next();
+                    let end_pos = self.pos.clone();
+                    self.next();
                     self.mode = Mode::Text;
-                    Some(Symbol::EndPoint)
+                    Some(Symbol::new( SymbolType::EndPoint, start_pos, end_pos))
                 }
                 _ => {
                     None
diff --git a/src/lib.rs b/src/lib.rs
index 0f6d01d..7d0e7f2 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -8,6 +8,20 @@ use std::io::Read;
 mod lexer;
 mod parser;
 
+#[derive(Debug,Copy,Clone)]
+pub struct Position {
+    pub line: u32,
+    pub column: u32,
+}
+
+impl Position {
+    pub fn new( line: u32, column: u32 ) -> Self {
+        Self {
+            line, column,
+        }
+    }
+}
+
 pub struct Crimtag {
     fragments: HashMap<String, Fragment>,
     sources: Vec<FragmentSource>,
@@ -104,13 +118,15 @@ impl Crimtag {
 mod tests {
     use super::*;
 
+    use crate::lexer::*;
+
     #[test]
     fn lexing() {
         let data = r#"Leading comment: [|fragment "basic" something="yeup"|>Hello there <|fragment|] Trailing text"#;
         let ll = lexer::Lexer::new( &data );
         for sym in ll {
-            if let lexer::Symbol::Error{line,row,what} = sym {
-                println!("Error {}:{}: {:?}", line, row, what );
+            if let SymbolType::Error{what} = sym.symbol() {
+                println!("Error {}:{}: {:?}", sym.start().line, sym.start().column, what );
                 break;
             } else {
                 println!("Symbol: {:?}", sym );
@@ -121,6 +137,8 @@ mod tests {
     #[test]
     fn parsing() {
         let mut ct = Crimtag::new();
-        ct.load_static(r#"Here is a sample [|frament "index" theme="standard"|><html><body>Hi</body></html><|fragment|] That was fun!"#);
+        ct.load_static(r#"Here is a sample
+[|frament "index" theme="standard"|><html><body>Hi</body></html><|fragment|]
+That was fun!"#);
     }
 }
diff --git a/src/parser.rs b/src/parser.rs
index 54b77ac..e5ea1f5 100644
--- a/src/parser.rs
+++ b/src/parser.rs
@@ -6,8 +6,7 @@ use std::fmt;
 
 #[derive(Debug)]
 struct ParseError {
-    line: u32,
-    row: u32,
+    position: Position,
     what: String,
 }
 
@@ -138,15 +137,15 @@ impl Parser {
     }
 
     fn lex_error(&self, error: &Symbol) -> Result<(),Box<dyn Error>> {
-        if let Symbol::Error{line,row,what} = error {
+        if let SymbolType::Error{what} = error.symbol() {
             Err(Box::new(ParseError {
-                line: *line,
-                row: *row,
+                position: error.start().clone(),
                 what: format!("What: {:?}", *what)
             }))
         } else {
             Err(Box::new(ParseError {
-                line: 0, row: 0, what: "Not an error?".to_string()
+                position: Position::new(0,0),
+                what: "Not an error?".to_string(),
             }))
         }
     }
@@ -166,12 +165,12 @@ impl Parser {
             if ctx.cur().is_none() {
                 break;
             }
-            match ctx.cur() {
-                Some(Symbol::Text(_)) => { /* Skip top level text */ }
-                Some(Symbol::StartFlat) => {
+            match ctx.cur().unwrap().symbol() {
+                SymbolType::Text(_) => { /* Skip top level text */ }
+                SymbolType::StartFlat => {
                     self.parse_tag( ctx );
                 }
-                Some(Symbol::Error{..}) => {
+                SymbolType::Error{..} => {
                     return self.lex_error( &ctx.cur().unwrap() );
                 }
                 _ => {
@@ -188,10 +187,14 @@ impl Parser {
 
         let start_sym = ctx.cur().unwrap();
 
-        if let Some(Symbol::Token(s)) = ctx.next() {
-            tb.set_name( s.to_string() );
+        if ctx.next().is_some() {
+            if let SymbolType::Token(s) = ctx.next().unwrap().symbol() {
+                tb.set_name( s.to_string() );
+            } else {
+                return Err(Box::new(ParseError{position: ctx.cur().unwrap().start().clone(), what: "Unexpecetd symbol".to_string()}));
+            }
         } else {
-            return Err(Box::new(ParseError{line: 0, row: 0, what: "Unexpecetd symbol".to_string()}));
+            return Err(Box::new(ParseError{position: Position::new(0,0), what: "Unexpecetd end of stream".to_string()}));
         }
 
         self.parse_tag_params( ctx, &mut tb )?;
@@ -202,10 +205,15 @@ impl Parser {
 
     fn parse_tag_params(&mut self, ctx: &mut Context, tb: &mut TagBuilder ) -> Result<(), Box<dyn Error>> {
         loop {
-            if let Some(Symbol::Token(s)) = ctx.cur() {
-                if matches!(ctx.peek(), Some(Symbol::Equals)) {
-                    // Done with parameters
-                    break;
+            if ctx.cur().is_none() {
+
+                return Err(Box::new(ParseError{position: Position::new(0,0), what: "Unexpecetd end of stream".to_string()}));
+            }
+            if let SymbolType::Token(s) = ctx.cur().unwrap().symbol() {
+                if let Some(p) = ctx.peek() {
+                    if p.check_type(|t| matches!(t, SymbolType::Equals)) {
+                        break;
+                    }
                 }
                 else {
                     tb.add_param( s.to_string() );
@@ -218,16 +226,20 @@ impl Parser {
     
     fn parse_tag_props(&mut self, ctx: &mut Context, tb: &mut TagBuilder ) -> Result<(), Box<dyn Error>> {
         loop {
-            if let Some(Symbol::Token(s)) = ctx.cur() {
-                if matches!(ctx.peek(), Some(Symbol::Equals)) {
+            if ctx.cur().is_none() {
+                return Err(Box::new(ParseError{position: Position::new(0,0), what: "Unexpecetd end of stream".to_string()}));
+            }
+            if let SymbolType::Token(s) = ctx.cur().unwrap().symbol() {
+                if ctx.peek().is_some_and(|s|s.check_type(|t| matches!(t,SymbolType::Equals))) {
                     ctx.next();
-                    if let Some(Symbol::Literal(lv)) = ctx.next() {
+                    if let Some(sym) = ctx.next() &&
+                        let SymbolType::Literal(lv) = sym.symbol() {
                         tb.add_prop( s.to_string(), lv.to_string() );
                     } else {
-                        return Err(Box::new(ParseError{line: 0, row: 0, what: "Expected quoted literal string".to_string()}));
+                        return Err(Box::new(ParseError{position: Position::new(0,0), what: "Expected quoted literal string".to_string()}));
                     }
                 } else {
-                    return Err(Box::new(ParseError{line: 0, row: 0, what: "Expected = ".to_string()}));
+                    return Err(Box::new(ParseError{position:Position::new(0,0), what: "Expected = ".to_string()}));
                 }
             } else {
                 break;