2 files changed, 333 insertions, 0 deletions

diff --git a/src/lexer.rs b/src/lexer.rs
new file mode 100644
index 0000000..d6806d8
--- /dev/null
+++ b/src/lexer.rs
@@ -0,0 +1,258 @@
+use std::iter::Iterator;
+//use core::error::Error;
+use std::str::CharIndices;
+
+#[derive(Debug)]
+pub enum Symbol<'a> {
+    StartFlat,
+    StartPoint,
+    EndFlat,
+    EndPoint,
+    Fragment,
+    Section,
+    Output,
+    Equals,
+    Token(&'a str),
+    Literal(&'a str),
+    Text(&'a str),
+    Error{ line: u32, row: u32, what: String },
+    EOS,
+}
+
+enum Mode {
+    Text,
+    InTag,
+}
+
+pub struct Lexer<'a> {
+    data: &'a str,
+    chars: CharIndices<'a>,
+    cur: [Option<(usize, char)>;2],
+    icur: usize,
+    mode: Mode,
+    line: u32,
+    row: u32,
+}
+
+impl<'a> Lexer<'a> {
+    pub fn new( data: &'a str ) -> Lexer<'a> {
+        let mut chars = data.char_indices();
+        let cur = chars.next();
+        let cur2 = chars.next();
+        //println!(" - cur: {:?}, peek: {:?}", cur, cur2 );
+        Lexer {
+            data,
+            chars,
+            cur: [cur, cur2],
+            icur: 0,
+            mode: Mode::Text,
+            line: 0,
+            row: 0,
+        }
+    }
+
+    fn next(&mut self) -> Option<char> {
+        self.cur[self.icur] = self.chars.next();
+        self.icur = (self.icur+1)%2;
+        //println!(" - cur: {:?}, peek: {:?}", self.cur(), self.peek() );
+        if let Some((_,chr)) = self.cur[self.icur] {
+            Some(chr)
+        } else {
+            None
+        }
+    }
+
+    fn cur(&self) -> Option<char> {
+        if let Some((_, chr)) = self.cur[self.icur] {
+            Some(chr)
+        } else {
+            None
+        }
+    }
+
+    fn cur_index(&self) -> usize {
+        if let Some((idx, _)) = self.cur[self.icur] {
+            idx
+        } else {
+            self.chars.offset()
+        }
+    }
+
+    fn peek(&self) -> Option<char> {
+        if let Some((_,chr)) = self.cur[(self.icur+1)%2] {
+            Some(chr)
+        } else {
+            None
+        }
+    }
+
+    fn peek_index(&self) -> usize {
+        if let Some((idx, _)) = self.cur[(self.icur+1)%2] {
+            idx
+        } else {
+            self.chars.offset()
+        }
+    }
+
+    fn error( &self, what: String ) -> Option<Symbol<'a>> {
+        Some(Symbol::Error {
+            line: self.line,
+            row: self.row,
+            what: what
+        })
+    }
+
+    fn skip_ws( &mut self ) {
+        while self.cur().is_some_and(|x|x.is_whitespace()) {
+            self.next();
+        }
+    }
+
+    fn next_symbol(&mut self) -> Option<Symbol<'a>> {
+        // If we hit the end then we're already done.
+        if self.cur().is_none() {
+            return None;
+        }
+
+        match self.mode {
+            Mode::Text => {
+                if let Some(sym) = self.parse_start_tag() {
+                    Some(sym)
+                } else {
+                    self.parse_text()
+                }
+            }
+            Mode::InTag => {
+                if let Some(sym) = self.parse_end_tag() {
+                    Some(sym)
+                } else {
+                    self.parse_token()
+                }
+            }
+        }
+    }
+
+    fn parse_text(&mut self) -> Option<Symbol<'a>> {
+        let start = self.cur_index();
+        while self.next().is_some() && !self.is_start_tag() { }
+        let end = self.cur_index();
+        let s = &self.data[start..end];
+        //println!(" text: >>>{}<<<", s);
+        if start == end {
+            None
+        } else {
+            Some(Symbol::Text(s))
+        }
+    }
+
+    fn parse_token(&mut self) -> Option<Symbol<'a>> {
+        self.skip_ws();
+        if let Some(chr) = self.cur() && chr == '"' {
+            // Literal string
+            return self.parse_literal_str();
+        }
+        let start = self.cur_index();
+        
+        while self.next().is_some_and(|ch| !ch.is_whitespace() ) &&
+            !self.is_end_tag() {}
+        let end = self.cur_index();
+        let s = &self.data[start..end];
+        if start == end {
+            None
+        } else {
+            Some(match s {
+                "fragment" => Symbol::Fragment,
+                "section" => Symbol::Section,
+                "output" => Symbol::Output,
+                _ => Symbol::Token(s)
+            })
+        }
+    }
+
+    fn parse_literal_str(&mut self) -> Option<Symbol<'a>> {
+        if let Some(chr) = self.cur() && chr != '"' {
+            return self.error(format!("Expected '\"' but found '{}'", chr));
+        }
+        let start = self.peek_index();
+        while self.next().is_some_and(|chr| chr != '"') { }
+        let end = self.cur_index();
+        self.next();
+        let s = &self.data[start..end];
+        Some(Symbol::Literal(s))
+    }
+
+    fn is_start_tag(&mut self) -> bool {
+        if let Some(cur) = self.cur() && (cur == '[' || cur == '<') &&
+            let Some(peek) = self.peek() && peek == '|' {
+            true
+        } else {
+            false
+        }
+    }
+
+    fn parse_start_tag(&mut self) -> Option<Symbol<'a>> {
+        match self.cur() {
+            Some('[') => {
+                if let Some(p) = self.peek() && p == '|' {
+                    self.next(); self.next();
+                    self.mode = Mode::InTag;
+                    Some(Symbol::StartFlat)
+                } else {
+                    None
+                }
+            }
+            Some('<') => {
+                if let Some(p) = self.peek() && p == '|' {
+                    self.next(); self.next();
+                    self.mode = Mode::InTag;
+                    Some(Symbol::StartPoint)
+                } else {
+                    None
+                }
+            }
+            _ => {
+                None
+            }
+        }
+    }
+    
+    fn is_end_tag(&mut self) -> bool {
+        if let Some(cur) = self.cur() && cur == '|' &&
+            let Some(peek) = self.peek() && (peek == '>' || peek == ']') {
+            true
+        } else {
+            false
+        }
+    }
+    
+    fn parse_end_tag(&mut self) -> Option<Symbol<'a>> {
+        if let Some(chr) = self.cur() && chr == '|' {
+            match self.peek() {
+                Some(']') => {
+                    self.next(); self.next();
+                    self.mode = Mode::Text;
+                    Some(Symbol::EndFlat)
+                }
+                Some('>') => {
+                    self.next(); self.next();
+                    self.mode = Mode::Text;
+                    Some(Symbol::EndPoint)
+                }
+                _ => {
+                    None
+                }
+            }
+        } else {
+            None
+        }
+    }
+}
+
+impl<'a> Iterator for Lexer<'a> {
+    type Item = Symbol<'a>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        self.next_symbol()
+    }
+}
+
diff --git a/src/lib.rs b/src/lib.rs
new file mode 100644
index 0000000..57f777e
--- /dev/null
+++ b/src/lib.rs
@@ -0,0 +1,75 @@
+use std::path::{PathBuf,Path};
+use std::collections::HashMap;
+use core::error::Error;
+
+mod lexer;
+
+pub struct Tx {
+    fragments: HashMap<String, Fragment>, 
+}
+
+enum FragmentSource {
+    File {
+        path: PathBuf,
+        loaded: usize,
+    }
+}
+
+struct Fragment {
+    name: String,
+    source: FragmentSource,
+    sections: HashMap<String, Section>,
+}
+
+struct Section {
+    name: String,
+    ast_root: Token,
+}
+
+type Properties = HashMap<String, String>;
+
+enum Token {
+    Root(Vec<Token>),
+    Fragment(String,Properties,Vec<Token>),
+    Section(String,Properties,Vec<Token>),
+    Text(String),
+
+}
+
+struct State {
+}
+
+impl Tx {
+    pub fn new() -> Self {
+        Self {
+            fragments: HashMap::new(),
+        }
+    }
+
+    pub fn load(path: &Path) -> Result<(),Box::<dyn Error>> {
+        Ok(())
+    }
+
+    pub fn parse(data: String) -> Result<(),Box::<dyn Error>> {
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn lexing() {
+        let data = r#"Leading comment: [|fragment "basic"|>Hello there <|fragment|] Trailing text"#;
+        let ll = lexer::Lexer::new( &data );
+        for sym in ll {
+            if let lexer::Symbol::Error{line,row,what} = sym {
+                println!("Error {}:{}: {}", line, row, what );
+                break;
+            } else {
+                println!("Symbol: {:?}", sym );
+            }
+        }
+    }
+}