Minor change to token parsing.

Tokens now can only have a-z, A-Z, or 0-9 in them, we can add more
later, _ and - maybe? but this lets us easily break on = and other
things we may want.

1 files changed, 11 insertions, 1 deletions

diff --git a/src/lexer.rs b/src/lexer.rs
index d6806d8..46651c4 100644
--- a/src/lexer.rs
+++ b/src/lexer.rs
@@ -34,6 +34,16 @@ pub struct Lexer<'a> {
     row: u32,
 }
 
+fn is_valid_token_char( c: char ) -> bool {
+    if c.is_whitespace() {
+        return false;
+    }
+    match c {
+        'a'..'z' | 'A'..'Z' | '0'..'9' => true,
+        _ => false
+    }
+}
+
 impl<'a> Lexer<'a> {
     pub fn new( data: &'a str ) -> Lexer<'a> {
         let mut chars = data.char_indices();
@@ -153,7 +163,7 @@ impl<'a> Lexer<'a> {
         }
         let start = self.cur_index();
         
-        while self.next().is_some_and(|ch| !ch.is_whitespace() ) &&
+        while self.next().is_some_and(|ch| is_valid_token_char(ch) ) &&
             !self.is_end_tag() {}
         let end = self.cur_index();
         let s = &self.data[start..end];