From f6b7dce7914f27e80da33c9b063827982e62e027 Mon Sep 17 00:00:00 2001
From: PaddiM8 <paddim8@pm.me>
Date: Sat, 6 Jun 2020 20:15:32 +0200
Subject: [PATCH] Added position data to tokens.

---
 kalk/src/lexer.rs        | 41 +++++++++++++++++++++-------------------
 kalk/src/test_helpers.rs |  1 +
 2 files changed, 23 insertions(+), 19 deletions(-)

diff --git a/kalk/src/lexer.rs b/kalk/src/lexer.rs
index 13b4481..ff9c2c5 100644
--- a/kalk/src/lexer.rs
+++ b/kalk/src/lexer.rs
@@ -29,6 +29,7 @@ pub enum TokenKind {
 pub struct Token {
     pub kind: TokenKind,
     pub value: String,
+    pub span: (usize, usize),
 }
 
 pub struct Lexer<'a> {
@@ -51,7 +52,7 @@ impl<'a> Lexer<'a> {
         // If there isn't already an EOF token, add it.
         if let TokenKind::EOF = tokens.last().unwrap().kind {
         } else {
-            tokens.push(build(TokenKind::EOF, ""));
+            tokens.push(build(TokenKind::EOF, "", (source.len(), source.len())));
         }
 
         tokens
@@ -64,7 +65,7 @@ impl<'a> Lexer<'a> {
             self.advance();
 
             if self.is_at_end() {
-                return build(TokenKind::EOF, "");
+                return build(TokenKind::EOF, "", (self.index, self.index));
             } else {
                 c = self.peek();
             }
@@ -78,19 +79,20 @@ impl<'a> Lexer<'a> {
             return self.next_identifier();
         }
 
+        let span = (self.index, self.index + 1);
         let token = match c {
-            '+' => build(TokenKind::Plus, ""),
-            '-' => build(TokenKind::Minus, ""),
-            '*' => build(TokenKind::Star, ""),
-            '/' => build(TokenKind::Slash, ""),
-            '^' => build(TokenKind::Power, ""),
-            '(' => build(TokenKind::OpenParenthesis, ""),
-            ')' => build(TokenKind::ClosedParenthesis, ""),
-            '|' => build(TokenKind::Pipe, ""),
-            '=' => build(TokenKind::Equals, ""),
-            '!' => build(TokenKind::Exclamation, ""),
-            ',' => build(TokenKind::Comma, ""),
-            _ => build(TokenKind::Unknown, ""),
+            '+' => build(TokenKind::Plus, "", span),
+            '-' => build(TokenKind::Minus, "", span),
+            '*' => build(TokenKind::Star, "", span),
+            '/' => build(TokenKind::Slash, "", span),
+            '^' => build(TokenKind::Power, "", span),
+            '(' => build(TokenKind::OpenParenthesis, "", span),
+            ')' => build(TokenKind::ClosedParenthesis, "", span),
+            '|' => build(TokenKind::Pipe, "", span),
+            '=' => build(TokenKind::Equals, "", span),
+            '!' => build(TokenKind::Exclamation, "", span),
+            ',' => build(TokenKind::Comma, "", span),
+            _ => build(TokenKind::Unknown, "", span),
         };
 
         self.advance();
@@ -110,9 +112,9 @@ impl<'a> Lexer<'a> {
         }
 
         if let Ok(value) = str::from_utf8(&self.source[start..end]) {
-            build(TokenKind::Literal, value)
+            build(TokenKind::Literal, value, (start, end))
         } else {
-            build(TokenKind::Unknown, "")
+            build(TokenKind::Unknown, "", (self.index, self.index))
         }
     }
 
@@ -132,9 +134,9 @@ impl<'a> Lexer<'a> {
                 _ => TokenKind::Identifier,
             };
 
-            build(kind, value)
+            build(kind, value, (start, end))
         } else {
-            build(TokenKind::Unknown, "")
+            build(TokenKind::Unknown, "", (self.index, self.index))
         }
     }
 
@@ -151,10 +153,11 @@ impl<'a> Lexer<'a> {
     }
 }
 
-fn build(kind: TokenKind, value: &str) -> Token {
+fn build(kind: TokenKind, value: &str, span: (usize, usize)) -> Token {
     Token {
         kind,
         value: value.to_string(),
+        span,
     }
 }
 
diff --git a/kalk/src/test_helpers.rs b/kalk/src/test_helpers.rs
index 0b1101c..66fa4e2 100644
--- a/kalk/src/test_helpers.rs
+++ b/kalk/src/test_helpers.rs
@@ -7,6 +7,7 @@ pub fn token(kind: TokenKind, value: &str) -> Token {
     Token {
         kind,
         value: value.into(),
+        span: (0, 0),
     }
 }