extern/kalker
1
0
Fork 0
mirror of https://github.com/PaddiM8/kalker.git synced 2025-01-08 22:49:07 +01:00
Code Issues Packages Projects Releases Wiki Activity

Added position data to tokens.

Browse Source
This commit is contained in:
PaddiM8 2020-06-06 20:15:32 +02:00
parent aeed8049fc
commit f6b7dce791
2 changed files with 23 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
kalk/src/lexer.rs
View File

@ -29,6 +29,7 @@ pub enum TokenKind {
pub struct Token { pub struct Token {
pub kind: TokenKind, pub kind: TokenKind,
pub value: String, pub value: String,
pub span: (usize, usize),
} }
pub struct Lexer<'a> { pub struct Lexer<'a> {
@ -51,7 +52,7 @@ impl<'a> Lexer<'a> {
// If there isn't already an EOF token, add it. // If there isn't already an EOF token, add it.
if let TokenKind::EOF = tokens.last().unwrap().kind { if let TokenKind::EOF = tokens.last().unwrap().kind {
} else { } else {
tokens.push(build(TokenKind::EOF, "")); tokens.push(build(TokenKind::EOF, "", (source.len(), source.len())));
} }
tokens tokens
@ -64,7 +65,7 @@ impl<'a> Lexer<'a> {
self.advance(); self.advance();
if self.is_at_end() { if self.is_at_end() {
return build(TokenKind::EOF, ""); return build(TokenKind::EOF, "", (self.index, self.index));
} else { } else {
c = self.peek(); c = self.peek();
} }
@ -78,19 +79,20 @@ impl<'a> Lexer<'a> {
return self.next_identifier(); return self.next_identifier();
} }
let span = (self.index, self.index + 1);
let token = match c { let token = match c {
'+' => build(TokenKind::Plus, ""), '+' => build(TokenKind::Plus, "", span),
'-' => build(TokenKind::Minus, ""), '-' => build(TokenKind::Minus, "", span),
'*' => build(TokenKind::Star, ""), '*' => build(TokenKind::Star, "", span),
'/' => build(TokenKind::Slash, ""), '/' => build(TokenKind::Slash, "", span),
'^' => build(TokenKind::Power, ""), '^' => build(TokenKind::Power, "", span),
'(' => build(TokenKind::OpenParenthesis, ""), '(' => build(TokenKind::OpenParenthesis, "", span),
')' => build(TokenKind::ClosedParenthesis, ""), ')' => build(TokenKind::ClosedParenthesis, "", span),
'|' => build(TokenKind::Pipe, ""), '|' => build(TokenKind::Pipe, "", span),
'=' => build(TokenKind::Equals, ""), '=' => build(TokenKind::Equals, "", span),
'!' => build(TokenKind::Exclamation, ""), '!' => build(TokenKind::Exclamation, "", span),
',' => build(TokenKind::Comma, ""), ',' => build(TokenKind::Comma, "", span),
_ => build(TokenKind::Unknown, ""), _ => build(TokenKind::Unknown, "", span),
}; };
self.advance(); self.advance();
@ -110,9 +112,9 @@ impl<'a> Lexer<'a> {
} }
if let Ok(value) = str::from_utf8(&self.source[start..end]) { if let Ok(value) = str::from_utf8(&self.source[start..end]) {
build(TokenKind::Literal, value) build(TokenKind::Literal, value, (start, end))
} else { } else {
build(TokenKind::Unknown, "") build(TokenKind::Unknown, "", (self.index, self.index))
} }
} }
@ -132,9 +134,9 @@ impl<'a> Lexer<'a> {
_ => TokenKind::Identifier, _ => TokenKind::Identifier,
}; };
build(kind, value) build(kind, value, (start, end))
} else { } else {
build(TokenKind::Unknown, "") build(TokenKind::Unknown, "", (self.index, self.index))
} }
} }
@ -151,10 +153,11 @@ impl<'a> Lexer<'a> {
} }
} }
fn build(kind: TokenKind, value: &str) -> Token { fn build(kind: TokenKind, value: &str, span: (usize, usize)) -> Token {
Token { Token {
kind, kind,
value: value.to_string(), value: value.to_string(),
span,
} }
} }

1
kalk/src/test_helpers.rs
View File

@ -7,6 +7,7 @@ pub fn token(kind: TokenKind, value: &str) -> Token {
Token { Token {
kind, kind,
value: value.into(), value: value.into(),
span: (0, 0),
} }
} }