extern/kalker
1
0
Fork 0
mirror of https://github.com/PaddiM8/kalker.git synced 2025-01-07 05:59:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

Added position data to tokens.

Browse Source
This commit is contained in:
PaddiM8 2020-06-06 20:15:32 +02:00
parent aeed8049fc
commit f6b7dce791
2 changed files with 23 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
kalk/src/lexer.rs
View File

@ -29,6 +29,7 @@ pub enum TokenKind {
pub struct Token {
pub kind: TokenKind,
pub value: String,
pub span: (usize, usize),
}
pub struct Lexer<'a> {
@ -51,7 +52,7 @@ impl<'a> Lexer<'a> {
// If there isn't already an EOF token, add it.
if let TokenKind::EOF = tokens.last().unwrap().kind {
} else {
tokens.push(build(TokenKind::EOF, ""));
tokens.push(build(TokenKind::EOF, "", (source.len(), source.len())));
}
tokens
@ -64,7 +65,7 @@ impl<'a> Lexer<'a> {
self.advance();
if self.is_at_end() {
return build(TokenKind::EOF, "");
return build(TokenKind::EOF, "", (self.index, self.index));
} else {
c = self.peek();
}
@ -78,19 +79,20 @@ impl<'a> Lexer<'a> {
return self.next_identifier();
}
let span = (self.index, self.index + 1);
let token = match c {
'+' => build(TokenKind::Plus, ""),
'-' => build(TokenKind::Minus, ""),
'*' => build(TokenKind::Star, ""),
'/' => build(TokenKind::Slash, ""),
'^' => build(TokenKind::Power, ""),
'(' => build(TokenKind::OpenParenthesis, ""),
')' => build(TokenKind::ClosedParenthesis, ""),
'|' => build(TokenKind::Pipe, ""),
'=' => build(TokenKind::Equals, ""),
'!' => build(TokenKind::Exclamation, ""),
',' => build(TokenKind::Comma, ""),
_ => build(TokenKind::Unknown, ""),
'+' => build(TokenKind::Plus, "", span),
'-' => build(TokenKind::Minus, "", span),
'*' => build(TokenKind::Star, "", span),
'/' => build(TokenKind::Slash, "", span),
'^' => build(TokenKind::Power, "", span),
'(' => build(TokenKind::OpenParenthesis, "", span),
')' => build(TokenKind::ClosedParenthesis, "", span),
'|' => build(TokenKind::Pipe, "", span),
'=' => build(TokenKind::Equals, "", span),
'!' => build(TokenKind::Exclamation, "", span),
',' => build(TokenKind::Comma, "", span),
_ => build(TokenKind::Unknown, "", span),
};
self.advance();
@ -110,9 +112,9 @@ impl<'a> Lexer<'a> {
}
if let Ok(value) = str::from_utf8(&self.source[start..end]) {
build(TokenKind::Literal, value)
build(TokenKind::Literal, value, (start, end))
} else {
build(TokenKind::Unknown, "")
build(TokenKind::Unknown, "", (self.index, self.index))
}
}
@ -132,9 +134,9 @@ impl<'a> Lexer<'a> {
_ => TokenKind::Identifier,
};
build(kind, value)
build(kind, value, (start, end))
} else {
build(TokenKind::Unknown, "")
build(TokenKind::Unknown, "", (self.index, self.index))
}
}
@ -151,10 +153,11 @@ impl<'a> Lexer<'a> {
}
}
fn build(kind: TokenKind, value: &str) -> Token {
fn build(kind: TokenKind, value: &str, span: (usize, usize)) -> Token {
Token {
kind,
value: value.to_string(),
span,
}
}

1
kalk/src/test_helpers.rs
View File

@ -7,6 +7,7 @@ pub fn token(kind: TokenKind, value: &str) -> Token {
Token {
kind,
value: value.into(),
span: (0, 0),
}
}