refactor(tazjin/rlox): Use &[char] instead of &str in scanner

This makes it easier to work with the Unicode issue. The original
string representation can be discarded.

Change-Id: I740be4cb9654679ea7950f3899c5c709b1e7a739
Reviewed-on: https://cl.tvl.fyi/c/depot/+/2160
Reviewed-by: tazjin <mail@tazj.in>
Tested-by: BuildkiteCI
This commit is contained in:
Vincent Ambo 2020-11-27 17:55:38 +01:00 committed by tazjin
parent b595553f63
commit 6363efbebf

View file

@ -55,13 +55,13 @@ pub enum TokenKind {
#[derive(Debug)]
pub struct Token<'a> {
kind: TokenKind,
lexeme: &'a str,
lexeme: &'a [char],
// literal: Object, // TODO(tazjin): Uhh?
line: usize,
}
struct Scanner<'a> {
source: &'a str,
source: &'a [char],
tokens: Vec<Token<'a>>,
errors: Vec<Error>,
start: usize, // offset of first character in current lexeme
@ -76,11 +76,7 @@ impl<'a> Scanner<'a> {
fn advance(&mut self) -> char {
self.current += 1;
// TODO(tazjin): Due to utf8-safety, this is a bit annoying.
// Since string iteration is not the point here I'm just
// leaving this as is for now.
self.source.chars().nth(self.current - 1).unwrap()
self.source[self.current-1]
}
fn add_token(&mut self, kind: TokenKind) {