/* This file is part of DarkFi (https://dark.fi)
*
* Copyright (C) 2020-2022 Dyne.org foundation
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see .
*/
use std::str::Chars;
use super::error::ErrorEmitter;
const SPECIAL_CHARS: [char; 7] = ['{', '}', '(', ')', ',', ';', '='];
fn is_letter(ch: char) -> bool {
('a'..='z').contains(&ch) || ('A'..='Z').contains(&ch) || ch == '_'
}
fn is_digit(ch: char) -> bool {
('0'..='9').contains(&ch)
}
#[derive(Copy, Clone, PartialEq, Eq, Debug)]
pub enum TokenType {
Symbol,
String,
Number,
LeftBrace,
RightBrace,
LeftParen,
RightParen,
Comma,
Semicolon,
Assign,
}
#[derive(Clone, Debug)]
pub struct Token {
pub token: String,
pub token_type: TokenType,
pub line: usize,
pub column: usize,
}
impl Token {
fn new(token: &str, token_type: TokenType, line: usize, column: usize) -> Self {
Self { token: token.to_string(), token_type, line, column }
}
}
pub struct Lexer<'a> {
source: Chars<'a>,
error: ErrorEmitter,
}
impl<'a> Lexer<'a> {
pub fn new(filename: &str, source: Chars<'a>) -> Self {
// For nice error reporting, we'll load everything into a string
// vector so we have references to lines.
let lines: Vec = source.as_str().lines().map(|x| x.to_string()).collect();
let error = ErrorEmitter::new("Lexer", filename, lines);
Self { source, error }
}
pub fn lex(&self) -> Vec {
let mut tokens = vec![];
let mut lineno = 1;
let mut column = 0;
// We use this as a buffer to store a single token, which is then
// reset after a token is pushed to the returning vec.
let mut buf = String::new();
// We use these to keep state when iterating.
let mut in_comment = false;
let mut in_string = false;
let mut in_number = false;
let mut in_symbol = false;
macro_rules! new_symbol {
() => {
tokens.push(Token::new(&buf, TokenType::Symbol, lineno, column - buf.len()));
in_symbol = false;
buf = String::new();
};
}
macro_rules! new_string {
() => {
tokens.push(Token::new(&buf, TokenType::String, lineno, column - buf.len()));
in_string = false;
buf = String::new();
};
}
macro_rules! new_number {
() => {
tokens.push(Token::new(&buf, TokenType::Number, lineno, column - buf.len()));
in_number = false;
buf = String::new();
};
}
#[allow(clippy::explicit_counter_loop)]
for c in self.source.clone() {
column += 1;
if c == '\n' {
if in_symbol {
new_symbol!();
}
if in_string {
self.error.abort("Strings can't contain newlines", lineno, column);
}
if in_number {
self.error.abort("Numbers can't contain newlines", lineno, column);
}
in_comment = false;
lineno += 1;
column = 0;
continue
}
if c == '#' || in_comment {
if in_symbol {
new_symbol!();
}
if in_number {
new_number!();
}
if in_string {
buf.push(c);
continue
}
in_comment = true;
continue
}
if c.is_whitespace() {
if in_symbol {
new_symbol!();
}
if in_number {
new_number!();
}
if in_string {
// TODO: Perhaps forbid whitespace.
buf.push(c);
}
continue
}
// Main cases, in_comment is already checked above.
if !in_number && !in_symbol && !in_string && is_digit(c) {
in_number = true;
buf.push(c);
continue
}
if in_number && !is_digit(c) {
new_number!();
}
if in_number && is_digit(c) {
buf.push(c);
continue
}
if !in_number && !in_symbol && !in_string && is_letter(c) {
in_symbol = true;
buf.push(c);
continue
}
if !in_number && !in_symbol && !in_string && c == '"' {
// " I need to fix my Rust vis lexer
in_string = true;
continue
}
if (in_symbol || in_string) && (is_letter(c) || is_digit(c)) {
buf.push(c);
continue
}
if in_string && c == '"' {
// " I need to fix my vis lexer
if buf.is_empty() {
self.error.abort("String cannot be empty", lineno, column);
}
new_string!();
continue
}
if SPECIAL_CHARS.contains(&c) {
if in_symbol {
new_symbol!();
}
if in_number {
new_number!();
}
if in_string {
// TODO: Perhaps forbid these chars inside strings.
}
match c {
'{' => {
tokens.push(Token::new("{", TokenType::LeftBrace, lineno, column));
continue
}
'}' => {
tokens.push(Token::new("}", TokenType::RightBrace, lineno, column));
continue
}
'(' => {
tokens.push(Token::new("(", TokenType::LeftParen, lineno, column));
continue
}
')' => {
tokens.push(Token::new(")", TokenType::RightParen, lineno, column));
continue
}
',' => {
tokens.push(Token::new(",", TokenType::Comma, lineno, column));
continue
}
';' => {
tokens.push(Token::new(";", TokenType::Semicolon, lineno, column));
continue
}
'=' => {
tokens.push(Token::new("=", TokenType::Assign, lineno, column));
continue
}
_ => self.error.abort(&format!("Invalid token `{}`", c), lineno, column - 1),
}
continue
}
self.error.abort(&format!("Invalid token `{}`", c), lineno, column - 1);
}
tokens
}
}