From c53bc78b4682365dd0858b0a5d5d8059915409f9 Mon Sep 17 00:00:00 2001 From: YannAhlgrim Date: Wed, 3 Jun 2026 16:35:43 +0200 Subject: [PATCH] finish parser + change token types to enum --- src/ast/mod.rs | 297 ++++++++++++++++++++++++++++++++--- src/lexer/mod.rs | 72 +++------ src/parser/mod.rs | 393 +++++++++++++++++++++++++++++++++++++++++----- src/repl/mod.rs | 5 +- src/token/mod.rs | 118 +++++++++----- 5 files changed, 742 insertions(+), 143 deletions(-) diff --git a/src/ast/mod.rs b/src/ast/mod.rs index c53e457..759d1c6 100644 --- a/src/ast/mod.rs +++ b/src/ast/mod.rs @@ -1,32 +1,65 @@ -use crate::token; +use crate::token::Token; pub trait NodeTrait { fn token_literal(&self) -> String; + fn string(&self) -> String; } pub trait StatementTrait: NodeTrait { fn statement_node(&self); } -trait ExpressionTrait: NodeTrait { - fn expression_node(&self); -} -pub struct Expression { - // +pub enum Expression { + Identifier(Identifier), + IntegerLiteral(IntegerLiteral), + PrefixExpression(PrefixExpression), + InfixExpression(InfixExpression), + Boolean(Boolean), + BlockStatement(BlockStatement), + IfExpression(IfExpression), + FunctionLiteral(FunctionLiteral), + CallExpression(CallExpression), } + impl NodeTrait for Expression { fn token_literal(&self) -> String { - todo!() + match self { + Expression::Identifier(i) => i.token_literal(), + Expression::IntegerLiteral(i) => i.token_literal(), + Expression::PrefixExpression(i) => i.token_literal(), + Expression::InfixExpression(i) => i.token_literal(), + Expression::Boolean(i) => i.token_literal(), + Expression::BlockStatement(i) => i.token_literal(), + Expression::IfExpression(i) => i.token_literal(), + Expression::FunctionLiteral(i) => i.token_literal(), + Expression::CallExpression(i) => i.token_literal(), + } + } + + fn string(&self) -> String { + match self { + Expression::Identifier(i) => i.string(), + Expression::IntegerLiteral(i) => i.string(), + Expression::PrefixExpression(i) => i.string(), + Expression::InfixExpression(i) => i.string(), + Expression::Boolean(i) => i.string(), + Expression::BlockStatement(i) => i.string(), + Expression::IfExpression(i) => i.string(), + Expression::FunctionLiteral(i) => i.string(), + Expression::CallExpression(i) => i.string(), + } } } -pub struct Statement { - // -} +pub struct Statement; impl NodeTrait for Statement { fn token_literal(&self) -> String { todo!() } + + fn string(&self) -> String { + todo!() + } } impl StatementTrait for Statement { @@ -45,28 +78,89 @@ impl NodeTrait for Program { String::new() } } + + fn string(&self) -> String { + let mut out = String::new(); + for stmt in &self.statements { + out.push_str(&stmt.string()); + } + out + } } pub struct LetStatement { - pub token: token::Token, + pub token: Token, pub name: Identifier, pub value: Expression, } +pub struct ReturnStatement { + pub token: Token, + pub return_value: Expression, +} + +pub struct ExpressionStatement { + pub token: Token, + pub expression: Expression, +} + impl NodeTrait for LetStatement { fn token_literal(&self) -> String { String::from(&self.token.literal) } -} -impl StatementTrait for LetStatement { - fn statement_node(&self) { - todo!() + fn string(&self) -> String { + let mut out = String::new(); + out.push_str(&self.token_literal()); + out.push_str(" "); + out.push_str(&self.name.string()); + out.push_str(" = "); + out.push_str(&self.value.string()); + out.push_str(";"); + out } } +impl StatementTrait for LetStatement { + fn statement_node(&self) {} +} + +impl NodeTrait for ReturnStatement { + fn token_literal(&self) -> String { + String::from(&self.token.literal) + } + + fn string(&self) -> String { + let mut out = String::new(); + out.push_str(&self.token_literal()); + out.push_str(" "); + out.push_str(&self.return_value.string()); + out.push_str(";"); + out + } +} + +impl StatementTrait for ReturnStatement { + fn statement_node(&self) {} +} + +impl NodeTrait for ExpressionStatement { + fn token_literal(&self) -> String { + String::from(&self.token.literal) + } + + fn string(&self) -> String { + self.expression.string() + } +} + +impl StatementTrait for ExpressionStatement { + fn statement_node(&self) {} +} + +#[derive(Debug, Default)] pub struct Identifier { - pub token: token::Token, + pub token: Token, pub value: String, } @@ -74,10 +168,173 @@ impl NodeTrait for Identifier { fn token_literal(&self) -> String { String::from(&self.token.literal) } -} -impl ExpressionTrait for Identifier { - fn expression_node(&self) { - todo!() + fn string(&self) -> String { + self.value.clone() + } +} + +#[derive(Debug)] +pub struct IntegerLiteral { + pub token: Token, + pub value: i64, +} + +impl NodeTrait for IntegerLiteral { + fn token_literal(&self) -> String { + String::from(&self.token.literal) + } + + fn string(&self) -> String { + self.token.literal.clone() + } +} + +pub struct PrefixExpression { + pub token: Token, + pub operator: String, + pub right: Box, +} + +impl NodeTrait for PrefixExpression { + fn token_literal(&self) -> String { + String::from(&self.token.literal) + } + + fn string(&self) -> String { + let mut out = String::new(); + out.push_str(&self.operator); + out.push_str(&self.right.string()); + out + } +} + +pub struct InfixExpression { + pub token: Token, + pub left: Box, + pub operator: String, + pub right: Box, +} + +impl NodeTrait for InfixExpression { + fn token_literal(&self) -> String { + String::from(&self.token.literal) + } + + fn string(&self) -> String { + let mut out = String::new(); + out.push_str("("); + out.push_str(&self.left.string()); + out.push_str(" "); + out.push_str(&self.operator); + out.push_str(" "); + out.push_str(&self.right.string()); + out.push_str(")"); + out + } +} + +pub struct Boolean { + pub token: Token, + pub value: bool, +} + +impl NodeTrait for Boolean { + fn token_literal(&self) -> String { + String::from(&self.token.literal) + } + + fn string(&self) -> String { + self.token.literal.clone() + } +} + +pub struct IfExpression { + pub token: Token, + pub condition: Box, + pub consequence: BlockStatement, + pub alternative: Option, +} + +impl NodeTrait for IfExpression { + fn token_literal(&self) -> String { + String::from(&self.token.literal) + } + + fn string(&self) -> String { + let mut out = String::new(); + out.push_str("if"); + out.push_str(&self.condition.string()); + out.push_str(" "); + out.push_str(&self.consequence.string()); + if let Some(alt) = &self.alternative { + out.push_str("else "); + out.push_str(&alt.string()); + } + out + } +} + +pub struct BlockStatement { + pub token: Token, + pub statements: Vec>, +} + +impl NodeTrait for BlockStatement { + fn token_literal(&self) -> String { + String::from(&self.token.literal) + } + + fn string(&self) -> String { + let mut out = String::new(); + for stmt in &self.statements { + out.push_str(&stmt.string()); + } + out + } +} + +pub struct FunctionLiteral { + pub token: Token, + pub parameters: Vec, + pub body: BlockStatement, +} + +impl NodeTrait for FunctionLiteral { + fn token_literal(&self) -> String { + String::from(&self.token.literal) + } + + fn string(&self) -> String { + let mut out = String::new(); + out.push_str(&self.token_literal()); + out.push_str("("); + let params: Vec = self.parameters.iter().map(|p| p.string()).collect(); + out.push_str(¶ms.join(", ")); + out.push_str(") "); + out.push_str(&self.body.string()); + out + } +} + +pub struct CallExpression { + pub token: Token, + pub function: Box, + pub arguments: Vec, +} + +impl NodeTrait for CallExpression { + fn token_literal(&self) -> String { + String::from(&self.token.literal) + } + + fn string(&self) -> String { + let mut out = String::new(); + out.push_str(&self.function.string()); + out.push_str("("); + let args: Vec = self.arguments.iter().map(|a| a.string()).collect(); + out.push_str(&args.join(", ")); + out.push_str(")"); + out } } diff --git a/src/lexer/mod.rs b/src/lexer/mod.rs index 4740f70..bcbba93 100644 --- a/src/lexer/mod.rs +++ b/src/lexer/mod.rs @@ -1,25 +1,5 @@ use crate::token::lookup_ident; - -use super::token::ASSIGN; -use super::token::ASTERISK; -use super::token::BANG; -use super::token::COMMA; -use super::token::EOF; -use super::token::EQ; -use super::token::GT; -use super::token::ILLEGAL; -use super::token::INT; -use super::token::LBRACE; -use super::token::LPAREN; -use super::token::LT; -use super::token::MINUS; -use super::token::NQ; -use super::token::PLUS; -use super::token::RBRACE; -use super::token::RPAREN; -use super::token::SEMICOLON; -use super::token::SLASH; -use super::token::Token; +use crate::token::{Token, TokenType}; pub struct Lexer { input: String, @@ -67,44 +47,46 @@ impl LexerTraits for Lexer { } fn next_token(&mut self) -> Token { - #[allow(unused_assignments)] - let mut tok = Token::default(); + let mut tok = Token { + type_: TokenType::Illegal, + literal: String::new(), + }; self.skip_whitespace(); let lit = self.ch.unwrap(); let c = char::from(lit); match c { - '\0' => tok = new_token(EOF, lit), + '\0' => tok = new_token(TokenType::Eof, lit), '=' => { if char::from(self.peek_char()) == '=' { self.read_char(); let lit = String::from("=="); - tok = new_token_from_str(EQ, lit); + tok = new_token_from_str(TokenType::Eq, lit); } else { - tok = new_token(ASSIGN, lit); + tok = new_token(TokenType::Assign, lit); } } - ';' => tok = new_token(SEMICOLON, lit), - '(' => tok = new_token(LPAREN, lit), - ')' => tok = new_token(RPAREN, lit), - ',' => tok = new_token(COMMA, lit), - '+' => tok = new_token(PLUS, lit), - '-' => tok = new_token(MINUS, lit), + ';' => tok = new_token(TokenType::Semicolon, lit), + '(' => tok = new_token(TokenType::Lparen, lit), + ')' => tok = new_token(TokenType::Rparen, lit), + ',' => tok = new_token(TokenType::Comma, lit), + '+' => tok = new_token(TokenType::Plus, lit), + '-' => tok = new_token(TokenType::Minus, lit), '!' => { if char::from(self.peek_char()) == '=' { self.read_char(); let lit = String::from("!="); - tok = new_token_from_str(NQ, lit); + tok = new_token_from_str(TokenType::Neq, lit); } else { - tok = new_token(BANG, lit); + tok = new_token(TokenType::Bang, lit); } } - '/' => tok = new_token(SLASH, lit), - '*' => tok = new_token(ASTERISK, lit), - '<' => tok = new_token(LT, lit), - '>' => tok = new_token(GT, lit), - '{' => tok = new_token(LBRACE, lit), - '}' => tok = new_token(RBRACE, lit), + '/' => tok = new_token(TokenType::Slash, lit), + '*' => tok = new_token(TokenType::Asterisk, lit), + '<' => tok = new_token(TokenType::Lt, lit), + '>' => tok = new_token(TokenType::Gt, lit), + '{' => tok = new_token(TokenType::Lbrace, lit), + '}' => tok = new_token(TokenType::Rbrace, lit), _ => { if c.is_alphabetic() { let lit = self.read_identifier(); @@ -112,9 +94,9 @@ impl LexerTraits for Lexer { return new_token_from_str(tok_type, lit); } else if c.is_ascii_digit() { let lit = self.read_number(); - return new_token_from_str(INT, lit); + return new_token_from_str(TokenType::Int, lit); } else { - tok = new_token(ILLEGAL, lit); + tok = new_token(TokenType::Illegal, lit); } } } @@ -166,17 +148,15 @@ impl LexerTraits for Lexer { } } -fn new_token(token_type: &str, ch: u8) -> Token { +fn new_token(token_type: TokenType, ch: u8) -> Token { let lit = String::from(char::from(ch)); - let token_type = String::from(token_type); Token { type_: token_type, literal: lit, } } -fn new_token_from_str(token_type: &str, lit: String) -> Token { - let token_type = String::from(token_type); +fn new_token_from_str(token_type: TokenType, lit: String) -> Token { Token { type_: token_type, literal: lit, diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 92be775..23b1346 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -1,52 +1,118 @@ use crate::{ ast::{self}, lexer::{self, LexerTraits}, - token::{self, EOF}, + token::{self, TokenType}, }; +use std::collections::HashMap; + +type PrefixParseFn = fn(&mut Parser) -> Option; +type InfixParseFn = fn(&mut Parser, ast::Expression) -> Option; pub struct Parser { lexer: lexer::Lexer, cur_token: token::Token, peek_token: token::Token, errors: Vec, -} - -pub trait ParserTrait { - fn next_token(&mut self); - fn parse_program(&mut self) -> ast::Program; - fn parse_statement(&mut self) -> Option>; - fn parse_let_statement(&mut self) -> Option>; - - fn cur_token_is(&self, t: String) -> bool; - fn peek_token_is(&self, t: String) -> bool; - fn expect_peek(&mut self, t: String) -> bool; - fn errors(&self) -> &Vec; - fn peek_error(&mut self, t: String); + prefix_parse_fns: HashMap, + infix_parse_fns: HashMap, } pub fn new_parser(l: lexer::Lexer) -> Parser { - let mut p: Parser = Parser { + let mut p = Parser { lexer: l, cur_token: token::Token { literal: String::new(), - type_: String::new(), + type_: TokenType::Illegal, }, peek_token: token::Token { literal: String::new(), - type_: String::new(), + type_: TokenType::Illegal, }, errors: Vec::new(), + prefix_parse_fns: HashMap::new(), + infix_parse_fns: HashMap::new(), }; + p.register_prefix(TokenType::Ident, Parser::parse_identifier); + p.register_prefix(TokenType::Int, Parser::parse_integer_literal); + p.register_prefix(TokenType::Bang, Parser::parse_prefix_expression); + p.register_prefix(TokenType::Minus, Parser::parse_prefix_expression); + p.register_infix(TokenType::Plus, Parser::parse_infix_expression); + p.register_infix(TokenType::Minus, Parser::parse_infix_expression); + p.register_infix(TokenType::Slash, Parser::parse_infix_expression); + p.register_infix(TokenType::Asterisk, Parser::parse_infix_expression); + p.register_infix(TokenType::Eq, Parser::parse_infix_expression); + p.register_infix(TokenType::Neq, Parser::parse_infix_expression); + p.register_infix(TokenType::Lt, Parser::parse_infix_expression); + p.register_infix(TokenType::Gt, Parser::parse_infix_expression); + p.register_prefix(TokenType::True, Parser::parse_boolean); + p.register_prefix(TokenType::False, Parser::parse_boolean); + p.register_prefix(TokenType::Lparen, Parser::parse_grouped_expression); + p.register_prefix(TokenType::If, Parser::parse_if_expression); + p.register_prefix(TokenType::Function, Parser::parse_function_literal); + p.register_infix(TokenType::Lparen, Parser::parse_call_expression); p.next_token(); p.next_token(); p } -impl ParserTrait for Parser { +impl Parser { + fn register_prefix(&mut self, t: TokenType, f: PrefixParseFn) { + self.prefix_parse_fns.insert(t, f); + } + + fn register_infix(&mut self, t: TokenType, f: InfixParseFn) { + self.infix_parse_fns.insert(t, f); + } + + fn parse_identifier(&mut self) -> Option { + Some(ast::Expression::Identifier(ast::Identifier { + token: self.cur_token.clone(), + value: self.cur_token.literal.clone(), + })) + } + + fn parse_integer_literal(&mut self) -> Option { + let value: i64 = self.cur_token.literal.parse().ok()?; + Some(ast::Expression::IntegerLiteral(ast::IntegerLiteral { + token: self.cur_token.clone(), + value, + })) + } + + fn no_prefix_parse_fn_error(&mut self) { + let msg = format!( + "no prefix parse function for {} found", + self.cur_token.type_ + ); + self.errors.push(msg); + } + + fn peek_precedence(&self) -> Precedence { + match self.peek_token.type_ { + TokenType::Eq | TokenType::Neq => Precedence::Equals, + TokenType::Lt | TokenType::Gt => Precedence::Lessgreater, + TokenType::Plus | TokenType::Minus => Precedence::Sum, + TokenType::Slash | TokenType::Asterisk => Precedence::Product, + TokenType::Lparen => Precedence::Call, + _ => Precedence::Lowest, + } + } + + fn cur_precedence(&self) -> Precedence { + match self.cur_token.type_ { + TokenType::Eq | TokenType::Neq => Precedence::Equals, + TokenType::Lt | TokenType::Gt => Precedence::Lessgreater, + TokenType::Plus | TokenType::Minus => Precedence::Sum, + TokenType::Slash | TokenType::Asterisk => Precedence::Product, + TokenType::Lparen => Precedence::Call, + _ => Precedence::Lowest, + } + } + fn next_token(&mut self) { self.cur_token = token::Token { literal: String::from(&self.peek_token.literal), - type_: String::from(&self.peek_token.type_), + type_: self.peek_token.type_, }; self.peek_token = self.lexer.next_token(); } @@ -55,7 +121,7 @@ impl ParserTrait for Parser { let mut program = ast::Program { statements: Vec::new(), }; - while &self.cur_token.type_ != EOF { + while self.cur_token.type_ != TokenType::Eof { if let Some(stmt) = self.parse_statement() { program.statements.push(stmt); } @@ -65,21 +131,22 @@ impl ParserTrait for Parser { } fn parse_statement(&mut self) -> Option> { - match self.cur_token.type_.as_str() { - token::LET => self.parse_let_statement(), - _ => None, + match self.cur_token.type_ { + TokenType::Let => self.parse_let_statement(), + TokenType::Return => self.parse_return_statement(), + _ => self.parse_expression_statement(), } } - fn cur_token_is(&self, t: String) -> bool { + fn cur_token_is(&self, t: TokenType) -> bool { self.cur_token.type_ == t } - fn peek_token_is(&self, t: String) -> bool { + fn peek_token_is(&self, t: TokenType) -> bool { self.peek_token.type_ == t } - fn expect_peek(&mut self, t: String) -> bool { + fn expect_peek(&mut self, t: TokenType) -> bool { if self.peek_token_is(t) { self.next_token(); true @@ -92,46 +159,291 @@ impl ParserTrait for Parser { let mut stmt = ast::LetStatement { token: token::Token { literal: String::from(&self.cur_token.literal), - type_: String::from(&self.cur_token.type_), + type_: self.cur_token.type_.clone(), }, name: ast::Identifier { token: token::Token { literal: String::new(), - type_: String::new(), + type_: TokenType::Illegal, }, value: String::new(), }, - value: ast::Expression {}, + value: ast::Expression::Identifier(ast::Identifier { + token: token::Token { + literal: String::new(), + type_: TokenType::Illegal, + }, + value: String::new(), + }), }; - if !self.expect_peek(token::IDENT.to_string()) { + if !self.expect_peek(TokenType::Ident) { return None; } stmt.name = ast::Identifier { token: token::Token { literal: String::from(&self.cur_token.literal), - type_: String::from(&self.cur_token.type_), + type_: self.cur_token.type_.clone(), }, value: String::from(&self.cur_token.literal), }; - if !self.expect_peek(token::ASSIGN.to_string()) { + if !self.expect_peek(TokenType::Assign) { return None; } - while !self.cur_token_is(token::SEMICOLON.to_string()) { + self.next_token(); + + stmt.value = self.parse_expression(Precedence::Lowest)?; + + if self.peek_token_is(TokenType::Semicolon) { self.next_token(); } - Some(Box::from(stmt)) + Some(Box::new(stmt)) + } + + fn parse_return_statement(&mut self) -> Option> { + let mut stmt = ast::ReturnStatement { + token: token::Token { + literal: String::from(&self.cur_token.literal), + type_: self.cur_token.type_.clone(), + }, + return_value: ast::Expression::Identifier(ast::Identifier { + token: token::Token { + literal: String::new(), + type_: TokenType::Illegal, + }, + value: String::new(), + }), + }; + + self.next_token(); + + stmt.return_value = self.parse_expression(Precedence::Lowest)?; + + if self.peek_token_is(TokenType::Semicolon) { + self.next_token(); + } + + Some(Box::new(stmt)) + } + + fn parse_expression_statement(&mut self) -> Option> { + let stmt = ast::ExpressionStatement { + token: token::Token { + literal: String::from(&self.cur_token.literal), + type_: self.cur_token.type_, + }, + expression: self.parse_expression(Precedence::Lowest).unwrap(), + }; + + if self.peek_token_is(TokenType::Semicolon) { + self.next_token(); + } + + Some(Box::new(stmt)) + } + + fn parse_expression(&mut self, precedence: Precedence) -> Option { + let prefix = self.prefix_parse_fns.get(&self.cur_token.type_).copied(); + let mut left = match prefix { + Some(f) => f(self)?, + None => { + self.no_prefix_parse_fn_error(); + return None; + } + }; + while !self.peek_token_is(TokenType::Semicolon) && precedence < self.peek_precedence() { + let infix = self.infix_parse_fns.get(&self.peek_token.type_).copied(); + match infix { + Some(f) => { + self.next_token(); + left = f(self, left)?; + } + None => return Some(left), + } + } + Some(left) + } + + fn parse_prefix_expression(&mut self) -> Option { + let token = self.cur_token.clone(); + let operator = self.cur_token.literal.clone(); + self.next_token(); + let right = self.parse_expression(Precedence::Prefix)?; + Some(ast::Expression::PrefixExpression(ast::PrefixExpression { + token, + operator, + right: Box::new(right), + })) + } + + fn parse_infix_expression(&mut self, left: ast::Expression) -> Option { + let token = self.cur_token.clone(); + let operator = self.cur_token.literal.clone(); + let precedence = self.cur_precedence(); + self.next_token(); + let right = self.parse_expression(precedence)?; + Some(ast::Expression::InfixExpression(ast::InfixExpression { + token, + left: Box::new(left), + operator, + right: Box::new(right), + })) + } + + fn parse_boolean(&mut self) -> Option { + Some(ast::Expression::Boolean(ast::Boolean { + token: self.cur_token.clone(), + value: self.cur_token_is(TokenType::True), + })) + } + + fn parse_grouped_expression(&mut self) -> Option { + self.next_token(); + let exp = self.parse_expression(Precedence::Lowest)?; + if !self.expect_peek(TokenType::Rparen) { + return None; + } + Some(exp) + } + + fn parse_if_expression(&mut self) -> Option { + let mut expression = ast::IfExpression { + token: self.cur_token.clone(), + condition: Box::new(ast::Expression::Identifier(ast::Identifier { + token: token::Token { + literal: String::new(), + type_: TokenType::Illegal, + }, + value: String::new(), + })), + consequence: ast::BlockStatement { + token: token::Token { + literal: String::new(), + type_: TokenType::Illegal, + }, + statements: Vec::new(), + }, + alternative: None, + }; + if !self.expect_peek(TokenType::Lparen) { + return None; + } + self.next_token(); + expression.condition = Box::new(self.parse_expression(Precedence::Lowest)?); + if !self.expect_peek(TokenType::Rparen) { + return None; + } + if !self.expect_peek(TokenType::Lbrace) { + return None; + } + expression.consequence = self.parse_block_statement()?; + + if self.peek_token_is(TokenType::Else) { + self.next_token(); + if !self.expect_peek(TokenType::Lbrace) { + return None; + } + expression.alternative = Some(self.parse_block_statement()?); + } + + return Some(ast::Expression::IfExpression(expression)); + } + + fn parse_block_statement(&mut self) -> Option { + let mut block = ast::BlockStatement { + token: self.cur_token.clone(), + statements: Vec::new(), + }; + self.next_token(); + while !self.cur_token_is(TokenType::Rbrace) && !self.cur_token_is(TokenType::Eof) { + if let Some(stmt) = self.parse_statement() { + block.statements.push(stmt); + } + self.next_token(); + } + Some(block) + } + + fn parse_function_literal(&mut self) -> Option { + let token = self.cur_token.clone(); + if !self.expect_peek(TokenType::Lparen) { + return None; + } + let parameters = self.parse_function_parameters()?; + if !self.expect_peek(TokenType::Lbrace) { + return None; + } + let body = self.parse_block_statement()?; + Some(ast::Expression::FunctionLiteral(ast::FunctionLiteral { + token, + parameters, + body, + })) + } + + fn parse_function_parameters(&mut self) -> Option> { + let mut identifiers = Vec::new(); + if self.peek_token_is(TokenType::Rparen) { + self.next_token(); + return Some(identifiers); + } + self.next_token(); + identifiers.push(ast::Identifier { + token: self.cur_token.clone(), + value: self.cur_token.literal.clone(), + }); + while self.peek_token_is(TokenType::Comma) { + self.next_token(); + self.next_token(); + identifiers.push(ast::Identifier { + token: self.cur_token.clone(), + value: self.cur_token.literal.clone(), + }); + } + if !self.expect_peek(TokenType::Rparen) { + return None; + } + Some(identifiers) + } + + fn parse_call_expression(&mut self, function: ast::Expression) -> Option { + let token = self.cur_token.clone(); + let arguments = self.parse_call_arguments()?; + Some(ast::Expression::CallExpression(ast::CallExpression { + token, + function: Box::new(function), + arguments, + })) + } + + fn parse_call_arguments(&mut self) -> Option> { + let mut args = Vec::new(); + if self.peek_token_is(TokenType::Rparen) { + self.next_token(); + return Some(args); + } + self.next_token(); + args.push(self.parse_expression(Precedence::Lowest)?); + while self.peek_token_is(TokenType::Comma) { + self.next_token(); + self.next_token(); + args.push(self.parse_expression(Precedence::Lowest)?); + } + if !self.expect_peek(TokenType::Rparen) { + return None; + } + Some(args) } fn errors(&self) -> &Vec { &self.errors } - fn peek_error(&mut self, t: String) { + fn peek_error(&mut self, t: TokenType) { let msg = format!( "expected next token to be {}, got {} instead", t, self.peek_token.type_ @@ -139,3 +451,14 @@ impl ParserTrait for Parser { self.errors.push(msg); } } + +#[derive(Debug, PartialEq, PartialOrd)] +pub enum Precedence { + Lowest, + Equals, + Lessgreater, + Sum, + Product, + Prefix, + Call, +} diff --git a/src/repl/mod.rs b/src/repl/mod.rs index 72d64dc..3bb7b54 100644 --- a/src/repl/mod.rs +++ b/src/repl/mod.rs @@ -1,3 +1,4 @@ +use crate::ast::NodeTrait; use crate::lexer::new; use crate::parser::Parser; use crate::parser::ParserTrait; @@ -20,8 +21,6 @@ pub fn start_repl() { let lexer = new(input.to_string()); let mut parser: Parser = new_parser(lexer); let program = parser.parse_program(); - for stmt in program.statements { - println!("{:?}", stmt.token_literal()); - } + println!("{}", program.string()); } } diff --git a/src/token/mod.rs b/src/token/mod.rs index 411df06..ed65960 100644 --- a/src/token/mod.rs +++ b/src/token/mod.rs @@ -1,46 +1,86 @@ -#[derive(Default)] +use std::fmt; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] +pub enum TokenType { + #[default] + Illegal, + Eof, + Ident, + Int, + Assign, + Plus, + Minus, + Bang, + Asterisk, + Slash, + Lt, + Gt, + Comma, + Semicolon, + Lparen, + Rparen, + Lbrace, + Rbrace, + Function, + Let, + True, + False, + If, + Else, + Return, + Eq, + Neq, +} + +impl fmt::Display for TokenType { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + TokenType::Illegal => write!(f, "ILLEGAL"), + TokenType::Eof => write!(f, "EOF"), + TokenType::Ident => write!(f, "IDENT"), + TokenType::Int => write!(f, "INT"), + TokenType::Assign => write!(f, "="), + TokenType::Plus => write!(f, "+"), + TokenType::Minus => write!(f, "-"), + TokenType::Bang => write!(f, "!"), + TokenType::Asterisk => write!(f, "*"), + TokenType::Slash => write!(f, "/"), + TokenType::Lt => write!(f, "<"), + TokenType::Gt => write!(f, ">"), + TokenType::Comma => write!(f, ","), + TokenType::Semicolon => write!(f, ";"), + TokenType::Lparen => write!(f, "("), + TokenType::Rparen => write!(f, ")"), + TokenType::Lbrace => write!(f, "{{"), + TokenType::Rbrace => write!(f, "}}"), + TokenType::Function => write!(f, "FUNCTION"), + TokenType::Let => write!(f, "LET"), + TokenType::True => write!(f, "TRUE"), + TokenType::False => write!(f, "FALSE"), + TokenType::If => write!(f, "IF"), + TokenType::Else => write!(f, "ELSE"), + TokenType::Return => write!(f, "RETURN"), + TokenType::Eq => write!(f, "=="), + TokenType::Neq => write!(f, "!="), + } + } +} + +#[derive(Debug, Clone, PartialEq, Default)] pub struct Token { - pub type_: String, + pub type_: TokenType, pub literal: String, } -pub const ILLEGAL: &str = "ILLEGAL"; -pub const EOF: &str = "EOF"; -pub const IDENT: &str = "IDENT"; -pub const INT: &str = "INT"; -pub const ASSIGN: &str = "="; -pub const PLUS: &str = "+"; -pub const MINUS: &str = "-"; -pub const BANG: &str = "!"; -pub const ASTERISK: &str = "*"; -pub const SLASH: &str = "/"; -pub const LT: &str = "<"; -pub const GT: &str = ">"; -pub const COMMA: &str = ","; -pub const SEMICOLON: &str = ";"; -pub const LPAREN: &str = "("; -pub const RPAREN: &str = ")"; -pub const LBRACE: &str = "{"; -pub const RBRACE: &str = "}"; -pub const FUNCTION: &str = "FUNCTION"; -pub const LET: &str = "LET"; -pub const TRUE: &str = "TRUE"; -pub const FALSE: &str = "FALSE"; -pub const IF: &str = "IF"; -pub const ELSE: &str = "ELSE"; -pub const RETURN: &str = "RETURN"; -pub const EQ: &str = "EQ"; -pub const NQ: &str = "NQ"; - -pub fn lookup_ident(ident: &str) -> &'static str { +pub fn lookup_ident(ident: &str) -> TokenType { match ident { - "fn" => FUNCTION, - "let" => LET, - "true" => TRUE, - "false" => FALSE, - "if" => IF, - "else" => ELSE, - "return" => RETURN, - _ => IDENT, + "fn" => TokenType::Function, + "let" => TokenType::Let, + "true" => TokenType::True, + "false" => TokenType::False, + "if" => TokenType::If, + "else" => TokenType::Else, + "return" => TokenType::Return, + _ => TokenType::Ident, } }