diff --git a/src/interpreter/ast/alter_table_statement.rs b/src/interpreter/ast/alter_table_statement.rs index 9f39cbd..1e79bb6 100644 --- a/src/interpreter/ast/alter_table_statement.rs +++ b/src/interpreter/ast/alter_table_statement.rs @@ -1,6 +1,7 @@ use crate::interpreter::{ ast::{parser::Parser, SqlStatement, AlterTableStatement, AlterTableAction}, - ast::helpers::common::{expect_token_type, get_table_name, token_to_data_type}, + ast::helpers::token::{expect_token_type, token_to_data_type}, + ast::helpers::common::get_table_name, tokenizer::token::TokenTypes, }; use crate::db::table::ColumnDefinition; diff --git a/src/interpreter/ast/create_statement.rs b/src/interpreter/ast/create_statement.rs index d78b475..aa21344 100644 --- a/src/interpreter/ast/create_statement.rs +++ b/src/interpreter/ast/create_statement.rs @@ -1,7 +1,8 @@ use crate::interpreter::{ ast::{ parser::Parser, CreateTableStatement, SqlStatement::{self, CreateTable}, ExistenceCheck, - helpers::common::{expect_token_type, get_table_name, exists_clause, token_to_data_type} + helpers::token::{expect_token_type, token_to_data_type}, + helpers::common::{get_table_name, exists_clause}, }, tokenizer::token::TokenTypes }; diff --git a/src/interpreter/ast/delete_statement.rs b/src/interpreter/ast/delete_statement.rs index 5bd8f56..becb9ba 100644 --- a/src/interpreter/ast/delete_statement.rs +++ b/src/interpreter/ast/delete_statement.rs @@ -2,7 +2,8 @@ use crate::interpreter::{ ast::{ parser::Parser, SqlStatement, DeleteStatement, helpers::{ - common::{expect_token_type, get_table_name}, + token::expect_token_type, + common::get_table_name, order_by_clause::get_order_by, where_clause::get_where_clause, limit_clause::get_limit } }, diff --git a/src/interpreter/ast/drop_statement.rs b/src/interpreter/ast/drop_statement.rs index c765eec..01b57c7 100644 --- a/src/interpreter/ast/drop_statement.rs +++ b/src/interpreter/ast/drop_statement.rs @@ -1,7 +1,8 @@ use crate::interpreter::{ ast::{ parser::Parser, SqlStatement, DropTableStatement, ExistenceCheck, - helpers::common::{expect_token_type, get_table_name, exists_clause} + helpers::token::expect_token_type, + helpers::common::{get_table_name, exists_clause}, }, tokenizer::token::TokenTypes }; diff --git a/src/interpreter/ast/helpers/common.rs b/src/interpreter/ast/helpers/common.rs index d5aa172..e1509ab 100644 --- a/src/interpreter/ast/helpers/common.rs +++ b/src/interpreter/ast/helpers/common.rs @@ -1,73 +1,7 @@ use crate::interpreter::{ast::{parser::Parser, ExistenceCheck}, tokenizer::token::TokenTypes}; +use crate::interpreter::ast::helpers::token::expect_token_type; -use crate::db::table::{Value, DataType}; -// Returns an error if the current token does not match the given token type -pub fn expect_token_type(parser: &Parser, token_type: TokenTypes) -> Result<(), String> { - let token = parser.current_token()?; - if token.token_type != token_type { - return Err(parser.format_error()); - } - Ok(()) -} - -pub fn token_to_value(parser: &Parser) -> Result { - let token = parser.current_token()?; - - match token.token_type { - TokenTypes::IntLiteral => { - let num = token.value.parse::() - .map_err(|_| parser.format_error())?; - Ok(Value::Integer(num)) - }, - TokenTypes::RealLiteral => { - let num = token.value.parse::() - .map_err(|_| parser.format_error())?; - Ok(Value::Real(num)) - }, - TokenTypes::String => Ok(Value::Text(token.value.to_string())), - TokenTypes::Blob => { - let bytes = decode(token.value) - .map_err(|_| parser.format_error())?; - Ok(Value::Blob(bytes)) - }, - TokenTypes::Null => Ok(Value::Null), - _ => Err(parser.format_error()), - } -} - -// Returns a list of Values from the tokens when they are formated as "value, value, ..." -pub fn tokens_to_value_list(parser: &mut Parser) -> Result, String> { - let mut values: Vec = vec![]; - loop { - values.push(token_to_value(parser)?); - parser.advance()?; - let token = parser.current_token()?; - if token.token_type != TokenTypes::Comma { - break; - } - parser.advance()?; - } - return Ok(values); -} - -// Returns a list of Strings from the tokens when they are formated as "identifier, identifier, ..." -pub fn tokens_to_identifier_list(parser: &mut Parser) -> Result, String> { - let mut identifiers: Vec = vec![]; - loop { - let token = parser.current_token()?; - expect_token_type(parser, TokenTypes::Identifier)?; - - identifiers.push(token.value.to_string()); - parser.advance()?; - let token = parser.current_token()?; - if token.token_type != TokenTypes::Comma { - break; - } - parser.advance()?; - } - return Ok(identifiers); -} pub fn get_table_name(parser: &mut Parser) -> Result { let token = parser.current_token()?; @@ -98,7 +32,7 @@ pub fn exists_clause(parser: &mut Parser, check_type: ExistenceCheck) -> Result< return Ok(None); } -fn decode(hex: &str) -> Result, String> { +pub fn hex_decode(hex: &str) -> Result, String> { if hex.len() % 2 != 0 { return Err("Hex string must have even length".to_string()); } @@ -109,41 +43,14 @@ fn decode(hex: &str) -> Result, String> { }).collect() } -pub fn token_to_data_type(parser: &mut Parser) -> Result { - let token = parser.current_token()?; - return match token.token_type { - TokenTypes::Integer => Ok(DataType::Integer), - TokenTypes::Real => Ok(DataType::Real), - TokenTypes::Text => Ok(DataType::Text), - TokenTypes::Blob => Ok(DataType::Blob), - TokenTypes::Null => Ok(DataType::Null), - _ => Err(parser.format_error()), - }; -} - #[cfg(test)] mod tests { use super::*; - use crate::interpreter::ast::test_utils::token; - use crate::interpreter::ast::parser::Parser; - use crate::interpreter::tokenizer::token::TokenTypes; - - #[test] - fn value_list_handles_single_value() { - // 1);... - let tokens = vec![ - token(TokenTypes::IntLiteral, "1"), - token(TokenTypes::RightParen, ")"), - ]; - let mut parser = Parser::new(tokens); - let result = tokens_to_value_list(&mut parser); - assert_eq!(result, Ok(vec![Value::Integer(1)])); - } #[test] fn decode_handles_valid_hex_string() { - let result = decode("0A1A3F"); + let result = hex_decode("0A1A3F"); assert!(result.is_ok()); let expected = vec![0x0A, 0x1A, 0x3F]; assert_eq!(expected, result.unwrap()); @@ -151,12 +58,12 @@ mod tests { #[test] fn decode_handles_invalid_hex_string() { - let result = decode("0AZA3A"); + let result = hex_decode("0AZA3A"); assert!(result.is_err()); let expected = "Invalid hex at 2: invalid digit found in string"; assert_eq!(expected, result.err().unwrap()); - let result = decode("0A1"); + let result = hex_decode("0A1"); assert!(result.is_err()); let expected = "Hex string must have even length"; assert_eq!(expected, result.err().unwrap()); diff --git a/src/interpreter/ast/helpers/limit_clause.rs b/src/interpreter/ast/helpers/limit_clause.rs index 481345a..7104257 100644 --- a/src/interpreter/ast/helpers/limit_clause.rs +++ b/src/interpreter/ast/helpers/limit_clause.rs @@ -1,7 +1,7 @@ use crate::interpreter::ast::{parser::Parser, LimitClause}; use crate::interpreter::tokenizer::token::TokenTypes; use crate::db::table::Value; -use crate::interpreter::ast::helpers::common::{expect_token_type, token_to_value}; +use crate::interpreter::ast::helpers::token::{expect_token_type, token_to_value}; pub fn get_limit(parser: &mut Parser) -> Result, String> { if expect_token_type(parser, TokenTypes::Limit).is_err() { diff --git a/src/interpreter/ast/helpers/mod.rs b/src/interpreter/ast/helpers/mod.rs index d640337..f972b11 100644 --- a/src/interpreter/ast/helpers/mod.rs +++ b/src/interpreter/ast/helpers/mod.rs @@ -2,4 +2,5 @@ pub mod where_clause; pub mod order_by_clause; pub mod limit_clause; pub mod common; -pub mod select_statement; \ No newline at end of file +pub mod select_statement; +pub mod token; \ No newline at end of file diff --git a/src/interpreter/ast/helpers/order_by_clause.rs b/src/interpreter/ast/helpers/order_by_clause.rs index e31e342..576f713 100644 --- a/src/interpreter/ast/helpers/order_by_clause.rs +++ b/src/interpreter/ast/helpers/order_by_clause.rs @@ -1,7 +1,7 @@ use crate::interpreter::ast::{parser::Parser, OrderByClause, OrderByDirection}; use crate::interpreter::tokenizer::token::TokenTypes; -use crate::interpreter::ast::helpers::common::expect_token_type; +use crate::interpreter::ast::helpers::token::expect_token_type; pub fn get_order_by(parser: &mut Parser) -> Result>, String> { if expect_token_type(parser, TokenTypes::Order).is_err() { diff --git a/src/interpreter/ast/helpers/select_statement.rs b/src/interpreter/ast/helpers/select_statement.rs index 782d7b0..fe487e5 100644 --- a/src/interpreter/ast/helpers/select_statement.rs +++ b/src/interpreter/ast/helpers/select_statement.rs @@ -2,7 +2,8 @@ use crate::{interpreter::{ ast::{ parser::Parser, SelectStatement, SelectStatementColumns, WhereStackElement, SelectMode, helpers::{ - common::{tokens_to_identifier_list, get_table_name, expect_token_type}, + token::{tokens_to_identifier_list, expect_token_type}, + common::get_table_name, order_by_clause::get_order_by, where_clause::get_where_clause, limit_clause::get_limit } }, diff --git a/src/interpreter/ast/helpers/token.rs b/src/interpreter/ast/helpers/token.rs new file mode 100644 index 0000000..9f49486 --- /dev/null +++ b/src/interpreter/ast/helpers/token.rs @@ -0,0 +1,148 @@ +use crate::interpreter::ast::parser::Parser; +use crate::interpreter::tokenizer::scanner::Token; +use crate::interpreter::tokenizer::token::TokenTypes; +use crate::db::table::{Value, DataType}; +use crate::interpreter::ast::helpers::common::hex_decode; + +// Returns an error if the current token does not match the given token type +pub fn expect_token_type(parser: &Parser, token_type: TokenTypes) -> Result<(), String> { + let token = parser.current_token()?; + if token.token_type != token_type { + return Err(parser.format_error()); + } + Ok(()) +} + +pub fn token_to_value(parser: &Parser) -> Result { + let token = parser.current_token()?; + + match token.token_type { + TokenTypes::IntLiteral => { + let num = token.value.parse::() + .map_err(|_| parser.format_error())?; + Ok(Value::Integer(num)) + }, + TokenTypes::RealLiteral => { + let num = token.value.parse::() + .map_err(|_| parser.format_error())?; + Ok(Value::Real(num)) + }, + TokenTypes::String => Ok(Value::Text(token.value.to_string())), + TokenTypes::Blob => { + let bytes = hex_decode(token.value) + .map_err(|_| parser.format_error())?; + Ok(Value::Blob(bytes)) + }, + TokenTypes::Null => Ok(Value::Null), + _ => Err(parser.format_error()), + } +} + +// Returns a list of Values from the tokens when they are formated as "value, value, ..." +pub fn tokens_to_value_list(parser: &mut Parser) -> Result, String> { + let mut values: Vec = vec![]; + loop { + values.push(token_to_value(parser)?); + parser.advance()?; + let token = parser.current_token()?; + if token.token_type != TokenTypes::Comma { + break; + } + parser.advance()?; + } + return Ok(values); +} + +// Returns a list of Strings from the tokens when they are formated as "identifier, identifier, ..." +pub fn tokens_to_identifier_list(parser: &mut Parser) -> Result, String> { + let mut identifiers: Vec = vec![]; + loop { + let token = parser.current_token()?; + expect_token_type(parser, TokenTypes::Identifier)?; + + identifiers.push(token.value.to_string()); + parser.advance()?; + let token = parser.current_token()?; + if token.token_type != TokenTypes::Comma { + break; + } + parser.advance()?; + } + return Ok(identifiers); +} + +pub fn token_to_data_type(parser: &mut Parser) -> Result { + let token = parser.current_token()?; + return match token.token_type { + TokenTypes::Integer => Ok(DataType::Integer), + TokenTypes::Real => Ok(DataType::Real), + TokenTypes::Text => Ok(DataType::Text), + TokenTypes::Blob => Ok(DataType::Blob), + TokenTypes::Null => Ok(DataType::Null), + _ => Err(parser.format_error()), + }; +} + +pub fn token_to_string(token: &Token) -> String { + match token.token_type { + TokenTypes::String => format!("'{}'", token.value), + TokenTypes::HexLiteral => format!("X'{}'", token.value), + TokenTypes::EOF | TokenTypes::SemiColon | TokenTypes::LeftParen | TokenTypes::RightParen => token.value.to_string(), + _ => token.value.to_string() + " ", + } +} + +// TODO: Improve this function and the related code. Parsing tokens back into a string is a messy. +// This should be guarenteed to only be hit if the statement is valid. +pub fn format_statement_tokens(tokens: &[Token]) -> String { + let mut result = String::new(); + for token in tokens { + result += &token_to_string(token); + } + result = result + .replace(" ;", ";") + .replace(" ,", ","); + return result; +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::interpreter::ast::test_utils::token; + use crate::interpreter::ast::parser::Parser; + use crate::interpreter::tokenizer::token::TokenTypes; + + #[test] + fn value_list_handles_single_value() { + // 1);... + let tokens = vec![ + token(TokenTypes::IntLiteral, "1"), + token(TokenTypes::RightParen, ")"), + ]; + let mut parser = Parser::new(tokens); + let result = tokens_to_value_list(&mut parser); + assert_eq!(result, Ok(vec![Value::Integer(1)])); + } + + #[test] + fn format_statement_tokens_handles_single_token() { + let tokens = vec![ + token(TokenTypes::SemiColon, ";"), + ]; + let result = format_statement_tokens(&tokens); + assert_eq!(";".to_string(), result); + } + + #[test] + fn format_statement_tokens_handles_multiple_tokens() { + let tokens = vec![ + token(TokenTypes::Select, "SELECT"), + token(TokenTypes::Asterisk, "*"), + token(TokenTypes::From, "FROM"), + token(TokenTypes::Identifier, "users"), + token(TokenTypes::SemiColon, ";"), + ]; + let result = format_statement_tokens(&tokens); + assert_eq!("SELECT * FROM users;".to_string(), result); + } +} diff --git a/src/interpreter/ast/helpers/where_clause/mod.rs b/src/interpreter/ast/helpers/where_clause/mod.rs index c789c36..2444b6a 100644 --- a/src/interpreter/ast/helpers/where_clause/mod.rs +++ b/src/interpreter/ast/helpers/where_clause/mod.rs @@ -4,7 +4,7 @@ mod where_stack_element; use expected_token_matches_current::{next_expected_token_from_current, WhereClauseExpectedNextToken}; use crate::interpreter::{ast::{ - helpers::{common::expect_token_type, where_clause::where_stack_element::get_where_stack_element}, + helpers::{token::expect_token_type, where_clause::where_stack_element::get_where_stack_element}, parser::Parser, WhereStackElement, WhereStackOperators, Parentheses}}; use crate::interpreter::tokenizer::token::TokenTypes; diff --git a/src/interpreter/ast/helpers/where_clause/where_condition.rs b/src/interpreter/ast/helpers/where_clause/where_condition.rs index 08df93b..d5066a7 100644 --- a/src/interpreter/ast/helpers/where_clause/where_condition.rs +++ b/src/interpreter/ast/helpers/where_clause/where_condition.rs @@ -1,6 +1,6 @@ use crate::interpreter::ast::{parser::Parser, WhereCondition, Operand, Operator}; use crate::interpreter::tokenizer::token::TokenTypes; -use crate::interpreter::ast::helpers::common::{expect_token_type, token_to_value, tokens_to_value_list}; +use crate::interpreter::ast::helpers::token::{expect_token_type, token_to_value, tokens_to_value_list}; pub fn get_condition(parser: &mut Parser) -> Result { diff --git a/src/interpreter/ast/insert_statement.rs b/src/interpreter/ast/insert_statement.rs index 1cf3a31..5e387d0 100644 --- a/src/interpreter/ast/insert_statement.rs +++ b/src/interpreter/ast/insert_statement.rs @@ -1,6 +1,7 @@ use crate::interpreter::{ ast::{ - helpers::common::{token_to_value, expect_token_type, get_table_name}, + helpers::token::{token_to_value, expect_token_type}, + helpers::common::get_table_name, parser::Parser, InsertIntoStatement, SqlStatement::{self, InsertInto} }, tokenizer::token::TokenTypes diff --git a/src/interpreter/ast/mod.rs b/src/interpreter/ast/mod.rs index 7567e15..81c3674 100644 --- a/src/interpreter/ast/mod.rs +++ b/src/interpreter/ast/mod.rs @@ -354,7 +354,7 @@ pub fn generate(tokens: Vec) -> Vec> Ok(DatabaseSqlStatement { sql_statement: sql_statement, line_num: line_num, - statement_text: "".to_string(), + statement_text: parser.get_sql_statement_text(), }) ); } @@ -393,6 +393,7 @@ mod tests { #[test] fn ast_handles_multiple_statements() { + // SELECT * FROM users; INSERT INTO users VALUES (1, "Alice"); let tokens = vec![ token(TokenTypes::Select, "SELECT"), token(TokenTypes::Asterisk, "*"), @@ -430,7 +431,7 @@ mod tests { limit_clause: None, }), line_num: 1, - statement_text: "".to_string(), + statement_text: "SELECT * FROM users;".to_string(), }), Ok(DatabaseSqlStatement { sql_statement: SqlStatement::InsertInto(InsertIntoStatement { @@ -441,7 +442,7 @@ mod tests { ], }), line_num: 1, - statement_text: "".to_string(), + statement_text: "INSERT INTO users VALUES (1, 'Alice');".to_string(), }), ]; assert_eq!(expected, result); @@ -479,7 +480,7 @@ mod tests { ], }), line_num: 1, - statement_text: "".to_string(), + statement_text: "INSERT INTO users VALUES (1, 'Alice');".to_string(), }), ]; assert_eq!(expected, result); @@ -524,7 +525,7 @@ mod tests { limit_clause: None, }), line_num: 1, - statement_text: "".to_string(), + statement_text: "SELECT * FROM users;".to_string(), }), Ok(DatabaseSqlStatement { sql_statement: SqlStatement::InsertInto(InsertIntoStatement { @@ -535,7 +536,7 @@ mod tests { ], }), line_num: 1, - statement_text: "".to_string(), + statement_text: "INSERT INTO users VALUES (1, 'Alice');".to_string(), }), ]; assert_eq!(expected, result); diff --git a/src/interpreter/ast/parser.rs b/src/interpreter/ast/parser.rs index 41a6b64..a2ba7d1 100644 --- a/src/interpreter/ast/parser.rs +++ b/src/interpreter/ast/parser.rs @@ -1,17 +1,20 @@ use crate::interpreter::{ ast::{SqlStatement, StatementBuilder}, - tokenizer::scanner::Token, tokenizer::token::TokenTypes, + ast::helpers::token::format_statement_tokens, + tokenizer::scanner::Token, tokenizer::token::TokenTypes }; pub struct Parser<'a> { tokens: Vec>, + start: usize, current: usize, } impl<'a> Parser<'a> { pub fn new(tokens: Vec>) -> Self { return Self { - tokens, + tokens, + start: 0, current: 0, }; } @@ -34,6 +37,10 @@ impl<'a> Parser<'a> { return Ok(&self.tokens[self.current + 1]); } + pub fn get_sql_statement_text(&self) -> String { + return format_statement_tokens(&self.tokens[self.start..self.current]); + } + pub fn advance(&mut self) -> Result<(), String> { if let Ok(token) = self.current_token() { if token.token_type == TokenTypes::SemiColon { @@ -79,6 +86,7 @@ impl<'a> Parser<'a> { } pub fn next_statement(&mut self, builder: &dyn StatementBuilder) -> Option> { + self.start = self.current; match (&self.current_token(), &self.peek_token()) { (Ok(token), Ok(peek_token)) => match (&token.token_type, &peek_token.token_type) { (TokenTypes::Create, _) => Some(builder.build_create(self)), diff --git a/src/interpreter/ast/update_statement.rs b/src/interpreter/ast/update_statement.rs index 157b925..3ab7b74 100644 --- a/src/interpreter/ast/update_statement.rs +++ b/src/interpreter/ast/update_statement.rs @@ -1,6 +1,7 @@ use crate::interpreter::ast::{ parser::Parser, SqlStatement, UpdateStatement, ColumnValue, - helpers::common::{expect_token_type, token_to_value, get_table_name}, + helpers::token::{expect_token_type, token_to_value}, + helpers::common::get_table_name, helpers::{order_by_clause::get_order_by, limit_clause::get_limit}, }; use crate::interpreter::ast::helpers::where_clause::get_where_clause;