AST - Statements contain SQL string

src/interpreter/ast/alter_table_statement.rs

@@ -1,6 +1,7 @@
 use crate::interpreter::{
     ast::{parser::Parser, SqlStatement, AlterTableStatement, AlterTableAction},
-    ast::helpers::common::{expect_token_type, get_table_name, token_to_data_type},
+    ast::helpers::token::{expect_token_type, token_to_data_type},
+    ast::helpers::common::get_table_name,
     tokenizer::token::TokenTypes,
 };
 use crate::db::table::ColumnDefinition;

src/interpreter/ast/create_statement.rs

@@ -1,7 +1,8 @@
 use crate::interpreter::{
     ast::{
         parser::Parser, CreateTableStatement, SqlStatement::{self, CreateTable}, ExistenceCheck,
-        helpers::common::{expect_token_type, get_table_name, exists_clause, token_to_data_type}
+        helpers::token::{expect_token_type, token_to_data_type},
+        helpers::common::{get_table_name, exists_clause},
     }, 
     tokenizer::token::TokenTypes
 };

src/interpreter/ast/delete_statement.rs

@@ -2,7 +2,8 @@ use crate::interpreter::{
     ast::{
         parser::Parser, SqlStatement, DeleteStatement, 
         helpers::{
-            common::{expect_token_type, get_table_name},
+            token::expect_token_type,
+            common::get_table_name,
             order_by_clause::get_order_by, where_clause::get_where_clause, limit_clause::get_limit
         }
     },

src/interpreter/ast/drop_statement.rs

@@ -1,7 +1,8 @@
 use crate::interpreter::{
     ast::{
         parser::Parser, SqlStatement, DropTableStatement, ExistenceCheck,
-        helpers::common::{expect_token_type, get_table_name, exists_clause}
+        helpers::token::expect_token_type,
+        helpers::common::{get_table_name, exists_clause},
     },
     tokenizer::token::TokenTypes
 };

src/interpreter/ast/helpers/common.rs

@@ -1,73 +1,7 @@
 use crate::interpreter::{ast::{parser::Parser, ExistenceCheck}, tokenizer::token::TokenTypes};
+use crate::interpreter::ast::helpers::token::expect_token_type;
 
-use crate::db::table::{Value, DataType};
 
-// Returns an error if the current token does not match the given token type
-pub fn expect_token_type(parser: &Parser, token_type: TokenTypes) -> Result<(), String> {
-    let token = parser.current_token()?;
-    if token.token_type != token_type {
-        return Err(parser.format_error());
-    }
-    Ok(())
-}
-
-pub fn token_to_value(parser: &Parser) -> Result<Value, String> {
-    let token = parser.current_token()?;
-
-    match token.token_type {
-        TokenTypes::IntLiteral => {
-            let num = token.value.parse::<i64>()
-                .map_err(|_| parser.format_error())?;
-            Ok(Value::Integer(num))
-        },
-        TokenTypes::RealLiteral => {
-            let num = token.value.parse::<f64>()
-                .map_err(|_| parser.format_error())?;
-            Ok(Value::Real(num))
-        },
-        TokenTypes::String => Ok(Value::Text(token.value.to_string())),
-        TokenTypes::Blob => {
-            let bytes = decode(token.value)
-                .map_err(|_| parser.format_error())?;
-            Ok(Value::Blob(bytes))
-        },
-        TokenTypes::Null => Ok(Value::Null),
-        _ => Err(parser.format_error()),
-    }
-}
-
-// Returns a list of Values from the tokens when they are formated as "value, value, ..."
-pub fn tokens_to_value_list(parser: &mut Parser) -> Result<Vec<Value>, String> {
-    let mut values: Vec<Value> = vec![];
-    loop {
-        values.push(token_to_value(parser)?);
-        parser.advance()?;
-        let token = parser.current_token()?;
-        if token.token_type != TokenTypes::Comma {
-            break;
-        }
-        parser.advance()?;
-    }
-    return Ok(values);
-}
-
-// Returns a list of Strings from the tokens when they are formated as "identifier, identifier, ..."
-pub fn tokens_to_identifier_list(parser: &mut Parser) -> Result<Vec<String>, String> {
-    let mut identifiers: Vec<String> = vec![];
-    loop {
-        let token = parser.current_token()?;
-        expect_token_type(parser, TokenTypes::Identifier)?;
-
-        identifiers.push(token.value.to_string());
-        parser.advance()?;
-        let token = parser.current_token()?;
-        if token.token_type != TokenTypes::Comma {
-            break;
-        }
-        parser.advance()?;
-    }
-    return Ok(identifiers);
-}
 
 pub fn get_table_name(parser: &mut Parser) -> Result<String, String> {
     let token = parser.current_token()?;
@@ -98,7 +32,7 @@ pub fn exists_clause(parser: &mut Parser, check_type: ExistenceCheck) -> Result<
     return Ok(None);
 }
 
-fn decode(hex: &str) -> Result<Vec<u8>, String> {
+pub fn hex_decode(hex: &str) -> Result<Vec<u8>, String> {
     if hex.len() % 2 != 0 {
         return Err("Hex string must have even length".to_string());
     }
@@ -109,54 +43,27 @@ fn decode(hex: &str) -> Result<Vec<u8>, String> {
         }).collect()
 }
 
-pub fn token_to_data_type(parser: &mut Parser) -> Result<DataType, String> {
-    let token = parser.current_token()?;
-    return match token.token_type {
-        TokenTypes::Integer => Ok(DataType::Integer),
-        TokenTypes::Real => Ok(DataType::Real),
-        TokenTypes::Text => Ok(DataType::Text),
-        TokenTypes::Blob => Ok(DataType::Blob),
-        TokenTypes::Null => Ok(DataType::Null),
-        _ => Err(parser.format_error()),
-    };
-}
-
 
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::interpreter::ast::test_utils::token;
-    use crate::interpreter::ast::parser::Parser;
-    use crate::interpreter::tokenizer::token::TokenTypes;
-
-    #[test]
-    fn value_list_handles_single_value() {
-        // 1);...
-        let tokens = vec![
-            token(TokenTypes::IntLiteral, "1"),
-            token(TokenTypes::RightParen, ")"),
-        ];
-        let mut parser = Parser::new(tokens);
-        let result = tokens_to_value_list(&mut parser);
-        assert_eq!(result, Ok(vec![Value::Integer(1)]));
-    }
 
     #[test]
     fn decode_handles_valid_hex_string() {
-        let result = decode("0A1A3F");
+        let result = hex_decode("0A1A3F");
         assert!(result.is_ok());
         let expected = vec![0x0A, 0x1A, 0x3F];
         assert_eq!(expected, result.unwrap());
     }
 
     #[test]
     fn decode_handles_invalid_hex_string() {
-        let result = decode("0AZA3A");
+        let result = hex_decode("0AZA3A");
         assert!(result.is_err());
         let expected = "Invalid hex at 2: invalid digit found in string";
         assert_eq!(expected, result.err().unwrap());
 
-        let result = decode("0A1");
+        let result = hex_decode("0A1");
         assert!(result.is_err());
         let expected = "Hex string must have even length";
         assert_eq!(expected, result.err().unwrap());

src/interpreter/ast/helpers/limit_clause.rs

@@ -1,7 +1,7 @@
 use crate::interpreter::ast::{parser::Parser, LimitClause};
 use crate::interpreter::tokenizer::token::TokenTypes;
 use crate::db::table::Value;
-use crate::interpreter::ast::helpers::common::{expect_token_type, token_to_value};
+use crate::interpreter::ast::helpers::token::{expect_token_type, token_to_value};
 
 pub fn get_limit(parser: &mut Parser) -> Result<Option<LimitClause>, String> { 
     if expect_token_type(parser, TokenTypes::Limit).is_err() {

src/interpreter/ast/helpers/mod.rs

@@ -2,4 +2,5 @@ pub mod where_clause;
 pub mod order_by_clause;
 pub mod limit_clause;
 pub mod common;
-pub mod select_statement;
+pub mod select_statement;
+pub mod token;

src/interpreter/ast/helpers/order_by_clause.rs

@@ -1,7 +1,7 @@
 use crate::interpreter::ast::{parser::Parser, OrderByClause, OrderByDirection};
 use crate::interpreter::tokenizer::token::TokenTypes;
 
-use crate::interpreter::ast::helpers::common::expect_token_type;
+use crate::interpreter::ast::helpers::token::expect_token_type;
 
 pub fn get_order_by(parser: &mut Parser) -> Result<Option<Vec<OrderByClause>>, String> {
     if expect_token_type(parser, TokenTypes::Order).is_err() {

src/interpreter/ast/helpers/select_statement.rs

@@ -2,7 +2,8 @@ use crate::{interpreter::{
     ast::{
         parser::Parser, SelectStatement, SelectStatementColumns, WhereStackElement, SelectMode,
         helpers::{
-            common::{tokens_to_identifier_list, get_table_name, expect_token_type},
+            token::{tokens_to_identifier_list, expect_token_type},
+            common::get_table_name,
             order_by_clause::get_order_by, where_clause::get_where_clause, limit_clause::get_limit
         }
     }, 

src/interpreter/ast/helpers/token.rs

@@ -0,0 +1,148 @@
+use crate::interpreter::ast::parser::Parser;
+use crate::interpreter::tokenizer::scanner::Token;
+use crate::interpreter::tokenizer::token::TokenTypes;
+use crate::db::table::{Value, DataType};
+use crate::interpreter::ast::helpers::common::hex_decode;
+
+// Returns an error if the current token does not match the given token type
+pub fn expect_token_type(parser: &Parser, token_type: TokenTypes) -> Result<(), String> {
+    let token = parser.current_token()?;
+    if token.token_type != token_type {
+        return Err(parser.format_error());
+    }
+    Ok(())
+}
+
+pub fn token_to_value(parser: &Parser) -> Result<Value, String> {
+    let token = parser.current_token()?;
+
+    match token.token_type {
+        TokenTypes::IntLiteral => {
+            let num = token.value.parse::<i64>()
+                .map_err(|_| parser.format_error())?;
+            Ok(Value::Integer(num))
+        },
+        TokenTypes::RealLiteral => {
+            let num = token.value.parse::<f64>()
+                .map_err(|_| parser.format_error())?;
+            Ok(Value::Real(num))
+        },
+        TokenTypes::String => Ok(Value::Text(token.value.to_string())),
+        TokenTypes::Blob => {
+            let bytes = hex_decode(token.value)
+                .map_err(|_| parser.format_error())?;
+            Ok(Value::Blob(bytes))
+        },
+        TokenTypes::Null => Ok(Value::Null),
+        _ => Err(parser.format_error()),
+    }
+}
+
+// Returns a list of Values from the tokens when they are formated as "value, value, ..."
+pub fn tokens_to_value_list(parser: &mut Parser) -> Result<Vec<Value>, String> {
+    let mut values: Vec<Value> = vec![];
+    loop {
+        values.push(token_to_value(parser)?);
+        parser.advance()?;
+        let token = parser.current_token()?;
+        if token.token_type != TokenTypes::Comma {
+            break;
+        }
+        parser.advance()?;
+    }
+    return Ok(values);
+}
+
+// Returns a list of Strings from the tokens when they are formated as "identifier, identifier, ..."
+pub fn tokens_to_identifier_list(parser: &mut Parser) -> Result<Vec<String>, String> {
+    let mut identifiers: Vec<String> = vec![];
+    loop {
+        let token = parser.current_token()?;
+        expect_token_type(parser, TokenTypes::Identifier)?;
+
+        identifiers.push(token.value.to_string());
+        parser.advance()?;
+        let token = parser.current_token()?;
+        if token.token_type != TokenTypes::Comma {
+            break;
+        }
+        parser.advance()?;
+    }
+    return Ok(identifiers);
+}
+
+pub fn token_to_data_type(parser: &mut Parser) -> Result<DataType, String> {
+    let token = parser.current_token()?;
+    return match token.token_type {
+        TokenTypes::Integer => Ok(DataType::Integer),
+        TokenTypes::Real => Ok(DataType::Real),
+        TokenTypes::Text => Ok(DataType::Text),
+        TokenTypes::Blob => Ok(DataType::Blob),
+        TokenTypes::Null => Ok(DataType::Null),
+        _ => Err(parser.format_error()),
+    };
+}
+
+pub fn token_to_string(token: &Token) -> String {
+    match token.token_type {
+        TokenTypes::String => format!("'{}'", token.value),
+        TokenTypes::HexLiteral => format!("X'{}'", token.value),
+        TokenTypes::EOF | TokenTypes::SemiColon | TokenTypes::LeftParen | TokenTypes::RightParen => token.value.to_string(),
+        _ => token.value.to_string() + " ",
+    }
+}
+
+// TODO: Improve this function and the related code. Parsing tokens back into a string is a messy.
+// This should be guarenteed to only be hit if the statement is valid.
+pub fn format_statement_tokens(tokens: &[Token]) -> String {
+    let mut result = String::new();
+    for token in tokens {
+        result += &token_to_string(token);
+    }
+    result = result
+        .replace(" ;", ";")
+        .replace(" ,", ",");
+    return result;
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::interpreter::ast::test_utils::token;
+    use crate::interpreter::ast::parser::Parser;
+    use crate::interpreter::tokenizer::token::TokenTypes;
+
+    #[test]
+    fn value_list_handles_single_value() {
+        // 1);...
+        let tokens = vec![
+            token(TokenTypes::IntLiteral, "1"),
+            token(TokenTypes::RightParen, ")"),
+        ];
+        let mut parser = Parser::new(tokens);
+        let result = tokens_to_value_list(&mut parser);
+        assert_eq!(result, Ok(vec![Value::Integer(1)]));
+    }
+
+    #[test]
+    fn format_statement_tokens_handles_single_token() {
+        let tokens = vec![
+            token(TokenTypes::SemiColon, ";"),
+        ];
+        let result = format_statement_tokens(&tokens);
+        assert_eq!(";".to_string(), result);
+    }
+
+    #[test]
+    fn format_statement_tokens_handles_multiple_tokens() {
+        let tokens = vec![
+            token(TokenTypes::Select, "SELECT"),
+            token(TokenTypes::Asterisk, "*"),
+            token(TokenTypes::From, "FROM"),
+            token(TokenTypes::Identifier, "users"),
+            token(TokenTypes::SemiColon, ";"),
+        ];
+        let result = format_statement_tokens(&tokens);
+        assert_eq!("SELECT * FROM users;".to_string(), result);
+    }
+}

src/interpreter/ast/helpers/where_clause/mod.rs

@@ -4,7 +4,7 @@ mod where_stack_element;
 use expected_token_matches_current::{next_expected_token_from_current, WhereClauseExpectedNextToken};
 
 use crate::interpreter::{ast::{
-    helpers::{common::expect_token_type, where_clause::where_stack_element::get_where_stack_element}, 
+    helpers::{token::expect_token_type, where_clause::where_stack_element::get_where_stack_element}, 
     parser::Parser, WhereStackElement, WhereStackOperators, Parentheses}};
 use crate::interpreter::tokenizer::token::TokenTypes;
 

src/interpreter/ast/helpers/where_clause/where_condition.rs

@@ -1,6 +1,6 @@
 use crate::interpreter::ast::{parser::Parser, WhereCondition, Operand, Operator};
 use crate::interpreter::tokenizer::token::TokenTypes;
-use crate::interpreter::ast::helpers::common::{expect_token_type, token_to_value, tokens_to_value_list};
+use crate::interpreter::ast::helpers::token::{expect_token_type, token_to_value, tokens_to_value_list};
 
 
 pub fn get_condition(parser: &mut Parser) -> Result<WhereCondition, String> {

src/interpreter/ast/insert_statement.rs

@@ -1,6 +1,7 @@
 use crate::interpreter::{
     ast::{ 
-        helpers::common::{token_to_value, expect_token_type, get_table_name}, 
+        helpers::token::{token_to_value, expect_token_type},
+        helpers::common::get_table_name,
         parser::Parser, InsertIntoStatement, SqlStatement::{self, InsertInto}
     }, 
     tokenizer::token::TokenTypes