feat: arrays and type system consolidation

prql-compiler/prqlc/src/cli.rs

@@ -9,7 +9,7 @@ use std::process::exit;
 use std::str::FromStr;
 
 use prql_compiler::semantic::{self, reporting::*};
-use prql_compiler::{ast::pl::Frame, pl_to_prql};
+use prql_compiler::{ast::pl::Lineage, pl_to_prql};
 use prql_compiler::{downcast, Options, Target};
 use prql_compiler::{pl_to_rq_tree, prql_to_pl, prql_to_pl_tree, rq_to_sql, FileTree, Span};
 
@@ -326,7 +326,7 @@ impl Command {
     }
 }
 
-fn combine_prql_and_frames(source: &str, frames: Vec<(Span, Frame)>) -> String {
+fn combine_prql_and_frames(source: &str, frames: Vec<(Span, Lineage)>) -> String {
     let source = Source::from(source);
     let lines = source.lines().collect_vec();
     let width = lines.iter().map(|l| l.len()).max().unwrap_or(0);

prql-compiler/src/ast/pl/expr.rs

@@ -3,6 +3,7 @@ use std::fmt::{Display, Write};
 
 use anyhow::{anyhow, Result};
 use enum_as_inner::EnumAsInner;
+use itertools::Itertools;
 use semver::VersionReq;
 
 use serde::{Deserialize, Serialize};
@@ -31,10 +32,18 @@ pub struct Expr {
     #[serde(skip_serializing_if = "Vec::is_empty", default)]
     pub target_ids: Vec<usize>,
 
-    /// Type of expression this node represents. [None] means type has not yet been determined.
+    /// Type of expression this node represents.
+    /// [None] means that type should be inferred.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub ty: Option<Ty>,
 
+    /// Information about where data of this expression will come from.
+    ///
+    /// Currently, this is used to infer relational pipeline frames.
+    /// Must always exists if ty is a relation.
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub lineage: Option<Lineage>,
+
     #[serde(skip)]
     pub needs_window: bool,
 
@@ -43,6 +52,7 @@ pub struct Expr {
 
     /// When true on [ExprKind::List], this list will be flattened when placed
     /// in some other list.
+    // TODO: maybe we should have a special ExprKind instead of this flag?
     #[serde(skip)]
     pub flatten: bool,
 }
@@ -56,7 +66,10 @@ pub enum ExprKind {
     },
     Literal(Literal),
     Pipeline(Pipeline),
+
+    /// Also known as tuple or struct
     List(Vec<Expr>),
+    Array(Vec<Expr>),
     Range(Range),
     Binary {
         left: Box<Expr>,
@@ -77,7 +90,8 @@ pub enum ExprKind {
         name: String,
         args: Vec<Expr>,
     },
-    Type(TypeExpr),
+
+    Type(TyKind),
 
     /// a placeholder for values provided after query is compiled
     Param(String),
@@ -340,6 +354,24 @@ pub enum TransformKind {
     Loop(Box<Expr>),
 }
 
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub struct ColumnSort<T = Expr> {
+    pub direction: SortDirection,
+    pub column: T,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
+pub enum SortDirection {
+    Asc,
+    Desc,
+}
+
+impl Default for SortDirection {
+    fn default() -> Self {
+        SortDirection::Asc
+    }
+}
+
 #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]
 pub enum WindowKind {
     Rows,
@@ -422,6 +454,7 @@ impl From<ExprKind> for Expr {
             target_id: None,
             target_ids: Vec::new(),
             ty: None,
+            lineage: None,
             needs_window: false,
             alias: None,
             flatten: false,
@@ -525,6 +558,10 @@ impl Display for Expr {
                     f.write_str("]")?;
                 }
             }
+            ExprKind::Array(items) => {
+                let items = items.iter().map(|x| x.to_string()).join(", ");
+                write!(f, "{{{items}}}")?;
+            }
             ExprKind::Range(r) => {
                 if let Some(start) = &r.start {
                     write!(f, "{}", start)?;

prql-compiler/src/ast/pl/fold.rs

@@ -95,6 +95,7 @@ pub fn fold_expr_kind<T: ?Sized + AstFold>(fold: &mut T, expr_kind: ExprKind) ->
             expr: Box::new(fold.fold_expr(*expr)?),
         },
         List(items) => List(fold.fold_exprs(items)?),
+        Array(items) => Array(fold.fold_exprs(items)?),
         Range(range) => Range(fold_range(fold, range)?),
         Pipeline(p) => Pipeline(fold.fold_pipeline(p)?),
         SString(items) => SString(

prql-compiler/src/ast/pl/frame.rs

@@ -1,29 +1,27 @@
-use std::{
-    collections::HashSet,
-    fmt::{Debug, Display, Formatter},
-};
+use std::collections::HashSet;
+use std::fmt::{Debug, Display, Formatter};
 
 use enum_as_inner::EnumAsInner;
 use itertools::{Itertools, Position};
 use serde::{Deserialize, Serialize};
 
-use super::{Expr, Ident};
+use super::Ident;
 
 /// Represents the object that is manipulated by the pipeline transforms.
 /// Similar to a view in a database or a data frame.
-#[derive(Clone, Default, Eq, PartialEq, Serialize, Deserialize)]
-pub struct Frame {
-    pub columns: Vec<FrameColumn>,
+#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
+pub struct Lineage {
+    pub columns: Vec<LineageColumn>,
 
-    pub inputs: Vec<FrameInput>,
+    pub inputs: Vec<LineageInput>,
 
     // A hack that allows name retention when applying `ExprKind::All { except }`
     #[serde(skip)]
-    pub prev_columns: Vec<FrameColumn>,
+    pub prev_columns: Vec<LineageColumn>,
 }
 
-#[derive(Clone, Eq, Debug, PartialEq, Serialize, Deserialize)]
-pub struct FrameInput {
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct LineageInput {
     /// Id of the node in AST that declares this input.
     pub id: usize,
 
@@ -35,7 +33,7 @@ pub struct FrameInput {
 }
 
 #[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize, EnumAsInner)]
-pub enum FrameColumn {
+pub enum LineageColumn {
     /// All columns (including unknown ones) from an input (i.e. `foo_table.*`)
     All {
         input_name: String,
@@ -48,59 +46,34 @@ pub enum FrameColumn {
     },
 }
 
-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
-pub struct ColumnSort<T = Expr> {
-    pub direction: SortDirection,
-    pub column: T,
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
-pub enum SortDirection {
-    Asc,
-    Desc,
-}
-
-impl Default for SortDirection {
-    fn default() -> Self {
-        SortDirection::Asc
-    }
-}
-
-impl Display for Frame {
-    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        display_frame(self, f, false)
-    }
-}
-
-impl Debug for Frame {
+impl Display for Lineage {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        display_frame(self, f, true)?;
-        std::fmt::Debug::fmt(&self.inputs, f)
+        display_lineage(self, f, false)
     }
 }
 
-fn display_frame(frame: &Frame, f: &mut Formatter, display_ids: bool) -> std::fmt::Result {
+fn display_lineage(lineage: &Lineage, f: &mut Formatter, display_ids: bool) -> std::fmt::Result {
     write!(f, "[")?;
-    for col in frame.columns.iter().with_position() {
+    for col in lineage.columns.iter().with_position() {
         let is_last = matches!(col, Position::Last(_) | Position::Only(_));
-        display_frame_column(col.into_inner(), f, display_ids)?;
+        display_lineage_column(col.into_inner(), f, display_ids)?;
         if !is_last {
             write!(f, ", ")?;
         }
     }
     write!(f, "]")
 }
 
-fn display_frame_column(
-    col: &FrameColumn,
+fn display_lineage_column(
+    col: &LineageColumn,
     f: &mut Formatter,
     display_ids: bool,
 ) -> std::fmt::Result {
     match col {
-        FrameColumn::All { input_name, .. } => {
+        LineageColumn::All { input_name, .. } => {
             write!(f, "{input_name}.*")?;
         }
-        FrameColumn::Single { name, expr_id } => {
+        LineageColumn::Single { name, expr_id } => {
             if let Some(name) = name {
                 write!(f, "{name}")?
             } else {

prql-compiler/src/ast/pl/literal.rs

@@ -15,7 +15,6 @@ pub enum Literal {
     Time(String),
     Timestamp(String),
     ValueAndUnit(ValueAndUnit),
-    Relation(RelationLiteral),
 }
 
 // Compound units, such as "2 days 3 hours" can be represented as `2days + 3hours`
@@ -88,10 +87,6 @@ impl Display for Literal {
             Literal::ValueAndUnit(i) => {
                 write!(f, "{}{}", i.n, i.unit)?;
             }
-
-            Literal::Relation(_) => {
-                write!(f, "<unimplemented relation>")?;
-            }
         }
         Ok(())
     }