From 515864a23b166e002ae0394a3ea7f2d314170f72 Mon Sep 17 00:00:00 2001 From: Simon Vandel Sillesen Date: Thu, 21 Aug 2025 21:15:40 +0200 Subject: [PATCH 01/12] support WHERE pipe operator --- datafusion/sql/src/query.rs | 39 +++++++++++++++-- datafusion/sql/src/select.rs | 2 +- .../sqllogictest/test_files/pipe_operator.slt | 42 +++++++++++++++++++ 3 files changed, 79 insertions(+), 4 deletions(-) create mode 100644 datafusion/sqllogictest/test_files/pipe_operator.slt diff --git a/datafusion/sql/src/query.rs b/datafusion/sql/src/query.rs index c06147e08f87a..a88ca0c555f47 100644 --- a/datafusion/sql/src/query.rs +++ b/datafusion/sql/src/query.rs @@ -27,8 +27,8 @@ use datafusion_expr::{ CreateMemoryTable, DdlStatement, Distinct, Expr, LogicalPlan, LogicalPlanBuilder, }; use sqlparser::ast::{ - Expr as SQLExpr, Ident, LimitClause, OrderBy, OrderByExpr, OrderByKind, Query, - SelectInto, SetExpr, + Expr as SQLExpr, Ident, LimitClause, OrderBy, OrderByExpr, OrderByKind, PipeOperator, + Query, SelectInto, SetExpr, }; use sqlparser::tokenizer::Span; @@ -48,8 +48,10 @@ impl SqlToRel<'_, S> { self.plan_with_clause(with, planner_context)?; } + let pipe_operators = query.pipe_operators.clone(); + let set_expr = *query.body; - match set_expr { + let plan = match set_expr { SetExpr::Select(mut select) => { let select_into = select.into.take(); let plan = @@ -78,6 +80,37 @@ impl SqlToRel<'_, S> { let plan = self.order_by(plan, order_by_rex)?; self.limit(plan, query.limit_clause, planner_context) } + }?; + + self.pipe_operators(plan, pipe_operators, planner_context) + } + + /// Apply pipe operators to a plan + fn pipe_operators( + &self, + plan: LogicalPlan, + pipe_operators: Vec, + planner_context: &mut PlannerContext, + ) -> Result { + let mut plan = plan; + for pipe_operator in pipe_operators { + plan = self.pipe_operator(plan, pipe_operator, planner_context)?; + } + Ok(plan) + } + + /// Apply a pipe operator to a plan + fn pipe_operator( + &self, + plan: LogicalPlan, + pipe_operator: PipeOperator, + planner_context: &mut PlannerContext, + ) -> Result { + match pipe_operator { + PipeOperator::Where { expr } => { + self.plan_selection(Some(expr), plan, planner_context) + } + x => not_impl_err!("{x} pipe operator is not supported yet"), } } diff --git a/datafusion/sql/src/select.rs b/datafusion/sql/src/select.rs index 1a90e5e09b778..bcc6da3318134 100644 --- a/datafusion/sql/src/select.rs +++ b/datafusion/sql/src/select.rs @@ -574,7 +574,7 @@ impl SqlToRel<'_, S> { Ok((intermediate_plan, intermediate_select_exprs)) } - fn plan_selection( + pub(crate) fn plan_selection( &self, selection: Option, plan: LogicalPlan, diff --git a/datafusion/sqllogictest/test_files/pipe_operator.slt b/datafusion/sqllogictest/test_files/pipe_operator.slt new file mode 100644 index 0000000000000..d72480c06e1b3 --- /dev/null +++ b/datafusion/sqllogictest/test_files/pipe_operator.slt @@ -0,0 +1,42 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# BigQuery supports the pipe operator syntax +# TODO: Make the Generic dialect support the pipe operator syntax +statement ok +set datafusion.sql_parser.dialect = 'BigQuery'; + +statement ok +CREATE TABLE test( + a INT, + b FLOAT, + c VARCHAR, + n VARCHAR +) AS VALUES + (1, 1.1, 'a', NULL), + (2, 2.2, 'b', NULL), + (3, 3.3, 'c', NULL) +; + +# WHERE pipe +query IRTT +SELECT * +FROM test +|> WHERE a > 1 +---- +2 2.2 b NULL +3 3.3 c NULL From 848dc08d83b922b66604bb0c5b8e0df4a8866e9a Mon Sep 17 00:00:00 2001 From: Simon Vandel Sillesen Date: Thu, 21 Aug 2025 21:23:16 +0200 Subject: [PATCH 02/12] support order by --- datafusion/sql/src/query.rs | 10 ++++++++++ datafusion/sqllogictest/test_files/pipe_operator.slt | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/datafusion/sql/src/query.rs b/datafusion/sql/src/query.rs index a88ca0c555f47..54cc2b7605d60 100644 --- a/datafusion/sql/src/query.rs +++ b/datafusion/sql/src/query.rs @@ -110,6 +110,16 @@ impl SqlToRel<'_, S> { PipeOperator::Where { expr } => { self.plan_selection(Some(expr), plan, planner_context) } + PipeOperator::OrderBy { exprs } => { + let sort_exprs = self.order_by_to_sort_expr( + exprs, + plan.schema(), + planner_context, + true, + None, + )?; + self.order_by(plan, sort_exprs) + } x => not_impl_err!("{x} pipe operator is not supported yet"), } } diff --git a/datafusion/sqllogictest/test_files/pipe_operator.slt b/datafusion/sqllogictest/test_files/pipe_operator.slt index d72480c06e1b3..42522b7aeb23b 100644 --- a/datafusion/sqllogictest/test_files/pipe_operator.slt +++ b/datafusion/sqllogictest/test_files/pipe_operator.slt @@ -40,3 +40,13 @@ FROM test ---- 2 2.2 b NULL 3 3.3 c NULL + +# ORDER BY pipe +query IRTT +SELECT * +FROM test +|> ORDER BY a DESC +---- +3 3.3 c NULL +2 2.2 b NULL +1 1.1 a NULL From ac64a79be6936e5682f12cf5e0392338ade7c86e Mon Sep 17 00:00:00 2001 From: Simon Vandel Sillesen Date: Thu, 21 Aug 2025 21:36:50 +0200 Subject: [PATCH 03/12] support limit --- datafusion/sql/src/query.rs | 16 ++++++++++++++-- .../sqllogictest/test_files/pipe_operator.slt | 9 +++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/datafusion/sql/src/query.rs b/datafusion/sql/src/query.rs index 54cc2b7605d60..50bbcf92c4c4f 100644 --- a/datafusion/sql/src/query.rs +++ b/datafusion/sql/src/query.rs @@ -27,8 +27,8 @@ use datafusion_expr::{ CreateMemoryTable, DdlStatement, Distinct, Expr, LogicalPlan, LogicalPlanBuilder, }; use sqlparser::ast::{ - Expr as SQLExpr, Ident, LimitClause, OrderBy, OrderByExpr, OrderByKind, PipeOperator, - Query, SelectInto, SetExpr, + Expr as SQLExpr, Ident, LimitClause, Offset, OffsetRows, OrderBy, OrderByExpr, + OrderByKind, PipeOperator, Query, SelectInto, SetExpr, }; use sqlparser::tokenizer::Span; @@ -120,6 +120,18 @@ impl SqlToRel<'_, S> { )?; self.order_by(plan, sort_exprs) } + PipeOperator::Limit { expr, offset } => self.limit( + plan, + Some(LimitClause::LimitOffset { + limit: Some(expr), + offset: offset.map(|offset| Offset { + value: offset, + rows: OffsetRows::None, + }), + limit_by: vec![], + }), + planner_context, + ), x => not_impl_err!("{x} pipe operator is not supported yet"), } } diff --git a/datafusion/sqllogictest/test_files/pipe_operator.slt b/datafusion/sqllogictest/test_files/pipe_operator.slt index 42522b7aeb23b..c3d174959d5f9 100644 --- a/datafusion/sqllogictest/test_files/pipe_operator.slt +++ b/datafusion/sqllogictest/test_files/pipe_operator.slt @@ -50,3 +50,12 @@ FROM test 3 3.3 c NULL 2 2.2 b NULL 1 1.1 a NULL + +# ORDER BY pipe, limit +query IRTT +SELECT * +FROM test +|> ORDER BY a DESC +|> LIMIT 1 +---- +3 3.3 c NULL From 3f7d647df6b83896cb7f512a84d195ca1c828849 Mon Sep 17 00:00:00 2001 From: Simon Vandel Sillesen Date: Thu, 21 Aug 2025 21:44:18 +0200 Subject: [PATCH 04/12] select pipe --- datafusion/sql/src/query.rs | 8 +++++++- datafusion/sql/src/select.rs | 8 ++++++-- datafusion/sqllogictest/test_files/pipe_operator.slt | 10 ++++++++++ 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/datafusion/sql/src/query.rs b/datafusion/sql/src/query.rs index 50bbcf92c4c4f..d68df64963ae5 100644 --- a/datafusion/sql/src/query.rs +++ b/datafusion/sql/src/query.rs @@ -132,7 +132,13 @@ impl SqlToRel<'_, S> { }), planner_context, ), - x => not_impl_err!("{x} pipe operator is not supported yet"), + PipeOperator::Select { exprs } => { + let empty_from = matches!(plan, LogicalPlan::EmptyRelation(_)); + let select_exprs = + self.prepare_select_exprs(&plan, exprs, empty_from, planner_context)?; + self.project(plan, select_exprs) + } + x => not_impl_err!("`{x}` pipe operator is not supported yet"), } } diff --git a/datafusion/sql/src/select.rs b/datafusion/sql/src/select.rs index bcc6da3318134..bb6d4f96ba583 100644 --- a/datafusion/sql/src/select.rs +++ b/datafusion/sql/src/select.rs @@ -655,7 +655,7 @@ impl SqlToRel<'_, S> { } /// Returns the `Expr`'s corresponding to a SQL query's SELECT expressions. - fn prepare_select_exprs( + pub(crate) fn prepare_select_exprs( &self, plan: &LogicalPlan, projection: Vec, @@ -823,7 +823,11 @@ impl SqlToRel<'_, S> { } /// Wrap a plan in a projection - fn project(&self, input: LogicalPlan, expr: Vec) -> Result { + pub(crate) fn project( + &self, + input: LogicalPlan, + expr: Vec, + ) -> Result { // convert to Expr for validate_schema_satisfies_exprs let exprs = expr .iter() diff --git a/datafusion/sqllogictest/test_files/pipe_operator.slt b/datafusion/sqllogictest/test_files/pipe_operator.slt index c3d174959d5f9..a28553dbb39a5 100644 --- a/datafusion/sqllogictest/test_files/pipe_operator.slt +++ b/datafusion/sqllogictest/test_files/pipe_operator.slt @@ -59,3 +59,13 @@ FROM test |> LIMIT 1 ---- 3 3.3 c NULL + +# SELECT pipe +query I +SELECT * +FROM test +|> SELECT a +---- +1 +2 +3 From 420bf001a9feeca21f5d9cd01ee48bfc9eb3e8c6 Mon Sep 17 00:00:00 2001 From: Simon Vandel Sillesen Date: Thu, 21 Aug 2025 21:59:32 +0200 Subject: [PATCH 05/12] extend support --- datafusion/sql/src/query.rs | 14 ++++++++++++++ .../sqllogictest/test_files/pipe_operator.slt | 11 +++++++++++ 2 files changed, 25 insertions(+) diff --git a/datafusion/sql/src/query.rs b/datafusion/sql/src/query.rs index d68df64963ae5..28ebf54695455 100644 --- a/datafusion/sql/src/query.rs +++ b/datafusion/sql/src/query.rs @@ -23,6 +23,7 @@ use crate::stack::StackGuard; use datafusion_common::{not_impl_err, Constraints, DFSchema, Result}; use datafusion_expr::expr::Sort; +use datafusion_expr::select_expr::SelectExpr; use datafusion_expr::{ CreateMemoryTable, DdlStatement, Distinct, Expr, LogicalPlan, LogicalPlanBuilder, }; @@ -138,6 +139,19 @@ impl SqlToRel<'_, S> { self.prepare_select_exprs(&plan, exprs, empty_from, planner_context)?; self.project(plan, select_exprs) } + PipeOperator::Extend { exprs } => { + let empty_from = matches!(plan, LogicalPlan::EmptyRelation(_)); + let extend_exprs = + self.prepare_select_exprs(&plan, exprs, empty_from, planner_context)?; + let all_exprs = plan + .expressions() + .into_iter() + .map(SelectExpr::Expression) + .chain(extend_exprs) + .collect(); + self.project(plan, all_exprs) + } + x => not_impl_err!("`{x}` pipe operator is not supported yet"), } } diff --git a/datafusion/sqllogictest/test_files/pipe_operator.slt b/datafusion/sqllogictest/test_files/pipe_operator.slt index a28553dbb39a5..debe7d1fd8d27 100644 --- a/datafusion/sqllogictest/test_files/pipe_operator.slt +++ b/datafusion/sqllogictest/test_files/pipe_operator.slt @@ -69,3 +69,14 @@ FROM test 1 2 3 + +# EXTEND pipe +query IRR +SELECT * +FROM test +|> SELECT a, b +|> EXTEND a + b AS a_plus_b +---- +1 1.1 2.1 +2 2.2 4.2 +3 3.3 6.3 From 88343e8e9aefd74cdd4037f91a95cf82a07c6d11 Mon Sep 17 00:00:00 2001 From: Simon Vandel Sillesen Date: Fri, 22 Aug 2025 16:49:41 +0200 Subject: [PATCH 06/12] document supported pipe operators in user guide --- docs/source/user-guide/sql/operators.md | 88 +++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/docs/source/user-guide/sql/operators.md b/docs/source/user-guide/sql/operators.md index b63f552396211..3e3a2796def22 100644 --- a/docs/source/user-guide/sql/operators.md +++ b/docs/source/user-guide/sql/operators.md @@ -613,3 +613,91 @@ bar") | bar | +-----------------+ ``` + +## Pipe operators +Some SQL dialects (e.g. BigQuery) support the pipe operator `|>`. +The SQL dialect can be set like this: +```sql +set datafusion.sql_parser.dialect = 'BigQuery'; +``` + +DataFusion currently supports the following pipe operators: +- [WHERE](#pipe_where) +- [ORDER BY](#pipe_order_by) +- [LIMIT](#pipe_limit) +- [SELECT](#pipe_select) +- [EXTEND](#pipe_extend) + + +(pipe_where)= +### WHERE + +```sql +> select * from range(0,10) +|> where value < 2; ++-------+ +| value | ++-------+ +| 0 | +| 1 | ++-------+ +``` + +(pipe_order_by)= +### ORDER BY + +```sql +> select * from range(0,3) +|> order by value desc; ++-------+ +| value | ++-------+ +| 2 | +| 1 | +| 0 | ++-------+ +``` + +(pipe_limit)= +### LIMIT + +```sql +> select * from range(0,3) +|> order by value desc +|> limit 1; ++-------+ +| value | ++-------+ +| 2 | ++-------+ +``` + +(pipe_select)= +### SELECT + +```sql +> select * from range(0,3) +|> select value + 10; ++---------------------------+ +| range().value + Int64(10) | ++---------------------------+ +| 10 | +| 11 | +| 12 | ++---------------------------+ +``` + +(pipe_extend)= +### EXTEND + +```sql +> select * from range(0,3) +|> extend -value AS minus_value; ++-------+-------------+ +| value | minus_value | ++-------+-------------+ +| 0 | 0 | +| 1 | -1 | +| 2 | -2 | ++-------+-------------+ +``` From ca453b232154749ceed8c3f88fb55eecd7626770 Mon Sep 17 00:00:00 2001 From: Simon Vandel Sillesen Date: Fri, 22 Aug 2025 21:48:50 +0200 Subject: [PATCH 07/12] fmt --- docs/source/user-guide/sql/operators.md | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/source/user-guide/sql/operators.md b/docs/source/user-guide/sql/operators.md index 3e3a2796def22..7bbfa9aad8c0c 100644 --- a/docs/source/user-guide/sql/operators.md +++ b/docs/source/user-guide/sql/operators.md @@ -615,21 +615,24 @@ bar | ``` ## Pipe operators + Some SQL dialects (e.g. BigQuery) support the pipe operator `|>`. The SQL dialect can be set like this: + ```sql set datafusion.sql_parser.dialect = 'BigQuery'; ``` DataFusion currently supports the following pipe operators: + - [WHERE](#pipe_where) - [ORDER BY](#pipe_order_by) - [LIMIT](#pipe_limit) - [SELECT](#pipe_select) - [EXTEND](#pipe_extend) - (pipe_where)= + ### WHERE ```sql @@ -644,6 +647,7 @@ DataFusion currently supports the following pipe operators: ``` (pipe_order_by)= + ### ORDER BY ```sql @@ -659,6 +663,7 @@ DataFusion currently supports the following pipe operators: ``` (pipe_limit)= + ### LIMIT ```sql @@ -673,6 +678,7 @@ DataFusion currently supports the following pipe operators: ``` (pipe_select)= + ### SELECT ```sql @@ -688,6 +694,7 @@ DataFusion currently supports the following pipe operators: ``` (pipe_extend)= + ### EXTEND ```sql From 1e0b8db994f020b2bc5a3c8d50826f86a8862b6d Mon Sep 17 00:00:00 2001 From: Simon Vandel Sillesen Date: Mon, 25 Aug 2025 22:57:12 +0200 Subject: [PATCH 08/12] fix where pipe before extend --- datafusion/sql/src/query.rs | 12 +++++------- datafusion/sqllogictest/test_files/pipe_operator.slt | 9 +++++++++ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/datafusion/sql/src/query.rs b/datafusion/sql/src/query.rs index 28ebf54695455..d13578a219b27 100644 --- a/datafusion/sql/src/query.rs +++ b/datafusion/sql/src/query.rs @@ -21,7 +21,7 @@ use crate::planner::{ContextProvider, PlannerContext, SqlToRel}; use crate::stack::StackGuard; use datafusion_common::{not_impl_err, Constraints, DFSchema, Result}; -use datafusion_expr::expr::Sort; +use datafusion_expr::expr::{Sort, WildcardOptions}; use datafusion_expr::select_expr::SelectExpr; use datafusion_expr::{ @@ -143,12 +143,10 @@ impl SqlToRel<'_, S> { let empty_from = matches!(plan, LogicalPlan::EmptyRelation(_)); let extend_exprs = self.prepare_select_exprs(&plan, exprs, empty_from, planner_context)?; - let all_exprs = plan - .expressions() - .into_iter() - .map(SelectExpr::Expression) - .chain(extend_exprs) - .collect(); + let all_exprs = + std::iter::once(SelectExpr::Wildcard(WildcardOptions::default())) + .chain(extend_exprs) + .collect(); self.project(plan, all_exprs) } diff --git a/datafusion/sqllogictest/test_files/pipe_operator.slt b/datafusion/sqllogictest/test_files/pipe_operator.slt index debe7d1fd8d27..6b92df9431385 100644 --- a/datafusion/sqllogictest/test_files/pipe_operator.slt +++ b/datafusion/sqllogictest/test_files/pipe_operator.slt @@ -80,3 +80,12 @@ FROM test 1 1.1 2.1 2 2.2 4.2 3 3.3 6.3 + +query IRR +SELECT * +FROM test +|> SELECT a, b +|> where a = 1 +|> EXTEND a + b AS a_plus_b +---- +1 1.1 2.1 From 6457150cb7aa5c9967c03bdf5e79ebafa7a95dbc Mon Sep 17 00:00:00 2001 From: Simon Vandel Sillesen Date: Sun, 21 Sep 2025 08:57:56 +0200 Subject: [PATCH 09/12] don't rebind --- datafusion/sql/src/query.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/datafusion/sql/src/query.rs b/datafusion/sql/src/query.rs index d13578a219b27..443601cf7f34f 100644 --- a/datafusion/sql/src/query.rs +++ b/datafusion/sql/src/query.rs @@ -89,11 +89,10 @@ impl SqlToRel<'_, S> { /// Apply pipe operators to a plan fn pipe_operators( &self, - plan: LogicalPlan, + mut plan: LogicalPlan, pipe_operators: Vec, planner_context: &mut PlannerContext, ) -> Result { - let mut plan = plan; for pipe_operator in pipe_operators { plan = self.pipe_operator(plan, pipe_operator, planner_context)?; } From dbcc0c0d4bdd8effae6a099cbffc0f220dad2846 Mon Sep 17 00:00:00 2001 From: Simon Vandel Sillesen Date: Sun, 21 Sep 2025 09:00:38 +0200 Subject: [PATCH 10/12] remove clone --- datafusion/sql/src/query.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/datafusion/sql/src/query.rs b/datafusion/sql/src/query.rs index 443601cf7f34f..9f8b483b8fef0 100644 --- a/datafusion/sql/src/query.rs +++ b/datafusion/sql/src/query.rs @@ -49,8 +49,6 @@ impl SqlToRel<'_, S> { self.plan_with_clause(with, planner_context)?; } - let pipe_operators = query.pipe_operators.clone(); - let set_expr = *query.body; let plan = match set_expr { SetExpr::Select(mut select) => { @@ -83,7 +81,7 @@ impl SqlToRel<'_, S> { } }?; - self.pipe_operators(plan, pipe_operators, planner_context) + self.pipe_operators(plan, query.pipe_operators, planner_context) } /// Apply pipe operators to a plan From 19d450d466903eff485760bbd30d55c490dc3dcf Mon Sep 17 00:00:00 2001 From: Simon Vandel Sillesen Date: Sun, 21 Sep 2025 09:06:48 +0200 Subject: [PATCH 11/12] move docs into select.md --- docs/source/user-guide/sql/operators.md | 95 ------------------------ docs/source/user-guide/sql/select.md | 96 +++++++++++++++++++++++++ 2 files changed, 96 insertions(+), 95 deletions(-) diff --git a/docs/source/user-guide/sql/operators.md b/docs/source/user-guide/sql/operators.md index 7bbfa9aad8c0c..b63f552396211 100644 --- a/docs/source/user-guide/sql/operators.md +++ b/docs/source/user-guide/sql/operators.md @@ -613,98 +613,3 @@ bar") | bar | +-----------------+ ``` - -## Pipe operators - -Some SQL dialects (e.g. BigQuery) support the pipe operator `|>`. -The SQL dialect can be set like this: - -```sql -set datafusion.sql_parser.dialect = 'BigQuery'; -``` - -DataFusion currently supports the following pipe operators: - -- [WHERE](#pipe_where) -- [ORDER BY](#pipe_order_by) -- [LIMIT](#pipe_limit) -- [SELECT](#pipe_select) -- [EXTEND](#pipe_extend) - -(pipe_where)= - -### WHERE - -```sql -> select * from range(0,10) -|> where value < 2; -+-------+ -| value | -+-------+ -| 0 | -| 1 | -+-------+ -``` - -(pipe_order_by)= - -### ORDER BY - -```sql -> select * from range(0,3) -|> order by value desc; -+-------+ -| value | -+-------+ -| 2 | -| 1 | -| 0 | -+-------+ -``` - -(pipe_limit)= - -### LIMIT - -```sql -> select * from range(0,3) -|> order by value desc -|> limit 1; -+-------+ -| value | -+-------+ -| 2 | -+-------+ -``` - -(pipe_select)= - -### SELECT - -```sql -> select * from range(0,3) -|> select value + 10; -+---------------------------+ -| range().value + Int64(10) | -+---------------------------+ -| 10 | -| 11 | -| 12 | -+---------------------------+ -``` - -(pipe_extend)= - -### EXTEND - -```sql -> select * from range(0,3) -|> extend -value AS minus_value; -+-------+-------------+ -| value | minus_value | -+-------+-------------+ -| 0 | 0 | -| 1 | -1 | -| 2 | -2 | -+-------+-------------+ -``` diff --git a/docs/source/user-guide/sql/select.md b/docs/source/user-guide/sql/select.md index eb8bca7a75ef0..895ada74871c9 100644 --- a/docs/source/user-guide/sql/select.md +++ b/docs/source/user-guide/sql/select.md @@ -40,6 +40,7 @@ DataFusion supports the following syntax for queries: [ [ORDER BY](#order-by-clause) expression [ ASC | DESC ][, ...] ]
[ [LIMIT](#limit-clause) count ]
[ [EXCLUDE | EXCEPT](#exclude-and-except-clause) ]
+[Pipe operators](#pipe-operators)
@@ -341,3 +342,98 @@ FROM table; SELECT * EXCLUDE(age, person) FROM table; ``` + +## Pipe operators + +Some SQL dialects (e.g. BigQuery) support the pipe operator `|>`. +The SQL dialect can be set like this: + +```sql +set datafusion.sql_parser.dialect = 'BigQuery'; +``` + +DataFusion currently supports the following pipe operators: + +- [WHERE](#pipe_where) +- [ORDER BY](#pipe_order_by) +- [LIMIT](#pipe_limit) +- [SELECT](#pipe_select) +- [EXTEND](#pipe_extend) + +(pipe_where)= + +### WHERE + +```sql +> select * from range(0,10) +|> where value < 2; ++-------+ +| value | ++-------+ +| 0 | +| 1 | ++-------+ +``` + +(pipe_order_by)= + +### ORDER BY + +```sql +> select * from range(0,3) +|> order by value desc; ++-------+ +| value | ++-------+ +| 2 | +| 1 | +| 0 | ++-------+ +``` + +(pipe_limit)= + +### LIMIT + +```sql +> select * from range(0,3) +|> order by value desc +|> limit 1; ++-------+ +| value | ++-------+ +| 2 | ++-------+ +``` + +(pipe_select)= + +### SELECT + +```sql +> select * from range(0,3) +|> select value + 10; ++---------------------------+ +| range().value + Int64(10) | ++---------------------------+ +| 10 | +| 11 | +| 12 | ++---------------------------+ +``` + +(pipe_extend)= + +### EXTEND + +```sql +> select * from range(0,3) +|> extend -value AS minus_value; ++-------+-------------+ +| value | minus_value | ++-------+-------------+ +| 0 | 0 | +| 1 | -1 | +| 2 | -2 | ++-------+-------------+ +``` From 1b699d39d126662f8d65f131e4fcecf66e8a5925 Mon Sep 17 00:00:00 2001 From: Simon Vandel Sillesen Date: Sun, 21 Sep 2025 09:07:41 +0200 Subject: [PATCH 12/12] avoid confusion by removing `>` in examples --- docs/source/user-guide/sql/select.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/source/user-guide/sql/select.md b/docs/source/user-guide/sql/select.md index 895ada74871c9..5cc75ec6cc39e 100644 --- a/docs/source/user-guide/sql/select.md +++ b/docs/source/user-guide/sql/select.md @@ -365,7 +365,7 @@ DataFusion currently supports the following pipe operators: ### WHERE ```sql -> select * from range(0,10) +select * from range(0,10) |> where value < 2; +-------+ | value | @@ -380,7 +380,7 @@ DataFusion currently supports the following pipe operators: ### ORDER BY ```sql -> select * from range(0,3) +select * from range(0,3) |> order by value desc; +-------+ | value | @@ -396,7 +396,7 @@ DataFusion currently supports the following pipe operators: ### LIMIT ```sql -> select * from range(0,3) +select * from range(0,3) |> order by value desc |> limit 1; +-------+ @@ -411,7 +411,7 @@ DataFusion currently supports the following pipe operators: ### SELECT ```sql -> select * from range(0,3) +select * from range(0,3) |> select value + 10; +---------------------------+ | range().value + Int64(10) | @@ -427,7 +427,7 @@ DataFusion currently supports the following pipe operators: ### EXTEND ```sql -> select * from range(0,3) +select * from range(0,3) |> extend -value AS minus_value; +-------+-------------+ | value | minus_value |