From d0253895e7a9758f6f9aee4bcd866df1831e6e4d Mon Sep 17 00:00:00 2001 From: Igor Izvekov Date: Wed, 8 Mar 2023 22:48:55 +0300 Subject: [PATCH 1/4] feat: add the similar optimization function for bitwise negative --- .../simplify_expressions/expr_simplifier.rs | 5 +++ .../src/simplify_expressions/utils.rs | 34 ++++++++++++++++++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs index 220d532b03b14..a87f84cc05578 100644 --- a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs +++ b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs @@ -1002,6 +1002,11 @@ impl<'a, S: SimplifyInfo> ExprRewriter for Simplifier<'a, S> { // Expr::Not(inner) => negate_clause(*inner), + // + // Rules for Negative + // + Expr::Negative(inner) => negative_clause(*inner), + // // Rules for Case // diff --git a/datafusion/optimizer/src/simplify_expressions/utils.rs b/datafusion/optimizer/src/simplify_expressions/utils.rs index 352674c3a68e8..4d0aa870b2af0 100644 --- a/datafusion/optimizer/src/simplify_expressions/utils.rs +++ b/datafusion/optimizer/src/simplify_expressions/utils.rs @@ -20,7 +20,7 @@ use datafusion_common::{DataFusionError, Result, ScalarValue}; use datafusion_expr::{ expr::{Between, BinaryExpr}, - expr_fn::{and, concat_ws, or}, + expr_fn::{and, concat_ws, or, bitwise_or, bitwise_and}, lit, BuiltinScalarFunction, Expr, Like, Operator, }; @@ -311,6 +311,38 @@ pub fn negate_clause(expr: Expr) -> Expr { } } +/// bitwise negate a Negative clause +/// input is the clause to be bitwise negated.(args for Negative clause) +/// For BinaryExpr: +/// ~(A & B) ===> ~A | ~B +/// ~(A | B) ===> ~A & ~B +/// ~(~A) ===> A +/// For others, use Negative clause +pub fn negative_clause(expr: Expr) -> Expr { + match expr { + Expr::BinaryExpr(BinaryExpr { left, op, right }) => { + match op { + Operator::BitwiseAnd => { + let left = negative_clause(*left); + let right = negative_clause(*right); + + bitwise_or(left, right); + } + Operator::BitwiseOr => { + let left = negative_clause(*left); + let right = negative_clause(*right); + + bitwise_and(left, right); + } + _ => Expr::Negative(Box::new(Expr::BinaryExpr(BinaryExpr::new( + left, op, right, + )))), + } + }, + _ => Expr::Negative(Box::new(expr)), + } +} + /// Simplify the `concat` function by /// 1. filtering out all `null` literals /// 2. concatenating contiguous literal arguments From e1d8e278bc83322f3503fa06b6685b32b36d2f35 Mon Sep 17 00:00:00 2001 From: Igor Izvekov Date: Wed, 8 Mar 2023 23:14:08 +0300 Subject: [PATCH 2/4] fix: some feature fixes --- .../optimizer/src/simplify_expressions/utils.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/datafusion/optimizer/src/simplify_expressions/utils.rs b/datafusion/optimizer/src/simplify_expressions/utils.rs index 4d0aa870b2af0..b404c3e0ae589 100644 --- a/datafusion/optimizer/src/simplify_expressions/utils.rs +++ b/datafusion/optimizer/src/simplify_expressions/utils.rs @@ -20,7 +20,7 @@ use datafusion_common::{DataFusionError, Result, ScalarValue}; use datafusion_expr::{ expr::{Between, BinaryExpr}, - expr_fn::{and, concat_ws, or, bitwise_or, bitwise_and}, + expr_fn::{and, bitwise_and, bitwise_or, concat_ws, or}, lit, BuiltinScalarFunction, Expr, Like, Operator, }; @@ -322,23 +322,28 @@ pub fn negative_clause(expr: Expr) -> Expr { match expr { Expr::BinaryExpr(BinaryExpr { left, op, right }) => { match op { + // ~(A & B) ===> ~A | ~B Operator::BitwiseAnd => { let left = negative_clause(*left); let right = negative_clause(*right); - bitwise_or(left, right); + bitwise_or(left, right) } + // ~(A | B) ===> ~A & ~B Operator::BitwiseOr => { let left = negative_clause(*left); let right = negative_clause(*right); - bitwise_and(left, right); + bitwise_and(left, right) } + // use negative clause _ => Expr::Negative(Box::new(Expr::BinaryExpr(BinaryExpr::new( left, op, right, )))), } - }, + } + // ~(~A) ===> A + Expr::Negative(expr) => *expr, _ => Expr::Negative(Box::new(expr)), } } From 97f92468b2101469600c3948dd182a502f896a69 Mon Sep 17 00:00:00 2001 From: Igor Izvekov Date: Sat, 11 Mar 2023 22:17:11 +0300 Subject: [PATCH 3/4] fix: change name from "negative_clause" to "distribute_negation" --- .../optimizer/src/simplify_expressions/expr_simplifier.rs | 2 +- datafusion/optimizer/src/simplify_expressions/utils.rs | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs index 40901da4ac9f0..32b20ea6ff859 100644 --- a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs +++ b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs @@ -1005,7 +1005,7 @@ impl<'a, S: SimplifyInfo> ExprRewriter for Simplifier<'a, S> { // // Rules for Negative // - Expr::Negative(inner) => negative_clause(*inner), + Expr::Negative(inner) => distribute_negation(*inner), // // Rules for Case diff --git a/datafusion/optimizer/src/simplify_expressions/utils.rs b/datafusion/optimizer/src/simplify_expressions/utils.rs index b404c3e0ae589..ff8fb4fdcb249 100644 --- a/datafusion/optimizer/src/simplify_expressions/utils.rs +++ b/datafusion/optimizer/src/simplify_expressions/utils.rs @@ -318,7 +318,7 @@ pub fn negate_clause(expr: Expr) -> Expr { /// ~(A | B) ===> ~A & ~B /// ~(~A) ===> A /// For others, use Negative clause -pub fn negative_clause(expr: Expr) -> Expr { +pub fn distribute_negation(expr: Expr) -> Expr { match expr { Expr::BinaryExpr(BinaryExpr { left, op, right }) => { match op { @@ -344,6 +344,7 @@ pub fn negative_clause(expr: Expr) -> Expr { } // ~(~A) ===> A Expr::Negative(expr) => *expr, + // use negative clause _ => Expr::Negative(Box::new(expr)), } } From 133f56681e1c86bfe2efe14e5351de6c07aedc8f Mon Sep 17 00:00:00 2001 From: Igor Izvekov Date: Sat, 11 Mar 2023 23:09:12 +0300 Subject: [PATCH 4/4] feat: add tests for De Morgan's laws --- .../simplify_expressions/expr_simplifier.rs | 36 ++++++++++++++++--- .../src/simplify_expressions/utils.rs | 9 ++--- 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs index 32b20ea6ff859..497ad34c99adb 100644 --- a/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs +++ b/datafusion/optimizer/src/simplify_expressions/expr_simplifier.rs @@ -2127,6 +2127,37 @@ mod tests { assert_eq!(simplify(expr), expected); } + #[test] + fn test_simplify_by_de_morgan_laws() { + // Laws with logical operations + // !(c3 AND c4) --> !c3 OR !c4 + let expr = and(col("c3"), col("c4")).not(); + let expected = or(col("c3").not(), col("c4").not()); + assert_eq!(simplify(expr), expected); + // !(c3 OR c4) --> !c3 AND !c4 + let expr = or(col("c3"), col("c4")).not(); + let expected = and(col("c3").not(), col("c4").not()); + assert_eq!(simplify(expr), expected); + // !(!c3) --> c3 + let expr = col("c3").not().not(); + let expected = col("c3"); + assert_eq!(simplify(expr), expected); + + // Laws with bitwise operations + // !(c3 & c4) --> !c3 | !c4 + let expr = -bitwise_and(col("c3"), col("c4")); + let expected = bitwise_or(-col("c3"), -col("c4")); + assert_eq!(simplify(expr), expected); + // !(c3 | c4) --> !c3 & !c4 + let expr = -bitwise_or(col("c3"), col("c4")); + let expected = bitwise_and(-col("c3"), -col("c4")); + assert_eq!(simplify(expr), expected); + // !(!c3) --> c3 + let expr = -(-col("c3")); + let expected = col("c3"); + assert_eq!(simplify(expr), expected); + } + #[test] fn test_simplify_null_and_false() { let expr = and(lit_bool_null(), lit(false)); @@ -2430,11 +2461,6 @@ mod tests { ) } - #[test] - fn simplify_expr_not_not() { - assert_eq!(simplify(col("c2").not().not().not()), col("c2").not(),); - } - #[test] fn simplify_expr_null_comparison() { // x = null is always null diff --git a/datafusion/optimizer/src/simplify_expressions/utils.rs b/datafusion/optimizer/src/simplify_expressions/utils.rs index ff8fb4fdcb249..8b3f437dc233e 100644 --- a/datafusion/optimizer/src/simplify_expressions/utils.rs +++ b/datafusion/optimizer/src/simplify_expressions/utils.rs @@ -316,6 +316,7 @@ pub fn negate_clause(expr: Expr) -> Expr { /// For BinaryExpr: /// ~(A & B) ===> ~A | ~B /// ~(A | B) ===> ~A & ~B +/// For Negative: /// ~(~A) ===> A /// For others, use Negative clause pub fn distribute_negation(expr: Expr) -> Expr { @@ -324,15 +325,15 @@ pub fn distribute_negation(expr: Expr) -> Expr { match op { // ~(A & B) ===> ~A | ~B Operator::BitwiseAnd => { - let left = negative_clause(*left); - let right = negative_clause(*right); + let left = distribute_negation(*left); + let right = distribute_negation(*right); bitwise_or(left, right) } // ~(A | B) ===> ~A & ~B Operator::BitwiseOr => { - let left = negative_clause(*left); - let right = negative_clause(*right); + let left = distribute_negation(*left); + let right = distribute_negation(*right); bitwise_and(left, right) }