Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
510b16c
Tmp
mustafasrepo Aug 7, 2024
6ef4369
Minor changes
mustafasrepo Aug 7, 2024
c3efafc
Minor changes
mustafasrepo Aug 7, 2024
2bf220d
Minor changes
mustafasrepo Aug 7, 2024
eb83917
Implement top down recursion with delete check
mustafasrepo Aug 7, 2024
0b66b15
Minor changes
mustafasrepo Aug 7, 2024
c769f9f
Minor changes
mustafasrepo Aug 7, 2024
0ad7063
Address reviews
mustafasrepo Aug 7, 2024
3661f06
Update comments
mustafasrepo Aug 7, 2024
60967c1
Minor changes
mustafasrepo Aug 7, 2024
6b87c4c
Make test deterministic
mustafasrepo Aug 7, 2024
8dd7e0a
Add fetch info to the statistics
mustafasrepo Aug 8, 2024
15423ae
Enforce distribution use inexact count estimate also.
mustafasrepo Aug 8, 2024
94fb83d
Minor changes
mustafasrepo Aug 8, 2024
9053b9f
Minor changes
mustafasrepo Aug 8, 2024
1171584
Minor changes
mustafasrepo Aug 8, 2024
711038d
Do not add unnecessary hash partitioning
mustafasrepo Aug 9, 2024
7e598e5
Minor changes
mustafasrepo Aug 9, 2024
12ad2c2
Add config option to use inexact row number estimates during planning
mustafasrepo Aug 9, 2024
2e3cc5d
Update config
mustafasrepo Aug 9, 2024
34af8ba
Minor changes
mustafasrepo Aug 9, 2024
98760bc
Minor changes
mustafasrepo Aug 9, 2024
1e4dada
Final review
ozankabak Aug 9, 2024
9fc4f3d
Address reviews
mustafasrepo Aug 9, 2024
1116058
Add handling for sort removal with fetch
mustafasrepo Aug 9, 2024
44dc292
Fix linter errors
mustafasrepo Aug 9, 2024
c6d2de6
Minor changes
mustafasrepo Aug 9, 2024
c7c85f4
Update config
mustafasrepo Aug 9, 2024
7c8967d
Cleanup stats under fetch
ozankabak Aug 9, 2024
ed35660
Update SLT comment
ozankabak Aug 10, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Update comments
  • Loading branch information
mustafasrepo committed Aug 7, 2024
commit 3661f06810c1319377ceec8ce383e769113a434b
7 changes: 4 additions & 3 deletions datafusion/core/src/physical_optimizer/enforce_sorting.rs
Original file line number Diff line number Diff line change
Expand Up @@ -483,9 +483,10 @@ fn adjust_window_sort_removal(
Ok(window_tree)
}

/// Removes the [`CoalescePartitionsExec`] from the plan in `node`.After the removal of
/// the `CoalescePartitionsExec` from the plan. Some of the `RepartitionExec`s might turn into
/// redundant. Removes those `RepartitionExec`s from the plan on the way also.
/// Removes parallelization-reducing, avoidable [`CoalescePartitionsExec`]s from the plan in `node`.
/// After the removal of such `CoalescePartitionsExec`s from the plan, some of the
/// `RepartitionExec`s might become redundant. Removes those `RepartitionExec`s from the plan as
/// well.
fn remove_bottleneck_in_subplan(
mut requirements: PlanWithCorrespondingCoalescePartitions,
) -> Result<PlanWithCorrespondingCoalescePartitions> {
Expand Down
14 changes: 9 additions & 5 deletions datafusion/core/src/physical_optimizer/sort_pushdown.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,7 @@ use std::fmt::Debug;
use std::sync::Arc;

use super::utils::{add_sort_above, is_sort};
use crate::physical_optimizer::utils::{
is_limit, is_sort_preserving_merge, is_union, is_window,
};
use crate::physical_optimizer::utils::{is_sort_preserving_merge, is_union, is_window};
use crate::physical_plan::filter::FilterExec;
use crate::physical_plan::joins::utils::calculate_join_output_ordering;
use crate::physical_plan::joins::{HashJoinExec, SortMergeJoinExec};
Expand Down Expand Up @@ -188,11 +186,17 @@ fn pushdown_requirement_to_children(
} else {
Ok(None)
}
} else if is_limit(plan) {
} else if plan.fetch().is_some()
&& plan.maintains_input_order().len() == 1
&& plan.maintains_input_order()[0]
&& plan.supports_limit_pushdown()
{
let output_req = PhysicalSortRequirement::from_sort_exprs(
plan.properties().output_ordering().unwrap_or(&[]),
);
// Push down through limit only when requirement is aligned with output ordering.
// Push down through operator with fetch when:
// - requirement is aligned with output ordering
// - it preserves ordering during execution
if plan
.properties()
.eq_properties
Expand Down