From a8e96dfdce65c15287623ebb3d92fd8dd0989741 Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Fri, 24 Apr 2026 07:47:13 +0100 Subject: [PATCH 01/11] Remove incorrect comments about Rvalue::Aggregate not being allowed This is no longer true as of #107267. --- compiler/rustc_middle/src/mir/syntax.rs | 4 ---- compiler/rustc_public/src/mir/body.rs | 3 --- 2 files changed, 7 deletions(-) diff --git a/compiler/rustc_middle/src/mir/syntax.rs b/compiler/rustc_middle/src/mir/syntax.rs index 8b015e6cecaae..794b044d4c233 100644 --- a/compiler/rustc_middle/src/mir/syntax.rs +++ b/compiler/rustc_middle/src/mir/syntax.rs @@ -124,7 +124,6 @@ pub enum RuntimePhase { /// disallowed: /// * [`TerminatorKind::Yield`] /// * [`TerminatorKind::CoroutineDrop`] - /// * [`Rvalue::Aggregate`] for any `AggregateKind` except `Array` /// * [`Rvalue::CopyForDeref`] /// * [`PlaceElem::OpaqueCast`] /// * [`LocalInfo::DerefTemp`](super::LocalInfo::DerefTemp) @@ -1442,9 +1441,6 @@ pub enum Rvalue<'tcx> { /// This is needed because dataflow analysis needs to distinguish /// `dest = Foo { x: ..., y: ... }` from `dest.x = ...; dest.y = ...;` in the case that `Foo` /// has a destructor. - /// - /// Disallowed after deaggregation for all aggregate kinds except `Array` and `Coroutine`. After - /// coroutine lowering, `Coroutine` aggregate kinds are disallowed too. Aggregate(Box>, IndexVec>), /// A CopyForDeref is equivalent to a read from a place at the diff --git a/compiler/rustc_public/src/mir/body.rs b/compiler/rustc_public/src/mir/body.rs index f9b5f9af951e5..ff73f74237139 100644 --- a/compiler/rustc_public/src/mir/body.rs +++ b/compiler/rustc_public/src/mir/body.rs @@ -499,9 +499,6 @@ pub enum Rvalue { /// This is needed because dataflow analysis needs to distinguish /// `dest = Foo { x: ..., y: ... }` from `dest.x = ...; dest.y = ...;` in the case that `Foo` /// has a destructor. - /// - /// Disallowed after deaggregation for all aggregate kinds except `Array` and `Coroutine`. After - /// coroutine lowering, `Coroutine` aggregate kinds are disallowed too. Aggregate(AggregateKind, Vec), /// * `Offset` has the same semantics as `<*const T>::offset`, except that the second From 2fc24d428731b4cceb18a1257567edec2555a20b Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Tue, 28 Apr 2026 09:37:26 +0200 Subject: [PATCH 02/11] Pass block to `visit_block_start` and `visit_block_end` --- compiler/rustc_mir_dataflow/src/framework/direction.rs | 8 ++++---- compiler/rustc_mir_dataflow/src/framework/graphviz.rs | 4 ++-- compiler/rustc_mir_dataflow/src/framework/visitor.rs | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/compiler/rustc_mir_dataflow/src/framework/direction.rs b/compiler/rustc_mir_dataflow/src/framework/direction.rs index f8d0885a2808f..a73abc30c03ea 100644 --- a/compiler/rustc_mir_dataflow/src/framework/direction.rs +++ b/compiler/rustc_mir_dataflow/src/framework/direction.rs @@ -212,7 +212,7 @@ impl Direction for Backward { ) where A: Analysis<'tcx>, { - vis.visit_block_end(state); + vis.visit_block_end(state, block); let loc = Location { block, statement_index: block_data.statements.len() }; let term = block_data.terminator(); @@ -229,7 +229,7 @@ impl Direction for Backward { vis.visit_after_primary_statement_effect(analysis, state, stmt, loc); } - vis.visit_block_start(state); + vis.visit_block_start(state, block); } } @@ -391,7 +391,7 @@ impl Direction for Forward { ) where A: Analysis<'tcx>, { - vis.visit_block_start(state); + vis.visit_block_start(state, block); for (statement_index, stmt) in block_data.statements.iter().enumerate() { let loc = Location { block, statement_index }; @@ -408,6 +408,6 @@ impl Direction for Forward { analysis.apply_primary_terminator_effect(state, term, loc); vis.visit_after_primary_terminator_effect(analysis, state, term, loc); - vis.visit_block_end(state); + vis.visit_block_end(state, block); } } diff --git a/compiler/rustc_mir_dataflow/src/framework/graphviz.rs b/compiler/rustc_mir_dataflow/src/framework/graphviz.rs index 6c0f2e8d73058..28320c29ce2dd 100644 --- a/compiler/rustc_mir_dataflow/src/framework/graphviz.rs +++ b/compiler/rustc_mir_dataflow/src/framework/graphviz.rs @@ -660,13 +660,13 @@ where A: Analysis<'tcx>, A::Domain: DebugWithContext, { - fn visit_block_start(&mut self, state: &A::Domain) { + fn visit_block_start(&mut self, state: &A::Domain, _block: BasicBlock) { if A::Direction::IS_FORWARD { self.prev_state.clone_from(state); } } - fn visit_block_end(&mut self, state: &A::Domain) { + fn visit_block_end(&mut self, state: &A::Domain, _block: BasicBlock) { if A::Direction::IS_BACKWARD { self.prev_state.clone_from(state); } diff --git a/compiler/rustc_mir_dataflow/src/framework/visitor.rs b/compiler/rustc_mir_dataflow/src/framework/visitor.rs index 46940c6ab62fc..f5693bcffd891 100644 --- a/compiler/rustc_mir_dataflow/src/framework/visitor.rs +++ b/compiler/rustc_mir_dataflow/src/framework/visitor.rs @@ -46,7 +46,7 @@ pub trait ResultsVisitor<'tcx, A> where A: Analysis<'tcx>, { - fn visit_block_start(&mut self, _state: &A::Domain) {} + fn visit_block_start(&mut self, _state: &A::Domain, _block: BasicBlock) {} /// Called after the "early" effect of the given statement is applied to `state`. fn visit_after_early_statement_effect( @@ -90,5 +90,5 @@ where ) { } - fn visit_block_end(&mut self, _state: &A::Domain) {} + fn visit_block_end(&mut self, _state: &A::Domain, _block: BasicBlock) {} } From 8af1fb3eae8fff9b88e580ef871afed1afa43638 Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Sun, 3 May 2026 00:43:02 +0100 Subject: [PATCH 03/11] Various changes to `SparseIntervalMatrix` - Changed `iter_intervals` to return `RangeInclusive` instead of `Range` - Added `clear_row` and `disjoint_rows` methods --- compiler/rustc_index/src/interval.rs | 43 ++++++++++++++++------ compiler/rustc_index/src/interval/tests.rs | 2 +- 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/compiler/rustc_index/src/interval.rs b/compiler/rustc_index/src/interval.rs index 716dd81d08fd4..72cc899eda36e 100644 --- a/compiler/rustc_index/src/interval.rs +++ b/compiler/rustc_index/src/interval.rs @@ -1,6 +1,7 @@ use std::iter::Step; use std::marker::PhantomData; -use std::ops::{Bound, Range, RangeBounds}; +use std::ops::{Bound, RangeBounds}; +use std::range::RangeInclusive; use smallvec::SmallVec; @@ -59,11 +60,14 @@ impl IntervalSet { } /// Iterates through intervals stored in the set, in order. - pub fn iter_intervals(&self) -> impl Iterator> + pub fn iter_intervals(&self) -> impl Iterator> where I: Step, { - self.map.iter().map(|&(start, end)| I::new(start as usize)..I::new(end as usize + 1)) + self.map.iter().map(|&(start, end)| RangeInclusive { + start: I::new(start as usize), + last: I::new(end as usize), + }) } /// Returns true if we increased the number of elements present. @@ -210,11 +214,12 @@ impl IntervalSet { { let mut sup_iter = self.iter_intervals(); let mut current = None; - let contains = |sup: Range, sub: Range, current: &mut Option>| { - if sup.end < sub.start { - // if `sup.end == sub.start`, the next sup doesn't contain `sub.start` + let contains = |sup: RangeInclusive, + sub: RangeInclusive, + current: &mut Option>| { + if sup.last < sub.start { None // continue to the next sup - } else if sup.end >= sub.end && sup.start <= sub.start { + } else if sup.last >= sub.last && sup.start <= sub.start { *current = Some(sup); // save the current sup Some(true) } else { @@ -224,8 +229,8 @@ impl IntervalSet { other.iter_intervals().all(|sub| { current .take() - .and_then(|sup| contains(sup, sub.clone(), &mut current)) - .or_else(|| sup_iter.find_map(|sup| contains(sup, sub.clone(), &mut current))) + .and_then(|sup| contains(sup, sub, &mut current)) + .or_else(|| sup_iter.find_map(|sup| contains(sup, sub, &mut current))) .unwrap_or(false) }) } @@ -242,11 +247,11 @@ impl IntervalSet { let mut other_current = other_iter.next()?; loop { - if self_current.end <= other_current.start { + if self_current.last < other_current.start { self_current = self_iter.next()?; continue; } - if other_current.end <= self_current.start { + if other_current.last < self_current.start { other_current = other_iter.next()?; continue; } @@ -370,6 +375,12 @@ impl SparseIntervalMatrix { self.rows.get(row) } + pub fn clear_row(&mut self, row: R) { + if let Some(row) = self.rows.get_mut(row) { + row.clear(); + } + } + fn ensure_row(&mut self, row: R) -> &mut IntervalSet { self.rows.ensure_contains_elem(row, || IntervalSet::new(self.column_size)) } @@ -393,6 +404,16 @@ impl SparseIntervalMatrix { write_row.union(read_row) } + pub fn disjoint_rows(&self, a: R, b: R) -> bool + where + C: Step, + { + let (Some(a), Some(b)) = (self.rows.get(a), self.rows.get(b)) else { + return true; + }; + a.disjoint(b) + } + pub fn insert_all_into_row(&mut self, row: R) { self.ensure_row(row).insert_all(); } diff --git a/compiler/rustc_index/src/interval/tests.rs b/compiler/rustc_index/src/interval/tests.rs index 375af60f66207..cf3222e6c6572 100644 --- a/compiler/rustc_index/src/interval/tests.rs +++ b/compiler/rustc_index/src/interval/tests.rs @@ -5,7 +5,7 @@ fn insert_collapses() { let mut set = IntervalSet::::new(10000); set.insert_range(9831..=9837); set.insert_range(43..=9830); - assert_eq!(set.iter_intervals().collect::>(), [43..9838]); + assert_eq!(set.iter_intervals().collect::>(), [(43..=9837).into()]); } #[test] From 62e9181b9ccdd1e190191ac01ab95b01e6911569 Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Sun, 3 May 2026 00:51:40 +0100 Subject: [PATCH 04/11] WIP move elimination --- compiler/rustc_index/src/bit_set.rs | 26 + compiler/rustc_index/src/interval.rs | 20 + compiler/rustc_mir_dataflow/src/impls/mod.rs | 4 + .../src/impls/precise_liveness.rs | 560 +++++++++++ compiler/rustc_mir_dataflow/src/points.rs | 9 + compiler/rustc_mir_transform/src/dest_prop.rs | 2 +- compiler/rustc_mir_transform/src/lib.rs | 2 + .../src/move_elimination.rs | 909 ++++++++++++++++++ compiler/rustc_mir_transform/src/patch.rs | 15 + 9 files changed, 1546 insertions(+), 1 deletion(-) create mode 100644 compiler/rustc_mir_dataflow/src/impls/precise_liveness.rs create mode 100644 compiler/rustc_mir_transform/src/move_elimination.rs diff --git a/compiler/rustc_index/src/bit_set.rs b/compiler/rustc_index/src/bit_set.rs index 2910ba7c46851..698a12e084fb4 100644 --- a/compiler/rustc_index/src/bit_set.rs +++ b/compiler/rustc_index/src/bit_set.rs @@ -324,6 +324,15 @@ impl DenseBitSet { // out-of-domain bits, so we need to clear them. self.clear_excess_bits(); } + + /// Sets `self = self & (a | b)` without allocating a temporary for `a | b`. + /// + /// Returns `true` if `self` changed. + pub fn intersect_with_union(&mut self, a: &DenseBitSet, b: &DenseBitSet) -> bool { + assert_eq!(self.domain_size, a.domain_size); + assert_eq!(self.domain_size, b.domain_size); + bitwise3(&mut self.words, &a.words, &b.words, |s, a, b| s & (a | b)) + } } // dense REL dense @@ -1084,6 +1093,23 @@ where changed != 0 } +#[inline] +fn bitwise3(out_vec: &mut [Word], in_vec1: &[Word], in_vec2: &[Word], op: Op) -> bool +where + Op: Fn(Word, Word, Word) -> Word, +{ + assert_eq!(out_vec.len(), in_vec1.len()); + assert_eq!(out_vec.len(), in_vec2.len()); + let mut changed = 0; + for ((out_elem, in_elem1), in_elem2) in iter::zip(iter::zip(out_vec, in_vec1), in_vec2) { + let old_val = *out_elem; + let new_val = op(old_val, *in_elem1, *in_elem2); + *out_elem = new_val; + changed |= old_val ^ new_val; + } + changed != 0 +} + /// Returns true if a call to [`update_words`] would modify `lhs`, i.e. /// `lhs[i] != op(lhs[i], rhs[i])` for some `i`. #[inline] diff --git a/compiler/rustc_index/src/interval.rs b/compiler/rustc_index/src/interval.rs index 72cc899eda36e..91da30a0d2d51 100644 --- a/compiler/rustc_index/src/interval.rs +++ b/compiler/rustc_index/src/interval.rs @@ -208,6 +208,26 @@ impl IntervalSet { needle <= *prev_end } + /// Returns whether any point in `range` is contained in the set. + pub fn intersects_range(&self, range: impl RangeBounds + Clone) -> bool { + let start = inclusive_start(range.clone()); + let Some(end) = inclusive_end(self.domain, range) else { + // empty range + return false; + }; + if start > end { + return false; + } + + // Find the last interval whose start is <= end. + let Some(last) = self.map.partition_point(|r| r.0 <= end).checked_sub(1) else { + // All ranges in the map start after the new range's end + return false; + }; + let (_, prev_end) = &self.map[last]; + start <= *prev_end + } + pub fn superset(&self, other: &IntervalSet) -> bool where I: Step, diff --git a/compiler/rustc_mir_dataflow/src/impls/mod.rs b/compiler/rustc_mir_dataflow/src/impls/mod.rs index 6d573e1c00e1c..507932c79ffaa 100644 --- a/compiler/rustc_mir_dataflow/src/impls/mod.rs +++ b/compiler/rustc_mir_dataflow/src/impls/mod.rs @@ -1,6 +1,7 @@ mod borrowed_locals; mod initialized; mod liveness; +mod precise_liveness; mod storage_liveness; pub use self::borrowed_locals::{MaybeBorrowedLocals, borrowed_locals}; @@ -12,6 +13,9 @@ pub use self::liveness::{ DefUse, MaybeLiveLocals, MaybeTransitiveLiveLocals, TransferFunction as LivenessTransferFunction, }; +pub use self::precise_liveness::{ + SplitPointEffect, SplitPointIndex, dump_liveness_matrix, liveness_matrix, +}; pub use self::storage_liveness::{ MaybeRequiresStorage, MaybeStorageDead, MaybeStorageLive, always_storage_live_locals, }; diff --git a/compiler/rustc_mir_dataflow/src/impls/precise_liveness.rs b/compiler/rustc_mir_dataflow/src/impls/precise_liveness.rs new file mode 100644 index 0000000000000..f2d91dea742b4 --- /dev/null +++ b/compiler/rustc_mir_dataflow/src/impls/precise_liveness.rs @@ -0,0 +1,560 @@ +use std::fmt; + +use rustc_index::IndexVec; +use rustc_index::bit_set::DenseBitSet; +use rustc_index::interval::SparseIntervalMatrix; +use rustc_middle::mir::visit::{ + MutatingUseContext, NonMutatingUseContext, PlaceContext, VisitPlacesWith, Visitor, +}; +use rustc_middle::mir::{self, BasicBlock, Local, Location, MirDumper, PassWhere, Place}; +use rustc_middle::ty::TyCtxt; +use tracing::trace; + +use crate::fmt::DebugWithContext; +use crate::impls::{DefUse, MaybeBorrowedLocals, MaybeLiveLocals}; +use crate::points::{DenseLocationMap, PointIndex}; +use crate::{Analysis, GenKill, JoinSemiLattice, ResultsVisitor, visit_reachable_results}; + +struct KillPointsVisitor<'a> { + kill_points: &'a mut Vec<(Local, Location)>, + live_on_entry: &'a mut IndexVec>, +} + +impl<'tcx> ResultsVisitor<'tcx, MaybeLiveLocals> for KillPointsVisitor<'_> { + fn visit_block_start(&mut self, state: &DenseBitSet, block: BasicBlock) { + self.live_on_entry[block].clone_from(state); + } + + fn visit_after_early_statement_effect( + &mut self, + _analysis: &MaybeLiveLocals, + state: &DenseBitSet, + statement: &mir::Statement<'tcx>, + location: Location, + ) { + VisitPlacesWith(|place: Place<'tcx>, ctxt| { + // Ignore non-uses. + match ctxt { + PlaceContext::NonMutatingUse(_) | PlaceContext::MutatingUse(_) => {} + PlaceContext::NonUse(_) => return, + } + + // If a local is used in a statement but is dead after it then this + // location is a kill point. + if !state.contains(place.local) { + self.kill_points.push((place.local, location)); + } + }) + .visit_statement(statement, location); + } + + fn visit_after_early_terminator_effect( + &mut self, + _analysis: &MaybeLiveLocals, + state: &DenseBitSet, + terminator: &mir::Terminator<'tcx>, + location: Location, + ) { + VisitPlacesWith(|place: Place<'tcx>, ctxt| { + // Ignore non-uses (they don't do anything) and edge uses (kill + // points for those go at the start of the corresponding successor). + match ctxt { + PlaceContext::MutatingUse( + MutatingUseContext::AsmOutput + | MutatingUseContext::Call + | MutatingUseContext::Yield, + ) + | PlaceContext::NonUse(_) => return, + PlaceContext::NonMutatingUse(_) | PlaceContext::MutatingUse(_) => {} + } + + // If a local is used in a terminator but is dead after it then this + // location is a kill point. + if !state.contains(place.local) { + self.kill_points.push((place.local, location)); + } + }) + .visit_terminator(terminator, location); + } +} + +#[derive(Debug, PartialEq, Eq)] +struct Domain { + maybe_live: DenseBitSet, + maybe_borrowed: DenseBitSet, +} + +impl Clone for Domain { + fn clone(&self) -> Self { + Domain { maybe_live: self.maybe_live.clone(), maybe_borrowed: self.maybe_borrowed.clone() } + } + + // Data flow engine when possible uses `clone_from` for domain values. + // Providing an implementation will avoid some intermediate memory allocations. + fn clone_from(&mut self, other: &Self) { + self.maybe_live.clone_from(&other.maybe_live); + self.maybe_borrowed.clone_from(&other.maybe_borrowed); + } +} + +impl JoinSemiLattice for Domain { + fn join(&mut self, other: &Self) -> bool { + self.maybe_live.join(&other.maybe_live) | self.maybe_borrowed.join(&other.maybe_borrowed) + } +} + +impl DebugWithContext for Domain { + fn fmt_with(&self, ctxt: &C, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("maybe_live: ")?; + self.maybe_live.fmt_with(ctxt, f)?; + f.write_str("maybe_borrowed: ")?; + self.maybe_borrowed.fmt_with(ctxt, f)?; + Ok(()) + } + + fn fmt_diff_with(&self, old: &Self, ctxt: &C, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self == old { + return Ok(()); + } + + if self.maybe_live != old.maybe_live { + f.write_str("maybe_live: ")?; + self.maybe_live.fmt_diff_with(&old.maybe_live, ctxt, f)?; + f.write_str("\n")?; + } + + if self.maybe_borrowed != old.maybe_borrowed { + f.write_str("maybe_borrowed: ")?; + self.maybe_borrowed.fmt_diff_with(&old.maybe_borrowed, ctxt, f)?; + f.write_str("\n")?; + } + + Ok(()) + } +} + +struct PreciseLiveness<'a> { + kill_point_map: &'a IndexVec, + live_on_entry: &'a IndexVec>, + points: &'a DenseLocationMap, +} + +impl PreciseLiveness<'_> { + fn apply_block_start_effect(&self, state: &mut Domain, block: BasicBlock) { + // Only keep locals that are either live or borrowed. + // + // Notably this kills any dead results produced by a predecessor's + // terminator. + state.maybe_live.intersect_with_union(&self.live_on_entry[block], &state.maybe_borrowed); + } +} + +impl<'tcx> Analysis<'tcx> for PreciseLiveness<'_> { + type Domain = Domain; + + const NAME: &'static str = "precise_liveness"; + + fn bottom_value(&self, body: &mir::Body<'tcx>) -> Domain { + Domain { + maybe_live: DenseBitSet::new_empty(body.local_decls.len()), + maybe_borrowed: DenseBitSet::new_empty(body.local_decls.len()), + } + } + + fn initialize_start_block(&self, body: &mir::Body<'tcx>, state: &mut Domain) { + // Function arguments start out as live. + for arg in body.args_iter() { + state.maybe_live.gen_(arg); + } + } + + fn apply_primary_statement_effect( + &self, + state: &mut Domain, + statement: &mir::Statement<'tcx>, + location: Location, + ) { + if location.statement_index == 0 { + self.apply_block_start_effect(state, location.block); + } + + // StorageDead always kills a local, even if it has been borrowed. + if let mir::StatementKind::StorageDead(local) = statement.kind { + state.maybe_live.kill(local); + state.maybe_borrowed.kill(local); + return; + } + + MaybeBorrowedLocals::transfer_function(&mut state.maybe_borrowed) + .visit_statement(statement, location); + + // Kill moved operands if the whole local was moved. + VisitPlacesWith(|place: Place<'tcx>, ctxt| { + if ctxt == PlaceContext::NonMutatingUse(NonMutatingUseContext::Move) { + if let Some(local) = place.as_local() { + state.maybe_live.kill(local); + state.maybe_borrowed.kill(local); + } + } + }) + .visit_statement(statement, location); + + // Gen destination places. + VisitPlacesWith(|place: Place<'tcx>, ctxt| match DefUse::for_place(place, ctxt) { + DefUse::Def | DefUse::PartialWrite => state.maybe_live.gen_(place.local), + DefUse::Use | DefUse::NonUse => {} + }) + .visit_statement(statement, location); + + // Apply kill points at this statement: if a variable is dead + // then it doesn't need storage, *except* if its address has been taken. + let point = self.points.point_from_location(location); + for &(local, _) in self.kill_point_map[point] { + if !state.maybe_borrowed.contains(local) { + state.maybe_live.kill(local); + } + } + } + + fn apply_primary_terminator_effect<'mir>( + &self, + state: &mut Domain, + terminator: &'mir mir::Terminator<'tcx>, + location: Location, + ) -> mir::TerminatorEdges<'mir, 'tcx> { + if location.statement_index == 0 { + self.apply_block_start_effect(state, location.block); + } + + MaybeBorrowedLocals::transfer_function(&mut state.maybe_borrowed) + .visit_terminator(terminator, location); + + // Kill moved operands if the whole local was moved. Also kill dropped + // places if the entire local was dropped. + VisitPlacesWith(|place: Place<'tcx>, ctxt| { + if let PlaceContext::NonMutatingUse(NonMutatingUseContext::Move) + | PlaceContext::MutatingUse(MutatingUseContext::Drop) = ctxt + { + if let Some(local) = place.as_local() { + state.maybe_live.kill(local); + state.maybe_borrowed.kill(local); + } + } + }) + .visit_terminator(terminator, location); + + // Gen destination places. + VisitPlacesWith(|place: Place<'tcx>, ctxt| { + // These are handled through `apply_call_return_effect`. + if let PlaceContext::MutatingUse( + MutatingUseContext::AsmOutput + | MutatingUseContext::Call + | MutatingUseContext::Yield, + ) = ctxt + { + return; + } + + match DefUse::for_place(place, ctxt) { + DefUse::Def | DefUse::PartialWrite => state.maybe_live.gen_(place.local), + DefUse::Use | DefUse::NonUse => {} + } + }) + .visit_terminator(terminator, location); + + terminator.edges() + } + + fn apply_call_return_effect( + &self, + state: &mut Domain, + _block: BasicBlock, + return_places: mir::CallReturnPlaces<'_, 'tcx>, + ) { + return_places.for_each(|place| state.maybe_live.gen_(place.local)); + } +} + +/// Different "phases" of a single MIR statement, used to describe how +/// overlapping operands are handled. +/// +/// As a general rule, source operands are read in the `Early` phase and +/// destination places are written in the `Late` phase. +#[derive(Copy, Clone, Debug)] +pub enum SplitPointEffect { + Early = 0, + Late = 1, +} + +rustc_index::newtype_index! { + /// A `PointIndex` with the lower bit encoding early/late inside a statement. + /// + /// This is used to model overlap constraints within a MIR statement: if a + /// source/destination are allowed to overlap then the source is read in + /// `SplitPointEffect::Early` and the write is done in + /// `SplitPointEffect::Late`. + #[orderable] + #[debug_format = "SplitPointIndex({})"] + pub struct SplitPointIndex {} +} + +impl SplitPointIndex { + pub fn new(point: PointIndex, effect: SplitPointEffect) -> SplitPointIndex { + let index = (point.as_u32() << 1) | (effect as u32); + SplitPointIndex::from_u32(index) + } + + pub fn point(self) -> PointIndex { + PointIndex::from_u32(self.as_u32() >> 1) + } + + pub fn effect(self) -> SplitPointEffect { + match self.as_u32() & 1 { + 0 => SplitPointEffect::Early, + 1 => SplitPointEffect::Late, + _ => unreachable!(), + } + } +} + +fn compute_kill_points<'tcx>( + tcx: TyCtxt<'tcx>, + body: &mir::Body<'tcx>, + pass_name: Option<&'static str>, +) -> (Vec<(Local, Location)>, IndexVec>) { + let maybe_live_locals = MaybeLiveLocals.iterate_to_fixpoint(tcx, body, pass_name); + let mut kill_points = vec![]; + let mut live_on_entry = IndexVec::from_elem_n( + DenseBitSet::new_empty(body.local_decls.len()), + body.basic_blocks.len(), + ); + let mut visitor = + KillPointsVisitor { kill_points: &mut kill_points, live_on_entry: &mut live_on_entry }; + visit_reachable_results(body, &maybe_live_locals, &mut visitor); + trace!(?kill_points); + trace!(?live_on_entry); + (kill_points, live_on_entry) +} + +fn kill_point_map<'a>( + kill_points: &'a [(Local, Location)], + points: &DenseLocationMap, +) -> IndexVec { + let mut out = IndexVec::from_elem_n(&[][..], points.num_points()); + for chunk in kill_points.chunk_by(|a, b| a.1 == b.1) { + let point = points.point_from_location(chunk[0].1); + trace!("Kill points at {:?}: {:?}", chunk[0].1, chunk); + out[point] = chunk; + } + out +} + +/// Helper type to construct a `SparseIntervalMatrix`. +struct MatrixBuilder { + matrix: SparseIntervalMatrix, + range_start: IndexVec>, +} + +impl MatrixBuilder { + fn gen_(&mut self, local: Local, point: PointIndex, effect: SplitPointEffect) { + let split_point = SplitPointIndex::new(point, effect); + + // No-op if the local is already live. + self.range_start[local].get_or_insert(split_point); + } + + fn kill(&mut self, local: Local, point: PointIndex, effect: SplitPointEffect) { + let end = SplitPointIndex::new(point, effect); + + // No-op if the local is already dead. + if let Some(start) = self.range_start[local].take() { + debug_assert!(end >= start); + self.matrix.append_range(local, start..=end); + } + } +} + +pub fn liveness_matrix<'tcx>( + tcx: TyCtxt<'tcx>, + body: &mir::Body<'tcx>, + points: &DenseLocationMap, + pass_name: Option<&'static str>, +) -> SparseIntervalMatrix { + let (kill_points, live_on_entry) = compute_kill_points(tcx, body, pass_name); + let kill_point_map = &kill_point_map(&kill_points, points); + let mut results = PreciseLiveness { kill_point_map, live_on_entry: &live_on_entry, points } + .iterate_to_fixpoint(tcx, body, pass_name); + + let mut builder = MatrixBuilder { + matrix: SparseIntervalMatrix::new(points.num_points() * 2), + range_start: IndexVec::from_elem_n(None, body.local_decls.len()), + }; + for (block, block_data) in body.basic_blocks.iter_enumerated() { + // We can mutate the state in-place since we're not using it any more + // after this point. + let state = &mut results.entry_states[block]; + + // Only keep locals that are either live or borrowed. + // + // Notably this kills any dead results produced by a predecessor's + // terminator. + state.maybe_live.intersect_with_union(&live_on_entry[block], &state.maybe_borrowed); + + for local in state.maybe_live.iter() { + builder.gen_(local, points.entry_point(block), SplitPointEffect::Early); + } + + for (statement_index, statement) in block_data.statements.iter().enumerate() { + let location = Location { block, statement_index }; + let point = points.point_from_location(location); + + // StorageDead always kills a local, even if it has been borrowed. + if let mir::StatementKind::StorageDead(local) = statement.kind { + builder.kill(local, point, SplitPointEffect::Late); + state.maybe_borrowed.kill(local); + continue; + } + + MaybeBorrowedLocals::transfer_function(&mut state.maybe_borrowed) + .visit_statement(statement, location); + + // Kill moved operands if the whole local was moved. + VisitPlacesWith(|place: Place<'tcx>, ctxt| { + if ctxt == PlaceContext::NonMutatingUse(NonMutatingUseContext::Move) { + if let Some(local) = place.as_local() { + builder.kill(local, point, SplitPointEffect::Early); + state.maybe_borrowed.kill(local); + } + } + }) + .visit_statement(statement, location); + + // Kill any locals which are no longer used after this statement, + // but only if they have not been borrowed. + for &(local, _) in kill_point_map[point] { + if !state.maybe_borrowed.contains(local) { + builder.kill(local, point, SplitPointEffect::Early); + } + } + + // Gen destination places. + VisitPlacesWith(|place: Place<'tcx>, ctxt| match DefUse::for_place(place, ctxt) { + DefUse::Def | DefUse::PartialWrite => { + builder.gen_(place.local, point, SplitPointEffect::Late) + } + DefUse::Use | DefUse::NonUse => {} + }) + .visit_statement(statement, location); + + // Kill any dead destination places: they will only appear at + // the late point of the statement they are generated in, which is + // sufficient for determining overlap. + for &(local, _) in kill_point_map[point] { + if !state.maybe_borrowed.contains(local) { + builder.kill(local, point, SplitPointEffect::Late); + } + } + } + + let location = Location { block, statement_index: block_data.statements.len() }; + let point = points.point_from_location(location); + let terminator = block_data.terminator(); + + MaybeBorrowedLocals::transfer_function(&mut state.maybe_borrowed) + .visit_terminator(terminator, location); + + // Kill moved operands if the whole local was moved. Also kill dropped + // places if the entire local was dropped. + VisitPlacesWith(|place: Place<'tcx>, ctxt| { + if let PlaceContext::NonMutatingUse(NonMutatingUseContext::Move) + | PlaceContext::MutatingUse(MutatingUseContext::Drop) = ctxt + { + if let Some(local) = place.as_local() { + builder.kill(local, point, SplitPointEffect::Early); + state.maybe_borrowed.kill(local); + } + } + }) + .visit_terminator(terminator, location); + + // Kill any locals which are no longer used after this terminator, + // but only if they have not been borrowed. + for &(local, _) in kill_point_map[point] { + if !state.maybe_borrowed.contains(local) { + builder.kill(local, point, SplitPointEffect::Early); + } + } + + // Gen destination places. + VisitPlacesWith(|place: Place<'tcx>, ctxt| match DefUse::for_place(place, ctxt) { + DefUse::Def | DefUse::PartialWrite => { + builder.gen_(place.local, point, SplitPointEffect::Late) + } + DefUse::Use | DefUse::NonUse => {} + }) + .visit_terminator(terminator, location); + + // Move arguments to a call are treated specially: the place that they + // represent is passed directly to the callee, which means that they are + // not allowed to alias any other move operand or the destination place. + // This is represented here by extending their live range to the late + // part, making it overlap with that of the destination place. + // + // Notably, this *doesn't* apply to TailCall. + if let mir::TerminatorKind::Call { + func: _, + args, + destination: _, + target: _, + unwind: _, + call_source: _, + fn_span: _, + } = &terminator.kind + { + for arg in args { + if let mir::Operand::Move(place) = arg.node { + builder.gen_(place.local, point, SplitPointEffect::Late); + builder.kill(place.local, point, SplitPointEffect::Late); + } + } + } + + // End the lifetimes of all locals at the end of the block. Successor + // blocks (which may not be continuous in the index space!) will + // initialize the lifetimes again from their entry state. + for local in builder.range_start.indices() { + builder.kill(local, point, SplitPointEffect::Late); + } + } + + builder.matrix +} + +pub fn dump_liveness_matrix<'tcx>( + tcx: TyCtxt<'tcx>, + body: &mir::Body<'tcx>, + pass_name: &'static str, + points: &DenseLocationMap, + matrix: &SparseIntervalMatrix, +) { + let locals_live_at = |split_point| { + matrix.rows().filter(|&r| matrix.contains(r, split_point)).collect::>() + }; + + if let Some(dumper) = MirDumper::new(tcx, pass_name, body) { + let extra_data = &|pass_where, w: &mut dyn std::io::Write| { + if let PassWhere::BeforeLocation(loc) = pass_where { + let point = points.point_from_location(loc); + let split_point = SplitPointIndex::new(point, SplitPointEffect::Early); + let live = locals_live_at(split_point); + writeln!(w, " // {loc:?}-early => {live:?}")?; + let split_point = SplitPointIndex::new(point, SplitPointEffect::Late); + let live = locals_live_at(split_point); + writeln!(w, " // {loc:?}-late => {live:?}")?; + } + Ok(()) + }; + + dumper.set_extra_data(extra_data).dump_mir(body) + } +} diff --git a/compiler/rustc_mir_dataflow/src/points.rs b/compiler/rustc_mir_dataflow/src/points.rs index e3d1e04a319ba..92513b552410e 100644 --- a/compiler/rustc_mir_dataflow/src/points.rs +++ b/compiler/rustc_mir_dataflow/src/points.rs @@ -56,6 +56,15 @@ impl DenseLocationMap { PointIndex::new(start_index) } + /// Returns the `PointIndex` for the terminator in the given `BasicBlock`. O(1). + #[inline] + pub fn terminator(&self, block: BasicBlock) -> PointIndex { + let next_block = BasicBlock::new(block.index() + 1); + let next_start_index = + *self.statements_before_block.get(next_block).unwrap_or(&self.num_points); + PointIndex::new(next_start_index - 1) + } + /// Return the PointIndex for the block start of this index. #[inline] pub fn to_block_start(&self, index: PointIndex) -> PointIndex { diff --git a/compiler/rustc_mir_transform/src/dest_prop.rs b/compiler/rustc_mir_transform/src/dest_prop.rs index f6c5ae1e43f60..8bde214b45877 100644 --- a/compiler/rustc_mir_transform/src/dest_prop.rs +++ b/compiler/rustc_mir_transform/src/dest_prop.rs @@ -153,7 +153,7 @@ pub(super) struct DestinationPropagation; impl<'tcx> crate::MirPass<'tcx> for DestinationPropagation { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { - sess.mir_opt_level() >= 2 + false && sess.mir_opt_level() >= 2 } #[tracing::instrument(level = "trace", skip(self, tcx, body))] diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index 17c5ee110b86a..e02d8b8745b5a 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -160,6 +160,7 @@ declare_passes! { mod lower_slice_len : LowerSliceLenCalls; mod match_branches : MatchBranchSimplification; mod mentioned_items : MentionedItems; + mod move_elimination : MoveElimination; mod multiple_return_terminators : MultipleReturnTerminators; mod post_drop_elaboration : CheckLiveDrops; mod prettify : ReorderBasicBlocks, ReorderLocals; @@ -759,6 +760,7 @@ pub(crate) fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<' ©_prop::CopyProp, &dead_store_elimination::DeadStoreElimination::Final, &dest_prop::DestinationPropagation, + &move_elimination::MoveElimination, &simplify::SimplifyLocals::Final, &multiple_return_terminators::MultipleReturnTerminators, &large_enums::EnumSizeOpt { discrepancy: 128 }, diff --git a/compiler/rustc_mir_transform/src/move_elimination.rs b/compiler/rustc_mir_transform/src/move_elimination.rs new file mode 100644 index 0000000000000..ea79b6ed97725 --- /dev/null +++ b/compiler/rustc_mir_transform/src/move_elimination.rs @@ -0,0 +1,909 @@ +use rustc_abi::{FieldIdx, VariantIdx}; +use rustc_const_eval::util::most_packed_projection; +use rustc_data_structures::fx::FxHashMap; +use rustc_index::IndexVec; +use rustc_index::bit_set::DenseBitSet; +use rustc_index::interval::SparseIntervalMatrix; +use rustc_middle::mir::visit::{MutVisitor, NonUseContext, PlaceContext, VisitPlacesWith, Visitor}; +use rustc_middle::mir::*; +use rustc_middle::ty::{Ty, TyCtxt}; +use rustc_mir_dataflow::impls::{ + SplitPointEffect, SplitPointIndex, dump_liveness_matrix, liveness_matrix, +}; +use rustc_mir_dataflow::points::DenseLocationMap; +use tracing::{debug, trace}; + +use crate::patch::MirPatch; + +pub(super) struct MoveElimination; + +impl<'tcx> crate::MirPass<'tcx> for MoveElimination { + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + true && sess.mir_opt_level() >= 2 + } + + #[tracing::instrument(level = "trace", skip(self, tcx, body))] + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + let def_id = body.source.def_id(); + trace!(?def_id); + + let points = DenseLocationMap::new(body); + let mut liveness_matrix = + liveness_matrix(tcx, body, &points, Some("MoveElimination.liveness")); + + dump_liveness_matrix(tcx, body, "MoveElimination.pre-liveness", &points, &liveness_matrix); + + let mut unprojectable_locals = UnprojectableLocals::find(body); + trace!(?unprojectable_locals); + + let remapped_locals = + PlaceUnification::run(tcx, body, &mut liveness_matrix, &mut unprojectable_locals); + + apply_mappings(tcx, body, &remapped_locals); + + dump_liveness_matrix(tcx, body, "MoveElimination.post-liveness", &points, &liveness_matrix); + + if tcx.sess.emit_lifetime_markers() { + reconstruct_storage(body, &points, &liveness_matrix); + } + + apply_alias_fixup(tcx, body); + } + + fn is_required(&self) -> bool { + false + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Unprojectable locals + +/// Set of locals which can only be replaced with another local, instead of +/// an arbitrary place. This is usually because it is used directly as a +/// `Local` outside of a place (e.g. `Index` projections). +#[derive(Debug)] +struct UnprojectableLocals { + locals: DenseBitSet, +} + +impl UnprojectableLocals { + fn find(body: &Body<'_>) -> DenseBitSet { + let mut out = Self { locals: DenseBitSet::new_empty(body.local_decls.len()) }; + + // Arguments and return places have fixed roles and cannot be replaced + // with projected locals. + out.locals.insert(RETURN_PLACE); + for arg in body.args_iter() { + out.locals.insert(arg); + } + + out.visit_body(body); + out.locals + } +} + +impl<'tcx> Visitor<'tcx> for UnprojectableLocals { + fn visit_place(&mut self, place: &Place<'tcx>, context: PlaceContext, location: Location) { + // We can't add more projections before a first position Deref projection. + if place.is_indirect() { + trace!( + "unprojectable local {:?} due to use as deref base at {location:?}", + place.local + ); + self.locals.insert(place.local); + } + + // Only call visit_local for projections, not the base local. + self.visit_projection(place.as_ref(), context, location); + } + + fn visit_local(&mut self, local: Local, context: PlaceContext, location: Location) { + // Ignore uses in storage statements, we're going to remove all of those + // anyways. + if let PlaceContext::NonUse(NonUseContext::StorageLive | NonUseContext::StorageDead) = + context + { + return; + } + + // If this is reached, it means that this is a bare local used outside + // of a place, which means it cannot be replaced with a projection of + // another local. + trace!("unprojectable local {local:?} at {location:?} ({context:?})"); + self.locals.insert(local); + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Local unification + +struct PlaceUnification<'a, 'tcx> { + tcx: TyCtxt<'tcx>, + body: &'a Body<'tcx>, + liveness_matrix: &'a mut SparseIntervalMatrix, + unprojectable_locals: &'a mut DenseBitSet, + remapped_locals: IndexVec>>, +} + +impl<'tcx> PlaceUnification<'_, 'tcx> { + fn run( + tcx: TyCtxt<'tcx>, + body: &Body<'tcx>, + liveness_matrix: &mut SparseIntervalMatrix, + unprojectable_locals: &mut DenseBitSet, + ) -> IndexVec>> { + let mut visitor = PlaceUnification { + tcx, + body, + liveness_matrix, + unprojectable_locals, + remapped_locals: IndexVec::from_elem_n(None, body.local_decls.len()), + }; + visitor.visit_body(body); + + // Finalize the mappings by transitively resolving all locals to their + // new final place. + for local in visitor.remapped_locals.indices() { + if let Some(place) = visitor.remapped_locals[local] { + let place = visitor.resolve_place(place); + visitor.remapped_locals[local] = Some(place); + trace!("Remapped {local:?} to {place:?}"); + } + } + + visitor.remapped_locals + } + + #[tracing::instrument(ret, level = "trace", skip(self))] + fn resolve_place(&self, mut place: Place<'tcx>) -> Place<'tcx> { + while let Some(new_place) = self.remapped_locals[place.local] { + place = new_place.project_deeper(place.projection, self.tcx); + } + place + } + + #[tracing::instrument(ret, level = "trace", skip(self))] + fn can_unify_places(&self, a: Place<'tcx>, b: Place<'tcx>) -> Option<(Local, Place<'tcx>)> { + let a = self.resolve_place(a); + let b = self.resolve_place(b); + + if a.local == b.local { + if a.projection != b.projection { + trace!("cannot unify same local with different projections"); + } + return None; + } + + let (local, place) = match (a.as_local(), b.as_local()) { + (None, None) => { + trace!("cannot unify 2 places that both have projections"); + return None; + } + (None, Some(b)) => { + if self.unprojectable_locals.contains(b) { + trace!("cannot unify {b:?} which cannot be projected"); + return None; + } + (b, a) + } + (Some(a), None) => { + if self.unprojectable_locals.contains(a) { + trace!("cannot unify {a:?} which cannot be projected"); + return None; + } + (a, b) + } + (Some(a), Some(b)) => match (self.body.local_kind(a), self.body.local_kind(b)) { + ( + LocalKind::Arg | LocalKind::ReturnPointer, + LocalKind::Arg | LocalKind::ReturnPointer, + ) => { + trace!("cannot unify {a:?} and {b:?} which are both arguments or return place"); + return None; + } + (LocalKind::Arg | LocalKind::ReturnPointer, LocalKind::Temp) => (b, a.into()), + (LocalKind::Temp, _) => (a, b.into()), + }, + }; + + if most_packed_projection(self.tcx, &self.body.local_decls, place).is_some() { + trace!("cannot unify {place:?} which has packed field projections"); + return None; + } + + if !self.liveness_matrix.disjoint_rows(local, place.local) { + trace!("cannot unify {a:?} and {b:?} which have overlapping live ranges"); + return None; + } + + // FIXME(#112651): This can be removed afterwards. + let local_ty = self.body.local_decls[local].ty; + let place_ty = place.ty(&self.body.local_decls, self.tcx).ty; + if local_ty != place_ty { + trace!( + "cannot unify {a:?} and {b:?} which have different types due to subtyping ({local_ty:?} vs {place_ty:?})" + ); + return None; + } + + Some((local, place)) + } + + #[tracing::instrument(level = "trace", skip(self))] + fn remap_local(&mut self, local: Local, place: Place<'tcx>) { + self.remapped_locals[local] = Some(place); + + self.liveness_matrix.union_rows(local, place.local); + self.liveness_matrix.clear_row(local); + + // If the original local was unprojectable then this now also applies to + // the mapped local. + if self.unprojectable_locals.contains(local) { + debug_assert!(place.projection.is_empty()); + self.unprojectable_locals.insert(place.local); + } + } + + fn visit_aggregate_assign( + &mut self, + dest: Place<'tcx>, + project_field: impl Fn(TyCtxt<'tcx>, Place<'tcx>, FieldIdx, Ty<'tcx>) -> Place<'tcx>, + operands: &IndexVec>, + location: Location, + ) { + // Attempt to unify each field operand with the corresponding field in + // the destination place. + let mut candidates = vec![]; + for (idx, operand) in operands.iter_enumerated() { + let (Operand::Copy(src) | Operand::Move(src)) = *operand else { + continue; + }; + let Some(src) = src.as_local() else { + continue; + }; + let dest = project_field(self.tcx, dest, idx, self.body.local_decls[src].ty); + trace!("Attempting to unify {dest:?} and {src:?} at {location:?}"); + if let Some((local, place)) = self.can_unify_places(dest, src.into()) { + candidates.push((local, place)); + } + } + + // Do the actual remapping *after* checking for live range overlaps. + // This is necessary because the input operands necessarily have + // overlapping live ranges. + for (local, place) in candidates { + self.remap_local(local, place); + } + } +} + +/// Since we are replacing all uses of a local with another place, we need to +/// ensure that the projections on that place are stable no matter where it is +/// used in the body. Additional this local may be used in debuginfo, so ensure +/// that the projections are compatible with usage in debuginfo. +fn check_projections(place: Place<'_>) -> bool { + place.projection.iter().all(|elem| elem.is_stable_offset() && elem.can_use_in_debuginfo()) +} + +impl<'tcx> Visitor<'tcx> for PlaceUnification<'_, 'tcx> { + fn visit_assign(&mut self, dest: &Place<'tcx>, rvalue: &Rvalue<'tcx>, location: Location) { + if !check_projections(*dest) { + return; + } + match rvalue { + Rvalue::Use(Operand::Copy(src) | Operand::Move(src), _) => { + if !check_projections(*src) { + return; + } + + trace!("Attempting to unify {dest:?} and {src:?} at {location:?}"); + if let Some((local, place)) = self.can_unify_places(*src, *dest) { + self.remap_local(local, place); + } + } + Rvalue::Aggregate(aggregate_kind, operands) => match *aggregate_kind { + AggregateKind::Array(_) => self.visit_aggregate_assign( + *dest, + |tcx, place, field_idx, _field_ty| { + place.project_deeper( + &[PlaceElem::ConstantIndex { + offset: field_idx.as_u32().into(), + min_length: field_idx.as_u32() as u64 + 1, + from_end: false, + }], + tcx, + ) + }, + operands, + location, + ), + AggregateKind::Tuple => self.visit_aggregate_assign( + *dest, + |tcx, place, field_idx, field_ty| { + place.project_deeper(&[PlaceElem::Field(field_idx, field_ty)], tcx) + }, + operands, + location, + ), + AggregateKind::Adt(_, _, _, _, Some(union_field_idx)) => { + debug_assert_eq!(operands.len(), 1); + self.visit_aggregate_assign( + *dest, + |tcx, place, _, field_ty| { + place + .project_deeper(&[PlaceElem::Field(union_field_idx, field_ty)], tcx) + }, + operands, + location, + ) + } + AggregateKind::Adt(adt_did, var_idx, _, _, None) => { + let def = self.tcx.adt_def(adt_did); + if def.repr().simd() { + // MCP#838 banned projections into SIMD types. + return; + } + self.visit_aggregate_assign( + *dest, + |tcx, place, field_idx, field_ty| { + if def.is_enum() { + place.project_deeper( + &[ + PlaceElem::Downcast(None, var_idx), + PlaceElem::Field(field_idx, field_ty), + ], + tcx, + ) + } else { + place.project_deeper(&[PlaceElem::Field(field_idx, field_ty)], tcx) + } + }, + operands, + location, + ) + } + _ => {} + }, + _ => {} + }; + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Apply place mappings to the MIR body. + +fn apply_mappings<'tcx>( + tcx: TyCtxt<'tcx>, + body: &mut Body<'tcx>, + remapped_locals: &IndexVec>>, +) { + let mut rewriter = PlaceUpdater { tcx, remapped_locals }; + rewriter.visit_body_preserves_cfg(body); +} + +struct PlaceUpdater<'a, 'tcx> { + tcx: TyCtxt<'tcx>, + remapped_locals: &'a IndexVec>>, +} + +impl<'tcx> MutVisitor<'tcx> for PlaceUpdater<'_, 'tcx> { + fn tcx(&self) -> TyCtxt<'tcx> { + self.tcx + } + + fn visit_local(&mut self, local: &mut Local, context: PlaceContext, location: Location) { + if let Some(new_place) = self.remapped_locals[*local] { + trace!("replacing {local:?} with {new_place:?} at {location:?} ({context:?})"); + *local = new_place.as_local().expect("mapped place shouldn't have projections"); + } + } + + fn visit_place(&mut self, place: &mut Place<'tcx>, context: PlaceContext, location: Location) { + if let Some(new_place) = self.remapped_locals[place.local] { + trace!("replacing {place:?} with {new_place:?} at {location:?} ({context:?})"); + *place = new_place.project_deeper(place.projection, self.tcx) + } + + // Only call visit_local for projections, not the base local. + if let Some(new_projection) = self.process_projection(&place.projection, location) { + place.projection = self.tcx().mk_place_elems(&new_projection); + } + } + + fn visit_statement(&mut self, statement: &mut Statement<'tcx>, location: Location) { + match statement.kind { + // Remove *all* storage statements. These are rebuilt from liveness + // information later. Also, since we've preserved StorageDead in + // unwind paths until now, we will want to remove those since they + // hurt LLVM's codegen. + StatementKind::StorageDead(_) | StatementKind::StorageLive(_) => { + statement.make_nop(true); + return; + } + _ => {} + } + + self.super_statement(statement, location); + } +} + +//////////////////////////////////////////////////////////////////////////////// +// Storage reconstruction + +/// Helper function to split a critical edge if necessary. +fn get_or_split_edge<'tcx>( + patcher: &mut MirPatch<'tcx>, + body: &Body<'tcx>, + split_edges: &mut FxHashMap<(BasicBlock, BasicBlock), BasicBlock>, + pred: BasicBlock, + succ: BasicBlock, +) -> BasicBlock { + if let Some(&split_bb) = split_edges.get(&(pred, succ)) { + return split_bb; + } + let source_info = body.basic_blocks[pred].terminator().source_info; + let split_bb = patcher.new_block(BasicBlockData::new( + Some(Terminator { source_info, kind: TerminatorKind::Goto { target: succ } }), + body.basic_blocks[succ].is_cleanup, + )); + patcher.mutate_terminator(body, pred, |kind| { + kind.successors_mut(|t| { + if *t == succ { + *t = split_bb; + } + }); + }); + split_edges.insert((pred, succ), split_bb); + split_bb +} + +/// Don't insert storage statements in cleanup blocks and in unreachable blocks. +fn should_insert_storage<'tcx>(block_data: &BasicBlockData<'tcx>) -> bool { + !block_data.is_cleanup && !matches!(block_data.terminator().kind, TerminatorKind::Unreachable) +} + +/// Re-constructs storage statements for all locals. +fn reconstruct_storage<'tcx>( + body: &mut Body<'tcx>, + points: &DenseLocationMap, + liveness_matrix: &SparseIntervalMatrix, +) { + let mut patcher = MirPatch::new(body); + let mut split_edges: FxHashMap<(BasicBlock, BasicBlock), BasicBlock> = Default::default(); + + for local in body.local_decls.indices() { + // Arguments and return values don't use storage statements. + match body.local_kind(local) { + LocalKind::Arg | LocalKind::ReturnPointer => continue, + LocalKind::Temp => {} + } + + // Ignore dead locals. + let Some(row) = liveness_matrix.row(local) else { continue }; + if row.is_empty() { + continue; + } + + // Helper functions to emit storage statements in block predecessors and + // successors. + let mut emit_storage_live_in_preds = + |body: &mut Body<'tcx>, + patcher: &mut MirPatch<'tcx>, + local: Local, + block: BasicBlock| { + for &pred in &body.basic_blocks.predecessors()[block].clone() { + // If the local is live at any point in the predecessor's + // terminator then no StorageLive is needed. + let term_early = + SplitPointIndex::new(points.terminator(pred), SplitPointEffect::Early); + let term_late = + SplitPointIndex::new(points.terminator(pred), SplitPointEffect::Late); + if !row.intersects_range(term_early..=term_late) { + // The local must be live on at least one predecessor, + // so if this is the only one then there is nothing to + // do. + debug_assert!(body.basic_blocks.predecessors()[block].len() > 1); + + // If the predecessor block has multiple successors then + // we need to split the critical edge before inserting + // StorageLive, otherwise the local would end up live on + // paths where it is supposed to be dead. + let loc = if body.basic_blocks[pred].terminator().successors().count() > 1 { + get_or_split_edge(patcher, body, &mut split_edges, pred, block) + .start_location() + } else { + body.terminator_loc(pred) + }; + patcher.add_statement(loc, StatementKind::StorageLive(local)); + } + } + }; + let emit_storage_dead_in_succs = + |body: &mut Body<'tcx>, + patcher: &mut MirPatch<'tcx>, + local: Local, + block: BasicBlock| { + for succ in body.basic_blocks[block].terminator().successors() { + if !should_insert_storage(&body.basic_blocks[succ]) { + return; + } + + if !row.contains(SplitPointIndex::new( + points.entry_point(succ), + SplitPointEffect::Early, + )) { + // We don't care about critical edges here: if the local + // is already dead in the successor then it doesn't + // matter if we emit a redundant StorageDead. + + patcher.add_statement( + succ.start_location(), + StatementKind::StorageDead(local), + ); + } + } + }; + + // Iterate through the live range of the local and insert `StorageLive` + // and `StorageDead` at the points where it transitions from dead to + // live and vice versa. + // + // Note that the range here is an *inclusive range*. + for range in row.iter_intervals() { + let start_block = points.to_location(range.start.point()).block; + let end_block = points.to_location(range.last.point()).block; + + // If the live range starts at the `Early` point then it means that + // the value came from a predecessor block. A write from the first + // statement would happen at the `Late` point instead. + if should_insert_storage(&body.basic_blocks[start_block]) { + if range.start + == SplitPointIndex::new( + points.entry_point(start_block), + SplitPointEffect::Early, + ) + { + // If the local is dead at the end of any predecessor block then + // emit a `StorageLive` before the terminator. + emit_storage_live_in_preds(body, &mut patcher, local, start_block); + } else { + // Otherwise just add `StorageLive` before the statement that + // starts the live range. + patcher.add_statement( + points.to_location(range.start.point()), + StatementKind::StorageLive(local), + ); + } + } + + // The live range may span multiple blocks because + // `SparseIntervalMatrix` will coalesce adjacent ranges. If this + // happens then we need to repeat the start of block logic (see + // above) and end of block logic (see below) at each block boundary. + let mut current_block = start_block; + debug_assert!(start_block <= end_block); + while current_block != end_block { + if should_insert_storage(&body.basic_blocks[current_block]) { + emit_storage_dead_in_succs(body, &mut patcher, local, current_block); + } + current_block = BasicBlock::from_usize(current_block.index() + 1); + if should_insert_storage(&body.basic_blocks[current_block]) { + emit_storage_live_in_preds(body, &mut patcher, local, current_block); + } + } + + // We need to insert `StorageDead` after the last statement that + // uses a local. If this is a terminator then we need to instead + // insert it at the start of every successor block where the local + // is dead on entry. + if should_insert_storage(&body.basic_blocks[end_block]) { + if range.last.point() == points.terminator(end_block) { + emit_storage_dead_in_succs(body, &mut patcher, local, current_block); + } else { + // Don't emit StorageDead in cleanup blocks. + if !body.basic_blocks[end_block].is_cleanup { + patcher.add_statement( + points.to_location(range.last.point()).successor_within_block(), + StatementKind::StorageDead(local), + ); + } + } + } + } + } + + patcher.apply(body); +} + +//////////////////////////////////////////////////////////////////////////////// +// Aliasing assignment fixup +// +// MIR assignments currently do not allow source and destination to alias, so +// fix this in post-processing. + +fn apply_alias_fixup<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + let mut patcher = MirPatch::new(body); + let mut fixup = AliasFixup { tcx, local_decls: &body.local_decls, patcher: &mut patcher }; + for (block, data) in body.basic_blocks.as_mut_preserves_cfg().iter_enumerated_mut() { + fixup.visit_basic_block_data(block, data); + } + patcher.apply(body); +} + +fn places_alias<'tcx>( + tcx: TyCtxt<'tcx>, + local_decls: &IndexVec>, + a: Place<'tcx>, + b: Place<'tcx>, +) -> bool { + // Indirect places don't overlap because we assume they didn't overlap in + // the input MIR. + if a.local != b.local || a.is_indirect_first_projection() || b.is_indirect_first_projection() { + return false; + } + + for ((prefix, elem_a), (_, elem_b)) in a.iter_projections().zip(b.iter_projections()) { + // Continue until we find the first mismatching projection. + if elem_a == elem_b { + continue; + } + + match (elem_a, elem_b) { + // Disjoint fields don't alias except if they are union fields. + (PlaceElem::Field(_, _), PlaceElem::Field(_, _)) => { + let ty = prefix.ty(local_decls, tcx).ty; + return ty.is_union(); + } + + // Disjoint slice elements don't alias. + ( + PlaceElem::ConstantIndex { offset: offset_a, min_length: _, from_end: from_end_a }, + PlaceElem::ConstantIndex { offset: offset_b, min_length: _, from_end: from_end_b }, + ) if from_end_a == from_end_b && offset_a != offset_b => { + return false; + } + + // Conservatively assume the places may alias. + _ => return true, + } + } + + // If the projections are identical *or* one is a prefix of the other then + // the places alias. + true +} + +struct AliasFixup<'a, 'tcx> { + tcx: TyCtxt<'tcx>, + local_decls: &'a IndexVec>, + patcher: &'a mut MirPatch<'tcx>, +} + +impl<'tcx> AliasFixup<'_, 'tcx> { + fn isolate_rvalue_to_local( + &mut self, + rvalue: Rvalue<'tcx>, + source_info: SourceInfo, + location: Location, + ) -> Place<'tcx> { + let ty = rvalue.ty(self.local_decls, self.tcx); + let temp = Place::from(self.patcher.new_temp(ty, source_info.span)); + trace!("isolating {rvalue:?} to {temp:?} due to conflict"); + self.patcher.add_statement(location, StatementKind::StorageLive(temp.local)); + self.patcher.add_assign(location, Place::from(temp), rvalue); + self.patcher.add_statement( + location.successor_within_block(), + StatementKind::StorageDead(temp.local), + ); + temp + } + + fn visit_aggregate_assign( + &mut self, + dest: Place<'tcx>, + enum_variant: Option, + project_field: impl Fn(TyCtxt<'tcx>, Place<'tcx>, FieldIdx, Ty<'tcx>) -> Place<'tcx>, + operands: &IndexVec>, + source_info: SourceInfo, + location: Location, + ) { + // Fast path: if no operand alias the destination, we're done. + let has_any_alias = operands.iter().any(|op| match op { + Operand::Copy(src) | Operand::Move(src) => { + places_alias(self.tcx, self.local_decls, dest, *src) + } + Operand::Constant(_) | Operand::RuntimeChecks(_) => false, + }); + if !has_any_alias { + return; + } + + debug!("splitting aggregate assignment at {location:?}"); + + // Split into per-field assignments. + let mut assignments = vec![]; + for (idx, op) in operands.iter_enumerated() { + let field_ty = op.ty(self.local_decls, self.tcx); + let dest_field = project_field(self.tcx, dest, idx, field_ty); + + let emit_op = match op { + Operand::Copy(src) | Operand::Move(src) => { + if *src == dest_field { + // Skip identity assignments. + continue; + } else if places_alias(self.tcx, self.local_decls, dest, *src) { + // Partial alias: hoist the source to a temp first so + // the per-field write no longer overlaps the dest. + Operand::Move(self.isolate_rvalue_to_local( + Rvalue::Use(op.clone(), WithRetag::No), + source_info, + location, + )) + } else { + op.clone() + } + } + Operand::Constant(_) | Operand::RuntimeChecks(_) => op.clone(), + }; + assignments.push((dest_field, emit_op)); + } + + // Perform assignments *after* all aliasing fields have been read into + // temporary locals. + for (dest_field, emit_op) in assignments { + self.patcher.add_assign(location, dest_field, Rvalue::Use(emit_op, WithRetag::No)); + } + + // Delete the original aggregate assignment. + self.patcher.nop_statement(location); + + // For enum variants, set the discriminant after all field writes. + if let Some(variant_index) = enum_variant { + self.patcher.add_statement( + location, + StatementKind::SetDiscriminant { place: Box::new(dest), variant_index }, + ); + } + } +} + +impl<'tcx> MutVisitor<'tcx> for AliasFixup<'_, 'tcx> { + fn tcx(&self) -> TyCtxt<'tcx> { + self.tcx + } + + fn visit_statement(&mut self, statement: &mut Statement<'tcx>, location: Location) { + // Fixup the MIR to remove aliasing assignments. + if let StatementKind::Assign((dest, rvalue)) = &mut statement.kind { + match *rvalue { + Rvalue::Use(Operand::Copy(src) | Operand::Move(src), with_retag) => { + if places_alias(self.tcx, self.local_decls, *dest, src) { + if src == *dest { + debug!("{:?} turned into self-assignment, deleting", location); + statement.make_nop(true); + } else { + let temp = self.isolate_rvalue_to_local( + rvalue.clone(), + statement.source_info, + location, + ); + *rvalue = Rvalue::Use(Operand::Move(temp), with_retag); + } + } + } + Rvalue::Aggregate(AggregateKind::Array(_), ref mut operands) => self + .visit_aggregate_assign( + *dest, + None, + |tcx, place, field_idx, _field_ty| { + place.project_deeper( + &[PlaceElem::ConstantIndex { + offset: field_idx.as_u32().into(), + min_length: field_idx.as_u32() as u64 + 1, + from_end: false, + }], + tcx, + ) + }, + operands, + statement.source_info, + location, + ), + Rvalue::Aggregate(AggregateKind::Tuple, ref mut operands) => self + .visit_aggregate_assign( + *dest, + None, + |tcx, place, field_idx, field_ty| { + place.project_deeper(&[PlaceElem::Field(field_idx, field_ty)], tcx) + }, + operands, + statement.source_info, + location, + ), + Rvalue::Aggregate( + AggregateKind::Adt(_, _, _, _, Some(union_field_idx)), + ref mut operands, + ) => { + debug_assert_eq!(operands.len(), 1); + self.visit_aggregate_assign( + *dest, + None, + |tcx, place, _, field_ty| { + place + .project_deeper(&[PlaceElem::Field(union_field_idx, field_ty)], tcx) + }, + operands, + statement.source_info, + location, + ) + } + Rvalue::Aggregate( + AggregateKind::Adt(adt_did, var_idx, _, _, None), + ref mut operands, + ) => { + let def = self.tcx.adt_def(adt_did); + if def.repr().simd() { + // MCP#838 banned projections into SIMD types. + return; + } + self.visit_aggregate_assign( + *dest, + def.is_enum().then_some(var_idx), + |tcx, place, field_idx, field_ty| { + if def.is_enum() { + place.project_deeper( + &[ + PlaceElem::Downcast(None, var_idx), + PlaceElem::Field(field_idx, field_ty), + ], + tcx, + ) + } else { + place.project_deeper(&[PlaceElem::Field(field_idx, field_ty)], tcx) + } + }, + operands, + statement.source_info, + location, + ) + } + + // For other rvalues, don't try to split them into components + // and instead just introduce a temporary if there is any + // aliasing + Rvalue::Aggregate(..) + | Rvalue::Repeat(..) + | Rvalue::Cast(..) + | Rvalue::CopyForDeref(..) + | Rvalue::WrapUnsafeBinder(..) => { + let mut overlaps_dest = false; + VisitPlacesWith(|place, _ctxt| { + if places_alias(self.tcx, self.local_decls, *dest, place) { + overlaps_dest = true; + } + }) + .visit_rvalue(rvalue, location); + if overlaps_dest { + let temp = self.isolate_rvalue_to_local( + rvalue.clone(), + statement.source_info, + location, + ); + *rvalue = Rvalue::Use(Operand::Move(temp), WithRetag::No); + } + } + + // These permit either cannot have aliasing, or allow it because + // they only operate on scalar backend types. + Rvalue::Use(Operand::Constant(..) | Operand::RuntimeChecks(..), _) + | Rvalue::Ref(..) + | Rvalue::ThreadLocalRef(..) + | Rvalue::BinaryOp(..) + | Rvalue::UnaryOp(..) + | Rvalue::Discriminant(..) + | Rvalue::RawPtr(..) + | Rvalue::Reborrow(..) => {} + } + } + } +} diff --git a/compiler/rustc_mir_transform/src/patch.rs b/compiler/rustc_mir_transform/src/patch.rs index 015bae56cf57e..b8d25ab01ae14 100644 --- a/compiler/rustc_mir_transform/src/patch.rs +++ b/compiler/rustc_mir_transform/src/patch.rs @@ -215,6 +215,21 @@ impl<'tcx> MirPatch<'tcx> { self.term_patch_map.insert(block, new); } + /// Modifies the terminator of a block, reading the existing patch if one exists or + /// cloning from the body otherwise. + pub(crate) fn mutate_terminator( + &mut self, + body: &Body<'tcx>, + bb: BasicBlock, + f: impl FnOnce(&mut TerminatorKind<'tcx>), + ) { + let kind = self + .term_patch_map + .entry(bb) + .or_insert_with(|| body.basic_blocks[bb].terminator().kind.clone()); + f(kind); + } + /// Mark given statement to be replaced by a `Nop`. /// /// This method only works on statements from the initial body, and cannot be used to remove From 490bdecace695cb66a93035aead66464ef4f7b2d Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Thu, 21 May 2026 09:12:42 +0200 Subject: [PATCH 05/11] Relax MIR lint to allow use of a maybe-dead local --- compiler/rustc_mir_transform/src/lint.rs | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/compiler/rustc_mir_transform/src/lint.rs b/compiler/rustc_mir_transform/src/lint.rs index 41614462a81d4..f0cf83e43bd85 100644 --- a/compiler/rustc_mir_transform/src/lint.rs +++ b/compiler/rustc_mir_transform/src/lint.rs @@ -9,7 +9,7 @@ use rustc_index::bit_set::DenseBitSet; use rustc_middle::mir::visit::{PlaceContext, VisitPlacesWith, Visitor}; use rustc_middle::mir::*; use rustc_middle::ty::TyCtxt; -use rustc_mir_dataflow::impls::{MaybeStorageDead, MaybeStorageLive, always_storage_live_locals}; +use rustc_mir_dataflow::impls::{MaybeStorageLive, always_storage_live_locals}; use rustc_mir_dataflow::{Analysis, ResultsCursor}; pub(super) fn lint_body<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>, when: String) { @@ -19,10 +19,6 @@ pub(super) fn lint_body<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>, when: String .iterate_to_fixpoint(tcx, body, None) .into_results_cursor(body); - let maybe_storage_dead = MaybeStorageDead::new(Cow::Borrowed(always_live_locals)) - .iterate_to_fixpoint(tcx, body, None) - .into_results_cursor(body); - let mut lint = Lint { tcx, when, @@ -30,7 +26,6 @@ pub(super) fn lint_body<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>, when: String is_fn_like: tcx.def_kind(body.source.def_id()).is_fn_like(), always_live_locals, maybe_storage_live, - maybe_storage_dead, places: Default::default(), }; for (bb, data) in traversal::reachable(body) { @@ -45,7 +40,6 @@ struct Lint<'a, 'tcx> { is_fn_like: bool, always_live_locals: &'a DenseBitSet, maybe_storage_live: ResultsCursor<'a, 'tcx, MaybeStorageLive<'a>>, - maybe_storage_dead: ResultsCursor<'a, 'tcx, MaybeStorageDead<'a>>, places: FxHashSet>, } @@ -75,8 +69,8 @@ fn places_conflict_for_assignment<'tcx>(dest: Place<'tcx>, src: Place<'tcx>) -> impl<'a, 'tcx> Visitor<'tcx> for Lint<'a, 'tcx> { fn visit_local(&mut self, local: Local, context: PlaceContext, location: Location) { if context.is_use() { - self.maybe_storage_dead.seek_after_primary_effect(location); - if self.maybe_storage_dead.get().contains(local) { + self.maybe_storage_live.seek_after_primary_effect(location); + if !self.maybe_storage_live.get().contains(local) { self.fail(location, format!("use of local {local:?}, which has no storage here")); } } From df40d2fac49d95920945fa8a9c4f1f1a5a92a7bb Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Thu, 28 May 2026 23:04:39 +0100 Subject: [PATCH 06/11] Speed up handling of block end in precise_liveness --- .../src/impls/precise_liveness.rs | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/compiler/rustc_mir_dataflow/src/impls/precise_liveness.rs b/compiler/rustc_mir_dataflow/src/impls/precise_liveness.rs index f2d91dea742b4..2bad6cb64e34c 100644 --- a/compiler/rustc_mir_dataflow/src/impls/precise_liveness.rs +++ b/compiler/rustc_mir_dataflow/src/impls/precise_liveness.rs @@ -353,6 +353,11 @@ fn kill_point_map<'a>( struct MatrixBuilder { matrix: SparseIntervalMatrix, range_start: IndexVec>, + + // Track locals that have been live at any point in a block so that at the + // end of a block we don't need to iterate over all locals. This + // significantly speeds up matrix building. + maybe_live_locals: Vec, } impl MatrixBuilder { @@ -360,7 +365,10 @@ impl MatrixBuilder { let split_point = SplitPointIndex::new(point, effect); // No-op if the local is already live. - self.range_start[local].get_or_insert(split_point); + if self.range_start[local].is_none() { + self.range_start[local] = Some(split_point); + self.maybe_live_locals.push(local); + } } fn kill(&mut self, local: Local, point: PointIndex, effect: SplitPointEffect) { @@ -372,6 +380,12 @@ impl MatrixBuilder { self.matrix.append_range(local, start..=end); } } + + fn kill_all(&mut self, point: PointIndex, effect: SplitPointEffect) { + while let Some(local) = self.maybe_live_locals.pop() { + self.kill(local, point, effect); + } + } } pub fn liveness_matrix<'tcx>( @@ -388,6 +402,7 @@ pub fn liveness_matrix<'tcx>( let mut builder = MatrixBuilder { matrix: SparseIntervalMatrix::new(points.num_points() * 2), range_start: IndexVec::from_elem_n(None, body.local_decls.len()), + maybe_live_locals: Vec::new(), }; for (block, block_data) in body.basic_blocks.iter_enumerated() { // We can mutate the state in-place since we're not using it any more @@ -522,9 +537,7 @@ pub fn liveness_matrix<'tcx>( // End the lifetimes of all locals at the end of the block. Successor // blocks (which may not be continuous in the index space!) will // initialize the lifetimes again from their entry state. - for local in builder.range_start.indices() { - builder.kill(local, point, SplitPointEffect::Late); - } + builder.kill_all(point, SplitPointEffect::Late); } builder.matrix From c1a4988f17e626e1c6042bd1a6657e501981d981 Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Sat, 30 May 2026 01:26:45 +0100 Subject: [PATCH 07/11] Don't use kill points for locals that are ever borrowed --- compiler/rustc_index/src/bit_set.rs | 26 -- .../src/impls/precise_liveness.rs | 277 ++++++++---------- 2 files changed, 116 insertions(+), 187 deletions(-) diff --git a/compiler/rustc_index/src/bit_set.rs b/compiler/rustc_index/src/bit_set.rs index 698a12e084fb4..2910ba7c46851 100644 --- a/compiler/rustc_index/src/bit_set.rs +++ b/compiler/rustc_index/src/bit_set.rs @@ -324,15 +324,6 @@ impl DenseBitSet { // out-of-domain bits, so we need to clear them. self.clear_excess_bits(); } - - /// Sets `self = self & (a | b)` without allocating a temporary for `a | b`. - /// - /// Returns `true` if `self` changed. - pub fn intersect_with_union(&mut self, a: &DenseBitSet, b: &DenseBitSet) -> bool { - assert_eq!(self.domain_size, a.domain_size); - assert_eq!(self.domain_size, b.domain_size); - bitwise3(&mut self.words, &a.words, &b.words, |s, a, b| s & (a | b)) - } } // dense REL dense @@ -1093,23 +1084,6 @@ where changed != 0 } -#[inline] -fn bitwise3(out_vec: &mut [Word], in_vec1: &[Word], in_vec2: &[Word], op: Op) -> bool -where - Op: Fn(Word, Word, Word) -> Word, -{ - assert_eq!(out_vec.len(), in_vec1.len()); - assert_eq!(out_vec.len(), in_vec2.len()); - let mut changed = 0; - for ((out_elem, in_elem1), in_elem2) in iter::zip(iter::zip(out_vec, in_vec1), in_vec2) { - let old_val = *out_elem; - let new_val = op(old_val, *in_elem1, *in_elem2); - *out_elem = new_val; - changed |= old_val ^ new_val; - } - changed != 0 -} - /// Returns true if a call to [`update_words`] would modify `lhs`, i.e. /// `lhs[i] != op(lhs[i], rhs[i])` for some `i`. #[inline] diff --git a/compiler/rustc_mir_dataflow/src/impls/precise_liveness.rs b/compiler/rustc_mir_dataflow/src/impls/precise_liveness.rs index 2bad6cb64e34c..ad2fb1bfb3474 100644 --- a/compiler/rustc_mir_dataflow/src/impls/precise_liveness.rs +++ b/compiler/rustc_mir_dataflow/src/impls/precise_liveness.rs @@ -1,4 +1,21 @@ -use std::fmt; +//! Computes the points where each local must have a distinct allocation. +//! +//! The result is a [`SparseIntervalMatrix`] with one row per local. Two locals +//! may share the same address only if their rows are disjoint. To model MIR +//! statements where a source operand and destination place may share an address, +//! each statement and terminator is split into an early point, where operands +//! are read, and a late point, where destinations are written. +//! +//! A local live range starts at the late point of any statement or terminator +//! that writes to it without a `Deref` projection. It ends at the early point of +//! a `StorageDead`, a whole-local `Drop`, a whole-local move operand, or the +//! last use of that local on a control-flow path (only for locals whose +//! address is never observed). +//! +//! `Call` terminators are handled specially: move operands are kept live +//! through the late point of the terminator so they conflict with each other and +//! with the destination place. This matches the runtime behavior where the place +//! is donated to the callee for the duration of the call. use rustc_index::IndexVec; use rustc_index::bit_set::DenseBitSet; @@ -10,19 +27,72 @@ use rustc_middle::mir::{self, BasicBlock, Local, Location, MirDumper, PassWhere, use rustc_middle::ty::TyCtxt; use tracing::trace; -use crate::fmt::DebugWithContext; -use crate::impls::{DefUse, MaybeBorrowedLocals, MaybeLiveLocals}; +use crate::impls::{DefUse, MaybeLiveLocals, borrowed_locals}; use crate::points::{DenseLocationMap, PointIndex}; -use crate::{Analysis, GenKill, JoinSemiLattice, ResultsVisitor, visit_reachable_results}; +use crate::{Analysis, GenKill, ResultsVisitor, visit_reachable_results}; + +// Backward dataflow pass +// ====================== +// +// This pass computes "kill points" for each local, indicating the location of +// their last use in a particular control flow branch. These are later used in +// the forward pass later to end the live range of locals that are never +// borrowed at their last direct use. +// +// Borrowed locals are treated as always live by this pass since those need to +// remain allocated until `StorageDead` or a whole-local move. +// +// This pass has 2 outputs: a set of kill points that mark the last use +// locations of locals and a per-block bitset indicating which locals are live +// on entry to that block. + +fn compute_kill_points<'tcx>( + tcx: TyCtxt<'tcx>, + body: &mir::Body<'tcx>, + pass_name: Option<&'static str>, +) -> (Vec<(Local, Location)>, IndexVec>) { + let maybe_live_locals = MaybeLiveLocals.iterate_to_fixpoint(tcx, body, pass_name); + let borrowed_locals = borrowed_locals(body); + let mut kill_points = vec![]; + // Initialize all borrowed locals as live on entry. + let mut live_on_entry = IndexVec::from_elem_n(borrowed_locals.clone(), body.basic_blocks.len()); + let mut visitor = KillPointsVisitor { + kill_points: &mut kill_points, + live_on_entry: &mut live_on_entry, + borrowed_locals: &borrowed_locals, + }; + visit_reachable_results(body, &maybe_live_locals, &mut visitor); + trace!(?kill_points); + trace!(?live_on_entry); + (kill_points, live_on_entry) +} + +/// Creates a mapping of `PointIndex` to the set of killed locals at that location. +fn kill_point_map<'a>( + kill_points: &'a [(Local, Location)], + points: &DenseLocationMap, +) -> IndexVec { + let mut out = IndexVec::from_elem_n(&[][..], points.num_points()); + for chunk in kill_points.chunk_by(|a, b| a.1 == b.1) { + let point = points.point_from_location(chunk[0].1); + trace!("Kill points at {:?}: {:?}", chunk[0].1, chunk); + out[point] = chunk; + } + out +} struct KillPointsVisitor<'a> { kill_points: &'a mut Vec<(Local, Location)>, live_on_entry: &'a mut IndexVec>, + borrowed_locals: &'a DenseBitSet, } impl<'tcx> ResultsVisitor<'tcx, MaybeLiveLocals> for KillPointsVisitor<'_> { fn visit_block_start(&mut self, state: &DenseBitSet, block: BasicBlock) { - self.live_on_entry[block].clone_from(state); + // Borrowed locals are already marked as live when live_on_entry was + // initialized. This adds the non-borrowed locals that we have + // determined are live on entry to this block. + self.live_on_entry[block].union(state); } fn visit_after_early_statement_effect( @@ -40,8 +110,9 @@ impl<'tcx> ResultsVisitor<'tcx, MaybeLiveLocals> for KillPointsVisitor<'_> { } // If a local is used in a statement but is dead after it then this - // location is a kill point. - if !state.contains(place.local) { + // location is a kill point. Don't emit a kill point for borrowed + // locals. + if !state.contains(place.local) && !self.borrowed_locals.contains(place.local) { self.kill_points.push((place.local, location)); } }) @@ -56,8 +127,9 @@ impl<'tcx> ResultsVisitor<'tcx, MaybeLiveLocals> for KillPointsVisitor<'_> { location: Location, ) { VisitPlacesWith(|place: Place<'tcx>, ctxt| { - // Ignore non-uses (they don't do anything) and edge uses (kill - // points for those go at the start of the corresponding successor). + // Ignore non-uses (they don't do anything) and edge uses (implicitly + // killed though live_on_entry at the start of the corresponding + // successor). match ctxt { PlaceContext::MutatingUse( MutatingUseContext::AsmOutput @@ -69,8 +141,9 @@ impl<'tcx> ResultsVisitor<'tcx, MaybeLiveLocals> for KillPointsVisitor<'_> { } // If a local is used in a terminator but is dead after it then this - // location is a kill point. - if !state.contains(place.local) { + // location is a kill point. Don't emit a kill point for borrowed + // locals. + if !state.contains(place.local) && !self.borrowed_locals.contains(place.local) { self.kill_points.push((place.local, location)); } }) @@ -78,60 +151,8 @@ impl<'tcx> ResultsVisitor<'tcx, MaybeLiveLocals> for KillPointsVisitor<'_> { } } -#[derive(Debug, PartialEq, Eq)] -struct Domain { - maybe_live: DenseBitSet, - maybe_borrowed: DenseBitSet, -} - -impl Clone for Domain { - fn clone(&self) -> Self { - Domain { maybe_live: self.maybe_live.clone(), maybe_borrowed: self.maybe_borrowed.clone() } - } - - // Data flow engine when possible uses `clone_from` for domain values. - // Providing an implementation will avoid some intermediate memory allocations. - fn clone_from(&mut self, other: &Self) { - self.maybe_live.clone_from(&other.maybe_live); - self.maybe_borrowed.clone_from(&other.maybe_borrowed); - } -} - -impl JoinSemiLattice for Domain { - fn join(&mut self, other: &Self) -> bool { - self.maybe_live.join(&other.maybe_live) | self.maybe_borrowed.join(&other.maybe_borrowed) - } -} - -impl DebugWithContext for Domain { - fn fmt_with(&self, ctxt: &C, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str("maybe_live: ")?; - self.maybe_live.fmt_with(ctxt, f)?; - f.write_str("maybe_borrowed: ")?; - self.maybe_borrowed.fmt_with(ctxt, f)?; - Ok(()) - } - - fn fmt_diff_with(&self, old: &Self, ctxt: &C, f: &mut fmt::Formatter<'_>) -> fmt::Result { - if self == old { - return Ok(()); - } - - if self.maybe_live != old.maybe_live { - f.write_str("maybe_live: ")?; - self.maybe_live.fmt_diff_with(&old.maybe_live, ctxt, f)?; - f.write_str("\n")?; - } - - if self.maybe_borrowed != old.maybe_borrowed { - f.write_str("maybe_borrowed: ")?; - self.maybe_borrowed.fmt_diff_with(&old.maybe_borrowed, ctxt, f)?; - f.write_str("\n")?; - } - - Ok(()) - } -} +// Forward dataflow pass +// ===================== struct PreciseLiveness<'a> { kill_point_map: &'a IndexVec, @@ -140,37 +161,31 @@ struct PreciseLiveness<'a> { } impl PreciseLiveness<'_> { - fn apply_block_start_effect(&self, state: &mut Domain, block: BasicBlock) { - // Only keep locals that are either live or borrowed. - // - // Notably this kills any dead results produced by a predecessor's - // terminator. - state.maybe_live.intersect_with_union(&self.live_on_entry[block], &state.maybe_borrowed); + fn apply_block_start_effect(&self, state: &mut DenseBitSet, block: BasicBlock) { + // Notably this kills any dead results produced by a predecessor's terminator. + state.intersect(&self.live_on_entry[block]); } } impl<'tcx> Analysis<'tcx> for PreciseLiveness<'_> { - type Domain = Domain; + type Domain = DenseBitSet; const NAME: &'static str = "precise_liveness"; - fn bottom_value(&self, body: &mir::Body<'tcx>) -> Domain { - Domain { - maybe_live: DenseBitSet::new_empty(body.local_decls.len()), - maybe_borrowed: DenseBitSet::new_empty(body.local_decls.len()), - } + fn bottom_value(&self, body: &mir::Body<'tcx>) -> DenseBitSet { + DenseBitSet::new_empty(body.local_decls.len()) } - fn initialize_start_block(&self, body: &mir::Body<'tcx>, state: &mut Domain) { + fn initialize_start_block(&self, body: &mir::Body<'tcx>, state: &mut DenseBitSet) { // Function arguments start out as live. for arg in body.args_iter() { - state.maybe_live.gen_(arg); + state.gen_(arg); } } fn apply_primary_statement_effect( &self, - state: &mut Domain, + state: &mut DenseBitSet, statement: &mir::Statement<'tcx>, location: Location, ) { @@ -180,20 +195,15 @@ impl<'tcx> Analysis<'tcx> for PreciseLiveness<'_> { // StorageDead always kills a local, even if it has been borrowed. if let mir::StatementKind::StorageDead(local) = statement.kind { - state.maybe_live.kill(local); - state.maybe_borrowed.kill(local); + state.kill(local); return; } - MaybeBorrowedLocals::transfer_function(&mut state.maybe_borrowed) - .visit_statement(statement, location); - // Kill moved operands if the whole local was moved. VisitPlacesWith(|place: Place<'tcx>, ctxt| { if ctxt == PlaceContext::NonMutatingUse(NonMutatingUseContext::Move) { if let Some(local) = place.as_local() { - state.maybe_live.kill(local); - state.maybe_borrowed.kill(local); + state.kill(local); } } }) @@ -201,24 +211,22 @@ impl<'tcx> Analysis<'tcx> for PreciseLiveness<'_> { // Gen destination places. VisitPlacesWith(|place: Place<'tcx>, ctxt| match DefUse::for_place(place, ctxt) { - DefUse::Def | DefUse::PartialWrite => state.maybe_live.gen_(place.local), + DefUse::Def | DefUse::PartialWrite => state.gen_(place.local), DefUse::Use | DefUse::NonUse => {} }) .visit_statement(statement, location); // Apply kill points at this statement: if a variable is dead - // then it doesn't need storage, *except* if its address has been taken. + // then it doesn't need storage. let point = self.points.point_from_location(location); for &(local, _) in self.kill_point_map[point] { - if !state.maybe_borrowed.contains(local) { - state.maybe_live.kill(local); - } + state.kill(local); } } fn apply_primary_terminator_effect<'mir>( &self, - state: &mut Domain, + state: &mut DenseBitSet, terminator: &'mir mir::Terminator<'tcx>, location: Location, ) -> mir::TerminatorEdges<'mir, 'tcx> { @@ -226,9 +234,6 @@ impl<'tcx> Analysis<'tcx> for PreciseLiveness<'_> { self.apply_block_start_effect(state, location.block); } - MaybeBorrowedLocals::transfer_function(&mut state.maybe_borrowed) - .visit_terminator(terminator, location); - // Kill moved operands if the whole local was moved. Also kill dropped // places if the entire local was dropped. VisitPlacesWith(|place: Place<'tcx>, ctxt| { @@ -236,8 +241,7 @@ impl<'tcx> Analysis<'tcx> for PreciseLiveness<'_> { | PlaceContext::MutatingUse(MutatingUseContext::Drop) = ctxt { if let Some(local) = place.as_local() { - state.maybe_live.kill(local); - state.maybe_borrowed.kill(local); + state.kill(local); } } }) @@ -256,7 +260,7 @@ impl<'tcx> Analysis<'tcx> for PreciseLiveness<'_> { } match DefUse::for_place(place, ctxt) { - DefUse::Def | DefUse::PartialWrite => state.maybe_live.gen_(place.local), + DefUse::Def | DefUse::PartialWrite => state.gen_(place.local), DefUse::Use | DefUse::NonUse => {} } }) @@ -267,14 +271,17 @@ impl<'tcx> Analysis<'tcx> for PreciseLiveness<'_> { fn apply_call_return_effect( &self, - state: &mut Domain, + state: &mut DenseBitSet, _block: BasicBlock, return_places: mir::CallReturnPlaces<'_, 'tcx>, ) { - return_places.for_each(|place| state.maybe_live.gen_(place.local)); + return_places.for_each(|place| state.gen_(place.local)); } } +// Matrix construction +// =================== + /// Different "phases" of a single MIR statement, used to describe how /// overlapping operands are handled. /// @@ -317,38 +324,6 @@ impl SplitPointIndex { } } -fn compute_kill_points<'tcx>( - tcx: TyCtxt<'tcx>, - body: &mir::Body<'tcx>, - pass_name: Option<&'static str>, -) -> (Vec<(Local, Location)>, IndexVec>) { - let maybe_live_locals = MaybeLiveLocals.iterate_to_fixpoint(tcx, body, pass_name); - let mut kill_points = vec![]; - let mut live_on_entry = IndexVec::from_elem_n( - DenseBitSet::new_empty(body.local_decls.len()), - body.basic_blocks.len(), - ); - let mut visitor = - KillPointsVisitor { kill_points: &mut kill_points, live_on_entry: &mut live_on_entry }; - visit_reachable_results(body, &maybe_live_locals, &mut visitor); - trace!(?kill_points); - trace!(?live_on_entry); - (kill_points, live_on_entry) -} - -fn kill_point_map<'a>( - kill_points: &'a [(Local, Location)], - points: &DenseLocationMap, -) -> IndexVec { - let mut out = IndexVec::from_elem_n(&[][..], points.num_points()); - for chunk in kill_points.chunk_by(|a, b| a.1 == b.1) { - let point = points.point_from_location(chunk[0].1); - trace!("Kill points at {:?}: {:?}", chunk[0].1, chunk); - out[point] = chunk; - } - out -} - /// Helper type to construct a `SparseIntervalMatrix`. struct MatrixBuilder { matrix: SparseIntervalMatrix, @@ -409,13 +384,10 @@ pub fn liveness_matrix<'tcx>( // after this point. let state = &mut results.entry_states[block]; - // Only keep locals that are either live or borrowed. - // - // Notably this kills any dead results produced by a predecessor's - // terminator. - state.maybe_live.intersect_with_union(&live_on_entry[block], &state.maybe_borrowed); + // Notably this kills any dead results produced by a predecessor's terminator. + state.intersect(&live_on_entry[block]); - for local in state.maybe_live.iter() { + for local in state.iter() { builder.gen_(local, points.entry_point(block), SplitPointEffect::Early); } @@ -426,30 +398,22 @@ pub fn liveness_matrix<'tcx>( // StorageDead always kills a local, even if it has been borrowed. if let mir::StatementKind::StorageDead(local) = statement.kind { builder.kill(local, point, SplitPointEffect::Late); - state.maybe_borrowed.kill(local); continue; } - MaybeBorrowedLocals::transfer_function(&mut state.maybe_borrowed) - .visit_statement(statement, location); - // Kill moved operands if the whole local was moved. VisitPlacesWith(|place: Place<'tcx>, ctxt| { if ctxt == PlaceContext::NonMutatingUse(NonMutatingUseContext::Move) { if let Some(local) = place.as_local() { builder.kill(local, point, SplitPointEffect::Early); - state.maybe_borrowed.kill(local); } } }) .visit_statement(statement, location); - // Kill any locals which are no longer used after this statement, - // but only if they have not been borrowed. + // Kill any locals which are no longer used after this statement. for &(local, _) in kill_point_map[point] { - if !state.maybe_borrowed.contains(local) { - builder.kill(local, point, SplitPointEffect::Early); - } + builder.kill(local, point, SplitPointEffect::Early); } // Gen destination places. @@ -465,9 +429,7 @@ pub fn liveness_matrix<'tcx>( // the late point of the statement they are generated in, which is // sufficient for determining overlap. for &(local, _) in kill_point_map[point] { - if !state.maybe_borrowed.contains(local) { - builder.kill(local, point, SplitPointEffect::Late); - } + builder.kill(local, point, SplitPointEffect::Late); } } @@ -475,9 +437,6 @@ pub fn liveness_matrix<'tcx>( let point = points.point_from_location(location); let terminator = block_data.terminator(); - MaybeBorrowedLocals::transfer_function(&mut state.maybe_borrowed) - .visit_terminator(terminator, location); - // Kill moved operands if the whole local was moved. Also kill dropped // places if the entire local was dropped. VisitPlacesWith(|place: Place<'tcx>, ctxt| { @@ -486,18 +445,14 @@ pub fn liveness_matrix<'tcx>( { if let Some(local) = place.as_local() { builder.kill(local, point, SplitPointEffect::Early); - state.maybe_borrowed.kill(local); } } }) .visit_terminator(terminator, location); - // Kill any locals which are no longer used after this terminator, - // but only if they have not been borrowed. + // Kill any locals which are no longer used after this terminator. for &(local, _) in kill_point_map[point] { - if !state.maybe_borrowed.contains(local) { - builder.kill(local, point, SplitPointEffect::Early); - } + builder.kill(local, point, SplitPointEffect::Early); } // Gen destination places. From c837d9c795560265266d02399cd41f54acbf9ee1 Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Sat, 30 May 2026 23:48:50 +0100 Subject: [PATCH 08/11] Add tests for move elimination --- ..._fixup.aggregate_swap.MoveElimination.diff | 84 ++++++++++ ...ed_aggregate_aliasing.MoveElimination.diff | 90 ++++++++++ tests/mir-opt/move-elimination/alias_fixup.rs | 67 ++++++++ ....simple_partial_alias.MoveElimination.diff | 52 ++++++ ...basic.array_aggregate.MoveElimination.diff | 67 ++++++++ .../basic.enum_aggregate.MoveElimination.diff | 57 +++++++ .../basic.nrvo_borrowed.MoveElimination.diff | 53 ++++++ ...basic.nrvo_unborrowed.MoveElimination.diff | 27 +++ tests/mir-opt/move-elimination/basic.rs | 80 +++++++++ ...asic.struct_aggregate.MoveElimination.diff | 49 ++++++ .../exclusions.dse_guard.MoveElimination.diff | 97 +++++++++++ ...x_local_not_projected.MoveElimination.diff | 50 ++++++ ...overlapping_lifetimes.MoveElimination.diff | 120 ++++++++++++++ ..._fields_not_projected.MoveElimination.diff | 49 ++++++ tests/mir-opt/move-elimination/exclusions.rs | 113 +++++++++++++ ...d_field_not_projected.MoveElimination.diff | 30 ++++ ...d_storage_dead_at_end.MoveElimination.diff | 62 +++++++ ...ed_to_last_direct_use.MoveElimination.diff | 71 ++++++++ ...plit_for_storage_live.MoveElimination.diff | 36 ++++ tests/mir-opt/move-elimination/storage.rs | 154 ++++++++++++++++++ ....shorten_non_borrowed.MoveElimination.diff | 56 +++++++ ..._live_moved_to_branch.MoveElimination.diff | 63 +++++++ ...age_dead_in_successor.MoveElimination.diff | 57 +++++++ 23 files changed, 1584 insertions(+) create mode 100644 tests/mir-opt/move-elimination/alias_fixup.aggregate_swap.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/alias_fixup.mixed_aggregate_aliasing.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/alias_fixup.rs create mode 100644 tests/mir-opt/move-elimination/alias_fixup.simple_partial_alias.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/basic.array_aggregate.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/basic.enum_aggregate.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/basic.nrvo_borrowed.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/basic.nrvo_unborrowed.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/basic.rs create mode 100644 tests/mir-opt/move-elimination/basic.struct_aggregate.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/exclusions.dse_guard.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/exclusions.index_local_not_projected.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/exclusions.overlapping_lifetimes.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/exclusions.packed_fields_not_projected.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/exclusions.rs create mode 100644 tests/mir-opt/move-elimination/exclusions.simd_field_not_projected.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/storage.address_observed_storage_dead_at_end.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/storage.borrowed_not_shortened_to_last_direct_use.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/storage.critical_edge_split_for_storage_live.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/storage.rs create mode 100644 tests/mir-opt/move-elimination/storage.shorten_non_borrowed.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/storage.storage_live_moved_to_branch.MoveElimination.diff create mode 100644 tests/mir-opt/move-elimination/storage.terminator_end_storage_dead_in_successor.MoveElimination.diff diff --git a/tests/mir-opt/move-elimination/alias_fixup.aggregate_swap.MoveElimination.diff b/tests/mir-opt/move-elimination/alias_fixup.aggregate_swap.MoveElimination.diff new file mode 100644 index 0000000000000..481d3ef60feb8 --- /dev/null +++ b/tests/mir-opt/move-elimination/alias_fixup.aggregate_swap.MoveElimination.diff @@ -0,0 +1,84 @@ +- // MIR for `aggregate_swap` before MoveElimination ++ // MIR for `aggregate_swap` after MoveElimination + + fn aggregate_swap(_1: u8, _2: u8) -> (u8, u8) { + debug x => _1; + debug y => _2; + let mut _0: (u8, u8); + let mut _3: (u8, u8); + let mut _4: u8; + let mut _5: u8; + let mut _8: u8; + let mut _9: u8; ++ let mut _10: u8; + scope 1 { +- debug pair => _3; ++ debug pair => _0; + let _6: u8; + scope 2 { +- debug a => _6; ++ debug a => _9; + let _7: u8; + scope 3 { +- debug b => _7; ++ debug b => (_0.1: u8); + } + } + } + + bb0: { +- StorageLive(_3); +- StorageLive(_4); +- _4 = copy _1; +- StorageLive(_5); +- _5 = copy _2; +- _3 = (move _4, move _5); +- StorageDead(_5); +- StorageDead(_4); +- StorageLive(_6); +- _6 = copy (_3.0: u8); +- StorageLive(_7); +- _7 = copy (_3.1: u8); +- StorageLive(_8); +- _8 = copy _7; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ _0 = (move _1, move _2); ++ nop; ++ nop; ++ nop; + StorageLive(_9); +- _9 = copy _6; +- _3 = (move _8, move _9); ++ _9 = copy (_0.0: u8); ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ StorageLive(_10); ++ _10 = no_retag move (_0.1: u8); ++ (_0.0: u8) = no_retag move _10; ++ (_0.1: u8) = no_retag move _9; ++ nop; ++ StorageDead(_10); + StorageDead(_9); +- StorageDead(_8); +- _0 = copy _3; +- StorageDead(_7); +- StorageDead(_6); +- StorageDead(_3); ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/alias_fixup.mixed_aggregate_aliasing.MoveElimination.diff b/tests/mir-opt/move-elimination/alias_fixup.mixed_aggregate_aliasing.MoveElimination.diff new file mode 100644 index 0000000000000..27f6c10549040 --- /dev/null +++ b/tests/mir-opt/move-elimination/alias_fixup.mixed_aggregate_aliasing.MoveElimination.diff @@ -0,0 +1,90 @@ +- // MIR for `mixed_aggregate_aliasing` before MoveElimination ++ // MIR for `mixed_aggregate_aliasing` after MoveElimination + + fn mixed_aggregate_aliasing(_1: bool, _2: u8) -> Triple { + debug flag => _1; + debug z => _2; + let mut _0: Triple; + let _3: Triple; + let mut _5: bool; + let mut _6: u8; + let mut _7: u8; + let mut _8: u8; ++ let mut _9: u8; + scope 1 { +- debug input => _3; ++ debug input => _0; + let _4: Triple; + scope 2 { +- debug out => _4; ++ debug out => _0; + } + } + + bb0: { +- StorageLive(_3); +- _3 = opaque_triple() -> [return: bb1, unwind unreachable]; ++ nop; ++ _0 = opaque_triple() -> [return: bb1, unwind unreachable]; + } + + bb1: { +- StorageLive(_4); +- StorageLive(_5); +- _5 = copy _1; +- switchInt(move _5) -> [0: bb3, otherwise: bb2]; ++ nop; ++ nop; ++ nop; ++ switchInt(move _1) -> [0: bb3, otherwise: bb2]; + } + + bb2: { +- _4 = copy _3; ++ nop; + goto -> bb4; + } + + bb3: { ++ nop; + StorageLive(_6); +- _6 = copy (_3.1: u8); +- StorageLive(_7); +- _7 = copy (_3.0: u8); +- StorageLive(_8); +- _8 = copy _2; +- _4 = Triple(move _6, move _7, move _8); +- StorageDead(_8); +- StorageDead(_7); ++ _6 = copy (_0.1: u8); ++ nop; ++ nop; ++ nop; ++ nop; ++ StorageLive(_9); ++ _9 = no_retag move (_0.0: u8); ++ (_0.0: u8) = no_retag move _6; ++ (_0.1: u8) = no_retag move _9; ++ (_0.2: u8) = no_retag move _2; ++ nop; ++ StorageDead(_9); + StorageDead(_6); ++ nop; ++ nop; ++ nop; + goto -> bb4; + } + + bb4: { +- StorageDead(_5); +- _0 = copy _4; +- StorageDead(_4); +- StorageDead(_3); ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/alias_fixup.rs b/tests/mir-opt/move-elimination/alias_fixup.rs new file mode 100644 index 0000000000000..4a5debf6e715f --- /dev/null +++ b/tests/mir-opt/move-elimination/alias_fixup.rs @@ -0,0 +1,67 @@ +//@ test-mir-pass: MoveElimination +//@ compile-flags: -Cpanic=abort + +#![allow(dead_code)] + +#[derive(Copy, Clone)] +pub struct Triple(u8, u8, u8); + +pub union U { + a: [u8; 4], + b: [u8; 4], +} + +unsafe extern "C" { + safe fn opaque_triple() -> Triple; +} + +// EMIT_MIR alias_fixup.mixed_aggregate_aliasing.MoveElimination.diff +pub fn mixed_aggregate_aliasing(flag: bool, z: u8) -> Triple { + // This checks an aggregate assignment on one branch after the other branch + // remaps the input into the return place: overlapping field reads are + // hoisted through temporaries before writing back into the return place. + // CHECK-LABEL: fn mixed_aggregate_aliasing( + // CHECK: debug z => _2; + // CHECK: debug input => _0; + // CHECK: debug out => _0; + // CHECK: [[field1:_.*]] = copy (_0.1: u8); + // CHECK: [[field0:_.*]] = no_retag move (_0.0: u8); + // CHECK: (_0.0: u8) = no_retag move [[field1]]; + // CHECK: (_0.1: u8) = no_retag move [[field0]]; + // CHECK: (_0.2: u8) = no_retag move _2; + let input = opaque_triple(); + let out = if flag { input } else { Triple(input.1, input.0, z) }; + out +} + +// EMIT_MIR alias_fixup.aggregate_swap.MoveElimination.diff +pub fn aggregate_swap(x: u8, y: u8) -> (u8, u8) { + // This checks that an aggregate swap-like assignment is safe after any + // remapping that makes source fields share storage with destination fields. + // CHECK-LABEL: fn aggregate_swap( + // CHECK: debug pair => _0; + // CHECK: [[saved:_.*]] = copy (_0.0: u8); + // CHECK: [[tmp:_.*]] = no_retag move (_0.1: u8); + // CHECK: (_0.0: u8) = no_retag move [[tmp]]; + // CHECK: (_0.1: u8) = no_retag move [[saved]]; + let mut pair = (x, y); + let a = pair.0; + let b = pair.1; + pair = (b, a); + pair +} + +// EMIT_MIR alias_fixup.simple_partial_alias.MoveElimination.diff +pub fn simple_partial_alias(x: [u8; 4]) -> U { + // This checks a non-aggregate assignment involving two same-typed union + // fields, which are conservatively treated as aliasing. + // CHECK-LABEL: fn simple_partial_alias( + // CHECK: debug u => _0; + // CHECK: _0 = U { a: move _1 }; + // CHECK: [[tmp:_.*]] = copy (_0.0: [u8; 4]); + // CHECK: (_0.1: [u8; 4]) = move [[tmp]]; + let mut u = U { a: x }; + let tmp = unsafe { u.a }; + u.b = tmp; + u +} diff --git a/tests/mir-opt/move-elimination/alias_fixup.simple_partial_alias.MoveElimination.diff b/tests/mir-opt/move-elimination/alias_fixup.simple_partial_alias.MoveElimination.diff new file mode 100644 index 0000000000000..af47eb7347d2a --- /dev/null +++ b/tests/mir-opt/move-elimination/alias_fixup.simple_partial_alias.MoveElimination.diff @@ -0,0 +1,52 @@ +- // MIR for `simple_partial_alias` before MoveElimination ++ // MIR for `simple_partial_alias` after MoveElimination + + fn simple_partial_alias(_1: [u8; 4]) -> U { + debug x => _1; + let mut _0: U; + let mut _2: U; + let mut _3: [u8; 4]; + let mut _5: [u8; 4]; + scope 1 { +- debug u => _2; ++ debug u => _0; + let _4: [u8; 4]; + scope 2 { +- debug tmp => _4; ++ debug tmp => _5; + } + } + + bb0: { +- StorageLive(_2); +- StorageLive(_3); +- _3 = copy _1; +- _2 = U { a: move _3 }; +- StorageDead(_3); +- StorageLive(_4); +- _4 = copy (_2.0: [u8; 4]); ++ nop; ++ nop; ++ nop; ++ _0 = U { a: move _1 }; ++ nop; ++ nop; + StorageLive(_5); +- _5 = copy _4; +- (_2.1: [u8; 4]) = move _5; ++ _5 = copy (_0.0: [u8; 4]); ++ nop; ++ nop; ++ (_0.1: [u8; 4]) = move _5; + StorageDead(_5); +- _0 = move _2; +- StorageDead(_4); +- StorageDead(_2); ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/basic.array_aggregate.MoveElimination.diff b/tests/mir-opt/move-elimination/basic.array_aggregate.MoveElimination.diff new file mode 100644 index 0000000000000..962975645a9dd --- /dev/null +++ b/tests/mir-opt/move-elimination/basic.array_aggregate.MoveElimination.diff @@ -0,0 +1,67 @@ +- // MIR for `array_aggregate` before MoveElimination ++ // MIR for `array_aggregate` after MoveElimination + + fn array_aggregate() -> [[u8; 8]; 3] { + let mut _0: [[u8; 8]; 3]; + let _1: [u8; 8]; + let mut _4: [u8; 8]; + let mut _5: [u8; 8]; + let mut _6: [u8; 8]; + scope 1 { +- debug a => _1; ++ debug a => _0[0 of 1]; + let _2: [u8; 8]; + scope 2 { +- debug b => _2; ++ debug b => _0[1 of 2]; + let _3: [u8; 8]; + scope 3 { +- debug c => _3; ++ debug c => _0[2 of 3]; + } + } + } + + bb0: { +- StorageLive(_1); +- _1 = [const 1_u8; 8]; +- StorageLive(_2); +- _2 = [const 2_u8; 8]; +- StorageLive(_3); +- _3 = [const 3_u8; 8]; +- StorageLive(_4); +- _4 = copy _1; +- StorageLive(_5); +- _5 = copy _2; +- StorageLive(_6); +- _6 = copy _3; +- _0 = [move _4, move _5, move _6]; +- StorageDead(_6); +- StorageDead(_5); +- StorageDead(_4); +- StorageDead(_3); +- StorageDead(_2); +- StorageDead(_1); ++ nop; ++ _0[0 of 1] = [const 1_u8; 8]; ++ nop; ++ _0[1 of 2] = [const 2_u8; 8]; ++ nop; ++ _0[2 of 3] = [const 3_u8; 8]; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/basic.enum_aggregate.MoveElimination.diff b/tests/mir-opt/move-elimination/basic.enum_aggregate.MoveElimination.diff new file mode 100644 index 0000000000000..6953dbdb66c21 --- /dev/null +++ b/tests/mir-opt/move-elimination/basic.enum_aggregate.MoveElimination.diff @@ -0,0 +1,57 @@ +- // MIR for `enum_aggregate` before MoveElimination ++ // MIR for `enum_aggregate` after MoveElimination + + fn enum_aggregate() -> Result<([u8; 8], [u8; 8]), ()> { + let mut _0: std::result::Result<([u8; 8], [u8; 8]), ()>; + let _1: [u8; 8]; + let mut _3: ([u8; 8], [u8; 8]); + let mut _4: [u8; 8]; + let mut _5: [u8; 8]; + scope 1 { +- debug a => _1; ++ debug a => (((_0 as variant#0).0: ([u8; 8], [u8; 8])).0: [u8; 8]); + let _2: [u8; 8]; + scope 2 { +- debug b => _2; ++ debug b => (((_0 as variant#0).0: ([u8; 8], [u8; 8])).1: [u8; 8]); + } + } + + bb0: { +- StorageLive(_1); +- _1 = [const 1_u8; 8]; +- StorageLive(_2); +- _2 = [const 2_u8; 8]; +- StorageLive(_3); +- StorageLive(_4); +- _4 = copy _1; +- StorageLive(_5); +- _5 = copy _2; +- _3 = (move _4, move _5); +- StorageDead(_5); +- StorageDead(_4); +- _0 = Result::<([u8; 8], [u8; 8]), ()>::Ok(move _3); +- StorageDead(_3); +- StorageDead(_2); +- StorageDead(_1); ++ nop; ++ (((_0 as variant#0).0: ([u8; 8], [u8; 8])).0: [u8; 8]) = [const 1_u8; 8]; ++ nop; ++ (((_0 as variant#0).0: ([u8; 8], [u8; 8])).1: [u8; 8]) = [const 2_u8; 8]; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ discriminant(_0) = 0; ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/basic.nrvo_borrowed.MoveElimination.diff b/tests/mir-opt/move-elimination/basic.nrvo_borrowed.MoveElimination.diff new file mode 100644 index 0000000000000..42adcc7204422 --- /dev/null +++ b/tests/mir-opt/move-elimination/basic.nrvo_borrowed.MoveElimination.diff @@ -0,0 +1,53 @@ +- // MIR for `nrvo_borrowed` before MoveElimination ++ // MIR for `nrvo_borrowed` after MoveElimination + + fn nrvo_borrowed() -> String { + let mut _0: std::string::String; + let mut _1: std::string::String; + let _2: (); + let mut _3: &mut std::string::String; + let mut _4: &mut std::string::String; + scope 1 { +- debug buf => _1; ++ debug buf => _0; + } + + bb0: { +- StorageLive(_1); +- _1 = String::new() -> [return: bb1, unwind unreachable]; ++ nop; ++ _0 = String::new() -> [return: bb1, unwind unreachable]; + } + + bb1: { +- StorageLive(_2); +- StorageLive(_3); ++ nop; ++ nop; ++ nop; + StorageLive(_4); +- _4 = &mut _1; ++ _4 = &mut _0; ++ StorageLive(_3); + _3 = &mut (*_4); ++ StorageLive(_2); ++ StorageDead(_4); + _2 = init(move _3) -> [return: bb2, unwind unreachable]; + } + + bb2: { +- StorageDead(_3); +- StorageDead(_4); + StorageDead(_2); +- _0 = move _1; +- StorageDead(_1); ++ StorageDead(_3); ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/basic.nrvo_unborrowed.MoveElimination.diff b/tests/mir-opt/move-elimination/basic.nrvo_unborrowed.MoveElimination.diff new file mode 100644 index 0000000000000..42966abcde85f --- /dev/null +++ b/tests/mir-opt/move-elimination/basic.nrvo_unborrowed.MoveElimination.diff @@ -0,0 +1,27 @@ +- // MIR for `nrvo_unborrowed` before MoveElimination ++ // MIR for `nrvo_unborrowed` after MoveElimination + + fn nrvo_unborrowed() -> String { + let mut _0: std::string::String; + let _1: std::string::String; + scope 1 { +- debug buf => _1; ++ debug buf => _0; + } + + bb0: { +- StorageLive(_1); +- _1 = String::new() -> [return: bb1, unwind unreachable]; ++ nop; ++ _0 = String::new() -> [return: bb1, unwind unreachable]; + } + + bb1: { +- _0 = move _1; +- StorageDead(_1); ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/basic.rs b/tests/mir-opt/move-elimination/basic.rs new file mode 100644 index 0000000000000..77ad40f7ecb2b --- /dev/null +++ b/tests/mir-opt/move-elimination/basic.rs @@ -0,0 +1,80 @@ +//@ test-mir-pass: MoveElimination +//@ compile-flags: -Cpanic=abort + +struct Pair { + a: [u8; 8], + b: [u8; 8], +} + +fn init(_: &mut String) {} + +// EMIT_MIR basic.nrvo_unborrowed.MoveElimination.diff +pub fn nrvo_unborrowed() -> String { + // This checks the simplest NRVO-style case: the local should be merged with + // the return place instead of being copied into it at the end. + // CHECK-LABEL: fn nrvo_unborrowed( + // CHECK: debug buf => _0; + // CHECK: _0 = String::new() + let buf = String::new(); + buf +} + +// EMIT_MIR basic.nrvo_borrowed.MoveElimination.diff +pub fn nrvo_borrowed() -> String { + // This checks that taking a temporary mutable borrow does not prevent + // merging a non-Copy local once the borrow has ended. + // CHECK-LABEL: fn nrvo_borrowed( + // CHECK: debug buf => _0; + // CHECK: _0 = String::new() + // CHECK: init(move {{_.*}}) + // CHECK-NOT: _0 = move + let mut buf = String::new(); + init(&mut buf); + buf +} + +// EMIT_MIR basic.struct_aggregate.MoveElimination.diff +pub fn struct_aggregate() -> Pair { + // This checks aggregate field remapping: the field locals can live directly + // in the return place's fields. + // CHECK-LABEL: fn struct_aggregate( + // CHECK: debug a => (_0.0: [u8; 8]); + // CHECK: debug b => (_0.1: [u8; 8]); + // CHECK: (_0.0: [u8; 8]) = [const 1_u8; 8]; + // CHECK: (_0.1: [u8; 8]) = [const 2_u8; 8]; + let a = [1; 8]; + let b = [2; 8]; + Pair { a, b } +} + +// EMIT_MIR basic.enum_aggregate.MoveElimination.diff +pub fn enum_aggregate() -> Result<([u8; 8], [u8; 8]), ()> { + // This checks aggregate field remapping for enums: the payload fields can + // be written directly and then the discriminant is set for the variant. + // CHECK-LABEL: fn enum_aggregate( + // CHECK: debug a => (((_0 as variant#0).0: ([u8; 8], [u8; 8])).0: [u8; 8]); + // CHECK: debug b => (((_0 as variant#0).0: ([u8; 8], [u8; 8])).1: [u8; 8]); + // CHECK: (((_0 as variant#0).0: ([u8; 8], [u8; 8])).0: [u8; 8]) = [const 1_u8; 8]; + // CHECK: (((_0 as variant#0).0: ([u8; 8], [u8; 8])).1: [u8; 8]) = [const 2_u8; 8]; + // CHECK: discriminant(_0) = 0; + let a = [1; 8]; + let b = [2; 8]; + Result::Ok((a, b)) +} + +// EMIT_MIR basic.array_aggregate.MoveElimination.diff +pub fn array_aggregate() -> [[u8; 8]; 3] { + // This checks aggregate remapping for arrays, which uses ConstantIndex + // projections rather than field projections. + // CHECK-LABEL: fn array_aggregate( + // CHECK: debug a => _0[0 of 1]; + // CHECK: debug b => _0[1 of 2]; + // CHECK: debug c => _0[2 of 3]; + // CHECK: _0[0 of 1] = [const 1_u8; 8]; + // CHECK: _0[1 of 2] = [const 2_u8; 8]; + // CHECK: _0[2 of 3] = [const 3_u8; 8]; + let a = [1; 8]; + let b = [2; 8]; + let c = [3; 8]; + [a, b, c] +} diff --git a/tests/mir-opt/move-elimination/basic.struct_aggregate.MoveElimination.diff b/tests/mir-opt/move-elimination/basic.struct_aggregate.MoveElimination.diff new file mode 100644 index 0000000000000..dab733b7009ba --- /dev/null +++ b/tests/mir-opt/move-elimination/basic.struct_aggregate.MoveElimination.diff @@ -0,0 +1,49 @@ +- // MIR for `struct_aggregate` before MoveElimination ++ // MIR for `struct_aggregate` after MoveElimination + + fn struct_aggregate() -> Pair { + let mut _0: Pair; + let _1: [u8; 8]; + let mut _3: [u8; 8]; + let mut _4: [u8; 8]; + scope 1 { +- debug a => _1; ++ debug a => (_0.0: [u8; 8]); + let _2: [u8; 8]; + scope 2 { +- debug b => _2; ++ debug b => (_0.1: [u8; 8]); + } + } + + bb0: { +- StorageLive(_1); +- _1 = [const 1_u8; 8]; +- StorageLive(_2); +- _2 = [const 2_u8; 8]; +- StorageLive(_3); +- _3 = copy _1; +- StorageLive(_4); +- _4 = copy _2; +- _0 = Pair { a: move _3, b: move _4 }; +- StorageDead(_4); +- StorageDead(_3); +- StorageDead(_2); +- StorageDead(_1); ++ nop; ++ (_0.0: [u8; 8]) = [const 1_u8; 8]; ++ nop; ++ (_0.1: [u8; 8]) = [const 2_u8; 8]; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/exclusions.dse_guard.MoveElimination.diff b/tests/mir-opt/move-elimination/exclusions.dse_guard.MoveElimination.diff new file mode 100644 index 0000000000000..895dab2802597 --- /dev/null +++ b/tests/mir-opt/move-elimination/exclusions.dse_guard.MoveElimination.diff @@ -0,0 +1,97 @@ +- // MIR for `dse_guard` before MoveElimination ++ // MIR for `dse_guard` after MoveElimination + + fn dse_guard() -> () { + let mut _0: (); + let mut _1: Fields; + let mut _3: Fields; + let mut _4: Fields; + let _5: (); + let mut _6: *const Fields; + let mut _7: Fields; + let _8: (); + let mut _9: *const Fields; + scope 1 { +- debug a => _1; ++ debug a => _7; + let mut _2: Fields; + scope 2 { + debug b => _2; + } + } + + bb0: { +- StorageLive(_1); ++ nop; ++ nop; ++ nop; + StorageLive(_2); +- StorageLive(_3); +- _3 = make_fields(const 0_u8) -> [return: bb1, unwind unreachable]; ++ _2 = make_fields(const 0_u8) -> [return: bb1, unwind unreachable]; + } + + bb1: { +- _2 = move _3; +- StorageDead(_3); +- StorageLive(_4); +- _4 = make_fields(const 1_u8) -> [return: bb2, unwind unreachable]; ++ nop; ++ nop; ++ nop; ++ StorageLive(_7); ++ _7 = make_fields(const 1_u8) -> [return: bb2, unwind unreachable]; + } + + bb2: { +- _1 = move _4; +- StorageDead(_4); +- StorageLive(_5); ++ nop; ++ nop; ++ nop; ++ nop; + StorageLive(_6); +- _6 = &raw const _1; ++ _6 = &raw const _7; ++ StorageLive(_5); + _5 = observe(move _6) -> [return: bb3, unwind unreachable]; + } + + bb3: { +- StorageDead(_6); + StorageDead(_5); +- StorageLive(_7); +- _7 = move _1; ++ StorageDead(_6); ++ nop; ++ nop; ++ nop; ++ nop; + _2 = move _7; + StorageDead(_7); +- StorageLive(_8); ++ nop; ++ nop; ++ nop; + StorageLive(_9); + _9 = &raw const _2; ++ StorageLive(_8); + _8 = observe(move _9) -> [return: bb4, unwind unreachable]; + } + + bb4: { +- StorageDead(_9); + StorageDead(_8); ++ StorageDead(_9); ++ nop; ++ nop; + _0 = const (); ++ nop; + StorageDead(_2); +- StorageDead(_1); ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/exclusions.index_local_not_projected.MoveElimination.diff b/tests/mir-opt/move-elimination/exclusions.index_local_not_projected.MoveElimination.diff new file mode 100644 index 0000000000000..883476c39e1d5 --- /dev/null +++ b/tests/mir-opt/move-elimination/exclusions.index_local_not_projected.MoveElimination.diff @@ -0,0 +1,50 @@ +- // MIR for `index_local_not_projected` before MoveElimination ++ // MIR for `index_local_not_projected` after MoveElimination + + fn index_local_not_projected(_1: [usize; 4], _2: usize) -> [usize; 1] { + debug a => _1; + debug i => _2; + let mut _0: [usize; 1]; + let _3: usize; + let _4: usize; + let mut _5: bool; + let mut _6: usize; + scope 1 { +- debug idx => _3; ++ debug idx => _2; + scope 2 { + } + } + + bb0: { +- StorageLive(_3); +- _3 = copy _2; ++ nop; ++ nop; ++ nop; + StorageLive(_4); +- _4 = copy _3; ++ _4 = copy _2; ++ StorageLive(_5); + _5 = Lt(copy _4, const 4_usize); + assert(move _5, "index out of bounds: the length is {} but the index is {}", const 4_usize, copy _4) -> [success: bb1, unwind unreachable]; + } + + bb1: { + StorageDead(_4); +- StorageLive(_6); +- _6 = copy _3; +- _0 = [move _6]; +- StorageDead(_6); +- StorageDead(_3); ++ StorageDead(_5); ++ nop; ++ nop; ++ nop; ++ _0 = [move _2]; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/exclusions.overlapping_lifetimes.MoveElimination.diff b/tests/mir-opt/move-elimination/exclusions.overlapping_lifetimes.MoveElimination.diff new file mode 100644 index 0000000000000..cb3f5d8afd220 --- /dev/null +++ b/tests/mir-opt/move-elimination/exclusions.overlapping_lifetimes.MoveElimination.diff @@ -0,0 +1,120 @@ +- // MIR for `overlapping_lifetimes` before MoveElimination ++ // MIR for `overlapping_lifetimes` after MoveElimination + + fn overlapping_lifetimes(_1: bool) -> Fields { + debug flag => _1; + let mut _0: Fields; + let _2: Fields; + let _4: (); + let mut _5: *const Fields; + let _6: (); + let mut _7: bool; + let mut _8: Fields; + let mut _9: Fields; + let _10: (); + let mut _11: *const Fields; + scope 1 { +- debug src => _2; ++ debug src => _9; + let mut _3: Fields; + scope 2 { +- debug dst => _3; ++ debug dst => _0; + scope 3 { + } + } + } + + bb0: { +- StorageLive(_2); +- _2 = make_fields(const 0_u8) -> [return: bb1, unwind unreachable]; ++ nop; ++ StorageLive(_9); ++ _9 = make_fields(const 0_u8) -> [return: bb1, unwind unreachable]; + } + + bb1: { +- StorageLive(_3); +- StorageLive(_4); ++ nop; ++ nop; ++ nop; + StorageLive(_5); +- _5 = &raw const _2; ++ _5 = &raw const _9; ++ StorageLive(_4); + _4 = observe(move _5) -> [return: bb2, unwind unreachable]; + } + + bb2: { +- StorageDead(_5); + StorageDead(_4); +- StorageLive(_7); +- _7 = copy _1; +- switchInt(move _7) -> [0: bb5, otherwise: bb3]; ++ StorageDead(_5); ++ nop; ++ nop; ++ nop; ++ nop; ++ switchInt(move _1) -> [0: bb5, otherwise: bb3]; + } + + bb3: { +- StorageLive(_8); +- _8 = make_fields(const 1_u8) -> [return: bb4, unwind unreachable]; ++ nop; ++ _0 = make_fields(const 1_u8) -> [return: bb4, unwind unreachable]; + } + + bb4: { +- _3 = move _8; +- StorageDead(_8); ++ nop; ++ nop; + goto -> bb6; + } + + bb5: { +- StorageLive(_9); +- _9 = move _2; +- _3 = move _9; ++ nop; ++ nop; ++ _0 = move _9; + StorageDead(_9); ++ nop; ++ StorageLive(_9); + goto -> bb6; + } + + bb6: { +- StorageDead(_7); +- StorageLive(_10); ++ nop; ++ nop; ++ nop; + StorageLive(_11); +- _11 = &raw const _3; ++ _11 = &raw const _0; ++ StorageLive(_10); + _10 = observe(move _11) -> [return: bb7, unwind unreachable]; + } + + bb7: { +- StorageDead(_11); + StorageDead(_10); +- _0 = move _3; +- StorageDead(_3); +- StorageDead(_2); ++ StorageDead(_11); ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ StorageDead(_9); + return; + } + } + diff --git a/tests/mir-opt/move-elimination/exclusions.packed_fields_not_projected.MoveElimination.diff b/tests/mir-opt/move-elimination/exclusions.packed_fields_not_projected.MoveElimination.diff new file mode 100644 index 0000000000000..af6d2b34a0439 --- /dev/null +++ b/tests/mir-opt/move-elimination/exclusions.packed_fields_not_projected.MoveElimination.diff @@ -0,0 +1,49 @@ +- // MIR for `packed_fields_not_projected` before MoveElimination ++ // MIR for `packed_fields_not_projected` after MoveElimination + + fn packed_fields_not_projected() -> Packed { + let mut _0: Packed; + let _1: [u8; 8]; + let mut _3: [u8; 8]; + let mut _4: [u8; 8]; + scope 1 { +- debug a => _1; ++ debug a => _3; + let _2: [u8; 8]; + scope 2 { +- debug b => _2; ++ debug b => _4; + } + } + + bb0: { +- StorageLive(_1); +- _1 = [const 1_u8; 8]; +- StorageLive(_2); +- _2 = [const 2_u8; 8]; ++ nop; + StorageLive(_3); +- _3 = copy _1; ++ _3 = [const 1_u8; 8]; ++ nop; + StorageLive(_4); +- _4 = copy _2; ++ _4 = [const 2_u8; 8]; ++ nop; ++ nop; ++ nop; ++ nop; + _0 = Packed { a: move _3, b: move _4 }; +- StorageDead(_4); + StorageDead(_3); +- StorageDead(_2); +- StorageDead(_1); ++ StorageDead(_4); ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/exclusions.rs b/tests/mir-opt/move-elimination/exclusions.rs new file mode 100644 index 0000000000000..80b4f4a77a02c --- /dev/null +++ b/tests/mir-opt/move-elimination/exclusions.rs @@ -0,0 +1,113 @@ +//@ test-mir-pass: MoveElimination +//@ compile-flags: -Cpanic=abort -Zmir-enable-passes=+DeadStoreElimination-initial + +#![feature(repr_simd)] + +pub struct Fields { + data: [u8; 8], + tag: u8, +} + +#[repr(packed)] +struct Packed { + a: [u8; 8], + b: [u8; 8], +} + +#[repr(simd)] +struct U32x4([u32; 4]); + +unsafe extern "C" { + safe fn observe(_: *const Fields); + safe fn make_fields(_: u8) -> Fields; +} + +// EMIT_MIR exclusions.index_local_not_projected.MoveElimination.diff +pub fn index_local_not_projected(a: [usize; 4], i: usize) -> [usize; 1] { + // This checks that a local used as an array index is kept as a bare local, + // because it cannot later be rewritten to a projection like `_0[0]`. + // CHECK-LABEL: fn index_local_not_projected( + // CHECK: debug idx => _2; + // CHECK: _4 = copy _2; + // CHECK: _0 = [move _2]; + let idx = i; + let _ = a[idx]; + [idx] +} + +// EMIT_MIR exclusions.packed_fields_not_projected.MoveElimination.diff +pub fn packed_fields_not_projected() -> Packed { + // This checks that aggregate fields are not remapped into packed struct + // fields, which could create unaligned projected places. + // CHECK-LABEL: fn packed_fields_not_projected( + // CHECK: debug a => [[a:_.*]]; + // CHECK: debug b => [[b:_.*]]; + // CHECK: _0 = Packed { a: move [[a]], b: move [[b]] }; + let a = [1; 8]; + let b = [2; 8]; + Packed { a, b } +} + +// EMIT_MIR exclusions.simd_field_not_projected.MoveElimination.diff +pub fn simd_field_not_projected() -> U32x4 { + // This checks that aggregate fields are not remapped into repr(simd) ADTs, + // since optimized MIR must not project into SIMD vectors. + // CHECK-LABEL: fn simd_field_not_projected( + // CHECK: debug lanes => [[lanes:_.*]]; + // CHECK: _0 = U32x4(move [[lanes]]); + let lanes = [1, 2, 3, 4]; + U32x4(lanes) +} + +// EMIT_MIR exclusions.overlapping_lifetimes.MoveElimination.diff +pub fn overlapping_lifetimes(flag: bool) -> Fields { + // This checks the liveness-matrix overlap test for an address-observed + // move-only local: `src` and `dst` only overlap on one branch, but that is + // enough to reject merging them for the whole function. + // CHECK-LABEL: fn overlapping_lifetimes( + // CHECK: debug flag => _1; + // CHECK: debug src => [[src:_[1-9][0-9]*]]; + // CHECK: debug dst => _0; + // CHECK: &raw const [[src]]; + // CHECK: observe + // CHECK: switchInt(move _1) + // CHECK: _0 = make_fields(const 1_u8) + // CHECK: _0 = move [[src]]; + // CHECK: &raw const _0; + // CHECK: observe + let src = make_fields(0); + let mut dst; + observe(&raw const src); + if flag { + dst = make_fields(1); + let _ = src.tag; + } else { + dst = src; + } + observe(&raw const dst); + dst +} + +// EMIT_MIR exclusions.dse_guard.MoveElimination.diff +pub fn dse_guard() { + // This guards the RFC soundness hazard: DSE must not remove the first write + // to `b`, because that write keeps `b`'s address-observed lifetime + // overlapping with `a` and prevents the later move from being eliminated. + // CHECK-LABEL: fn dse_guard( + // CHECK: StorageLive([[b:_.*]]); + // CHECK: [[b]] = make_fields(const 0_u8) + // CHECK: StorageLive([[a:_.*]]); + // CHECK: [[a]] = make_fields(const 1_u8) + // CHECK: observe(move + // CHECK: [[b]] = move [[a]] + // CHECK: observe(move + let mut a; + let mut b; + + b = make_fields(0); + + a = make_fields(1); + observe(&raw const a); + b = a; + observe(&raw const b); +} diff --git a/tests/mir-opt/move-elimination/exclusions.simd_field_not_projected.MoveElimination.diff b/tests/mir-opt/move-elimination/exclusions.simd_field_not_projected.MoveElimination.diff new file mode 100644 index 0000000000000..4844207200f34 --- /dev/null +++ b/tests/mir-opt/move-elimination/exclusions.simd_field_not_projected.MoveElimination.diff @@ -0,0 +1,30 @@ +- // MIR for `simd_field_not_projected` before MoveElimination ++ // MIR for `simd_field_not_projected` after MoveElimination + + fn simd_field_not_projected() -> U32x4 { + let mut _0: U32x4; + let _1: [u32; 4]; + let mut _2: [u32; 4]; + scope 1 { +- debug lanes => _1; ++ debug lanes => _2; + } + + bb0: { +- StorageLive(_1); +- _1 = [const 1_u32, const 2_u32, const 3_u32, const 4_u32]; ++ nop; + StorageLive(_2); +- _2 = copy _1; ++ _2 = [const 1_u32, const 2_u32, const 3_u32, const 4_u32]; ++ nop; ++ nop; + _0 = U32x4(move _2); + StorageDead(_2); +- StorageDead(_1); ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/storage.address_observed_storage_dead_at_end.MoveElimination.diff b/tests/mir-opt/move-elimination/storage.address_observed_storage_dead_at_end.MoveElimination.diff new file mode 100644 index 0000000000000..5156d4dc363b8 --- /dev/null +++ b/tests/mir-opt/move-elimination/storage.address_observed_storage_dead_at_end.MoveElimination.diff @@ -0,0 +1,62 @@ +- // MIR for `address_observed_storage_dead_at_end` before MoveElimination ++ // MIR for `address_observed_storage_dead_at_end` after MoveElimination + + fn address_observed_storage_dead_at_end(_1: bool) -> () { + debug flag => _1; + let mut _0: (); + let _2: u32; + let mut _3: bool; + let _4: *const u32; + let mut _5: *const u32; + scope 1 { + debug x => _2; + } + + bb0: { +- StorageLive(_2); +- StorageLive(_3); +- _3 = copy _1; +- switchInt(move _3) -> [0: bb3, otherwise: bb1]; ++ nop; ++ nop; ++ nop; ++ switchInt(move _1) -> [0: bb3, otherwise: bb1]; + } + + bb1: { ++ StorageLive(_2); + _2 = const 1_u32; +- StorageLive(_4); ++ nop; ++ nop; + StorageLive(_5); + _5 = &raw const _2; ++ StorageLive(_4); + _4 = opaque::<*const u32>(move _5) -> [return: bb2, unwind unreachable]; + } + + bb2: { +- StorageDead(_5); + StorageDead(_4); ++ StorageDead(_5); ++ nop; ++ nop; + _0 = const (); + goto -> bb4; + } + + bb3: { + _0 = const (); ++ StorageLive(_2); + goto -> bb4; + } + + bb4: { +- StorageDead(_3); ++ nop; ++ nop; + StorageDead(_2); + return; + } + } + diff --git a/tests/mir-opt/move-elimination/storage.borrowed_not_shortened_to_last_direct_use.MoveElimination.diff b/tests/mir-opt/move-elimination/storage.borrowed_not_shortened_to_last_direct_use.MoveElimination.diff new file mode 100644 index 0000000000000..45b7567ceee46 --- /dev/null +++ b/tests/mir-opt/move-elimination/storage.borrowed_not_shortened_to_last_direct_use.MoveElimination.diff @@ -0,0 +1,71 @@ +- // MIR for `borrowed_not_shortened_to_last_direct_use` before MoveElimination ++ // MIR for `borrowed_not_shortened_to_last_direct_use` after MoveElimination + + fn borrowed_not_shortened_to_last_direct_use(_1: u32) -> u32 { + debug x => _1; + let mut _0: u32; + let _2: u32; + let mut _5: u32; + let mut _6: u32; + let mut _7: u32; + scope 1 { +- debug a => _2; ++ debug a => _1; + let _3: &u32; + scope 2 { + debug r => _3; + let _4: u32; + scope 3 { +- debug out => _4; ++ debug out => _7; + } + } + } + + bb0: { +- StorageLive(_2); +- _2 = copy _1; ++ nop; ++ nop; ++ nop; + StorageLive(_3); +- _3 = &_2; +- StorageLive(_4); +- _4 = copy _2; +- StorageLive(_5); ++ _3 = &_1; ++ nop; ++ StorageLive(_7); ++ _7 = copy _1; ++ nop; ++ nop; + StorageLive(_6); + _6 = copy (*_3); +- StorageLive(_7); +- _7 = copy _4; ++ StorageDead(_3); ++ nop; ++ nop; ++ StorageLive(_5); + _5 = Add(move _6, move _7); +- StorageDead(_7); + StorageDead(_6); ++ StorageDead(_7); ++ nop; ++ nop; + _0 = opaque::(move _5) -> [return: bb1, unwind unreachable]; + } + + bb1: { + StorageDead(_5); +- StorageDead(_4); +- StorageDead(_3); +- StorageDead(_2); ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/storage.critical_edge_split_for_storage_live.MoveElimination.diff b/tests/mir-opt/move-elimination/storage.critical_edge_split_for_storage_live.MoveElimination.diff new file mode 100644 index 0000000000000..e564f76b1b610 --- /dev/null +++ b/tests/mir-opt/move-elimination/storage.critical_edge_split_for_storage_live.MoveElimination.diff @@ -0,0 +1,36 @@ +- // MIR for `critical_edge_split_for_storage_live` before MoveElimination ++ // MIR for `critical_edge_split_for_storage_live` after MoveElimination + + fn critical_edge_split_for_storage_live(_1: bool) -> () { + debug x => _2; + let mut _0: (); + let mut _2: u32; + let mut _3: *const u32; + + bb0: { +- switchInt(copy _1) -> [1: bb1, otherwise: bb2]; ++ switchInt(copy _1) -> [1: bb1, otherwise: bb3]; + } + + bb1: { ++ nop; + StorageLive(_2); + _2 = const 1_u32; ++ StorageLive(_3); + _3 = &raw const _2; + _3 = opaque::<*const u32>(copy _3) -> [return: bb2, unwind unreachable]; + } + + bb2: { ++ StorageDead(_3); ++ nop; + StorageDead(_2); + return; ++ } ++ ++ bb3: { ++ StorageLive(_2); ++ goto -> bb2; + } + } + diff --git a/tests/mir-opt/move-elimination/storage.rs b/tests/mir-opt/move-elimination/storage.rs new file mode 100644 index 0000000000000..9107045718296 --- /dev/null +++ b/tests/mir-opt/move-elimination/storage.rs @@ -0,0 +1,154 @@ +//@ test-mir-pass: MoveElimination +//@ compile-flags: -Cpanic=abort + +#![feature(custom_mir, core_intrinsics)] + +use std::intrinsics::mir::*; + +fn opaque(x: T) -> T { + x +} + +// EMIT_MIR storage.shorten_non_borrowed.MoveElimination.diff +pub fn shorten_non_borrowed(x: u32) -> u32 { + // This checks that reconstruction can shorten a non-borrowed local's + // storage to its last use instead of keeping lexical storage markers. + // CHECK-LABEL: fn shorten_non_borrowed( + // CHECK: debug a => [[short:_.*]]; + // CHECK: debug b => [[short]]; + // CHECK: StorageLive([[short]]); + // CHECK: [[short]] = opaque::(move _1) + // CHECK: opaque::(move [[short]]) -> [return: [[short_ret:bb.*]], + // CHECK: [[short_ret]]: { + // CHECK-NEXT: StorageDead([[short]]); + let a = opaque(x); + let b = a; + opaque(b) +} + +// EMIT_MIR storage.borrowed_not_shortened_to_last_direct_use.MoveElimination.diff +pub fn borrowed_not_shortened_to_last_direct_use(x: u32) -> u32 { + // This checks that a borrowed local is not shortened merely to its last + // direct use; the borrow keeps its storage live while the reference exists. + // CHECK-LABEL: fn borrowed_not_shortened_to_last_direct_use( + // CHECK: debug a => _1; + // CHECK: debug r => [[borrow_ref:_.*]]; + // CHECK: debug out => [[borrow_out:_.*]]; + // CHECK: StorageLive([[borrow_ref]]); + // CHECK-NEXT: [[borrow_ref]] = &_1; + // CHECK: StorageLive([[borrow_out]]); + // CHECK-NEXT: [[borrow_out]] = copy _1; + // CHECK: copy (*[[borrow_ref]]); + // CHECK-NEXT: StorageDead([[borrow_ref]]); + let a = x; + let r = &a; + let out = a; + opaque(*r + out) +} + +// EMIT_MIR storage.storage_live_moved_to_branch.MoveElimination.diff +pub fn storage_live_moved_to_branch(flag: bool) { + // This checks storage reconstruction can shrink a local declared before a + // branch so its storage is live only on the arm where it is initialized. + // CHECK-LABEL: fn storage_live_moved_to_branch( + // CHECK: debug x => [[branch_tmp:_.*]]; + // CHECK: switchInt(move _1) -> [0: bb3, otherwise: bb1]; + // CHECK: bb1: { + // CHECK: StorageLive([[branch_tmp]]); + // CHECK: [[branch_tmp]] = const 1_u32; + // CHECK: opaque::(move [[branch_tmp]]) -> [return: [[branch_ret:bb.*]], + // CHECK: [[branch_ret]]: { + // CHECK: StorageDead([[branch_tmp]]); + let x: u32; + if flag { + x = 1; + opaque(x); + } +} + +// EMIT_MIR storage.address_observed_storage_dead_at_end.MoveElimination.diff +pub fn address_observed_storage_dead_at_end(flag: bool) { + // This checks that an address-observed local declared before a branch still + // has StorageLive moved into the initialized arm, but StorageDead remains at + // the end of the function instead of being shortened to the last direct use. + // CHECK-LABEL: fn address_observed_storage_dead_at_end( + // CHECK: debug x => [[addr_tmp:_.*]]; + // CHECK: switchInt(move _1) -> [0: [[skip:bb.*]], otherwise: [[init:bb.*]]]; + // CHECK: [[init]]: { + // CHECK: StorageLive([[addr_tmp]]); + // CHECK: [[addr_tmp]] = const 1_u32; + // CHECK: &raw const [[addr_tmp]]; + // CHECK: opaque::<*const u32> + // CHECK-NOT: StorageDead([[addr_tmp]]); + // CHECK: [[skip]]: { + // CHECK: StorageLive([[addr_tmp]]); + // CHECK: {{bb.*}}: { + // CHECK: StorageDead([[addr_tmp]]); + let x: u32; + if flag { + x = 1; + opaque(&raw const x); + } +} + +// EMIT_MIR storage.terminator_end_storage_dead_in_successor.MoveElimination.diff +pub fn terminator_end_storage_dead_in_successor(x: u32) -> u32 { + // This checks storage reconstruction when the last use of a local is as a + // call argument in a terminator. + // CHECK-LABEL: fn terminator_end_storage_dead_in_successor( + // CHECK: debug tmp => [[term_tmp:_.*]]; + // CHECK: StorageLive([[term_tmp]]); + // CHECK: [[term_tmp]] = opaque::(move _1) + // CHECK: opaque::(move [[term_tmp]]) -> [return: [[term_ret:bb.*]], + // CHECK: [[term_ret]]: { + // CHECK-NEXT: StorageDead([[term_tmp]]); + let tmp = opaque(x); + let out = opaque(tmp); + out +} + +// EMIT_MIR storage.critical_edge_split_for_storage_live.MoveElimination.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn critical_edge_split_for_storage_live(flag: bool) { + // This checks storage reconstruction on a custom CFG where the `false` + // branch goes directly to the final block. Since that block also has an + // incoming edge from the initialized branch, inserting StorageLive precisely + // requires splitting the critical edge from the entry switch. + // CHECK-LABEL: fn critical_edge_split_for_storage_live( + // CHECK: debug x => [[crit_tmp:_.*]]; + // CHECK: switchInt(copy _1) -> [1: [[init:bb.*]], otherwise: [[split:bb.*]]]; + // CHECK: [[init]]: { + // CHECK: StorageLive([[crit_tmp]]); + // CHECK: [[crit_tmp]] = const 1_u32; + // CHECK: &raw const [[crit_tmp]]; + // CHECK: opaque::<*const u32>{{.*}} -> [return: [[ret:bb.*]], + // CHECK: [[ret:bb.*]]: { + // CHECK: StorageDead([[crit_tmp]]); + // CHECK: [[split]]: { + // CHECK-NEXT: StorageLive([[crit_tmp]]); + // CHECK-NEXT: goto -> [[ret]]; + mir! { + let x: u32; + let ptr: *const u32; + debug x => x; + + { + match flag { + true => init, + _ => ret, + } + } + + init = { + StorageLive(x); + x = 1; + ptr = &raw const x; + Call(ptr = opaque::<*const u32>(ptr), ReturnTo(ret), UnwindUnreachable()) + } + + ret = { + StorageDead(x); + Return() + } + } +} diff --git a/tests/mir-opt/move-elimination/storage.shorten_non_borrowed.MoveElimination.diff b/tests/mir-opt/move-elimination/storage.shorten_non_borrowed.MoveElimination.diff new file mode 100644 index 0000000000000..1dbcb6b5f4e38 --- /dev/null +++ b/tests/mir-opt/move-elimination/storage.shorten_non_borrowed.MoveElimination.diff @@ -0,0 +1,56 @@ +- // MIR for `shorten_non_borrowed` before MoveElimination ++ // MIR for `shorten_non_borrowed` after MoveElimination + + fn shorten_non_borrowed(_1: u32) -> u32 { + debug x => _1; + let mut _0: u32; + let _2: u32; + let mut _3: u32; + let mut _5: u32; + scope 1 { +- debug a => _2; ++ debug a => _5; + let _4: u32; + scope 2 { +- debug b => _4; ++ debug b => _5; + } + } + + bb0: { +- StorageLive(_2); +- StorageLive(_3); +- _3 = copy _1; +- _2 = opaque::(move _3) -> [return: bb1, unwind unreachable]; ++ nop; ++ nop; ++ nop; ++ StorageLive(_5); ++ _5 = opaque::(move _1) -> [return: bb1, unwind unreachable]; + } + + bb1: { +- StorageDead(_3); +- StorageLive(_4); +- _4 = copy _2; +- StorageLive(_5); +- _5 = copy _4; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; + _0 = opaque::(move _5) -> [return: bb2, unwind unreachable]; + } + + bb2: { + StorageDead(_5); +- StorageDead(_4); +- StorageDead(_2); ++ nop; ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/storage.storage_live_moved_to_branch.MoveElimination.diff b/tests/mir-opt/move-elimination/storage.storage_live_moved_to_branch.MoveElimination.diff new file mode 100644 index 0000000000000..9a7c3df9cb98c --- /dev/null +++ b/tests/mir-opt/move-elimination/storage.storage_live_moved_to_branch.MoveElimination.diff @@ -0,0 +1,63 @@ +- // MIR for `storage_live_moved_to_branch` before MoveElimination ++ // MIR for `storage_live_moved_to_branch` after MoveElimination + + fn storage_live_moved_to_branch(_1: bool) -> () { + debug flag => _1; + let mut _0: (); + let _2: u32; + let mut _3: bool; + let _4: u32; + let mut _5: u32; + scope 1 { +- debug x => _2; ++ debug x => _5; + } + + bb0: { +- StorageLive(_2); +- StorageLive(_3); +- _3 = copy _1; +- switchInt(move _3) -> [0: bb3, otherwise: bb1]; ++ nop; ++ nop; ++ nop; ++ switchInt(move _1) -> [0: bb3, otherwise: bb1]; + } + + bb1: { +- _2 = const 1_u32; +- StorageLive(_4); + StorageLive(_5); +- _5 = copy _2; ++ _5 = const 1_u32; ++ nop; ++ nop; ++ nop; ++ StorageLive(_4); + _4 = opaque::(move _5) -> [return: bb2, unwind unreachable]; + } + + bb2: { +- StorageDead(_5); + StorageDead(_4); ++ StorageDead(_5); ++ nop; ++ nop; + _0 = const (); + goto -> bb4; + } + + bb3: { + _0 = const (); + goto -> bb4; + } + + bb4: { +- StorageDead(_3); +- StorageDead(_2); ++ nop; ++ nop; + return; + } + } + diff --git a/tests/mir-opt/move-elimination/storage.terminator_end_storage_dead_in_successor.MoveElimination.diff b/tests/mir-opt/move-elimination/storage.terminator_end_storage_dead_in_successor.MoveElimination.diff new file mode 100644 index 0000000000000..ffc1f04c58097 --- /dev/null +++ b/tests/mir-opt/move-elimination/storage.terminator_end_storage_dead_in_successor.MoveElimination.diff @@ -0,0 +1,57 @@ +- // MIR for `terminator_end_storage_dead_in_successor` before MoveElimination ++ // MIR for `terminator_end_storage_dead_in_successor` after MoveElimination + + fn terminator_end_storage_dead_in_successor(_1: u32) -> u32 { + debug x => _1; + let mut _0: u32; + let _2: u32; + let mut _3: u32; + let mut _5: u32; + scope 1 { +- debug tmp => _2; ++ debug tmp => _5; + let _4: u32; + scope 2 { +- debug out => _4; ++ debug out => _0; + } + } + + bb0: { +- StorageLive(_2); +- StorageLive(_3); +- _3 = copy _1; +- _2 = opaque::(move _3) -> [return: bb1, unwind unreachable]; ++ nop; ++ nop; ++ nop; ++ StorageLive(_5); ++ _5 = opaque::(move _1) -> [return: bb1, unwind unreachable]; + } + + bb1: { +- StorageDead(_3); +- StorageLive(_4); +- StorageLive(_5); +- _5 = copy _2; +- _4 = opaque::(move _5) -> [return: bb2, unwind unreachable]; ++ nop; ++ nop; ++ nop; ++ nop; ++ _0 = opaque::(move _5) -> [return: bb2, unwind unreachable]; + } + + bb2: { + StorageDead(_5); +- _0 = copy _4; +- StorageDead(_4); +- StorageDead(_2); ++ nop; ++ nop; ++ nop; ++ nop; + return; + } + } + From 0c839045312f0de24b28e058695efdd915511558 Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Sun, 31 May 2026 01:45:12 +0100 Subject: [PATCH 09/11] Add a MIR pass which turns copies into moves just before a return --- compiler/rustc_mir_transform/src/lib.rs | 2 + .../src/tail_copy_to_move.rs | 240 +++++++++++++++++ ...basic.array_aggregate.MoveElimination.diff | 6 +- .../basic.enum_aggregate.MoveElimination.diff | 4 +- .../basic.nrvo_borrowed.MoveElimination.diff | 23 +- ...basic.nrvo_unborrowed.MoveElimination.diff | 15 +- tests/mir-opt/move-elimination/basic.rs | 20 +- ...asic.struct_aggregate.MoveElimination.diff | 4 +- ...copy_to_move.aggregate.TailCopyToMove.diff | 24 ++ ...ove.aggregate_operands.TailCopyToMove.diff | 13 + ...e.aggregate_with_deref.TailCopyToMove.diff | 16 ++ ...rrowed_dest_stops_tail.TailCopyToMove.diff | 16 ++ ...e.borrowed_source_tail.TailCopyToMove.diff | 15 ++ ...ail_copy_to_move.chain.TailCopyToMove.diff | 22 ++ ...il_copy_to_move.direct.TailCopyToMove.diff | 14 + ...opy_to_move.index_dest.TailCopyToMove.diff | 17 ++ ..._to_move.index_operand.TailCopyToMove.diff | 13 + ...ove.indirect_tail_read.TailCopyToMove.diff | 19 ++ ...copy_to_move.projected.TailCopyToMove.diff | 14 + ...to_move.projected_dest.TailCopyToMove.diff | 15 ++ ..._move.repeated_operand.TailCopyToMove.diff | 13 + tests/mir-opt/tail_copy_to_move.rs | 254 ++++++++++++++++++ ..._move.set_discriminant.TailCopyToMove.diff | 14 + ..._to_move.shared_return.TailCopyToMove.diff | 34 +++ ...e.unrelated_tail_store.TailCopyToMove.diff | 16 ++ 25 files changed, 804 insertions(+), 39 deletions(-) create mode 100644 compiler/rustc_mir_transform/src/tail_copy_to_move.rs create mode 100644 tests/mir-opt/tail_copy_to_move.aggregate.TailCopyToMove.diff create mode 100644 tests/mir-opt/tail_copy_to_move.aggregate_operands.TailCopyToMove.diff create mode 100644 tests/mir-opt/tail_copy_to_move.aggregate_with_deref.TailCopyToMove.diff create mode 100644 tests/mir-opt/tail_copy_to_move.borrowed_dest_stops_tail.TailCopyToMove.diff create mode 100644 tests/mir-opt/tail_copy_to_move.borrowed_source_tail.TailCopyToMove.diff create mode 100644 tests/mir-opt/tail_copy_to_move.chain.TailCopyToMove.diff create mode 100644 tests/mir-opt/tail_copy_to_move.direct.TailCopyToMove.diff create mode 100644 tests/mir-opt/tail_copy_to_move.index_dest.TailCopyToMove.diff create mode 100644 tests/mir-opt/tail_copy_to_move.index_operand.TailCopyToMove.diff create mode 100644 tests/mir-opt/tail_copy_to_move.indirect_tail_read.TailCopyToMove.diff create mode 100644 tests/mir-opt/tail_copy_to_move.projected.TailCopyToMove.diff create mode 100644 tests/mir-opt/tail_copy_to_move.projected_dest.TailCopyToMove.diff create mode 100644 tests/mir-opt/tail_copy_to_move.repeated_operand.TailCopyToMove.diff create mode 100644 tests/mir-opt/tail_copy_to_move.rs create mode 100644 tests/mir-opt/tail_copy_to_move.set_discriminant.TailCopyToMove.diff create mode 100644 tests/mir-opt/tail_copy_to_move.shared_return.TailCopyToMove.diff create mode 100644 tests/mir-opt/tail_copy_to_move.unrelated_tail_store.TailCopyToMove.diff diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index e02d8b8745b5a..c8d330dd908ad 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -202,6 +202,7 @@ declare_passes! { mod sroa : ScalarReplacementOfAggregates; mod strip_debuginfo : StripDebugInfo; mod ssa_range_prop: SsaRangePropagation; + mod tail_copy_to_move : TailCopyToMove; mod unreachable_enum_branching : UnreachableEnumBranching; mod unreachable_prop : UnreachablePropagation; mod validate : Validator; @@ -760,6 +761,7 @@ pub(crate) fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<' ©_prop::CopyProp, &dead_store_elimination::DeadStoreElimination::Final, &dest_prop::DestinationPropagation, + &tail_copy_to_move::TailCopyToMove, &move_elimination::MoveElimination, &simplify::SimplifyLocals::Final, &multiple_return_terminators::MultipleReturnTerminators, diff --git a/compiler/rustc_mir_transform/src/tail_copy_to_move.rs b/compiler/rustc_mir_transform/src/tail_copy_to_move.rs new file mode 100644 index 0000000000000..dc8e150ff16c8 --- /dev/null +++ b/compiler/rustc_mir_transform/src/tail_copy_to_move.rs @@ -0,0 +1,240 @@ +//! Rewrite final-use copies before return into moves. +//! +//! # The problem +//! +//! MIR building represents reads of values whose type is `Copy` using `Operand::Copy`, including +//! when such a local is returned. If that local's address has ever been observed, then the local's +//! allocation is semantically valid until its `StorageDead` or function exit. This keeps the local +//! live across `_0 = copy local`, so its live range overlaps with the return place and +//! `MoveElimination` cannot unify the source local with `_0`. +//! +//! # The solution +//! +//! At function return, all local allocations are about to become invalid anyway. After borrowck, +//! this pass can therefore turn a final-use `Copy` into a `Move`, as long as shortening the source +//! local's live range has no observable effect before the return happens. +//! Concretely, between the transformed copy (now a move) and the return, there may only be writes +//! to unborrowed locals, storage markers, nops, and gotos. +//! +//! # The algorithm +//! +//! Start from every `Return` terminator, with `_0` treated as used by the return. Then scan +//! predecessor blocks backward through `Goto` edges, forming a return-tail tree. The scan maintains +//! `used_after`, the set of locals accessed later on that path. +//! +//! A `Copy` operand is rewritten to a `Move` when its base local is not in `used_after`. Then any +//! locals touched by that operand, including index-projection locals, are added to `used_after` +//! before the backward scan continues. +//! +//! The scan stops when accessing an indirect place because that may access any borrowed local, +//! which would make the pass unable to prove any useful final uses. It also stops at writes to +//! borrowed locals, because those can create a new address-observed allocation range whose overlap +//! with an earlier borrowed local must be preserved. + +use rustc_index::bit_set::DenseBitSet; +use rustc_middle::mir::*; +use rustc_middle::ty::TyCtxt; +use rustc_mir_dataflow::impls::borrowed_locals; + +pub(super) struct TailCopyToMove; + +impl<'tcx> crate::MirPass<'tcx> for TailCopyToMove { + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.mir_opt_level() >= 2 + } + + #[tracing::instrument(level = "trace", skip(self, _tcx, body))] + fn run_pass(&self, _tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + let borrowed = borrowed_locals(body); + let predecessors = body.basic_blocks.predecessors().clone(); + let mut stack = Vec::new(); + + // A return terminator implicitly uses the return place. Walking backward through + // assignments records the locals accessed later on this path. + for (bb, data) in body.basic_blocks.iter_enumerated() { + if matches!(data.terminator().kind, TerminatorKind::Return) { + let mut used_after = DenseBitSet::new_empty(body.local_decls.len()); + used_after.insert(RETURN_PLACE); + stack.push(TailState { block: bb, used_after }); + } + } + + while let Some(mut state) = stack.pop() { + loop { + // `scan_block` rewrites final-use copies in this block and updates `used_after` to + // the locals whose allocation is accessed after the block starts. If the block is not + // pure tail code, this path is done. + if !scan_block(body, state.block, &mut state.used_after, &borrowed) { + break; + } + + // Continue through predecessor blocks only when the predecessor's terminator is a + // plain `Goto` to this block. Other terminators are control-flow or effect + // boundaries. Follow one predecessor in-place and push only sibling paths, so + // straight-line return tails don't clone the bitset at every block. + let mut next = None; + for pred in predecessors[state.block].iter().copied() { + let terminator = body.basic_blocks[pred].terminator(); + if let TerminatorKind::Goto { target } = terminator.kind { + debug_assert_eq!(target, state.block); + if next.is_none() { + next = Some(pred); + } else { + stack.push(TailState { + block: pred, + used_after: state.used_after.clone(), + }); + } + } + } + + let Some(next) = next else { + break; + }; + state.block = next; + } + } + } + + fn is_required(&self) -> bool { + false + } +} + +struct TailState { + block: BasicBlock, + used_after: DenseBitSet, +} + +/// Scan a block backward while the return-tail invariant still holds. +/// +/// The invariant is that a whole-local `Copy` can be changed to a `Move` only if this path has no +/// later access to that local's allocation before returning, and no later operation whose observable +/// behavior could depend on ending an address-observed local's allocation early. `used_after` +/// tracks those later local-allocation accesses. +fn scan_block<'tcx>( + body: &mut Body<'tcx>, + block: BasicBlock, + used_after: &mut DenseBitSet, + borrowed: &DenseBitSet, +) -> bool { + for statement in body.basic_blocks.as_mut_preserves_cfg()[block].statements.iter_mut().rev() { + match &mut statement.kind { + // Under the local lifetime semantics from RFC 3943, `StorageLive` does not allocate, + // and `StorageDead` has no effect if the local was already freed by a move. These + // markers therefore do not affect whether a copy can be treated as a final use. + StatementKind::StorageLive(_) | StatementKind::StorageDead(_) | StatementKind::Nop => {} + StatementKind::Assign(assign) => { + let dest = assign.0; + + // Accessing an indirect place may touch any borrowed local, so continuing would + // require treating all borrowed locals as used after this point. + if dest.is_indirect_first_projection() { + return false; + } + + // Writing to a borrowed local can start a new allocation range. Shortening an + // earlier borrowed local could remove an overlap with that new range. + if borrowed.contains(dest.local) { + return false; + } + + // A destination write accesses the base local, and evaluating the destination may + // also access projection locals, such as an index. + record_place_locals(dest, used_after); + + // This pass only models `Use` and `Aggregate` rvalues whose operands are direct. + // Other rvalues are outside the conservative return-tail shape handled here. + if !process_rvalue(&mut assign.1, used_after) { + return false; + } + } + StatementKind::SetDiscriminant { place, .. } => { + let dest = **place; + + // Accessing an indirect place may touch any borrowed local, so continuing would + // require treating all borrowed locals as used after this point. + if dest.is_indirect_first_projection() { + return false; + } + + // Writing to a borrowed local can start a new allocation range. Shortening an + // earlier borrowed local could remove an overlap with that new range. + if borrowed.contains(dest.local) { + return false; + } + + // `SetDiscriminant` has a validity invariant on the rest of the place, so treat + // the base local as accessed along with any projection locals. + record_place_locals(dest, used_after); + } + _ => { + // Anything else may perform effects or evaluate places in ways this pass does not + // model, so it is not part of the pure return tail. + return false; + } + } + } + + true +} + +/// Records all locals used in a place, including `Index` projections in `used_after`. +fn record_place_locals<'tcx>(place: Place<'tcx>, used_after: &mut DenseBitSet) { + for local in place.as_ref().accessed_locals() { + used_after.insert(local); + } +} + +/// Process the RHS of an assignment in a pure return tail. +fn process_rvalue<'tcx>(rvalue: &mut Rvalue<'tcx>, used_after: &mut DenseBitSet) -> bool { + match rvalue { + Rvalue::Use(operand, _) => process_operand(operand, used_after), + Rvalue::Aggregate(_, operands) => { + // Operands are evaluated left-to-right. We scan them right-to-left so `used_after` + // includes uses later in the same statement. If an operand accesses an indirect place, + // only earlier operands and earlier statements are outside the pure tail. + for operand in operands.iter_mut().rev() { + if !process_operand(operand, used_after) { + return false; + } + } + true + } + _ => { + // This pass doesn't model other rvalues, so they are not part of the pure return tail. + return false; + } + } +} + +/// Process one operand in an rvalue. +fn process_operand<'tcx>(operand: &mut Operand<'tcx>, used_after: &mut DenseBitSet) -> bool { + let place = match operand { + Operand::Copy(place) | Operand::Move(place) if place.is_indirect_first_projection() => { + // Accessing an indirect place may touch any borrowed local. Continuing would + // require treating all borrowed locals as used after this point, which would prevent + // the useful copy-to-move rewrites this pass is looking for. + return false; + } + Operand::Copy(place) => { + let place = *place; + // No later operation in the scanned tail accesses this local's allocation, so this + // copy is a final use on the current return path and can be represented as a move. + if !used_after.contains(place.local) { + *operand = Operand::Move(place); + } + Some(place) + } + Operand::Move(place) => Some(*place), + Operand::Constant(_) | Operand::RuntimeChecks(_) => None, + }; + + if let Some(place) = place { + // Reading an operand place accesses its base local, and evaluating its projections may + // access additional locals, such as the index local in `place[index]`. + record_place_locals(place, used_after); + } + + true +} diff --git a/tests/mir-opt/move-elimination/basic.array_aggregate.MoveElimination.diff b/tests/mir-opt/move-elimination/basic.array_aggregate.MoveElimination.diff index 962975645a9dd..01938d10da6c5 100644 --- a/tests/mir-opt/move-elimination/basic.array_aggregate.MoveElimination.diff +++ b/tests/mir-opt/move-elimination/basic.array_aggregate.MoveElimination.diff @@ -30,11 +30,11 @@ - StorageLive(_3); - _3 = [const 3_u8; 8]; - StorageLive(_4); -- _4 = copy _1; +- _4 = move _1; - StorageLive(_5); -- _5 = copy _2; +- _5 = move _2; - StorageLive(_6); -- _6 = copy _3; +- _6 = move _3; - _0 = [move _4, move _5, move _6]; - StorageDead(_6); - StorageDead(_5); diff --git a/tests/mir-opt/move-elimination/basic.enum_aggregate.MoveElimination.diff b/tests/mir-opt/move-elimination/basic.enum_aggregate.MoveElimination.diff index 6953dbdb66c21..5f40d8a3c82fd 100644 --- a/tests/mir-opt/move-elimination/basic.enum_aggregate.MoveElimination.diff +++ b/tests/mir-opt/move-elimination/basic.enum_aggregate.MoveElimination.diff @@ -24,9 +24,9 @@ - _2 = [const 2_u8; 8]; - StorageLive(_3); - StorageLive(_4); -- _4 = copy _1; +- _4 = move _1; - StorageLive(_5); -- _5 = copy _2; +- _5 = move _2; - _3 = (move _4, move _5); - StorageDead(_5); - StorageDead(_4); diff --git a/tests/mir-opt/move-elimination/basic.nrvo_borrowed.MoveElimination.diff b/tests/mir-opt/move-elimination/basic.nrvo_borrowed.MoveElimination.diff index 42adcc7204422..70f11d8614d83 100644 --- a/tests/mir-opt/move-elimination/basic.nrvo_borrowed.MoveElimination.diff +++ b/tests/mir-opt/move-elimination/basic.nrvo_borrowed.MoveElimination.diff @@ -1,12 +1,12 @@ - // MIR for `nrvo_borrowed` before MoveElimination + // MIR for `nrvo_borrowed` after MoveElimination - fn nrvo_borrowed() -> String { - let mut _0: std::string::String; - let mut _1: std::string::String; + fn nrvo_borrowed() -> [u8; 8] { + let mut _0: [u8; 8]; + let mut _1: [u8; 8]; let _2: (); - let mut _3: &mut std::string::String; - let mut _4: &mut std::string::String; + let mut _3: &mut [u8; 8]; + let mut _4: &mut [u8; 8]; scope 1 { - debug buf => _1; + debug buf => _0; @@ -14,15 +14,12 @@ bb0: { - StorageLive(_1); -- _1 = String::new() -> [return: bb1, unwind unreachable]; -+ nop; -+ _0 = String::new() -> [return: bb1, unwind unreachable]; - } - - bb1: { +- _1 = [const 1_u8; 8]; - StorageLive(_2); - StorageLive(_3); + nop; ++ _0 = [const 1_u8; 8]; ++ nop; + nop; + nop; StorageLive(_4); @@ -32,10 +29,10 @@ _3 = &mut (*_4); + StorageLive(_2); + StorageDead(_4); - _2 = init(move _3) -> [return: bb2, unwind unreachable]; + _2 = init(move _3) -> [return: bb1, unwind unreachable]; } - bb2: { + bb1: { - StorageDead(_3); - StorageDead(_4); StorageDead(_2); diff --git a/tests/mir-opt/move-elimination/basic.nrvo_unborrowed.MoveElimination.diff b/tests/mir-opt/move-elimination/basic.nrvo_unborrowed.MoveElimination.diff index 42966abcde85f..7a44d4b598211 100644 --- a/tests/mir-opt/move-elimination/basic.nrvo_unborrowed.MoveElimination.diff +++ b/tests/mir-opt/move-elimination/basic.nrvo_unborrowed.MoveElimination.diff @@ -1,9 +1,9 @@ - // MIR for `nrvo_unborrowed` before MoveElimination + // MIR for `nrvo_unborrowed` after MoveElimination - fn nrvo_unborrowed() -> String { - let mut _0: std::string::String; - let _1: std::string::String; + fn nrvo_unborrowed() -> [u8; 8] { + let mut _0: [u8; 8]; + let _1: [u8; 8]; scope 1 { - debug buf => _1; + debug buf => _0; @@ -11,15 +11,12 @@ bb0: { - StorageLive(_1); -- _1 = String::new() -> [return: bb1, unwind unreachable]; -+ nop; -+ _0 = String::new() -> [return: bb1, unwind unreachable]; - } - - bb1: { +- _1 = [const 1_u8; 8]; - _0 = move _1; - StorageDead(_1); + nop; ++ _0 = [const 1_u8; 8]; ++ nop; + nop; return; } diff --git a/tests/mir-opt/move-elimination/basic.rs b/tests/mir-opt/move-elimination/basic.rs index 77ad40f7ecb2b..96f1776bebef1 100644 --- a/tests/mir-opt/move-elimination/basic.rs +++ b/tests/mir-opt/move-elimination/basic.rs @@ -1,34 +1,34 @@ //@ test-mir-pass: MoveElimination -//@ compile-flags: -Cpanic=abort +//@ compile-flags: -Cpanic=abort -Zmir-enable-passes=+TailCopyToMove struct Pair { a: [u8; 8], b: [u8; 8], } -fn init(_: &mut String) {} +fn init(_: &mut [u8; 8]) {} // EMIT_MIR basic.nrvo_unborrowed.MoveElimination.diff -pub fn nrvo_unborrowed() -> String { +pub fn nrvo_unborrowed() -> [u8; 8] { // This checks the simplest NRVO-style case: the local should be merged with - // the return place instead of being copied into it at the end. + // the return place even though it has `Copy` type. // CHECK-LABEL: fn nrvo_unborrowed( // CHECK: debug buf => _0; - // CHECK: _0 = String::new() - let buf = String::new(); + // CHECK: _0 = [const 1_u8; 8] + let buf = [1; 8]; buf } // EMIT_MIR basic.nrvo_borrowed.MoveElimination.diff -pub fn nrvo_borrowed() -> String { +pub fn nrvo_borrowed() -> [u8; 8] { // This checks that taking a temporary mutable borrow does not prevent - // merging a non-Copy local once the borrow has ended. + // merging a `Copy` local once the borrow has ended. // CHECK-LABEL: fn nrvo_borrowed( // CHECK: debug buf => _0; - // CHECK: _0 = String::new() + // CHECK: _0 = [const 1_u8; 8] // CHECK: init(move {{_.*}}) // CHECK-NOT: _0 = move - let mut buf = String::new(); + let mut buf = [1; 8]; init(&mut buf); buf } diff --git a/tests/mir-opt/move-elimination/basic.struct_aggregate.MoveElimination.diff b/tests/mir-opt/move-elimination/basic.struct_aggregate.MoveElimination.diff index dab733b7009ba..b03c4d2f3cff9 100644 --- a/tests/mir-opt/move-elimination/basic.struct_aggregate.MoveElimination.diff +++ b/tests/mir-opt/move-elimination/basic.struct_aggregate.MoveElimination.diff @@ -22,9 +22,9 @@ - StorageLive(_2); - _2 = [const 2_u8; 8]; - StorageLive(_3); -- _3 = copy _1; +- _3 = move _1; - StorageLive(_4); -- _4 = copy _2; +- _4 = move _2; - _0 = Pair { a: move _3, b: move _4 }; - StorageDead(_4); - StorageDead(_3); diff --git a/tests/mir-opt/tail_copy_to_move.aggregate.TailCopyToMove.diff b/tests/mir-opt/tail_copy_to_move.aggregate.TailCopyToMove.diff new file mode 100644 index 0000000000000..8c4045d4efcea --- /dev/null +++ b/tests/mir-opt/tail_copy_to_move.aggregate.TailCopyToMove.diff @@ -0,0 +1,24 @@ +- // MIR for `aggregate` before TailCopyToMove ++ // MIR for `aggregate` after TailCopyToMove + + fn aggregate(_1: u32, _2: u32) -> Pair { + debug x => _1; + debug y => _2; + let mut _0: Pair; + let mut _3: u32; + let mut _4: u32; + + bb0: { + StorageLive(_3); +- _3 = copy _1; ++ _3 = move _1; + StorageLive(_4); +- _4 = copy _2; ++ _4 = move _2; + _0 = Pair { a: move _3, b: move _4 }; + StorageDead(_4); + StorageDead(_3); + return; + } + } + diff --git a/tests/mir-opt/tail_copy_to_move.aggregate_operands.TailCopyToMove.diff b/tests/mir-opt/tail_copy_to_move.aggregate_operands.TailCopyToMove.diff new file mode 100644 index 0000000000000..cf0c2d62d3960 --- /dev/null +++ b/tests/mir-opt/tail_copy_to_move.aggregate_operands.TailCopyToMove.diff @@ -0,0 +1,13 @@ +- // MIR for `aggregate_operands` before TailCopyToMove ++ // MIR for `aggregate_operands` after TailCopyToMove + + fn aggregate_operands(_1: u32, _2: u32) -> (u32, u32) { + let mut _0: (u32, u32); + + bb0: { +- _0 = (copy _1, copy _2); ++ _0 = (move _1, move _2); + return; + } + } + diff --git a/tests/mir-opt/tail_copy_to_move.aggregate_with_deref.TailCopyToMove.diff b/tests/mir-opt/tail_copy_to_move.aggregate_with_deref.TailCopyToMove.diff new file mode 100644 index 0000000000000..f4a70b1d9fabb --- /dev/null +++ b/tests/mir-opt/tail_copy_to_move.aggregate_with_deref.TailCopyToMove.diff @@ -0,0 +1,16 @@ +- // MIR for `aggregate_with_deref` before TailCopyToMove ++ // MIR for `aggregate_with_deref` after TailCopyToMove + + fn aggregate_with_deref(_1: u32) -> (u32, u32) { + let mut _0: (u32, u32); + let mut _2: *const u32; + let mut _3: u32; + + bb0: { + _2 = &raw const _1; + _3 = copy _1; + _0 = (copy _3, copy (*_2)); + return; + } + } + diff --git a/tests/mir-opt/tail_copy_to_move.borrowed_dest_stops_tail.TailCopyToMove.diff b/tests/mir-opt/tail_copy_to_move.borrowed_dest_stops_tail.TailCopyToMove.diff new file mode 100644 index 0000000000000..6d1ba9ea819eb --- /dev/null +++ b/tests/mir-opt/tail_copy_to_move.borrowed_dest_stops_tail.TailCopyToMove.diff @@ -0,0 +1,16 @@ +- // MIR for `borrowed_dest_stops_tail` before TailCopyToMove ++ // MIR for `borrowed_dest_stops_tail` after TailCopyToMove + + fn borrowed_dest_stops_tail(_1: u32, _2: u32) -> u32 { + let mut _0: u32; + let mut _3: u32; + let mut _4: *const u32; + + bb0: { + _4 = &raw const _3; + _0 = copy _1; + _3 = copy _2; + return; + } + } + diff --git a/tests/mir-opt/tail_copy_to_move.borrowed_source_tail.TailCopyToMove.diff b/tests/mir-opt/tail_copy_to_move.borrowed_source_tail.TailCopyToMove.diff new file mode 100644 index 0000000000000..fb24d156a7383 --- /dev/null +++ b/tests/mir-opt/tail_copy_to_move.borrowed_source_tail.TailCopyToMove.diff @@ -0,0 +1,15 @@ +- // MIR for `borrowed_source_tail` before TailCopyToMove ++ // MIR for `borrowed_source_tail` after TailCopyToMove + + fn borrowed_source_tail(_1: u32) -> u32 { + let mut _0: u32; + let mut _2: *const u32; + + bb0: { + _2 = &raw const _1; +- _0 = copy _1; ++ _0 = move _1; + return; + } + } + diff --git a/tests/mir-opt/tail_copy_to_move.chain.TailCopyToMove.diff b/tests/mir-opt/tail_copy_to_move.chain.TailCopyToMove.diff new file mode 100644 index 0000000000000..d354f9f23446f --- /dev/null +++ b/tests/mir-opt/tail_copy_to_move.chain.TailCopyToMove.diff @@ -0,0 +1,22 @@ +- // MIR for `chain` before TailCopyToMove ++ // MIR for `chain` after TailCopyToMove + + fn chain(_1: u32) -> u32 { + debug x => _1; + let mut _0: u32; + let _2: u32; + scope 1 { + debug t => _2; + } + + bb0: { + StorageLive(_2); +- _2 = copy _1; +- _0 = copy _2; ++ _2 = move _1; ++ _0 = move _2; + StorageDead(_2); + return; + } + } + diff --git a/tests/mir-opt/tail_copy_to_move.direct.TailCopyToMove.diff b/tests/mir-opt/tail_copy_to_move.direct.TailCopyToMove.diff new file mode 100644 index 0000000000000..84e7c4f9d42ca --- /dev/null +++ b/tests/mir-opt/tail_copy_to_move.direct.TailCopyToMove.diff @@ -0,0 +1,14 @@ +- // MIR for `direct` before TailCopyToMove ++ // MIR for `direct` after TailCopyToMove + + fn direct(_1: u32) -> u32 { + debug x => _1; + let mut _0: u32; + + bb0: { +- _0 = copy _1; ++ _0 = move _1; + return; + } + } + diff --git a/tests/mir-opt/tail_copy_to_move.index_dest.TailCopyToMove.diff b/tests/mir-opt/tail_copy_to_move.index_dest.TailCopyToMove.diff new file mode 100644 index 0000000000000..9aba859937410 --- /dev/null +++ b/tests/mir-opt/tail_copy_to_move.index_dest.TailCopyToMove.diff @@ -0,0 +1,17 @@ +- // MIR for `index_dest` before TailCopyToMove ++ // MIR for `index_dest` after TailCopyToMove + + fn index_dest(_1: [usize; 4], _2: usize) -> [usize; 4] { + let mut _0: [usize; 4]; + let mut _3: [usize; 4]; + + bb0: { +- _3 = copy _1; ++ _3 = move _1; + _3[_2] = copy _2; +- _0 = copy _3; ++ _0 = move _3; + return; + } + } + diff --git a/tests/mir-opt/tail_copy_to_move.index_operand.TailCopyToMove.diff b/tests/mir-opt/tail_copy_to_move.index_operand.TailCopyToMove.diff new file mode 100644 index 0000000000000..da6bbb117a846 --- /dev/null +++ b/tests/mir-opt/tail_copy_to_move.index_operand.TailCopyToMove.diff @@ -0,0 +1,13 @@ +- // MIR for `index_operand` before TailCopyToMove ++ // MIR for `index_operand` after TailCopyToMove + + fn index_operand(_1: [u32; 4], _2: usize) -> (usize, u32) { + let mut _0: (usize, u32); + + bb0: { +- _0 = (copy _2, copy _1[_2]); ++ _0 = (copy _2, move _1[_2]); + return; + } + } + diff --git a/tests/mir-opt/tail_copy_to_move.indirect_tail_read.TailCopyToMove.diff b/tests/mir-opt/tail_copy_to_move.indirect_tail_read.TailCopyToMove.diff new file mode 100644 index 0000000000000..16ac0dd1806ca --- /dev/null +++ b/tests/mir-opt/tail_copy_to_move.indirect_tail_read.TailCopyToMove.diff @@ -0,0 +1,19 @@ +- // MIR for `indirect_tail_read` before TailCopyToMove ++ // MIR for `indirect_tail_read` after TailCopyToMove + + fn indirect_tail_read(_1: u32) -> (u32, u32) { + let mut _0: (u32, u32); + let mut _2: *const u32; + let mut _3: u32; + let mut _4: u32; + + bb0: { + _2 = &raw const _1; + _3 = copy _1; + _4 = copy (*_2); +- _0 = (copy _3, copy _4); ++ _0 = (move _3, move _4); + return; + } + } + diff --git a/tests/mir-opt/tail_copy_to_move.projected.TailCopyToMove.diff b/tests/mir-opt/tail_copy_to_move.projected.TailCopyToMove.diff new file mode 100644 index 0000000000000..d19a551cfb737 --- /dev/null +++ b/tests/mir-opt/tail_copy_to_move.projected.TailCopyToMove.diff @@ -0,0 +1,14 @@ +- // MIR for `projected` before TailCopyToMove ++ // MIR for `projected` after TailCopyToMove + + fn projected(_1: Pair) -> u32 { + debug pair => _1; + let mut _0: u32; + + bb0: { +- _0 = copy (_1.0: u32); ++ _0 = move (_1.0: u32); + return; + } + } + diff --git a/tests/mir-opt/tail_copy_to_move.projected_dest.TailCopyToMove.diff b/tests/mir-opt/tail_copy_to_move.projected_dest.TailCopyToMove.diff new file mode 100644 index 0000000000000..14911f4313ebd --- /dev/null +++ b/tests/mir-opt/tail_copy_to_move.projected_dest.TailCopyToMove.diff @@ -0,0 +1,15 @@ +- // MIR for `projected_dest` before TailCopyToMove ++ // MIR for `projected_dest` after TailCopyToMove + + fn projected_dest(_1: u32, _2: u32) -> (u32, u32) { + let mut _0: (u32, u32); + + bb0: { +- (_0.0: u32) = copy _1; +- (_0.1: u32) = copy _2; ++ (_0.0: u32) = move _1; ++ (_0.1: u32) = move _2; + return; + } + } + diff --git a/tests/mir-opt/tail_copy_to_move.repeated_operand.TailCopyToMove.diff b/tests/mir-opt/tail_copy_to_move.repeated_operand.TailCopyToMove.diff new file mode 100644 index 0000000000000..59e775c20f898 --- /dev/null +++ b/tests/mir-opt/tail_copy_to_move.repeated_operand.TailCopyToMove.diff @@ -0,0 +1,13 @@ +- // MIR for `repeated_operand` before TailCopyToMove ++ // MIR for `repeated_operand` after TailCopyToMove + + fn repeated_operand(_1: u32) -> (u32, u32) { + let mut _0: (u32, u32); + + bb0: { +- _0 = (copy _1, copy _1); ++ _0 = (copy _1, move _1); + return; + } + } + diff --git a/tests/mir-opt/tail_copy_to_move.rs b/tests/mir-opt/tail_copy_to_move.rs new file mode 100644 index 0000000000000..12dfe10f656bb --- /dev/null +++ b/tests/mir-opt/tail_copy_to_move.rs @@ -0,0 +1,254 @@ +//@ test-mir-pass: TailCopyToMove +//@ compile-flags: -Cpanic=abort + +#![feature(custom_mir, core_intrinsics)] +#![allow(internal_features)] + +use std::intrinsics::mir::*; + +#[derive(Copy, Clone)] +pub struct Pair { + a: u32, + b: u32, +} + +#[derive(Copy, Clone)] +pub enum Choice { + A(u32), + B, +} + +// EMIT_MIR tail_copy_to_move.direct.TailCopyToMove.diff +pub fn direct(x: u32) -> u32 { + // Checks the simplest returned `Copy` local. + // CHECK-LABEL: fn direct( + // CHECK: _0 = move _1; + x +} + +// EMIT_MIR tail_copy_to_move.chain.TailCopyToMove.diff +pub fn chain(x: u32) -> u32 { + // Checks that the scan propagates through a temporary local. + // CHECK-LABEL: fn chain( + // CHECK: [[TMP:_.*]] = move _1; + // CHECK: _0 = move [[TMP]]; + let t = x; + t +} + +// EMIT_MIR tail_copy_to_move.aggregate.TailCopyToMove.diff +pub fn aggregate(x: u32, y: u32) -> Pair { + // Checks aggregate construction from returned `Copy` locals. + // CHECK-LABEL: fn aggregate( + // CHECK: [[A:_.*]] = move _1; + // CHECK: [[B:_.*]] = move _2; + // CHECK: _0 = Pair { a: move [[A]], b: move [[B]] }; + Pair { a: x, b: y } +} + +// EMIT_MIR tail_copy_to_move.aggregate_operands.TailCopyToMove.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn aggregate_operands(x: u32, y: u32) -> (u32, u32) { + // Checks aggregate operands that are already in the final assignment. + // CHECK-LABEL: fn aggregate_operands( + // CHECK: _0 = (move _1, move _2); + mir!({ + RET = (x, y); + Return() + }) +} + +// EMIT_MIR tail_copy_to_move.projected_dest.TailCopyToMove.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn projected_dest(x: u32, y: u32) -> (u32, u32) { + // Checks assignments to direct projections of the return place. + // CHECK-LABEL: fn projected_dest( + // CHECK: (_0.0: u32) = move _1; + // CHECK: (_0.1: u32) = move _2; + mir! { + type RET = (u32, u32); + { + RET.0 = x; + RET.1 = y; + Return() + } + } +} + +// EMIT_MIR tail_copy_to_move.projected.TailCopyToMove.diff +pub fn projected(pair: Pair) -> u32 { + // Checks that direct projected source copies are also rewritten. + // CHECK-LABEL: fn projected( + // CHECK: _0 = move (_1.0: u32); + pair.a +} + +// EMIT_MIR tail_copy_to_move.set_discriminant.TailCopyToMove.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn set_discriminant(choice: Choice) -> Choice { + // Checks that `SetDiscriminant` is accepted in the return tail. + // CHECK-LABEL: fn set_discriminant( + // CHECK: _0 = move _1; + // CHECK: discriminant(_0) = 1; + mir!({ + RET = choice; + SetDiscriminant(RET, 1); + Return() + }) +} + +// EMIT_MIR tail_copy_to_move.indirect_tail_read.TailCopyToMove.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn indirect_tail_read(x: u32) -> (u32, u32) { + // Checks that an indirect read stops the scan before earlier assignments. + // CHECK-LABEL: fn indirect_tail_read( + // CHECK: [[P:_.*]] = &raw const _1; + // CHECK: [[Q:_.*]] = copy _1; + // CHECK: [[S:_.*]] = copy (*[[P]]); + // CHECK: _0 = (move [[Q]], move [[S]]); + mir! { + let p: *const u32; + let q: u32; + let s: u32; + + { + p = &raw const x; + q = x; + s = *p; + RET = (q, s); + Return() + } + } +} + +// EMIT_MIR tail_copy_to_move.aggregate_with_deref.TailCopyToMove.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn aggregate_with_deref(x: u32) -> (u32, u32) { + // Checks that an indirect aggregate operand stops the aggregate scan. + // CHECK-LABEL: fn aggregate_with_deref( + // CHECK: [[P:_.*]] = &raw const _1; + // CHECK: [[Q:_.*]] = copy _1; + // CHECK: _0 = (copy [[Q]], copy (*[[P]])); + mir! { + let p: *const u32; + let q: u32; + + { + p = &raw const x; + q = x; + RET = (q, *p); + Return() + } + } +} + +// EMIT_MIR tail_copy_to_move.borrowed_dest_stops_tail.TailCopyToMove.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn borrowed_dest_stops_tail(x: u32, z: u32) -> u32 { + // Checks that writing to a borrowed local stops the scan. + // CHECK-LABEL: fn borrowed_dest_stops_tail( + // CHECK: _0 = copy _1; + // CHECK: [[Y:_.*]] = copy _2; + mir! { + let y: u32; + let p: *const u32; + + { + p = &raw const y; + RET = x; + y = z; + Return() + } + } +} + +// EMIT_MIR tail_copy_to_move.unrelated_tail_store.TailCopyToMove.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn unrelated_tail_store(x: u32, z: u32) -> u32 { + // Checks that writing to an unborrowed local remains in the tail. + // CHECK-LABEL: fn unrelated_tail_store( + // CHECK: _0 = move _1; + // CHECK: [[Y:_.*]] = move _2; + mir! { + let y: u32; + + { + RET = x; + y = z; + Return() + } + } +} + +// EMIT_MIR tail_copy_to_move.index_operand.TailCopyToMove.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn index_operand(arr: [u32; 4], idx: usize) -> (usize, u32) { + // Checks that index projection locals count as later uses. + // CHECK-LABEL: fn index_operand( + // CHECK: _0 = (copy _2, move _1[_2]); + mir! { + { + RET = (idx, arr[idx]); + Return() + } + } +} + +// EMIT_MIR tail_copy_to_move.index_dest.TailCopyToMove.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn index_dest(arr: [usize; 4], idx: usize) -> [usize; 4] { + // Checks that index locals in destination projections are recorded. + // CHECK-LABEL: fn index_dest( + // CHECK: [[ARR:_.*]] = move _1; + // CHECK: [[ARR]][_2] = copy _2; + // CHECK: _0 = move [[ARR]]; + mir! { + let a: [usize; 4]; + + { + a = arr; + a[idx] = idx; + RET = a; + Return() + } + } +} + +// EMIT_MIR tail_copy_to_move.repeated_operand.TailCopyToMove.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn repeated_operand(x: u32) -> (u32, u32) { + // Checks right-to-left aggregate scanning for repeated operands. + // CHECK-LABEL: fn repeated_operand( + // CHECK: _0 = (copy _1, move _1); + mir!({ + RET = (x, x); + Return() + }) +} + +// EMIT_MIR tail_copy_to_move.borrowed_source_tail.TailCopyToMove.diff +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +pub fn borrowed_source_tail(x: u32) -> u32 { + // Checks that a borrowed source can still move at its final use. + // CHECK-LABEL: fn borrowed_source_tail( + // CHECK: [[P:_.*]] = &raw const _1; + // CHECK: _0 = move _1; + mir! { + let p: *const u32; + + { + p = &raw const x; + RET = x; + Return() + } + } +} + +// EMIT_MIR tail_copy_to_move.shared_return.TailCopyToMove.diff +pub fn shared_return(x: u32, y: u32, take_x: bool) -> u32 { + // Checks branch arms that share a return block. + // CHECK-LABEL: fn shared_return( + // CHECK: _0 = move _1; + // CHECK: _0 = move _2; + if take_x { x } else { y } +} diff --git a/tests/mir-opt/tail_copy_to_move.set_discriminant.TailCopyToMove.diff b/tests/mir-opt/tail_copy_to_move.set_discriminant.TailCopyToMove.diff new file mode 100644 index 0000000000000..76f23dae83a91 --- /dev/null +++ b/tests/mir-opt/tail_copy_to_move.set_discriminant.TailCopyToMove.diff @@ -0,0 +1,14 @@ +- // MIR for `set_discriminant` before TailCopyToMove ++ // MIR for `set_discriminant` after TailCopyToMove + + fn set_discriminant(_1: Choice) -> Choice { + let mut _0: Choice; + + bb0: { +- _0 = copy _1; ++ _0 = move _1; + discriminant(_0) = 1; + return; + } + } + diff --git a/tests/mir-opt/tail_copy_to_move.shared_return.TailCopyToMove.diff b/tests/mir-opt/tail_copy_to_move.shared_return.TailCopyToMove.diff new file mode 100644 index 0000000000000..26c0879695ae2 --- /dev/null +++ b/tests/mir-opt/tail_copy_to_move.shared_return.TailCopyToMove.diff @@ -0,0 +1,34 @@ +- // MIR for `shared_return` before TailCopyToMove ++ // MIR for `shared_return` after TailCopyToMove + + fn shared_return(_1: u32, _2: u32, _3: bool) -> u32 { + debug x => _1; + debug y => _2; + debug take_x => _3; + let mut _0: u32; + let mut _4: bool; + + bb0: { + StorageLive(_4); + _4 = copy _3; + switchInt(move _4) -> [0: bb2, otherwise: bb1]; + } + + bb1: { +- _0 = copy _1; ++ _0 = move _1; + goto -> bb3; + } + + bb2: { +- _0 = copy _2; ++ _0 = move _2; + goto -> bb3; + } + + bb3: { + StorageDead(_4); + return; + } + } + diff --git a/tests/mir-opt/tail_copy_to_move.unrelated_tail_store.TailCopyToMove.diff b/tests/mir-opt/tail_copy_to_move.unrelated_tail_store.TailCopyToMove.diff new file mode 100644 index 0000000000000..ed53bb36b7829 --- /dev/null +++ b/tests/mir-opt/tail_copy_to_move.unrelated_tail_store.TailCopyToMove.diff @@ -0,0 +1,16 @@ +- // MIR for `unrelated_tail_store` before TailCopyToMove ++ // MIR for `unrelated_tail_store` after TailCopyToMove + + fn unrelated_tail_store(_1: u32, _2: u32) -> u32 { + let mut _0: u32; + let mut _3: u32; + + bb0: { +- _0 = copy _1; +- _3 = copy _2; ++ _0 = move _1; ++ _3 = move _2; + return; + } + } + From 86bb416359c0a64b92731d725d1390bfc65bdf6a Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Mon, 1 Jun 2026 11:56:37 +0100 Subject: [PATCH 10/11] Avoid merging into the tuples that are passed to "rust-call" --- .../src/move_elimination.rs | 66 ++++++++++++++-- tests/mir-opt/move-elimination/exclusions.rs | 17 ++++ ...l_tuple_not_projected.MoveElimination.diff | 77 +++++++++++++++++++ 3 files changed, 154 insertions(+), 6 deletions(-) create mode 100644 tests/mir-opt/move-elimination/exclusions.rust_call_tuple_not_projected.MoveElimination.diff diff --git a/compiler/rustc_mir_transform/src/move_elimination.rs b/compiler/rustc_mir_transform/src/move_elimination.rs index ea79b6ed97725..ed4344538263e 100644 --- a/compiler/rustc_mir_transform/src/move_elimination.rs +++ b/compiler/rustc_mir_transform/src/move_elimination.rs @@ -1,4 +1,4 @@ -use rustc_abi::{FieldIdx, VariantIdx}; +use rustc_abi::{ExternAbi, FieldIdx, VariantIdx}; use rustc_const_eval::util::most_packed_projection; use rustc_data_structures::fx::FxHashMap; use rustc_index::IndexVec; @@ -33,11 +33,19 @@ impl<'tcx> crate::MirPass<'tcx> for MoveElimination { dump_liveness_matrix(tcx, body, "MoveElimination.pre-liveness", &points, &liveness_matrix); - let mut unprojectable_locals = UnprojectableLocals::find(body); + let unprojectable_locals = UnprojectableLocals::find(body); trace!(?unprojectable_locals); - let remapped_locals = - PlaceUnification::run(tcx, body, &mut liveness_matrix, &mut unprojectable_locals); + let rust_call_tuples = find_rust_call_tuples(tcx, body); + trace!(?rust_call_tuples); + + let remapped_locals = PlaceUnification::run( + tcx, + body, + &mut liveness_matrix, + unprojectable_locals, + rust_call_tuples, + ); apply_mappings(tcx, body, &remapped_locals); @@ -121,7 +129,8 @@ struct PlaceUnification<'a, 'tcx> { tcx: TyCtxt<'tcx>, body: &'a Body<'tcx>, liveness_matrix: &'a mut SparseIntervalMatrix, - unprojectable_locals: &'a mut DenseBitSet, + unprojectable_locals: DenseBitSet, + rust_call_tuples: DenseBitSet, remapped_locals: IndexVec>>, } @@ -130,13 +139,15 @@ impl<'tcx> PlaceUnification<'_, 'tcx> { tcx: TyCtxt<'tcx>, body: &Body<'tcx>, liveness_matrix: &mut SparseIntervalMatrix, - unprojectable_locals: &mut DenseBitSet, + unprojectable_locals: DenseBitSet, + rust_call_tuples: DenseBitSet, ) -> IndexVec>> { let mut visitor = PlaceUnification { tcx, body, liveness_matrix, unprojectable_locals, + rust_call_tuples, remapped_locals: IndexVec::from_elem_n(None, body.local_decls.len()), }; visitor.visit_body(body); @@ -174,6 +185,11 @@ impl<'tcx> PlaceUnification<'_, 'tcx> { return None; } + if self.rust_call_tuples.contains(a.local) || self.rust_call_tuples.contains(b.local) { + trace!("cannot unify {a:?} and {b:?} involving a rust-call tuple argument"); + return None; + } + let (local, place) = match (a.as_local(), b.as_local()) { (None, None) => { trace!("cannot unify 2 places that both have projections"); @@ -277,6 +293,44 @@ impl<'tcx> PlaceUnification<'_, 'tcx> { } } +/// Search for tuple locals passed to calls using the "rust-call" ABI. +/// +/// For rust-call ABI calls, caller-side MIR passes the logical arguments as a +/// tuple operand. We want to avoid remapping other locals into fields of that +/// tuple, especially if one of those locals is borrowed. +/// +/// Since the tuple itself is never borrowed, it is trivial for LLVM alias +/// analysis to see that accesses to one argument do not affect the others, but +/// merging the arguments into tuple fields from the start can hide that +/// independence. +fn find_rust_call_tuples<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>) -> DenseBitSet { + let mut rust_call_tuples = DenseBitSet::new_empty(body.local_decls.len()); + + for block in body.basic_blocks.iter() { + let terminator = block.terminator(); + let (func, args) = match &terminator.kind { + TerminatorKind::Call { func, args, .. } + | TerminatorKind::TailCall { func, args, .. } => (func, args), + _ => continue, + }; + + let sig = func.ty(&body.local_decls, tcx).fn_sig(tcx); + if sig.abi() != ExternAbi::RustCall { + continue; + } + + let arg_tuple = args.last().expect("rust-call ABI requires a tuple argument"); + let (Operand::Copy(place) | Operand::Move(place)) = arg_tuple.node else { + continue; + }; + if let Some(local) = place.as_local() { + rust_call_tuples.insert(local); + } + } + + rust_call_tuples +} + /// Since we are replacing all uses of a local with another place, we need to /// ensure that the projections on that place are stable no matter where it is /// used in the body. Additional this local may be used in debuginfo, so ensure diff --git a/tests/mir-opt/move-elimination/exclusions.rs b/tests/mir-opt/move-elimination/exclusions.rs index 80b4f4a77a02c..f4a71dd6d3eea 100644 --- a/tests/mir-opt/move-elimination/exclusions.rs +++ b/tests/mir-opt/move-elimination/exclusions.rs @@ -111,3 +111,20 @@ pub fn dse_guard() { b = a; observe(&raw const b); } + +// EMIT_MIR exclusions.rust_call_tuple_not_projected.MoveElimination.diff +pub fn rust_call_tuple_not_projected(f: F) { + // This checks that locals are not remapped into the tuple argument passed + // to a rust-call ABI function. If the tuple itself is never borrowed, alias + // analysis can trivially see that accesses to one argument don't affect the + // others. Merging the arguments into tuple fields from the start can hide + // that independence. + // CHECK-LABEL: fn rust_call_tuple_not_projected( + // CHECK: debug a => [[a:_.*]]; + // CHECK: debug b => [[b:_.*]]; + // CHECK: [[tuple:_.*]] = (move [[a]], move [[b]]); + // CHECK: >::call_once(move _1, move [[tuple]]) + let a = [1; 8]; + let b = [2; 8]; + f(a, b); +} diff --git a/tests/mir-opt/move-elimination/exclusions.rust_call_tuple_not_projected.MoveElimination.diff b/tests/mir-opt/move-elimination/exclusions.rust_call_tuple_not_projected.MoveElimination.diff new file mode 100644 index 0000000000000..11ae6220fce9d --- /dev/null +++ b/tests/mir-opt/move-elimination/exclusions.rust_call_tuple_not_projected.MoveElimination.diff @@ -0,0 +1,77 @@ +- // MIR for `rust_call_tuple_not_projected` before MoveElimination ++ // MIR for `rust_call_tuple_not_projected` after MoveElimination + + fn rust_call_tuple_not_projected(_1: F) -> () { + debug f => _1; + let mut _0: (); + let _2: [u8; 8]; + let _4: (); + let mut _5: F; + let mut _6: ([u8; 8], [u8; 8]); + let mut _7: [u8; 8]; + let mut _8: [u8; 8]; + scope 1 { +- debug a => _2; ++ debug a => _7; + let _3: [u8; 8]; + scope 2 { +- debug b => _3; ++ debug b => _8; + } + } + + bb0: { +- StorageLive(_2); +- _2 = [const 1_u8; 8]; +- StorageLive(_3); +- _3 = [const 2_u8; 8]; +- StorageLive(_4); +- StorageLive(_5); +- _5 = move _1; +- StorageLive(_6); ++ nop; + StorageLive(_7); +- _7 = copy _2; ++ _7 = [const 1_u8; 8]; ++ nop; + StorageLive(_8); +- _8 = copy _3; ++ _8 = [const 2_u8; 8]; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; ++ StorageLive(_6); + _6 = (move _7, move _8); +- _4 = >::call_once(move _5, move _6) -> [return: bb1, unwind unreachable]; ++ StorageLive(_4); ++ StorageDead(_7); ++ StorageDead(_8); ++ _4 = >::call_once(move _1, move _6) -> [return: bb1, unwind unreachable]; + } + + bb1: { +- StorageDead(_8); +- StorageDead(_7); +- StorageDead(_6); +- StorageDead(_5); + StorageDead(_4); ++ StorageDead(_6); ++ nop; ++ nop; ++ nop; ++ nop; ++ nop; + _0 = const (); +- StorageDead(_3); +- StorageDead(_2); ++ nop; ++ nop; + return; + } + } + From 343333e5dc03200b5437d2d651109f48cd2da1c9 Mon Sep 17 00:00:00 2001 From: Amanieu d'Antras Date: Tue, 9 Jun 2026 20:29:41 +0100 Subject: [PATCH 11/11] Experiment: try disabling GVN --- compiler/rustc_mir_transform/src/gvn.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/rustc_mir_transform/src/gvn.rs b/compiler/rustc_mir_transform/src/gvn.rs index d185c11452dbe..f347398c99259 100644 --- a/compiler/rustc_mir_transform/src/gvn.rs +++ b/compiler/rustc_mir_transform/src/gvn.rs @@ -128,7 +128,7 @@ pub(super) struct GVN; impl<'tcx> crate::MirPass<'tcx> for GVN { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { - sess.mir_opt_level() >= 2 + false && sess.mir_opt_level() >= 2 } #[instrument(level = "trace", skip(self, tcx, body))]