@@ -19,8 +19,8 @@ use tracing::{info, instrument, trace};
1919
2020use super :: {
2121 CtfeProvenance , FnVal , ImmTy , InterpCx , InterpResult , MPlaceTy , Machine , OpTy , PlaceTy ,
22- Projectable , Provenance , ReturnAction , ReturnContinuation , Scalar , StackPopInfo , interp_ok ,
23- throw_ub , throw_ub_custom,
22+ Projectable , Provenance , ReturnAction , ReturnContinuation , Scalar , interp_ok , throw_ub ,
23+ throw_ub_custom,
2424} ;
2525use crate :: enter_trace_span;
2626use crate :: interpret:: EnteredTraceSpan ;
@@ -43,25 +43,22 @@ impl<'tcx, Prov: Provenance> FnArg<'tcx, Prov> {
4343 FnArg :: InPlace ( mplace) => & mplace. layout ,
4444 }
4545 }
46- }
4746
48- impl < ' tcx , M : Machine < ' tcx > > InterpCx < ' tcx , M > {
4947 /// Make a copy of the given fn_arg. Any `InPlace` are degenerated to copies, no protection of the
5048 /// original memory occurs.
51- pub fn copy_fn_arg ( & self , arg : & FnArg < ' tcx , M :: Provenance > ) -> OpTy < ' tcx , M :: Provenance > {
52- match arg {
49+ pub fn copy_fn_arg ( & self ) -> OpTy < ' tcx , Prov > {
50+ match self {
5351 FnArg :: Copy ( op) => op. clone ( ) ,
5452 FnArg :: InPlace ( mplace) => mplace. clone ( ) . into ( ) ,
5553 }
5654 }
55+ }
5756
57+ impl < ' tcx , M : Machine < ' tcx > > InterpCx < ' tcx , M > {
5858 /// Make a copy of the given fn_args. Any `InPlace` are degenerated to copies, no protection of the
5959 /// original memory occurs.
60- pub fn copy_fn_args (
61- & self ,
62- args : & [ FnArg < ' tcx , M :: Provenance > ] ,
63- ) -> Vec < OpTy < ' tcx , M :: Provenance > > {
64- args. iter ( ) . map ( |fn_arg| self . copy_fn_arg ( fn_arg) ) . collect ( )
60+ pub fn copy_fn_args ( args : & [ FnArg < ' tcx , M :: Provenance > ] ) -> Vec < OpTy < ' tcx , M :: Provenance > > {
61+ args. iter ( ) . map ( |fn_arg| fn_arg. copy_fn_arg ( ) ) . collect ( )
6562 }
6663
6764 /// Helper function for argument untupling.
@@ -319,7 +316,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
319316 // We work with a copy of the argument for now; if this is in-place argument passing, we
320317 // will later protect the source it comes from. This means the callee cannot observe if we
321318 // did in-place of by-copy argument passing, except for pointer equality tests.
322- let caller_arg_copy = self . copy_fn_arg ( caller_arg ) ;
319+ let caller_arg_copy = caller_arg . copy_fn_arg ( ) ;
323320 if !already_live {
324321 let local = callee_arg. as_local ( ) . unwrap ( ) ;
325322 let meta = caller_arg_copy. meta ( ) ;
@@ -616,7 +613,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
616613 if let Some ( fallback) = M :: call_intrinsic (
617614 self ,
618615 instance,
619- & self . copy_fn_args ( args) ,
616+ & Self :: copy_fn_args ( args) ,
620617 destination,
621618 target,
622619 unwind,
@@ -703,7 +700,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
703700 // An `InPlace` does nothing here, we keep the original receiver intact. We can't
704701 // really pass the argument in-place anyway, and we are constructing a new
705702 // `Immediate` receiver.
706- let mut receiver = self . copy_fn_arg ( & args[ 0 ] ) ;
703+ let mut receiver = args[ 0 ] . copy_fn_arg ( ) ;
707704 let receiver_place = loop {
708705 match receiver. layout . ty . kind ( ) {
709706 ty:: Ref ( ..) | ty:: RawPtr ( ..) => {
@@ -824,41 +821,50 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
824821 with_caller_location : bool ,
825822 ) -> InterpResult < ' tcx > {
826823 trace ! ( "init_fn_tail_call: {:#?}" , fn_val) ;
827-
828824 // This is the "canonical" implementation of tails calls,
829825 // a pop of the current stack frame, followed by a normal call
830826 // which pushes a new stack frame, with the return address from
831827 // the popped stack frame.
832828 //
833- // Note that we are using `pop_stack_frame_raw` and not `return_from_current_stack_frame`,
834- // as the latter "executes" the goto to the return block, but we don't want to,
829+ // Note that we cannot use `return_from_current_stack_frame`,
830+ // as that "executes" the goto to the return block, but we don't want to,
835831 // only the tail called function should return to the current return block.
836- let StackPopInfo { return_action, return_cont, return_place } =
837- self . pop_stack_frame_raw ( false , |_this, _return_place| {
838- // This function's return value is just discarded, the tail-callee will fill in the return place instead.
839- interp_ok ( ( ) )
840- } ) ?;
841832
842- assert_eq ! ( return_action, ReturnAction :: Normal ) ;
843-
844- // Take the "stack pop cleanup" info, and use that to initiate the next call.
845- let ReturnContinuation :: Goto { ret, unwind } = return_cont else {
846- bug ! ( "can't tailcall as root" ) ;
833+ // The arguments need to all be copied since the current stack frame will be removed
834+ // before the callee even starts executing.
835+ // FIXME(explicit_tail_calls,#144855): does this match what codegen does?
836+ let args = args. iter ( ) . map ( |fn_arg| FnArg :: Copy ( fn_arg. copy_fn_arg ( ) ) ) . collect :: < Vec < _ > > ( ) ;
837+ // Remove the frame from the stack.
838+ let frame = self . pop_stack_frame_raw ( ) ?;
839+ // Remember where this frame would have returned to.
840+ let ReturnContinuation :: Goto { ret, unwind } = frame. return_cont ( ) else {
841+ bug ! ( "can't tailcall as root of the stack" ) ;
847842 } ;
848-
843+ // There's no return value to deal with! Instead, we forward the old return place
844+ // to the new function.
849845 // FIXME(explicit_tail_calls):
850846 // we should check if both caller&callee can/n't unwind,
851847 // see <https://github.com/rust-lang/rust/pull/113128#issuecomment-1614979803>
852848
849+ // Now push the new stack frame.
853850 self . init_fn_call (
854851 fn_val,
855852 ( caller_abi, caller_fn_abi) ,
856- args,
853+ & * args,
857854 with_caller_location,
858- & return_place,
855+ frame . return_place ( ) ,
859856 ret,
860857 unwind,
861- )
858+ ) ?;
859+
860+ // Finally, clear the local variables. Has to be done after pushing to support
861+ // non-scalar arguments.
862+ // FIXME(explicit_tail_calls,#144855): revisit this once codegen supports indirect
863+ // arguments, to ensure the semantics are compatible.
864+ let return_action = self . cleanup_stack_frame ( /* unwinding */ false , frame) ?;
865+ assert_eq ! ( return_action, ReturnAction :: Normal ) ;
866+
867+ interp_ok ( ( ) )
862868 }
863869
864870 pub ( super ) fn init_drop_in_place_call (
@@ -953,14 +959,18 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
953959 // local's value out.
954960 let return_op =
955961 self . local_to_op ( mir:: RETURN_PLACE , None ) . expect ( "return place should always be live" ) ;
956- // Do the actual pop + copy.
957- let stack_pop_info = self . pop_stack_frame_raw ( unwinding, |this, return_place| {
958- this. copy_op_allow_transmute ( & return_op, return_place) ?;
959- trace ! ( "return value: {:?}" , this. dump_place( return_place) ) ;
960- interp_ok ( ( ) )
961- } ) ?;
962-
963- match stack_pop_info. return_action {
962+ // Remove the frame from the stack.
963+ let frame = self . pop_stack_frame_raw ( ) ?;
964+ // Copy the return value and remember the return continuation.
965+ if !unwinding {
966+ self . copy_op_allow_transmute ( & return_op, frame. return_place ( ) ) ?;
967+ trace ! ( "return value: {:?}" , self . dump_place( frame. return_place( ) ) ) ;
968+ }
969+ let return_cont = frame. return_cont ( ) ;
970+ // Finish popping the stack frame.
971+ let return_action = self . cleanup_stack_frame ( unwinding, frame) ?;
972+ // Jump to the next block.
973+ match return_action {
964974 ReturnAction :: Normal => { }
965975 ReturnAction :: NoJump => {
966976 // The hook already did everything.
@@ -978,7 +988,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
978988 // Normal return, figure out where to jump.
979989 if unwinding {
980990 // Follow the unwind edge.
981- match stack_pop_info . return_cont {
991+ match return_cont {
982992 ReturnContinuation :: Goto { unwind, .. } => {
983993 // This must be the very last thing that happens, since it can in fact push a new stack frame.
984994 self . unwind_to_block ( unwind)
@@ -989,7 +999,7 @@ impl<'tcx, M: Machine<'tcx>> InterpCx<'tcx, M> {
989999 }
9901000 } else {
9911001 // Follow the normal return edge.
992- match stack_pop_info . return_cont {
1002+ match return_cont {
9931003 ReturnContinuation :: Goto { ret, .. } => self . return_to_block ( ret) ,
9941004 ReturnContinuation :: Stop { .. } => {
9951005 assert ! (
0 commit comments