diff --git a/crates/cranelift/src/func_environ/gc/drc.rs b/crates/cranelift/src/func_environ/gc/drc.rs index fe75676554ab..8a9de06f630b 100644 --- a/crates/cranelift/src/func_environ/gc/drc.rs +++ b/crates/cranelift/src/func_environ/gc/drc.rs @@ -15,10 +15,6 @@ use wasmtime_environ::{ WasmValType, drc::DrcTypeLayouts, }; -// The minimum over-approximated stack roots list size for which we will trigger -// a GC. -const MIN_OVER_APPROX_STACK_ROOTS_GC_THRESHOLD: i64 = 1024; - #[derive(Default)] pub struct DrcCompiler { layouts: DrcTypeLayouts, @@ -190,9 +186,10 @@ impl DrcCompiler { ); let doubled_last_len = builder.ins().iadd(last_len, last_len); - let min_threshold = builder - .ins() - .iconst(ir::types::I32, MIN_OVER_APPROX_STACK_ROOTS_GC_THRESHOLD); + let min_threshold = builder.ins().iconst( + ir::types::I32, + wasmtime_environ::drc::MIN_OVER_APPROX_STACK_ROOTS_GC_THRESHOLD, + ); let threshold = builder.ins().umax(doubled_last_len, min_threshold); let should_gc = diff --git a/crates/cranelift/src/lib.rs b/crates/cranelift/src/lib.rs index 50c0b26cbe58..150726f70c8b 100644 --- a/crates/cranelift/src/lib.rs +++ b/crates/cranelift/src/lib.rs @@ -49,7 +49,7 @@ mod trap; use self::compiler::Compiler; const TRAP_INTERNAL_ASSERT: TrapCode = TrapCode::unwrap_user(1); -const TRAP_GC_HEAP_CORRUPT: TrapCode = TrapCode::unwrap_user(2); +pub const TRAP_GC_HEAP_CORRUPT: TrapCode = TrapCode::unwrap_user(2); const TRAP_OFFSET: u8 = 3; pub const TRAP_CANNOT_LEAVE_COMPONENT: TrapCode = TrapCode::unwrap_user(Trap::CannotLeaveComponent as u8 + TRAP_OFFSET); diff --git a/crates/environ/src/gc.rs b/crates/environ/src/gc.rs index 5a5c9d64bf90..a560e7f918f9 100644 --- a/crates/environ/src/gc.rs +++ b/crates/environ/src/gc.rs @@ -29,6 +29,19 @@ use core::alloc::Layout; /// enabled. pub const POISON: u8 = 0b00001111; +/// The bit within a `VMDrcHeader`'s reserved bits that is the mark +/// bit. Collectively, this bit in all the heap's objects' headers implements +/// the precise-stack-roots set. +pub const DRC_HEADER_MARK_BIT: u32 = 1 << 0; + +/// The bit within a `VMDrcHeader`'s reserved bits that is the +/// in-the-over-approximated-stack-roots list bit. +pub const DRC_HEADER_IN_OVER_APPROX_LIST_BIT: u32 = 1 << 1; + +/// The minimum length the over-approximated-stack-roots list must reach +/// before a read barrier considers forcing a GC. +pub const DRC_MIN_OVER_APPROX_STACK_ROOTS_GC_THRESHOLD: i64 = 1024; + /// Assert a condition, but only when `gc_zeal` is enabled. #[macro_export] macro_rules! gc_assert { diff --git a/crates/environ/src/gc/drc.rs b/crates/environ/src/gc/drc.rs index 9f557506304b..a24a21f66862 100644 --- a/crates/environ/src/gc/drc.rs +++ b/crates/environ/src/gc/drc.rs @@ -17,14 +17,9 @@ pub const EXCEPTION_TAG_INSTANCE_OFFSET: u32 = HEADER_SIZE; /// The offset of the tag-defined-index field in an exception header. pub const EXCEPTION_TAG_DEFINED_OFFSET: u32 = HEADER_SIZE + 4; -/// The bit within a `VMDrcHeader`'s reserved bits that is the mark -/// bit. Collectively, this bit in all the heap's objects' headers implements -/// the precise-stack-roots set. -pub const HEADER_MARK_BIT: u32 = 1 << 0; - -/// The bit within a `VMDrcHeader`'s reserved bits that is the -/// in-the-over-approximated-stack-roots list bit. -pub const HEADER_IN_OVER_APPROX_LIST_BIT: u32 = 1 << 1; +pub use super::DRC_HEADER_IN_OVER_APPROX_LIST_BIT as HEADER_IN_OVER_APPROX_LIST_BIT; +pub use super::DRC_HEADER_MARK_BIT as HEADER_MARK_BIT; +pub use super::DRC_MIN_OVER_APPROX_STACK_ROOTS_GC_THRESHOLD as MIN_OVER_APPROX_STACK_ROOTS_GC_THRESHOLD; /// The layout of Wasm GC objects in the deferred reference-counting collector. #[derive(Default)] diff --git a/crates/test-util/src/wast.rs b/crates/test-util/src/wast.rs index 835c41e4f2d3..a3146a77457a 100644 --- a/crates/test-util/src/wast.rs +++ b/crates/test-util/src/wast.rs @@ -562,16 +562,6 @@ impl WastTest { return true; } - // Winch supports GC types only under the barrier-free collectors; - // the deferred reference-counting collector is refused until Winch - // emits GC barriers. - if config.compiler == Compiler::Winch - && config.collector == Collector::DeferredReferenceCounting - && self.config.gc_types() - { - return true; - } - // Disable spec tests per target for proposals that Winch does not implement yet. if config.compiler == Compiler::Winch { // Common list for tests that fail in all targets supported by Winch. diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs index 98090ecfc4e5..09e394af0e84 100644 --- a/crates/wasmtime/src/config.rs +++ b/crates/wasmtime/src/config.rs @@ -2458,13 +2458,6 @@ impl Config { | WasmFeatures::LEGACY_EXCEPTIONS | WasmFeatures::STACK_SWITCHING; - // Winch supports GC types only under the barrier-free - // collectors; the deferred reference-counting collector - // requires GC barriers that Winch does not emit yet. - #[cfg(feature = "gc")] - if self.collector.not_auto() == Some(Collector::DeferredReferenceCounting) { - unsupported |= WasmFeatures::GC_TYPES; - } match self.compiler_target().architecture { target_lexicon::Architecture::Aarch64(_) => { unsupported |= WasmFeatures::THREADS; diff --git a/docs/stability-tiers.md b/docs/stability-tiers.md index bf14e93398c9..b214fb741776 100644 --- a/docs/stability-tiers.md +++ b/docs/stability-tiers.md @@ -340,10 +340,11 @@ emitting Pulley bytecode. | [`exception-handling`] | ✅ | ❌ | | [`stack-switching`] | ❌ | ❌ | -[^a]: Winch supports some features of the [`reference-types`] proposal such as - the change to support multiple tables and LEB-encoding table indices in - instructions, but it does not support GC types such as `externref` or the - new table opcodes in the [`reference-types`] proposal. +[^a]: Winch supports GC reference values such as `externref` through parameters, + results, locals, globals, and calls, including the required stack maps and + collector barriers. It also supports multiple tables and LEB-encoded table + indices, but does not yet support every table and element-segment case in the + [`reference-types`] proposal. [^b]: Pulley does not support the [`threads`] proposal because there is no known safe way to implement this with Rust's memory model. [^c]: Winch's support for aarch64 is complete for Core Wasm. diff --git a/tests/all/gc.rs b/tests/all/gc.rs index ac381877f7d4..5e3154002832 100644 --- a/tests/all/gc.rs +++ b/tests/all/gc.rs @@ -3839,6 +3839,217 @@ fn winch_externref_survives_gc_in_frame() -> Result<()> { Ok(()) } +/// The write barrier's decrement chain releases an object once a global stops +/// holding the last reference to it. +#[test] +#[cfg_attr(miri, ignore)] +fn winch_drc_write_barrier_drops_old_global_value() -> Result<()> { + let mut config = Config::new(); + config.strategy(Strategy::Winch); + config.collector(Collector::DeferredReferenceCounting); + let Ok(engine) = Engine::new(&config) else { + return Ok(()); + }; + let module = Module::new( + &engine, + r#" + (module + (global $g (mut externref) (ref.null extern)) + (func (export "set") (param externref) + (global.set $g (local.get 0)))) + "#, + )?; + let mut store = Store::new(&engine, ()); + let instance = Instance::new(&mut store, &module, &[])?; + let set = instance.get_func(&mut store, "set").unwrap(); + + let dropped = Arc::new(AtomicBool::new(false)); + { + let mut scope = RootScope::new(&mut store); + let r = ExternRef::new(&mut scope, SetFlagOnDrop(dropped.clone()))?; + set.call(&mut scope, &[Val::ExternRef(Some(r))], &mut [])?; + } + + // The global holds the only reference; nothing may be dropped yet. + store.gc(None)?; + assert!(!dropped.load(SeqCst)); + + // Overwriting the global decrements the count to zero and releases the + // old value. + set.call(&mut store, &[Val::ExternRef(None)], &mut [])?; + store.gc(None)?; + assert!(dropped.load(SeqCst)); + + Ok(()) +} + +/// The read barrier holds a count for references entering the stack, so +/// overwriting their last long-lived home cannot free them out from under +/// the frame that loaded them. +#[test] +#[cfg_attr(miri, ignore)] +fn winch_drc_read_barrier_keeps_loaded_ref_alive() -> Result<()> { + let mut config = Config::new(); + config.strategy(Strategy::Winch); + config.collector(Collector::DeferredReferenceCounting); + let Ok(engine) = Engine::new(&config) else { + return Ok(()); + }; + let module = Module::new( + &engine, + r#" + (module + (import "" "gc" (func $gc)) + (global $g (mut externref) (ref.null extern)) + (func (export "set") (param externref) + (global.set $g (local.get 0))) + (func (export "swap") (result externref) + (local $tmp externref) + (local.set $tmp (global.get $g)) + (global.set $g (ref.null extern)) + (call $gc) + (local.get $tmp))) + "#, + )?; + let mut store = Store::new(&engine, ()); + let gc = Func::wrap(&mut store, |mut cx: Caller<'_, ()>| { + let _ = cx.gc(None); + }); + let instance = Instance::new(&mut store, &module, &[gc.into()])?; + let set = instance.get_func(&mut store, "set").unwrap(); + let swap = instance.get_typed_func::<(), Option>>(&mut store, "swap")?; + + { + let mut scope = RootScope::new(&mut store); + let r = ExternRef::new(&mut scope, 0xDECAFu32)?; + set.call(&mut scope, &[Val::ExternRef(Some(r))], &mut [])?; + } + + // Settle the deferred unroot so the global truly holds the last count. + store.gc(None)?; + + // `swap` loads the reference onto the stack, overwrites the global, and + // collects while the stack copy is live. + let out = swap.call(&mut store, ())?.expect("must not be null"); + let got = out + .data(&store)? + .and_then(|d| d.downcast_ref::().copied()); + assert_eq!(got, Some(0xDECAF)); + + Ok(()) +} + +/// An `externref` may wrap an unboxed i31 created through the host API. Such +/// values do not have reference counts and must bypass both DRC barriers. +#[test] +#[cfg_attr(miri, ignore)] +fn winch_drc_i31_wrapped_as_externref_skips_global_barriers() -> Result<()> { + let mut config = Config::new(); + config.strategy(Strategy::Winch); + config.collector(Collector::DeferredReferenceCounting); + let Ok(engine) = Engine::new(&config) else { + return Ok(()); + }; + let module = Module::new( + &engine, + r#" + (module + (global $g (mut externref) (ref.null extern)) + (func (export "set") (param externref) + local.get 0 + global.set $g) + (func (export "get") (result externref) + global.get $g)) + "#, + )?; + let mut store = Store::new(&engine, ()); + let instance = Instance::new(&mut store, &module, &[])?; + let set = instance.get_func(&mut store, "set").unwrap(); + let get = instance.get_func(&mut store, "get").unwrap(); + + let anyref = AnyRef::from_i31(&mut store, I31::wrapping_u32(0x1234)); + let externref = ExternRef::convert_any(&mut store, anyref)?; + set.call(&mut store, &[Val::ExternRef(Some(externref))], &mut [])?; + + let mut results = [Val::null_extern_ref()]; + get.call(&mut store, &[], &mut results)?; + let externref = results[0] + .unwrap_externref() + .expect("global.get returned null"); + let anyref = AnyRef::convert_extern(&mut store, *externref)?; + assert_eq!(anyref.unwrap_i31(&store)?.get_u32(), 0x1234); + + // Replacing the i31-backed externref exercises the old-value side of the + // write barrier as well. + set.call(&mut store, &[Val::null_extern_ref()], &mut [])?; + Ok(()) +} + +/// Growing the over-approximated-stack-roots list to its threshold forces a +/// collection from the read barrier and preserves the reference whose load +/// triggered that collection. +#[test] +#[cfg_attr(miri, ignore)] +fn winch_drc_read_barrier_forces_gc_at_threshold() -> Result<()> { + let mut config = Config::new(); + config.strategy(Strategy::Winch); + config.collector(Collector::DeferredReferenceCounting); + let Ok(engine) = Engine::new(&config) else { + return Ok(()); + }; + + let num_refs = wasmtime_environ::DRC_MIN_OVER_APPROX_STACK_ROOTS_GC_THRESHOLD as usize; + let mut wat = "(module\n".to_string(); + for i in 0..num_refs { + wat.push_str(&format!( + r#"(global $g{i} (export "g{i}") (mut externref) (ref.null extern)) +"#, + )); + } + wat.push_str("(func (export \"drain\")\n"); + for i in 0..num_refs { + wat.push_str(&format!( + "(drop (global.get $g{i}))\n(global.set $g{i} (ref.null extern))\n" + )); + } + wat.push_str("))"); + + let module = Module::new(&engine, &wat)?; + let mut store = Store::new(&engine, ()); + let instance = Instance::new(&mut store, &module, &[])?; + let dropped = Arc::new(AtomicUsize::new(0)); + + { + let mut scope = RootScope::new(&mut store); + for i in 0..num_refs { + let gc_ref = ExternRef::new(&mut scope, CountDrops(dropped.clone()))?; + let global = instance + .get_global(&mut scope, &format!("g{i}")) + .expect("global must be exported"); + global.set(&mut scope, Val::ExternRef(Some(gc_ref)))?; + } + } + + // Settle the host roots. Each global is now the only long-lived home for + // its reference. + store.gc(None)?; + assert_eq!(dropped.load(SeqCst), 0); + + let drain = instance.get_typed_func::<(), ()>(&mut store, "drain")?; + drain.call(&mut store, ())?; + + // The final global.get reaches the threshold and forces a collection. + // The preceding references are no longer on the stack or in globals, but + // the triggering reference is still live in the global.get result slot. + assert_eq!(dropped.load(SeqCst), num_refs - 1); + + // The triggering reference becomes collectible after `drain` returns. + store.gc(None)?; + assert_eq!(dropped.load(SeqCst), num_refs); + + Ok(()) +} + /// Reference values crossing the ABI boundary in every position: stack-passed /// externref params and multi-value externref results (more than fit in registers) #[test] diff --git a/tests/all/winch_engine_features.rs b/tests/all/winch_engine_features.rs index 966588eb07ac..fe252842a14a 100644 --- a/tests/all/winch_engine_features.rs +++ b/tests/all/winch_engine_features.rs @@ -64,41 +64,3 @@ fn ensure_compatibility_between_winch_and_debug_native(config: &mut Config) -> R Ok(()) } - -#[wasmtime_test(strategies(only(Winch)))] -#[cfg_attr(miri, ignore)] -fn ensure_compatibility_between_winch_and_drc_collector(config: &mut Config) -> Result<()> { - config.collector(Collector::DeferredReferenceCounting); - config.gc_support(true); - let result = Engine::new(&config); - match result { - Ok(_) => { - wasmtime::bail!( - "Expected incompatibility between the deferred reference-counting \ - collector and Winch" - ) - } - Err(e) => { - assert_eq!( - e.to_string(), - "the wasm_gc_types feature is not supported on this compiler configuration" - ); - } - } - - Ok(()) -} - -#[wasmtime_test(strategies(only(Winch)))] -#[cfg_attr(miri, ignore)] -fn winch_with_drc_collector_disables_gc_types_by_default(config: &mut Config) -> Result<()> { - config.collector(Collector::DeferredReferenceCounting); - let engine = Engine::new(&config)?; - let result = Module::new( - &engine, - r#"(module (global (mut externref) (ref.null extern)))"#, - ); - assert!(result.is_err()); - - Ok(()) -} diff --git a/tests/disas/winch/aarch64/global-get-externref-drc.wat b/tests/disas/winch/aarch64/global-get-externref-drc.wat new file mode 100644 index 000000000000..9ec469e213ea --- /dev/null +++ b/tests/disas/winch/aarch64/global-get-externref-drc.wat @@ -0,0 +1,94 @@ +;;! target = "aarch64" +;;! test = "winch" +;;! flags = "-Ccollector=drc" + +(module + (global $g (mut externref) (ref.null extern)) + (func (export "get") (result externref) + (global.get $g))) +;; wasm[0]::function[0]: +;; stp x29, x30, [sp, #-0x10]! +;; mov x29, sp +;; str x28, [sp, #-0x10]! +;; mov x28, sp +;; ldur x16, [x0, #8] +;; ldur x16, [x16, #0x18] +;; mov x17, #0 +;; movk x17, #0x20 +;; add x16, x16, x17 +;; cmp sp, x16 +;; b.lo #0x148 +;; 2c: mov x9, x0 +;; sub x28, x28, #0x10 +;; mov sp, x28 +;; stur x0, [x28, #8] +;; stur x1, [x28] +;; ldur w0, [x9, #0x30] +;; sub x28, x28, #4 +;; mov sp, x28 +;; stur w0, [x28] +;; ldur w0, [x28] +;; tst w0, w0 +;; b.eq #0x124 +;; b #0x60 +;; 60: mov w16, w0 +;; and w16, w16, #1 +;; tst w16, w16 +;; b.ne #0x124 +;; b #0x74 +;; 74: ldur x1, [x9, #8] +;; ldur x2, [x1, #0x28] +;; ldur x1, [x1, #0x20] +;; mov x16, x0 +;; add x16, x16, #0x14 +;; cmp x16, x2, uxtx +;; sub sp, x28, #0xc +;; b.hi #0x14c +;; 94: mov sp, x28 +;; mov x2, x1 +;; add x2, x2, x0, uxtx +;; ldur w16, [x2] +;; and w16, w16, #2 +;; tst w16, w16 +;; b.ne #0x124 +;; b #0xb4 +;; b4: ldur x3, [x9, #0x20] +;; ldur w16, [x3] +;; stur w16, [x2, #0x10] +;; ldur w16, [x2] +;; orr w16, w16, #2 +;; stur w16, [x2] +;; ldur x4, [x2, #8] +;; add x4, x4, #1 +;; stur x4, [x2, #8] +;; stur w0, [x3] +;; ldur w4, [x3, #4] +;; add w4, w4, #1 +;; stur w4, [x3, #4] +;; ldur w16, [x3, #8] +;; add w16, w16, w16, uxtx +;; cmp w4, w16, uxtx +;; b.lo #0x124 +;; b #0xfc +;; fc: cmp w4, #0x400 +;; b.lo #0x124 +;; b #0x108 +;; 108: sub x28, x28, #0xc +;; mov sp, x28 +;; mov x0, x9 +;; bl #0x29c +;; 118: add x28, x28, #0xc +;; ╰─╼ stack_map: frame_size=48, frame_offsets=[12] +;; mov sp, x28 +;; ldur x9, [x28, #0xc] +;; ldur w0, [x28] +;; add x28, x28, #4 +;; mov sp, x28 +;; add x28, x28, #0x10 +;; mov sp, x28 +;; mov sp, x28 +;; ldr x28, [sp], #0x10 +;; ldp x29, x30, [sp], #0x10 +;; ret +;; 148: udf #0xc11f +;; 14c: udf #0xc11f diff --git a/tests/disas/winch/aarch64/global-set-externref-drc.wat b/tests/disas/winch/aarch64/global-set-externref-drc.wat new file mode 100644 index 000000000000..6e5875a579b3 --- /dev/null +++ b/tests/disas/winch/aarch64/global-set-externref-drc.wat @@ -0,0 +1,103 @@ +;;! target = "aarch64" +;;! test = "winch" +;;! flags = "-Ccollector=drc" + +(module + (global $g (mut externref) (ref.null extern)) + (func (param externref) + (global.set $g (local.get 0)))) +;; wasm[0]::function[0]: +;; stp x29, x30, [sp, #-0x10]! +;; mov x29, sp +;; str x28, [sp, #-0x10]! +;; mov x28, sp +;; ldur x16, [x0, #8] +;; ldur x16, [x16, #0x18] +;; mov x17, #0 +;; movk x17, #0x20 +;; add x16, x16, x17 +;; cmp sp, x16 +;; b.lo #0x168 +;; 2c: mov x9, x0 +;; sub x28, x28, #0x18 +;; mov sp, x28 +;; stur x0, [x28, #0x10] +;; stur x1, [x28, #8] +;; stur w2, [x28, #4] +;; ldur w16, [x28, #4] +;; sub x28, x28, #4 +;; mov sp, x28 +;; stur w16, [x28] +;; ldur w0, [x28] +;; add x28, x28, #4 +;; mov sp, x28 +;; ldur x1, [x9, #8] +;; ldur x2, [x1, #0x28] +;; ldur x1, [x1, #0x20] +;; ldur w3, [x9, #0x30] +;; tst w0, w0 +;; b.eq #0xbc +;; b #0x7c +;; 7c: mov w16, w0 +;; and w16, w16, #1 +;; tst w16, w16 +;; b.ne #0xbc +;; b #0x90 +;; 90: mov x16, x0 +;; add x16, x16, #0x10 +;; cmp x16, x2, uxtx +;; sub sp, x28, #8 +;; b.hi #0x16c +;; a4: mov sp, x28 +;; mov x4, x1 +;; add x4, x4, x0, uxtx +;; ldur x5, [x4, #8] +;; add x5, x5, #1 +;; stur x5, [x4, #8] +;; stur w0, [x9, #0x30] +;; tst w3, w3 +;; b.eq #0x150 +;; b #0xcc +;; cc: mov w16, w3 +;; and w16, w16, #1 +;; tst w16, w16 +;; b.ne #0x150 +;; b #0xe0 +;; e0: mov x16, x3 +;; add x16, x16, #0x10 +;; cmp x16, x2, uxtx +;; sub sp, x28, #8 +;; b.hi #0x170 +;; f4: mov sp, x28 +;; mov x4, x1 +;; add x4, x4, x3, uxtx +;; ldur x5, [x4, #8] +;; sub x5, x5, #1 +;; cmp x5, #0 +;; b.eq #0x11c +;; b #0x114 +;; 114: stur x5, [x4, #8] +;; b #0x150 +;; 11c: sub x28, x28, #4 +;; mov sp, x28 +;; stur w3, [x28] +;; sub x28, x28, #4 +;; mov sp, x28 +;; mov x0, x9 +;; ldur w1, [x28, #4] +;; bl #0x1ec +;; 13c: add x28, x28, #4 +;; ╰─╼ stack_map: frame_size=48, frame_offsets=[12] +;; mov sp, x28 +;; add x28, x28, #4 +;; mov sp, x28 +;; ldur x9, [x28, #0x10] +;; add x28, x28, #0x18 +;; mov sp, x28 +;; mov sp, x28 +;; ldr x28, [sp], #0x10 +;; ldp x29, x30, [sp], #0x10 +;; ret +;; 168: udf #0xc11f +;; 16c: udf #0xc11f +;; 170: udf #0xc11f diff --git a/tests/disas/winch/x64/global-get-externref-drc.wat b/tests/disas/winch/x64/global-get-externref-drc.wat new file mode 100644 index 000000000000..4d04893636ee --- /dev/null +++ b/tests/disas/winch/x64/global-get-externref-drc.wat @@ -0,0 +1,75 @@ +;;! target = "x86_64" +;;! test = "winch" +;;! flags = "-Ccollector=drc" + +(module + (global $g (mut externref) (ref.null extern)) + (func (export "get") (result externref) + (global.get $g))) +;; wasm[0]::function[0]: +;; pushq %rbp +;; movq %rsp, %rbp +;; movq 8(%rdi), %r11 +;; movq 0x18(%r11), %r11 +;; addq $0x20, %r11 +;; cmpq %rsp, %r11 +;; ja 0x112 +;; 1c: movq %rdi, %r14 +;; subq $0x10, %rsp +;; movq %rdi, 8(%rsp) +;; movq %rsi, (%rsp) +;; movl 0x30(%r14), %eax +;; subq $4, %rsp +;; movl %eax, (%rsp) +;; movl (%rsp), %eax +;; testl %eax, %eax +;; je 0xff +;; 48: movl %eax, %r11d +;; andl $1, %r11d +;; testl %r11d, %r11d +;; jne 0xff +;; 5b: movq 8(%r14), %rcx +;; movq 0x28(%rcx), %rdx +;; movq 0x20(%rcx), %rcx +;; movq %rax, %r11 +;; addq $0x14, %r11 +;; cmpq %rdx, %r11 +;; ja 0x114 +;; 7a: movq %rcx, %rdx +;; addq %rax, %rdx +;; movl (%rdx), %r11d +;; andl $2, %r11d +;; testl %r11d, %r11d +;; jne 0xff +;; 93: movq 0x20(%r14), %rbx +;; movl (%rbx), %r11d +;; movl %r11d, 0x10(%rdx) +;; movl (%rdx), %r11d +;; orl $2, %r11d +;; movl %r11d, (%rdx) +;; movq 8(%rdx), %rsi +;; addq $1, %rsi +;; movq %rsi, 8(%rdx) +;; movl %eax, (%rbx) +;; movl 4(%rbx), %esi +;; addl $1, %esi +;; movl %esi, 4(%rbx) +;; movl 8(%rbx), %r11d +;; addl %r11d, %r11d +;; cmpl %r11d, %esi +;; jb 0xff +;; d8: cmpl $0x400, %esi +;; jb 0xff +;; e4: subq $0xc, %rsp +;; movq %r14, %rdi +;; callq 0x21a +;; addq $0xc, %rsp +;; ╰─╼ stack_map: frame_size=32, frame_offsets=[12] +;; movq 0xc(%rsp), %r14 +;; movl (%rsp), %eax +;; addq $4, %rsp +;; addq $0x10, %rsp +;; popq %rbp +;; retq +;; 112: ud2 +;; 114: ud2 diff --git a/tests/disas/winch/x64/global-set-externref-drc.wat b/tests/disas/winch/x64/global-set-externref-drc.wat new file mode 100644 index 000000000000..d272b62c6c61 --- /dev/null +++ b/tests/disas/winch/x64/global-set-externref-drc.wat @@ -0,0 +1,80 @@ +;;! target = "x86_64" +;;! test = "winch" +;;! flags = "-Ccollector=drc" + +(module + (global $g (mut externref) (ref.null extern)) + (func (param externref) + (global.set $g (local.get 0)))) +;; wasm[0]::function[0]: +;; pushq %rbp +;; movq %rsp, %rbp +;; movq 8(%rdi), %r11 +;; movq 0x18(%r11), %r11 +;; addq $0x30, %r11 +;; cmpq %rsp, %r11 +;; ja 0x130 +;; 1c: movq %rdi, %r14 +;; subq $0x20, %rsp +;; movq %rdi, 0x18(%rsp) +;; movq %rsi, 0x10(%rsp) +;; movl %edx, 0xc(%rsp) +;; movl 0xc(%rsp), %r11d +;; subq $4, %rsp +;; movl %r11d, (%rsp) +;; movl (%rsp), %eax +;; addq $4, %rsp +;; movq 8(%r14), %rcx +;; movq 0x28(%rcx), %rdx +;; movq 0x20(%rcx), %rcx +;; movl 0x30(%r14), %ebx +;; testl %eax, %eax +;; je 0xa1 +;; 66: movl %eax, %r11d +;; andl $1, %r11d +;; testl %r11d, %r11d +;; jne 0xa1 +;; 79: movq %rax, %r11 +;; addq $0x10, %r11 +;; cmpq %rdx, %r11 +;; ja 0x132 +;; 8c: movq %rcx, %rsi +;; addq %rax, %rsi +;; movq 8(%rsi), %rdi +;; addq $1, %rdi +;; movq %rdi, 8(%rsi) +;; movl %eax, 0x30(%r14) +;; testl %ebx, %ebx +;; je 0x127 +;; ad: movl %ebx, %r11d +;; andl $1, %r11d +;; testl %r11d, %r11d +;; jne 0x127 +;; c0: movq %rbx, %r11 +;; addq $0x10, %r11 +;; cmpq %rdx, %r11 +;; ja 0x134 +;; d3: movq %rcx, %rsi +;; addq %rbx, %rsi +;; movq 8(%rsi), %rdi +;; subq $1, %rdi +;; cmpq $0, %rdi +;; je 0xf7 +;; ee: movq %rdi, 8(%rsi) +;; jmp 0x127 +;; f7: subq $4, %rsp +;; movl %ebx, (%rsp) +;; subq $0xc, %rsp +;; movq %r14, %rdi +;; movl 0xc(%rsp), %esi +;; callq 0x193 +;; addq $0xc, %rsp +;; ╰─╼ stack_map: frame_size=48, frame_offsets=[28] +;; addq $4, %rsp +;; movq 0x18(%rsp), %r14 +;; addq $0x20, %rsp +;; popq %rbp +;; retq +;; 130: ud2 +;; 132: ud2 +;; 134: ud2 diff --git a/tests/misc_testsuite/winch/gc-refs.wast b/tests/misc_testsuite/winch/gc-refs.wast index 4f242b02239e..d9ba79a13a97 100644 --- a/tests/misc_testsuite/winch/gc-refs.wast +++ b/tests/misc_testsuite/winch/gc-refs.wast @@ -2,8 +2,8 @@ ;;! gc_types = true ;; Winch reference-value support: externref flows through params, results, -;; locals, and globals, and survives calls (stack maps). Runs under the -;; barrier-free collectors; the DRC collector is refused at config time until Winch emits barriers. +;; locals, and globals, and survives calls (stack maps). Global reads and +;; writes use barriers when running under the DRC collector. (module (global $g (mut externref) (ref.null extern)) diff --git a/winch/codegen/src/codegen/drc.rs b/winch/codegen/src/codegen/drc.rs new file mode 100644 index 000000000000..d3eb52d3217c --- /dev/null +++ b/winch/codegen/src/codegen/drc.rs @@ -0,0 +1,420 @@ +use super::{Callee, CodeGen, CodeGenError, Emission, FnCall}; +use crate::{ + Result, + masm::{IntCmpKind, IntScratch, MacroAssembler, OperandSize, RegImm}, + reg::{Reg, writable}, + stack::{TypedReg, Val}, +}; +use cranelift_codegen::MachLabel; +use wasmtime_environ::{ + DRC_HEADER_IN_OVER_APPROX_LIST_BIT, DRC_MIN_OVER_APPROX_STACK_ROOTS_GC_THRESHOLD, PtrSize, + WasmValType, +}; + +#[derive(Clone, Copy)] +enum RefCountMutation { + Increment, + Decrement, +} + +impl<'a, 'translation, 'data, M> CodeGen<'a, 'translation, 'data, M, Emission> +where + M: MacroAssembler, +{ + /// Emits a DRC read barrier for a value loaded from `addr`. + /// + /// The loaded reference is first pushed and spilled so it is represented + /// in a stack map if this barrier calls `force_gc`. Null and i31 references + /// do not point into the GC heap, so they skip the heap access and barrier + /// bookkeeping. For a heap reference, the barrier bounds-checks the header + /// access and, unless the object is already present, links it into the + /// over-approximated stack-roots list, marks it as linked, and retains it. + /// Finally it forces a collection when the roots list reaches both the + /// proportional and absolute thresholds. + /// + /// Leaves the loaded reference on the value stack and marks + /// `storage_base` available for register reuse before returning. + pub(crate) fn emit_drc_read_barrier( + &mut self, + ty: WasmValType, + storage_base: Reg, + addr: M::Address, + ) -> Result<()> { + let gc_ref = self.context.reg_for_type(ty, self.masm)?; + self.masm.load(addr, writable!(gc_ref), ty.try_into()?)?; + self.context.stack.push(Val::reg(gc_ref, ty)); + + // Spill the loaded result into a stack-map-visible slot before the + // possible `force_gc` call, then reload a temporary copy for the + // barrier calculations. + self.context.spill(self.masm)?; + let slot = self + .context + .stack + .peek() + .ok_or_else(|| CodeGenError::missing_values_in_stack())? + .unwrap_mem() + .slot; + let ref_reg = self.context.any_gpr(self.masm)?; + self.masm.load( + self.masm.address_from_sp(slot.offset)?, + writable!(ref_reg), + OperandSize::S32, + )?; + + let skip_barrier = self.masm.get_label()?; + self.emit_skip_if_gc_ref_is_null_or_i31(ref_reg, skip_barrier)?; + + let (heap_reg, bound_reg) = self.emit_load_gc_heap_base_and_bound()?; + + // The read barrier accesses through the header's `next` field. + let header_extent = i64::from( + self.env + .vmoffsets + .vm_drc_header_next_over_approximated_stack_root(), + ) + 4; + self.emit_gc_ref_bounds_check(ref_reg, bound_reg, header_extent)?; + + // The bound is dead after the check. Free it before requesting the + // object address so the allocator can hand the same register back and + // keep register pressure low. + self.context.free_reg(bound_reg); + let object_addr = self.emit_gc_ref_addr(ref_reg, heap_reg)?; + + self.emit_skip_if_in_over_approx_stack_roots(object_addr, skip_barrier)?; + + let (heap_data_reg, roots_len) = + self.emit_push_over_approx_stack_root(ref_reg, object_addr)?; + + // Only the roots-list length and the heap data pointer are still live; + // both are consumed and freed by `emit_maybe_force_gc`. + self.context.free_reg(ref_reg); + self.context.free_reg(heap_reg); + self.context.free_reg(object_addr); + self.emit_maybe_force_gc(roots_len, heap_data_reg)?; + + self.masm.bind(skip_barrier)?; + self.context.free_reg(storage_base); + Ok(()) + } + + /// Emits a DRC write barrier for replacing the reference at `addr`. + /// + /// The barrier increments the new reference's count before decrementing the + /// old reference's count. It stores the new reference before a possible + /// out-of-line `drop_gc_ref` call. Null and i31 references skip their + /// respective retain and release operations. + /// + /// Consumes the new reference from the value stack and marks + /// `storage_base` available for register reuse before a possible runtime + /// call. + pub(crate) fn emit_drc_write_barrier( + &mut self, + ty: WasmValType, + storage_base: Reg, + addr: M::Address, + ) -> Result<()> { + self.context.spill(self.masm)?; + let new_ref = self.context.pop_to_reg(self.masm, None)?; + let (heap_reg, bound_reg) = self.emit_load_gc_heap_base_and_bound()?; + + let old_reg = self.context.any_gpr(self.masm)?; + self.masm.load(addr, writable!(old_reg), OperandSize::S32)?; + + let ref_count_offset = self.env.vmoffsets.vm_drc_header_ref_count(); + let header_extent = i64::from(ref_count_offset) + 8; + + // Retain the new heap reference before publishing it. This ordering + // keeps self-assignment from temporarily dropping the final owner. + let skip_inc = self.masm.get_label()?; + self.emit_skip_if_gc_ref_is_null_or_i31(new_ref.reg, skip_inc)?; + self.emit_gc_ref_bounds_check(new_ref.reg, bound_reg, header_extent)?; + let new_addr = self.emit_gc_ref_addr(new_ref.reg, heap_reg)?; + let count = self.emit_mutate_ref_count(new_addr, RefCountMutation::Increment)?; + self.emit_store_ref_count(new_addr, count)?; + self.context.free_reg(count); + self.context.free_reg(new_addr); + + self.masm.bind(skip_inc)?; + // Publish the new value before releasing the old one because the + // zero-count path below can make an out-of-line runtime call. + self.masm.store(new_ref.reg.into(), addr, ty.try_into()?)?; + + // Release the old heap reference, storing a nonzero count inline and + // delegating the zero-count case to `drop_gc_ref`. + let skip_dec = self.masm.get_label()?; + self.emit_skip_if_gc_ref_is_null_or_i31(old_reg, skip_dec)?; + self.emit_gc_ref_bounds_check(old_reg, bound_reg, header_extent)?; + let old_addr = self.emit_gc_ref_addr(old_reg, heap_reg)?; + let count = self.emit_mutate_ref_count(old_addr, RefCountMutation::Decrement)?; + + let drop_old = self.masm.get_label()?; + self.masm.branch( + IntCmpKind::Eq, + count, + RegImm::i64(0), + drop_old, + OperandSize::S64, + )?; + self.emit_store_ref_count(old_addr, count)?; + self.masm.jmp(skip_dec)?; + + self.context.free_reg(count); + self.context.free_reg(old_addr); + self.context.free_reg(bound_reg); + self.context.free_reg(heap_reg); + self.context.free_reg(new_ref.reg); + self.context.free_reg(storage_base); + + self.masm.bind(drop_old)?; + let drop_gc_ref = self.env.builtins.drop_gc_ref::()?; + self.context.stack.push(TypedReg::i32(old_reg).into()); + FnCall::emit::( + &mut self.env, + self.masm, + &mut self.context, + Callee::Builtin(drop_gc_ref), + )?; + + self.masm.bind(skip_dec) + } + + /// Branches to `skip` when `object_addr` is already linked into the + /// over-approximated stack-roots list. + /// + /// Uses the dedicated scratch register, so it must not be called from + /// inside another `with_scratch` scope. + fn emit_skip_if_in_over_approx_stack_roots( + &mut self, + object_addr: Reg, + skip: MachLabel, + ) -> Result<()> { + let reserved_offset = self.env.vmoffsets.vm_gc_header_reserved_bits(); + self.masm.with_scratch::(|masm, scratch| { + masm.load( + masm.address_at_reg(object_addr, reserved_offset)?, + scratch.writable(), + OperandSize::S32, + )?; + masm.and( + scratch.writable(), + scratch.inner(), + RegImm::i32(DRC_HEADER_IN_OVER_APPROX_LIST_BIT as i32), + OperandSize::S32, + )?; + masm.branch( + IntCmpKind::Ne, + scratch.inner(), + scratch.inner().into(), + skip, + OperandSize::S32, + ) + }) + } + + /// Adds `gc_ref` to the DRC heap's over-approximated stack-roots list. + /// + /// The object is linked at the head, marked as linked, and retained for + /// the list's ownership. + /// + /// Returns the DRC heap data pointer and the updated roots-list length, + /// both allocated here and owned by the caller, which must eventually free + /// them. The length feeds the collection-threshold check. + fn emit_push_over_approx_stack_root( + &mut self, + gc_ref: Reg, + object_addr: Reg, + ) -> Result<(Reg, Reg)> { + let heap_data_offset = self.env.vmoffsets.ptr.vmctx_gc_heap_data(); + let roots_head_offset = u32::from( + self.env + .vmoffsets + .ptr + .vmdrc_heap_data_over_approximated_stack_roots(), + ); + let roots_len_offset = u32::from( + self.env + .vmoffsets + .ptr + .vmdrc_heap_data_current_over_approximated_stack_roots_len(), + ); + let next_offset = self + .env + .vmoffsets + .vm_drc_header_next_over_approximated_stack_root(); + let reserved_offset = self.env.vmoffsets.vm_gc_header_reserved_bits(); + + let heap_data_reg = self.context.any_gpr(self.masm)?; + self.masm.load_ptr( + self.masm.address_at_vmctx(u32::from(heap_data_offset))?, + writable!(heap_data_reg), + )?; + + // Link this object to the old head of the over-approximated list, then + // mark it as present so subsequent reads can take the fast path. Both + // steps only need a temporary, so they share the scratch register. + self.masm.with_scratch::(|masm, scratch| { + masm.load( + masm.address_at_reg(heap_data_reg, roots_head_offset)?, + scratch.writable(), + OperandSize::S32, + )?; + masm.store( + scratch.inner().into(), + masm.address_at_reg(object_addr, next_offset)?, + OperandSize::S32, + )?; + + masm.load( + masm.address_at_reg(object_addr, reserved_offset)?, + scratch.writable(), + OperandSize::S32, + )?; + masm.or( + scratch.writable(), + scratch.inner(), + RegImm::i32(DRC_HEADER_IN_OVER_APPROX_LIST_BIT as i32), + OperandSize::S32, + )?; + masm.store( + scratch.inner().into(), + masm.address_at_reg(object_addr, reserved_offset)?, + OperandSize::S32, + ) + })?; + + // Retain the object for the list's ownership. + let count = self.emit_mutate_ref_count(object_addr, RefCountMutation::Increment)?; + self.emit_store_ref_count(object_addr, count)?; + self.context.free_reg(count); + + // Publish the new head, then the updated length. The length outlives + // this helper, so it gets its own allocated register. + self.masm.store( + gc_ref.into(), + self.masm.address_at_reg(heap_data_reg, roots_head_offset)?, + OperandSize::S32, + )?; + let roots_len = self.context.any_gpr(self.masm)?; + self.masm.load( + self.masm.address_at_reg(heap_data_reg, roots_len_offset)?, + writable!(roots_len), + OperandSize::S32, + )?; + self.masm.add( + writable!(roots_len), + roots_len, + RegImm::i32(1), + OperandSize::S32, + )?; + self.masm.store( + roots_len.into(), + self.masm.address_at_reg(heap_data_reg, roots_len_offset)?, + OperandSize::S32, + )?; + + Ok((heap_data_reg, roots_len)) + } + + /// Forces a collection when the over-approximated roots list has reached + /// both twice its post-GC length and the absolute minimum threshold. + /// + /// This method consumes and frees both register arguments before a + /// possible call to `force_gc`. + /// + /// The doubled post-GC length is only needed for the first comparison, so + /// it lives in the dedicated scratch register. This helper must therefore + /// not be called from inside another `with_scratch` scope. + fn emit_maybe_force_gc(&mut self, current_len: Reg, heap_data: Reg) -> Result<()> { + let last_len_offset = u32::from( + self.env + .vmoffsets + .ptr + .vmdrc_heap_data_over_approximated_stack_roots_len_after_last_gc(), + ); + let skip_gc = self.masm.get_label()?; + self.masm.with_scratch::(|masm, scratch| { + masm.load( + masm.address_at_reg(heap_data, last_len_offset)?, + scratch.writable(), + OperandSize::S32, + )?; + masm.add( + scratch.writable(), + scratch.inner(), + scratch.inner().into(), + OperandSize::S32, + )?; + masm.branch( + IntCmpKind::LtU, + current_len, + scratch.inner().into(), + skip_gc, + OperandSize::S32, + ) + })?; + let min_threshold = i32::try_from(DRC_MIN_OVER_APPROX_STACK_ROOTS_GC_THRESHOLD).unwrap(); + self.masm.branch( + IntCmpKind::LtU, + current_len, + RegImm::i32(min_threshold), + skip_gc, + OperandSize::S32, + )?; + + self.context.free_reg(heap_data); + self.context.free_reg(current_len); + + let force_gc = self.env.builtins.force_gc::()?; + FnCall::emit::( + &mut self.env, + self.masm, + &mut self.context, + Callee::Builtin(force_gc), + )?; + self.context.pop_and_free(self.masm)?; + + self.masm.bind(skip_gc) + } + + /// Loads a reference count and applies `mutation`, returning the register + /// holding the updated value. The caller decides whether and when to store + /// it, and must eventually free the returned register. + /// + /// The count is allocated rather than taken from the caller because the + /// decrement path inspects it after this helper returns, to decide between + /// storing it and calling `drop_gc_ref`. + fn emit_mutate_ref_count( + &mut self, + object_addr: Reg, + mutation: RefCountMutation, + ) -> Result { + let ref_count_offset = self.env.vmoffsets.vm_drc_header_ref_count(); + let count = self.context.any_gpr(self.masm)?; + self.masm.load( + self.masm.address_at_reg(object_addr, ref_count_offset)?, + writable!(count), + OperandSize::S64, + )?; + match mutation { + RefCountMutation::Increment => { + self.masm + .add(writable!(count), count, RegImm::i64(1), OperandSize::S64)? + } + RefCountMutation::Decrement => { + self.masm + .sub(writable!(count), count, RegImm::i64(1), OperandSize::S64)? + } + } + Ok(count) + } + + fn emit_store_ref_count(&mut self, object_addr: Reg, count: Reg) -> Result<()> { + let ref_count_offset = self.env.vmoffsets.vm_drc_header_ref_count(); + self.masm.store( + count.into(), + self.masm.address_at_reg(object_addr, ref_count_offset)?, + OperandSize::S64, + ) + } +} diff --git a/winch/codegen/src/codegen/gc.rs b/winch/codegen/src/codegen/gc.rs new file mode 100644 index 000000000000..a7f7ad86d509 --- /dev/null +++ b/winch/codegen/src/codegen/gc.rs @@ -0,0 +1,140 @@ +use super::{CodeGen, Emission}; +use crate::{ + Result, + masm::{IntCmpKind, IntScratch, MacroAssembler, OperandSize, RegImm}, + reg::{Reg, writable}, +}; +use cranelift_codegen::MachLabel; +use wasmtime_cranelift::TRAP_GC_HEAP_CORRUPT; +use wasmtime_environ::{I31_DISCRIMINANT, PtrSize}; + +impl<'a, 'translation, 'data, M> CodeGen<'a, 'translation, 'data, M, Emission> +where + M: MacroAssembler, +{ + /// Branches to `skip` when `gc_ref` is null or an unboxed i31 reference. + /// + /// A `VMGcRef` is null when all its bits are zero. The first branch tests + /// this by comparing the register with itself. The second test masks off + /// every bit except the i31 discriminant and skips the heap access when that bit is set. + /// + /// The discriminant test uses the dedicated scratch register rather than an + /// allocated temporary: the scratch register is non-allocatable, so it can + /// never alias `gc_ref`, which masking in place would otherwise destroy on + /// the fall-through path. Note that this means the helper must not be + /// called from inside another `with_scratch` scope. + pub(super) fn emit_skip_if_gc_ref_is_null_or_i31( + &mut self, + gc_ref: Reg, + skip: MachLabel, + ) -> Result<()> { + self.masm.branch( + IntCmpKind::Eq, + gc_ref, + gc_ref.into(), + skip, + OperandSize::S32, + )?; + self.masm.with_scratch::(|masm, scratch| { + masm.mov(scratch.writable(), gc_ref.into(), OperandSize::S32)?; + masm.and( + scratch.writable(), + scratch.inner(), + RegImm::i32(I31_DISCRIMINANT as i32), + OperandSize::S32, + )?; + masm.branch( + IntCmpKind::Ne, + scratch.inner(), + scratch.inner().into(), + skip, + OperandSize::S32, + ) + }) + } + + /// Loads the GC heap's base address and current length. + /// + /// Both returned registers are allocated from the code-generation context + /// and are owned by the caller. The caller must eventually free them. + pub(super) fn emit_load_gc_heap_base_and_bound(&mut self) -> Result<(Reg, Reg)> { + let store_context_offset = self.env.vmoffsets.ptr.vmctx_store_context(); + let gc_heap_base_offset = self.env.vmoffsets.ptr.vm_store_context().gc_heap_base(); + let gc_heap_len_offset = self + .env + .vmoffsets + .ptr + .vm_store_context() + .gc_heap_current_length(); + let heap_base = self.context.any_gpr(self.masm)?; + let heap_bound = self.context.any_gpr(self.masm)?; + + self.masm.load_ptr( + self.masm + .address_at_vmctx(u32::from(store_context_offset))?, + writable!(heap_base), + )?; + self.masm.load( + self.masm + .address_at_reg(heap_base, u32::from(gc_heap_len_offset))?, + writable!(heap_bound), + OperandSize::S64, + )?; + self.masm.load_ptr( + self.masm + .address_at_reg(heap_base, u32::from(gc_heap_base_offset))?, + writable!(heap_base), + )?; + + Ok((heap_base, heap_bound)) + } + + /// Checks that accessing `access_extent` bytes starting at `gc_ref` stays + /// within the GC heap. + /// + /// Callers must perform this check before converting a non-null, non-i31 + /// reference into a native address and accessing its object header. + /// + /// The end of the access is computed in the dedicated scratch register. It + /// is non-allocatable, so it can alias neither `gc_ref` — which would leave + /// the reference offset by `access_extent` — nor `heap_bound`, which would + /// turn the comparison into `cmp scratch, scratch` and silently disable the + /// check. Using it also guarantees nothing is emitted between `cmp` and + /// `trapif`. Note that this means the helper must not be called from inside + /// another `with_scratch` scope. + pub(super) fn emit_gc_ref_bounds_check( + &mut self, + gc_ref: Reg, + heap_bound: Reg, + access_extent: i64, + ) -> Result<()> { + self.masm.with_scratch::(|masm, scratch| { + masm.mov(scratch.writable(), gc_ref.into(), OperandSize::S64)?; + masm.add( + scratch.writable(), + scratch.inner(), + RegImm::i64(access_extent), + OperandSize::S64, + )?; + masm.cmp(scratch.inner(), heap_bound.into(), OperandSize::S64)?; + masm.trapif(IntCmpKind::GtU, TRAP_GC_HEAP_CORRUPT) + }) + } + + /// Converts a bounds-checked `VMGcRef` heap offset into a native address. + /// + /// The returned register is allocated from the code-generation context and + /// is owned by the caller, which must eventually free it. Allocating here + /// keeps the result from aliasing `gc_ref`, which the addition would + /// otherwise clobber. Callers that want the address to reuse a register + /// that is dead by this point should free that register first and let the + /// allocator hand it back. + pub(super) fn emit_gc_ref_addr(&mut self, gc_ref: Reg, heap_base: Reg) -> Result { + let dst = self.context.any_gpr(self.masm)?; + self.masm + .mov(writable!(dst), heap_base.into(), OperandSize::S64)?; + self.masm + .add(writable!(dst), dst, gc_ref.into(), OperandSize::S64)?; + Ok(dst) + } +} diff --git a/winch/codegen/src/codegen/mod.rs b/winch/codegen/src/codegen/mod.rs index 0df13d76d950..1577c091aeeb 100644 --- a/winch/codegen/src/codegen/mod.rs +++ b/winch/codegen/src/codegen/mod.rs @@ -40,6 +40,8 @@ pub(crate) use control::*; mod builtin; pub use builtin::*; pub(crate) mod bounds; +mod drc; +mod gc; use bounds::{Bounds, ImmOffset, Index}; @@ -490,6 +492,14 @@ where } } + /// Whether a GC barrier must be emitted when writing or reading a + /// reference of the given type through a collector-visible location. + pub fn gc_barrier_needed(&self, ty: &WasmValType) -> bool { + ty.is_vmgcref_type_and_not_i31() + && self.tunables.collector + == Some(wasmtime_environ::Collector::DeferredReferenceCounting) + } + /// Emits a a series of instructions that will type check a function reference call. pub fn emit_typecheck_funcref( &mut self, diff --git a/winch/codegen/src/visitor.rs b/winch/codegen/src/visitor.rs index 791f7354eea0..90deb3d394f5 100644 --- a/winch/codegen/src/visitor.rs +++ b/winch/codegen/src/visitor.rs @@ -2066,11 +2066,15 @@ where let index = GlobalIndex::from_u32(global_index); let (ty, base, offset) = self.emit_get_global_addr(index)?; let addr = self.masm.address_at_reg(base, offset)?; - let dst = self.context.reg_for_type(ty, self.masm)?; - self.masm.load(addr, writable!(dst), ty.try_into()?)?; - self.context.stack.push(Val::reg(dst, ty)); + if self.gc_barrier_needed(&ty) { + self.emit_drc_read_barrier(ty, base, addr)?; + } else { + let gc_ref = self.context.reg_for_type(ty, self.masm)?; + self.masm.load(addr, writable!(gc_ref), ty.try_into()?)?; + self.context.stack.push(Val::reg(gc_ref, ty)); - self.context.free_reg(base); + self.context.free_reg(base); + } Ok(()) } @@ -2080,11 +2084,15 @@ where let (ty, base, offset) = self.emit_get_global_addr(index)?; let addr = self.masm.address_at_reg(base, offset)?; - let typed_reg = self.context.pop_to_reg(self.masm, None)?; - self.masm - .store(typed_reg.reg.into(), addr, ty.try_into()?)?; - self.context.free_reg(typed_reg.reg); - self.context.free_reg(base); + if self.gc_barrier_needed(&ty) { + self.emit_drc_write_barrier(ty, base, addr)?; + } else { + let typed_reg = self.context.pop_to_reg(self.masm, None)?; + self.masm + .store(typed_reg.reg.into(), addr, ty.try_into()?)?; + self.context.free_reg(typed_reg.reg); + self.context.free_reg(base); + } Ok(()) }