gc: default-on evacuating young-gen scavenge (moving minor at precise safepoints)

crates/perry-codegen/src/collectors/pointer_locals.rs

@@ -191,12 +191,31 @@ pub fn collect_pointer_typed_locals(
         matches!(
             ty,
             Type::String
+                // A string-LITERAL type (`"foo"`, or a `"a" | "b"` discriminant
+                // union member) is a heap String at runtime — it needs a root
+                // slot exactly like `Type::String`, or the moving-GC precise scan
+                // reaps a live string → silent corruption.
+                | Type::StringLiteral(_)
                 | Type::Array(_)
                 | Type::Tuple(_)
                 | Type::Object(_)
                 | Type::Named(_)
+                // An unresolved generic type parameter (`T`) can bind to any
+                // heap type; treat it as a pointer (fail-safe).
+                | Type::TypeVar(_)
                 | Type::Promise(_)
                 | Type::Function(_)
+                // A generic instantiation (`Map<K,V>`, `Set<T>`, `WeakMap`,
+                // `Box<T>`, `Array<T>`, a user generic class, …) is always a
+                // heap-reference type. Without this, a `Map`/`Set`-typed local
+                // got NO shadow-stack slot, so the PRECISE moving-GC root scan
+                // never saw it — the object was reaped as dead while still live
+                // (crash: "grown Map must retain its side-allocation owner
+                // record"). The non-moving default GC hid this via its
+                // conservative C-stack scan. Treating a rare non-pointer generic
+                // value as a root is harmless: the GC decode rejects any slot
+                // value that isn't a live heap pointer.
+                | Type::Generic { .. }
                 | Type::BigInt
                 | Type::Any
                 | Type::Unknown

crates/perry-codegen/src/loop_purity.rs

@@ -52,6 +52,115 @@ pub(crate) fn body_needs_asm_barrier(body: &[Stmt]) -> bool {
     !body_writes_outside(body, &body_locals)
 }
 
+/// True when the loop body may allocate (or otherwise call into the runtime and
+/// trip a GC), so a moving-GC back-edge poll (`js_gc_loop_safepoint`) must be
+/// emitted to drain any deferred minor. Reuses the LLVM-purity whitelist: a body
+/// that is fully LLVM-pure performs no call / allocation / heap mutation, so it
+/// can never cross a nursery trigger and defer a collection — the poll would be
+/// a guaranteed no-op that only defeats vectorization. Conservative in the SAFE
+/// direction: anything not provably pure is treated as "may allocate" and gets
+/// the poll. A spurious poll costs a little vectorization; a missing one only
+/// delays a deferred minor to the next safepoint (bounded by the moving-GC hard
+/// cap) — never a correctness or UAF hazard.
+pub(crate) fn body_may_allocate(body: &[Stmt]) -> bool {
+    !body.iter().all(stmt_alloc_free)
+}
+
+/// Like `stmt_is_pure`, but the question is narrower — "can this allocate (or
+/// call into the runtime and trip a GC)?" — so it additionally accepts element
+/// ACCESS that never allocates: array/typed-array element READS and in-place
+/// numeric updates never allocate, and typed-array element WRITES store into a
+/// fixed-size backing buffer that never grows. Generic `IndexSet` is NOT
+/// accepted: a plain JS-array index write can grow (reallocate) the backing
+/// store. This lets a `for (…) acc += arr[i]` reduction stay poll-free (LLVM can
+/// vectorize) while `keep.push({…})` (a Call) still gets its poll.
+fn stmt_alloc_free(s: &Stmt) -> bool {
+    match s {
+        Stmt::Expr(e) => expr_alloc_free(e),
+        Stmt::Let { init, .. } => init.as_ref().is_none_or(expr_alloc_free),
+        Stmt::If {
+            condition,
+            then_branch,
+            else_branch,
+        } => {
+            expr_alloc_free(condition)
+                && then_branch.iter().all(stmt_alloc_free)
+                && else_branch
+                    .as_ref()
+                    .is_none_or(|b| b.iter().all(stmt_alloc_free))
+        }
+        Stmt::While { condition, body } => {
+            expr_alloc_free(condition) && body.iter().all(stmt_alloc_free)
+        }
+        Stmt::DoWhile { body, condition } => {
+            expr_alloc_free(condition) && body.iter().all(stmt_alloc_free)
+        }
+        Stmt::For {
+            init,
+            condition,
+            update,
+            body,
+        } => {
+            init.as_deref().is_none_or(stmt_alloc_free)
+                && condition.as_ref().is_none_or(expr_alloc_free)
+                && update.as_ref().is_none_or(expr_alloc_free)
+                && body.iter().all(stmt_alloc_free)
+        }
+        Stmt::Labeled { body, .. } => stmt_alloc_free(body),
+        Stmt::Break | Stmt::Continue | Stmt::LabeledBreak(_) | Stmt::LabeledContinue(_) => true,
+        _ => false,
+    }
+}
+
+fn expr_alloc_free(e: &Expr) -> bool {
+    // Everything LLVM-pure is allocation-free (literals, reads, arithmetic).
+    if expr_is_pure(e) {
+        return true;
+    }
+    match e {
+        // Element READS never allocate — they return an existing element / a
+        // number. Recurse so the object and index are themselves alloc-free.
+        Expr::IndexGet { object, index } => expr_alloc_free(object) && expr_alloc_free(index),
+        Expr::BufferIndexGet { buffer, index } => {
+            expr_alloc_free(buffer) && expr_alloc_free(index)
+        }
+        Expr::Uint8ArrayGet { array, index } => expr_alloc_free(array) && expr_alloc_free(index),
+        // `arr[i]++` / `--`: read-modify-write of an existing numeric slot, no
+        // growth, no allocation.
+        Expr::IndexUpdate { object, index, .. } => {
+            expr_alloc_free(object) && expr_alloc_free(index)
+        }
+        // Typed-array element WRITES store into a fixed-size backing buffer that
+        // never grows/reallocates. (Generic `IndexSet` is deliberately absent —
+        // a plain JS-array write can grow the array and allocate.)
+        Expr::BufferIndexSet {
+            buffer,
+            index,
+            value,
+        } => expr_alloc_free(buffer) && expr_alloc_free(index) && expr_alloc_free(value),
+        Expr::Uint8ArraySet {
+            array,
+            index,
+            value,
+        } => expr_alloc_free(array) && expr_alloc_free(index) && expr_alloc_free(value),
+        // Re-handle the composite arithmetic/assign forms so an alloc-free (but
+        // not LLVM-pure) operand — e.g. a `BufferIndexGet` — propagates through.
+        Expr::LocalSet(_, val) => expr_alloc_free(val),
+        Expr::Binary { left, right, .. }
+        | Expr::Compare { left, right, .. }
+        | Expr::Logical { left, right, .. } => expr_alloc_free(left) && expr_alloc_free(right),
+        Expr::Unary { operand, .. } | Expr::TypeOf(operand) | Expr::Void(operand) => {
+            expr_alloc_free(operand)
+        }
+        Expr::Conditional {
+            condition,
+            then_expr,
+            else_expr,
+        } => expr_alloc_free(condition) && expr_alloc_free(then_expr) && expr_alloc_free(else_expr),
+        _ => false,
+    }
+}
+
 fn stmt_is_pure(s: &Stmt) -> bool {
     match s {
         Stmt::Expr(e) => expr_is_pure(e),

crates/perry-codegen/src/stmt/loops.rs

@@ -5141,7 +5141,7 @@ fn lower_for_after_init_with_i32_bound(
         ctx.block().asm_sideeffect_barrier();
     }
     if !ctx.block().is_terminated() {
-        emit_gc_loop_safepoint(ctx);
+        emit_gc_loop_safepoint(ctx, body);
         ctx.block().br(&update_label);
     }
 
@@ -5225,10 +5225,15 @@ fn lower_for_after_init_with_i32_bound(
 fn moving_safepoint_polls_enabled() -> bool {
     use std::sync::OnceLock;
     static CACHED: OnceLock<bool> = OnceLock::new();
+    // DEFAULT ON (moving-nursery flip): emit the back-edge poll, but ONLY for
+    // allocating loop bodies (see the `body_may_allocate` gate in
+    // `emit_gc_loop_safepoint`) so numeric/vectorizable loops stay call-free.
+    // Kill switch: PERRY_GC_MOVING_LOOP_POLLS=0/off/false. Must match the runtime
+    // `gc_moving_loop_polls_enabled` (same env) so deferrals always have a drain.
     *CACHED.get_or_init(|| {
-        matches!(
+        !matches!(
             std::env::var("PERRY_GC_MOVING_LOOP_POLLS").as_deref(),
-            Ok("1") | Ok("on") | Ok("true")
+            Ok("0") | Ok("off") | Ok("false")
         )
     })
 }
@@ -5246,10 +5251,17 @@ fn moving_safepoint_polls_enabled() -> bool {
 /// loop that takes one of those paths won't drain a deferred moving minor until
 /// the next event-loop safepoint. Adding the poll to every back-edge across
 /// those paths is the remaining Phase 2 codegen work.
-pub(crate) fn emit_gc_loop_safepoint(ctx: &mut FnCtx<'_>) {
+pub(crate) fn emit_gc_loop_safepoint(ctx: &mut FnCtx<'_>, body: &[Stmt]) {
     if !moving_safepoint_polls_enabled() || ctx.block().is_terminated() {
         return;
     }
+    // Only an ALLOCATING loop body can defer a collection to this poll; skip the
+    // poll for pure (non-allocating) bodies so numeric/vectorizable loops stay
+    // call-free (a poll defeats LLVM auto-vectorization — measured ~2x on a tight
+    // scalar reduction). See `body_may_allocate` for the safe-direction rationale.
+    if !crate::loop_purity::body_may_allocate(body) {
+        return;
+    }
     ctx.block().call_void("js_gc_loop_safepoint", &[]);
 }
 
@@ -7030,7 +7042,7 @@ pub(crate) fn lower_while(
         ctx.block().asm_sideeffect_barrier();
     }
     if !ctx.block().is_terminated() {
-        emit_gc_loop_safepoint(ctx);
+        emit_gc_loop_safepoint(ctx, body);
         ctx.block().br(&cond_label);
     }
     ctx.active_region_id = previous_region_id;
@@ -7088,7 +7100,7 @@ pub(crate) fn lower_do_while(
         ctx.block().asm_sideeffect_barrier();
     }
     if !ctx.block().is_terminated() {
-        emit_gc_loop_safepoint(ctx);
+        emit_gc_loop_safepoint(ctx, body);
         ctx.block().br(&cond_label);
     }
 

crates/perry-runtime/src/gc/copying.rs

@@ -26,7 +26,19 @@ pub(crate) struct CopyingPointerSet {
 impl CopyingPointerSet {
     pub(super) fn new() -> Self {
         let (malloc_registry_available, malloc_registry_empty_at_start) = MALLOC_STATE.with(|s| {
-            let s = s.borrow();
+            let mut s = s.borrow_mut();
+            // Moving-nursery mode (`PERRY_GC_MOVING_LOOP_POLLS`): eagerly build the
+            // malloc registry so this copying minor can CLASSIFY malloc-tracked
+            // objects and evacuate, instead of hitting
+            // `MallocRegistryUnavailable` and falling back to a non-moving minor
+            // (which reclaims ~nothing on reallocation-heavy async/Map/generator
+            // code — measured: broad3 192 MiB / 100 fallbacks). The
+            // O(malloc-objects) rebuild is paid back by the RSS win. Default
+            // (non-moving) copied minors keep the lazy behavior — see
+            // `ensure_set_built`'s "keep copied-minor from rebuilding" note.
+            if super::gc_moving_loop_polls_enabled() && !s.objects.is_empty() {
+                super::malloc::ensure_set_built(&mut s);
+            }
             (s.malloc_registry_available(), s.objects.is_empty())
         });
         let malloc_registry_rebuild_count_start = MALLOC_REGISTRY_REBUILD_COUNT.with(|c| c.get());
@@ -446,14 +458,62 @@ impl CopyingNurseryCollector {
         })
     }
 
+    /// Follow the forwarding chain for a raw metadata key/address the SAME
+    /// way the evacuation verifier does (`verify::try_rewrite_raw_addr`), so
+    /// the post-copy rewrite pass and the verifier never DISAGREE about a
+    /// moved address (#scavenge-cause).
+    ///
+    /// The old body classified `addr` via `self.ptrs.classify()` and bailed to
+    /// `None` whenever that returned `None`. But the verifier follows the
+    /// forwarding pointer gated only by its live census, so any from-space key
+    /// the classifier rejected stayed *un-rekeyed* in a runtime mutable
+    /// metadata table (e.g. `shapes.entries`, keyed by keys-array heap address)
+    /// — and the verifier then aborted on that still-stale forwarded key
+    /// (`slot=0x0 ... in runtime mutable root scanner`). Because
+    /// `rewrite_raw_addr` is the single shared path for every metadata scanner
+    /// (shapes, map/set, symbol, proxy, weakref, descriptor/class registries,
+    /// …), the disagreement is fixed for all of them at once.
+    ///
+    /// Gate on a heap-region check instead of `classify`: `GC_FLAG_FORWARDED`
+    /// is set ONLY by `set_forwarding_address`, and during this rewrite pass
+    /// the from-space is still intact and page-registered
+    /// (`copying_reset_from_spaces_and_flip` runs strictly later — after both
+    /// this rewrite pass and the verify pass), so any address in a known heap
+    /// region carrying that flag IS genuinely forwarded. Mirrors
+    /// `try_rewrite_raw_addr`'s 64-hop cap and `next == 0 || next == current`
+    /// stops, returning `rewrote.then_some(current)` (Some only when the
+    /// address actually moved).
     pub(super) fn rewrite_raw_addr(&self, addr: usize) -> Option<usize> {
-        let ptr = self.ptrs.classify(addr)?;
-        unsafe {
-            if (*ptr.header).gc_flags & GC_FLAG_FORWARDED == 0 {
-                return None;
+        if addr < GC_HEADER_SIZE {
+            return None;
+        }
+        let mut current = addr;
+        let mut rewrote = false;
+        for _ in 0..64 {
+            if current < GC_HEADER_SIZE {
+                return rewrote.then_some(current);
+            }
+            let header_addr = current - GC_HEADER_SIZE;
+            if matches!(
+                crate::arena::classify_heap_space(header_addr),
+                crate::arena::HeapSpace::Unknown
+            ) {
+                return rewrote.then_some(current);
+            }
+            let header = header_addr as *mut GcHeader;
+            unsafe {
+                if (*header).gc_flags & GC_FLAG_FORWARDED == 0 {
+                    return rewrote.then_some(current);
+                }
+                let next = forwarding_address(header) as usize;
+                if next == 0 || next == current {
+                    return rewrote.then_some(current);
+                }
+                current = next;
+                rewrote = true;
             }
-            Some(forwarding_address(ptr.header) as usize)
         }
+        rewrote.then_some(current)
     }
 
     pub(super) fn mark_addr(&mut self, addr: usize) -> Option<usize> {

crates/perry-runtime/src/gc/mod.rs

@@ -247,6 +247,29 @@ fn gc_verify_evacuation_enabled() -> bool {
     )
 }
 
+/// Phase-1 de-risking flag (OFF by default). When set, the alloc-point
+/// nursery-churn arm (`gc_check_trigger`) runs its direct minor with the
+/// PRECISE shadow-stack roots instead of forcing the conservative native
+/// scan. The conservative scan makes the copying fast path ineligible
+/// (`CopiedMinorFallbackReason::ConservativeStack`), pinning the minor to the
+/// non-moving in-place sweep that cannot reclaim array-growth stubs; skipping
+/// it lets the evacuating scavenge run and reset the whole young arena in
+/// O(live). NOT sound as a production default yet — the alloc point can be
+/// register-imprecise — so it stays behind this flag for measurement +
+/// `PERRY_GC_VERIFY_EVACUATION` probing only. Pairs with
+/// `PERRY_GC_MAJOR_PACING_FLOOR_MB=0` so the #6939 pacing doesn't escalate the
+/// minor to a full before the copying path is reached.
+pub(super) fn gc_scavenge_enabled() -> bool {
+    use std::sync::OnceLock;
+    static CACHED: OnceLock<bool> = OnceLock::new();
+    *CACHED.get_or_init(|| {
+        matches!(
+            std::env::var("PERRY_GC_SCAVENGE").as_deref(),
+            Ok("1") | Ok("on") | Ok("true")
+        )
+    })
+}
+
 #[cfg(test)]
 fn gc_collect_inner() -> u64 {
     if defer_gc_request(DeferredGcRequest::Collect(GcTriggerKind::Direct)) {