diff --git a/cranelift/codegen/src/ir/function.rs b/cranelift/codegen/src/ir/function.rs index ee1d2debae1d..a194246ad852 100644 --- a/cranelift/codegen/src/ir/function.rs +++ b/cranelift/codegen/src/ir/function.rs @@ -473,6 +473,20 @@ impl Function { pub fn import_function(&mut self, data: ExtFuncData) -> FuncRef { self.stencil.dfg.ext_funcs.push(data) } + + /// Is the given block marked `cold` or otherwise effectively `cold` in + /// practice? + pub fn is_effectively_cold(&self, block: Block) -> bool { + if self.layout.is_cold(block) { + return true; + } + + // Blocks that unconditionally trap are effectively + // also cold. + self.layout + .last_inst(block) + .is_some_and(|inst| self.dfg.insts[inst].opcode() == ir::Opcode::Trap) + } } /// Wrapper type capable of displaying a `Function`. diff --git a/cranelift/codegen/src/machinst/blockorder.rs b/cranelift/codegen/src/machinst/blockorder.rs index 215b204eeee2..f19720e6c29a 100644 --- a/cranelift/codegen/src/machinst/blockorder.rs +++ b/cranelift/codegen/src/machinst/blockorder.rs @@ -257,7 +257,7 @@ impl BlockLoweringOrder { lowered_succ_indices .extend(block_succs[range].iter().map(|lb| lb_to_bindex[lb])); - if f.layout.is_cold(block) { + if f.is_effectively_cold(block) { cold_blocks.insert(bindex); } @@ -282,7 +282,7 @@ impl BlockLoweringOrder { // Edges inherit indirect branch and cold block metadata from their // successor. - if f.layout.is_cold(succ) { + if f.is_effectively_cold(succ) { cold_blocks.insert(bindex); } diff --git a/cranelift/filetests/filetests/isa/pulley64/trap_blocks_are_cold.clif b/cranelift/filetests/filetests/isa/pulley64/trap_blocks_are_cold.clif new file mode 100644 index 000000000000..a4d18639921f --- /dev/null +++ b/cranelift/filetests/filetests/isa/pulley64/trap_blocks_are_cold.clif @@ -0,0 +1,40 @@ +test compile precise-output +target pulley64 + +function %f(i8, i64) -> i64 { +block0(v0: i8, v1: i64): + brif v0, block1, block2 + +;; This block is implicitly cold because it ends in a trap; it should come last +;; in the lowered block order. +block1: + v2 = iconst.i32 42 + store notrap aligned v2, v1 + trap user123 + +block2: + v3 = iconst.i64 0 + return v3 +} + +; VCode: +; block0: +; zext8 x5, x0 +; br_if32 x5, label2; jump label1 +; block1: +; xzero x0 +; ret +; block2: +; xconst8 x7, 42 +; xstore32 x1+0, x7 // flags = notrap aligned +; trap // trap=TrapCode(123) +; +; Disassembled: +; zext8 x5, x0 +; br_if32 x5, 0x9 // target = 0xc +; xzero x0 +; ret +; xconst8 x7, 42 +; xstore32le_o32 x1, 0, x7 +; trap + diff --git a/tests/disas/debug-exceptions.wat b/tests/disas/debug-exceptions.wat index c903d304a57e..327084f2f5ca 100644 --- a/tests/disas/debug-exceptions.wat +++ b/tests/disas/debug-exceptions.wat @@ -31,35 +31,35 @@ ;; stur x0, [sp, #0x18] ;; mov x0, sp ;; cmp x0, x1 -;; b.lo #0x21c +;; b.lo #0x220 ;; 48: stur x2, [sp] ;; mov x0, x2 ;; stur x2, [sp, #0x10] ;; nop ;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x35, slot at FP-0xb0, locals , stack -;; ╰─╼ breakpoint patch: wasm PC 0x35, patch bytes [61, 1, 0, 148] +;; ╰─╼ breakpoint patch: wasm PC 0x35, patch bytes [62, 1, 0, 148] ;; ldur x0, [sp, #0x10] ;; nop ;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x37, slot at FP-0xb0, locals , stack -;; ╰─╼ breakpoint patch: wasm PC 0x37, patch bytes [59, 1, 0, 148] +;; ╰─╼ breakpoint patch: wasm PC 0x37, patch bytes [60, 1, 0, 148] ;; ldur x0, [sp, #0x10] ;; nop ;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x3d, slot at FP-0xb0, locals , stack -;; ╰─╼ breakpoint patch: wasm PC 0x3d, patch bytes [57, 1, 0, 148] +;; ╰─╼ breakpoint patch: wasm PC 0x3d, patch bytes [58, 1, 0, 148] ;; mov w1, #0x2a ;; stur w1, [sp, #8] ;; nop ;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x3f, slot at FP-0xb0, locals , stack I32 @ slot+0x8 -;; ╰─╼ breakpoint patch: wasm PC 0x3f, patch bytes [54, 1, 0, 148] +;; ╰─╼ breakpoint patch: wasm PC 0x3f, patch bytes [55, 1, 0, 148] ;; nop ;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x40, slot at FP-0xb0, locals , stack -;; ╰─╼ breakpoint patch: wasm PC 0x40, patch bytes [53, 1, 0, 148] +;; ╰─╼ breakpoint patch: wasm PC 0x40, patch bytes [54, 1, 0, 148] ;; stur w1, [sp, #8] ;; nop ;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x42, slot at FP-0xb0, locals , stack I32 @ slot+0x8 -;; ╰─╼ breakpoint patch: wasm PC 0x42, patch bytes [51, 1, 0, 148] +;; ╰─╼ breakpoint patch: wasm PC 0x42, patch bytes [52, 1, 0, 148] ;; ldur x2, [sp, #0x10] -;; bl #0x4b0 +;; bl #0x4b4 ;; 88: ldur x0, [sp, #0x10] ;; mov x19, x2 ;; ldr x1, [x0, #0x20] @@ -68,7 +68,7 @@ ;; add x2, x2, #0x20 ;; ldr w4, [x1, #4] ;; cmp x2, x4 -;; b.hi #0x1b4 +;; b.hi #0x1a0 ;; ac: ldur x0, [sp, #0x18] ;; add w4, w3, #0x20 ;; str w4, [x1] @@ -92,47 +92,41 @@ ;; mov w11, #0 ;; str w11, [x1, #0x14] ;; ldur x2, [sp, #0x10] -;; bl #0x4e8 +;; bl #0x4ec ;; ├─╼ exception frame offset: SP = FP - 0xb0 -;; ╰─╼ exception handler: tag=0, context at [SP+0x10], handler=0x124 -;; 10c: mov w3, #9 -;; ldur x2, [sp, #0x10] -;; bl #0x444 -;; 118: ldur x2, [sp, #0x10] -;; bl #0x47c -;; ╰─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x42, slot at FP-0xb0, locals , stack I32 @ slot+0x8 -;; 120: udf #0xc11f -;; mov w1, w0 +;; ╰─╼ exception handler: tag=0, context at [SP+0x10], handler=0x110 +;; b #0x1d8 +;; 110: mov w1, w0 ;; mov x2, #0x20 ;; adds x15, x1, x2 ;; cset x1, hs ;; tst w1, #0xff -;; b.ne #0x204 -;; 13c: ldur x2, [sp, #0x18] +;; b.ne #0x208 +;; 128: ldur x2, [sp, #0x18] ;; ldr x1, [x2, #0x28] ;; cmp x15, x1 -;; b.hi #0x1ec -;; 14c: ldr x1, [x2, #0x20] +;; b.hi #0x1f0 +;; 138: ldr x1, [x2, #0x20] ;; add x1, x1, #0x18 ;; ldr w0, [x1, w0, uxtw] ;; stur w0, [sp, #8] ;; ldur x0, [sp, #0x10] ;; nop ;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x48, slot at FP-0xb0, locals , stack I32 @ slot+0x8 -;; ╰─╼ breakpoint patch: wasm PC 0x48, patch bytes [250, 0, 0, 148] +;; ╰─╼ breakpoint patch: wasm PC 0x48, patch bytes [0, 1, 0, 148] ;; ldur x1, [sp, #0x10] ;; ldr x0, [x1, #0x38] ;; ldr x2, [x1, #0x48] ;; ldur x3, [sp, #0x10] ;; blr x0 ;; ╰─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x4a, slot at FP-0xb0, locals , stack I32 @ slot+0x8 -;; 178: ldur x0, [sp, #0x10] +;; 164: ldur x0, [sp, #0x10] ;; nop ;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x4a, slot at FP-0xb0, locals , stack I32 @ slot+0x8 -;; ╰─╼ breakpoint patch: wasm PC 0x4a, patch bytes [243, 0, 0, 148] +;; ╰─╼ breakpoint patch: wasm PC 0x4a, patch bytes [249, 0, 0, 148] ;; nop ;; ├─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x4b, slot at FP-0xb0, locals , stack -;; ╰─╼ breakpoint patch: wasm PC 0x4b, patch bytes [242, 0, 0, 148] +;; ╰─╼ breakpoint patch: wasm PC 0x4b, patch bytes [248, 0, 0, 148] ;; add sp, sp, #0x20 ;; ldp d8, d9, [sp], #0x10 ;; ldp d10, d11, [sp], #0x10 @@ -145,38 +139,45 @@ ;; ldp x27, x28, [sp], #0x10 ;; ldp x29, x30, [sp], #0x10 ;; ret -;; 1b4: mov w3, #2 -;; 1b8: movk w3, #0x400, lsl #16 -;; 1bc: ldur x0, [sp, #0x10] -;; 1c0: ldr x4, [x0, #0x28] -;; 1c4: ldr w4, [x4, #8] -;; 1c8: mov w5, #0x20 -;; 1cc: mov w6, #0x10 -;; 1d0: ldur x2, [sp, #0x10] -;; 1d4: bl #0x3dc -;; 1d8: ldur x0, [sp, #0x18] -;; 1dc: ldr x3, [x0, #0x20] -;; 1e0: add x1, x3, w2, uxtw -;; 1e4: mov x3, x2 -;; 1e8: b #0xec -;; 1ec: mov w3, #0xfe -;; 1f0: ldur x2, [sp, #0x10] -;; 1f4: bl #0x444 -;; 1f8: ldur x2, [sp, #0x10] -;; 1fc: bl #0x47c +;; 1a0: mov w3, #2 +;; 1a4: movk w3, #0x400, lsl #16 +;; 1a8: ldur x0, [sp, #0x10] +;; 1ac: ldr x4, [x0, #0x28] +;; 1b0: ldr w4, [x4, #8] +;; 1b4: mov w5, #0x20 +;; 1b8: mov w6, #0x10 +;; 1bc: ldur x2, [sp, #0x10] +;; 1c0: bl #0x3e0 +;; 1c4: ldur x0, [sp, #0x18] +;; 1c8: ldr x3, [x0, #0x20] +;; 1cc: add x1, x3, w2, uxtw +;; 1d0: mov x3, x2 +;; 1d4: b #0xec +;; 1d8: mov w3, #9 +;; 1dc: ldur x2, [sp, #0x10] +;; 1e0: bl #0x448 +;; 1e4: ldur x2, [sp, #0x10] +;; 1e8: bl #0x480 +;; ╰─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x42, slot at FP-0xb0, locals , stack I32 @ slot+0x8 +;; 1ec: udf #0xc11f +;; 1f0: mov w3, #0xfe +;; 1f4: ldur x2, [sp, #0x10] +;; 1f8: bl #0x448 +;; 1fc: ldur x2, [sp, #0x10] +;; 200: bl #0x480 ;; ╰─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x37, slot at FP-0xb0, locals , stack -;; 200: udf #0xc11f -;; 204: mov w3, #0xfe -;; 208: ldur x2, [sp, #0x10] -;; 20c: bl #0x444 -;; 210: ldur x2, [sp, #0x10] -;; 214: bl #0x47c +;; 204: udf #0xc11f +;; 208: mov w3, #0xfe +;; 20c: ldur x2, [sp, #0x10] +;; 210: bl #0x448 +;; 214: ldur x2, [sp, #0x10] +;; 218: bl #0x480 ;; ╰─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x37, slot at FP-0xb0, locals , stack -;; 218: udf #0xc11f -;; 21c: stur x2, [sp, #0x10] -;; 220: mov w3, #0 -;; 224: bl #0x444 -;; 228: ldur x2, [sp, #0x10] -;; 22c: bl #0x47c +;; 21c: udf #0xc11f +;; 220: stur x2, [sp, #0x10] +;; 224: mov w3, #0 +;; 228: bl #0x448 +;; 22c: ldur x2, [sp, #0x10] +;; 230: bl #0x480 ;; ╰─╼ debug frame state (after previous inst): func key DefinedWasmFunction(StaticModuleIndex(0), DefinedFuncIndex(0)), wasm PC 0x34, slot at FP-0xb0, locals , stack -;; 230: udf #0xc11f +;; 234: udf #0xc11f diff --git a/tests/disas/exceptions.wat b/tests/disas/exceptions.wat index 8af24433e7a8..efee2b2931a8 100644 --- a/tests/disas/exceptions.wat +++ b/tests/disas/exceptions.wat @@ -23,7 +23,7 @@ ;; movq 0x18(%r10), %r10 ;; addq $0x50, %r10 ;; cmpq %rsp, %r10 -;; ja 0xe7 +;; ja 0xec ;; 19: subq $0x40, %rsp ;; movq %rbx, 0x10(%rsp) ;; movq %r12, 0x18(%rsp) @@ -41,7 +41,7 @@ ;; leaq 0x30(%rcx), %rax ;; movl 4(%rdx), %edi ;; cmpq %rdi, %rax -;; ja 0xb3 +;; ja 0xb6 ;; 5f: leal 0x30(%rsi), %eax ;; movl %eax, (%rdx) ;; movq 8(%rbx), %rax @@ -63,8 +63,8 @@ ;; movq %rbx, %rdi ;; movq %rbx, (%rsp) ;; callq 0x418 -;; ud2 -;; movl $0x4000002, %esi +;; jmp 0xea +;; b6: movl $0x4000002, %esi ;; movq 0x28(%rbx), %rax ;; movl 0xc(%rax), %edx ;; movl $0x30, %ecx @@ -77,7 +77,8 @@ ;; movq %rax, %rsi ;; movq %r14, %rdi ;; jmp 0x8e -;; e7: ud2 +;; ea: ud2 +;; ud2 ;; ;; wasm[0]::function[1]::catch: ;; pushq %rbp