diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index ba84e2617404..31c8fbdb7ac0 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -663,85 +663,6 @@ impl<'module_environment> FuncEnvironment<'module_environment> { builder.switch_to_block(continuation_block); } - /// Manually insert a fuel check, as opposed to what already happens around - /// normal loops headers and function entries. - /// - /// This can be used for expensive opcodes, such as `array.copy`, where the - /// operation's runtime is a function of the runtime state. - fn manual_fuel_check(&mut self, builder: &mut FunctionBuilder<'_>, fuel_to_consume: ir::Value) { - self.fuel_increment_var(builder); - - let fuel = builder.use_var(self.fuel_var); - let fuel = builder.ins().iadd(fuel, fuel_to_consume); - builder.def_var(self.fuel_var, fuel); - - self.fuel_check(builder); - } - - /// Consumes `units * cost_per_unit` fuel, saturating the charge at - /// `i64::MAX` so that an oversized unsigned operand cannot wrap around and - /// add fuel instead. - fn consume_variable_fuel( - &mut self, - builder: &mut FunctionBuilder<'_>, - units: ir::Value, - cost_per_unit: u8, - ) { - let may_exceed_i64_max = match builder.func.dfg.value_type(units) { - I32 => false, - I64 => true, - ty => unreachable!("unsupported variable fuel unit type: {ty}"), - }; - self.consume_variable_fuel_impl(builder, units, cost_per_unit, may_exceed_i64_max); - } - - /// Like [`Self::consume_variable_fuel`], but for a unit count whose product - /// with `cost_per_unit` is statically known to fit in an `i64`. - fn consume_bounded_variable_fuel( - &mut self, - builder: &mut FunctionBuilder<'_>, - units: ir::Value, - cost_per_unit: u8, - ) { - self.consume_variable_fuel_impl(builder, units, cost_per_unit, false); - } - - fn consume_variable_fuel_impl( - &mut self, - builder: &mut FunctionBuilder<'_>, - units: ir::Value, - cost_per_unit: u8, - may_exceed_i64_max: bool, - ) { - if !self.tunables.consume_fuel || cost_per_unit == 0 { - return; - } - - let units = match builder.func.dfg.value_type(units) { - I32 => builder.ins().uextend(I64, units), - I64 => units, - ty => unreachable!("unsupported variable fuel unit type: {ty}"), - }; - let fuel = if cost_per_unit == 1 { - units - } else { - builder.ins().imul_imm_s(units, i64::from(cost_per_unit)) - }; - let fuel = if may_exceed_i64_max { - let max = builder.ins().iconst(I64, i64::MAX); - let max_units = builder - .ins() - .iconst(I64, i64::MAX / i64::from(cost_per_unit)); - let saturate = builder - .ins() - .icmp(IntCC::UnsignedGreaterThan, units, max_units); - builder.ins().select(saturate, max, fuel) - } else { - fuel - }; - self.manual_fuel_check(builder, fuel); - } - fn epoch_function_entry(&mut self, builder: &mut FunctionBuilder<'_>) { debug_assert!(self.epoch_deadline_var.is_reserved_value()); self.epoch_deadline_var = builder.declare_var(ir::types::I64); @@ -2684,17 +2605,19 @@ impl FuncEnvironment<'_> { delta: ir::Value, init_value: ir::Value, ) -> WasmResult { + let cost = self + .tunables + .operator_cost + .variable() + .table_grow_per_element; + self.pre_translate_bulk_op(builder, delta, cost)?; + let mut pos = builder.cursor(); let table = self.table(table_index); let (table_vmctx, defined_table_index) = self.table_vmctx_and_defined_index(&mut pos, table_index); let index_type = table.idx_type; let delta64 = self.cast_index_to_i64(&mut pos, delta, index_type); - let cost = self - .tunables - .operator_cost - .variable() - .table_grow_per_element; // Call out to the host to perform the actual growth of the underlying // table. This will initialize table slots as all null. Afterwards the @@ -2740,7 +2663,6 @@ impl FuncEnvironment<'_> { // A failed attempt performs no initialization loop, but still charge // for the requested growth so repeated failures are not free. builder.switch_to_block(failed_block); - self.consume_variable_fuel(builder, delta, cost); builder.ins().jump(done_block, &[]); builder.switch_to_block(fill_block); @@ -2748,12 +2670,11 @@ impl FuncEnvironment<'_> { builder, CheckedEntity::Table { table: table_index, - initialized: true, + initialized: false, }, result_idx, init_value, delta, - cost, )?; builder.ins().jump(done_block, &[]); @@ -2907,6 +2828,7 @@ impl FuncEnvironment<'_> { .operator_cost .variable() .table_fill_per_element; + self.pre_translate_bulk_op(builder, len, cost)?; self.translate_entity_fill( builder, CheckedEntity::Table { @@ -2916,7 +2838,6 @@ impl FuncEnvironment<'_> { dst, val, len, - cost, ) } @@ -3053,8 +2974,7 @@ impl FuncEnvironment<'_> { elem: ir::Value, len: ir::Value, ) -> WasmResult { - let cost = self.tunables.operator_cost.variable().array_new_per_element; - gc::translate_array_new(self, builder, array_type_index, elem, len, cost) + gc::translate_array_new(self, builder, array_type_index, elem, len) } pub fn translate_array_new_default( @@ -3063,12 +2983,7 @@ impl FuncEnvironment<'_> { array_type_index: TypeIndex, len: ir::Value, ) -> WasmResult { - let cost = self - .tunables - .operator_cost - .variable() - .array_new_default_per_element; - gc::translate_array_new_default(self, builder, array_type_index, len, cost) + gc::translate_array_new_default(self, builder, array_type_index, len) } pub fn translate_array_new_fixed( @@ -3151,6 +3066,7 @@ impl FuncEnvironment<'_> { .operator_cost .variable() .array_copy_per_element; + self.pre_translate_bulk_op(builder, len, cost)?; self.translate_entity_copy( builder, CheckedEntity::Array { @@ -3166,7 +3082,6 @@ impl FuncEnvironment<'_> { dst_index, src_index, len, - cost, ) } @@ -3185,6 +3100,7 @@ impl FuncEnvironment<'_> { .operator_cost .variable() .array_fill_per_element; + self.pre_translate_bulk_op(builder, len, cost)?; self.translate_entity_fill( builder, CheckedEntity::Array { @@ -3195,7 +3111,6 @@ impl FuncEnvironment<'_> { index, value, len, - cost, ) } @@ -3216,6 +3131,7 @@ impl FuncEnvironment<'_> { .operator_cost .variable() .array_init_data_per_element; + self.pre_translate_bulk_op(builder, len, cost)?; self.translate_entity_copy( builder, CheckedEntity::Array { @@ -3230,7 +3146,6 @@ impl FuncEnvironment<'_> { dst_index, data_offset, len, - cost, ) } @@ -3250,6 +3165,7 @@ impl FuncEnvironment<'_> { .operator_cost .variable() .array_init_elem_per_element; + self.pre_translate_bulk_op(builder, len, cost)?; self.translate_entity_copy( builder, CheckedEntity::Array { @@ -3261,7 +3177,6 @@ impl FuncEnvironment<'_> { dst_index, elem_offset, len, - cost, ) } @@ -3634,7 +3549,7 @@ impl FuncEnvironment<'_> { let index_type = self.memory(index).idx_type; let cost = self.tunables.operator_cost.variable().memory_grow_per_page; - self.consume_variable_fuel(builder, val, cost); + self.pre_translate_bulk_op(builder, val, cost)?; let mut pos = builder.cursor(); let val = self.cast_index_to_i64(&mut pos, val, index_type); let call_inst = pos @@ -3733,7 +3648,8 @@ impl FuncEnvironment<'_> { len: ir::Value, ) -> WasmResult<()> { let cost = self.tunables.operator_cost.variable().memory_copy_per_byte; - self.translate_entity_copy(builder, dst_index, src_index, dst, src, len, cost) + self.pre_translate_bulk_op(builder, len, cost)?; + self.translate_entity_copy(builder, dst_index, src_index, dst, src, len) } /// Perform a raw bulk-memory-like libcall. @@ -3741,7 +3657,11 @@ impl FuncEnvironment<'_> { /// The main purpose of this helper is to handle situations when fuel and /// epochs are enabled to break up the copy into a loop of chunks with /// preemption checks between them. - fn raw_bulk_memory_operation(&mut self, builder: &mut FunctionBuilder<'_>, mut op: BulkOp) { + fn raw_bulk_memory_operation( + &mut self, + builder: &mut FunctionBuilder<'_>, + op: BulkOp, + ) -> WasmResult<()> { // Fast path: a copy whose byte length is a small compile-time constant is // expanded inline (see `emit_inline_memcpy`), skipping the libcall's fixed // per-call cost (a wasm/host transition and an indirect call) that @@ -3758,259 +3678,33 @@ impl FuncEnvironment<'_> { const_len: Some(bytes), src_entity, dst_entity, - fuel, .. } = op { if bytes <= INLINE_COPY_MAX_BYTES { - if self.tunables.consume_fuel { - let units = bytes / u64::from(fuel.bytes_per_unit); - self.fuel_consumed += - i64::try_from(units * u64::from(fuel.cost_per_unit)).unwrap(); - } let src_region = self.bulk_copy_alias_region(builder.func, src_entity); let dst_region = self.bulk_copy_alias_region(builder.func, dst_entity); self.emit_inline_memcpy(builder, dst, src, bytes, src_region, dst_region); - return; + return Ok(()); } } - // Very scientifically chosen. Or, more seriously, this is just an - // arbitrary number for now. 100k copies of this size locally takes half - // a second, so seems like a reasonably large chunk size to not hit perf - // too much by chunking but also enable time slicing. - const UNINTERRUPTABLE_CHUNK_SIZE: i64 = 128 << 20; - let mut pos = builder.cursor(); let vmctx = self.vmctx_val(&mut pos); - let pointer_type = self.pointer_type(); // Performs a raw call to the actual libcall, as dictated by the - // provided `op`. This inserts configured epoch/fuel checks before the - // call. - let raw_call = - |env: &mut FuncEnvironment<'_>, builder: &mut FunctionBuilder<'_>, op: &BulkOp| { - if env.tunables.epoch_interruption { - env.epoch_check(builder); - } - let fuel = op.fuel(); - if env.tunables.consume_fuel && fuel.cost_per_unit != 0 { - let byte_len = op.len(); - debug_assert!(fuel.bytes_per_unit.is_power_of_two()); - let units = if fuel.bytes_per_unit == 1 { - byte_len - } else { - builder - .ins() - .ushr_imm_u(byte_len, i64::from(fuel.bytes_per_unit.trailing_zeros())) - }; - // With fuel enabled all calls emitted below are limited to - // `UNINTERRUPTABLE_CHUNK_SIZE`, so this multiplication - // cannot exceed `i64::MAX` even at the maximum `u8` rate. - env.consume_bounded_variable_fuel(builder, units, fuel.cost_per_unit); - } - match *op { - BulkOp::MemoryCopy { dst, src, len, .. } => { - let memory_copy = env.builtin_functions.memory_copy(&mut builder.func); - builder.ins().call(memory_copy, &[vmctx, dst, src, len]); - } - BulkOp::MemoryFill { dst, val, len, .. } => { - let memory_fill = env.builtin_functions.memory_fill(&mut builder.func); - builder.ins().call(memory_fill, &[vmctx, dst, val, len]); - } - } - }; - - // If epochs and fuel are disabled, then just call the libcall and - // return. No need for the loops below. - if !self.tunables.epoch_interruption && !self.tunables.consume_fuel { - raw_call(self, builder, &op); - return; - } - - // If fuel is enabled, first take all the pending fuel and flush it to - // our internal variable. This is necessary to avoid picking up all - // pending fuel on each turn of the loop below. - if self.tunables.consume_fuel { - self.fuel_increment_var(builder); - } - - let current_block = builder.current_block().unwrap(); - let chunk_block = builder.create_block(); - let last_chunk_block = builder.create_block(); - - builder.ensure_inserted_block(); - builder.insert_block_after(chunk_block, current_block); - builder.insert_block_after(last_chunk_block, chunk_block); - - let chunk = builder - .ins() - .iconst(pointer_type, UNINTERRUPTABLE_CHUNK_SIZE); - - // For `memcpy` when chunking this up we might need to do a backwards - // copy or a forwards copy. Determine that here and jump to the - // backwards copy if needed. - let backwards_block = if let BulkOp::MemoryCopy { dst, src, .. } = op { - let forwards = builder.ins().icmp(IntCC::UnsignedGreaterThan, src, dst); - let forwards_block = builder.create_block(); - let backwards_block = builder.create_block(); - builder - .ins() - .brif(forwards, forwards_block, &[], backwards_block, &[]); - builder.switch_to_block(forwards_block); - builder.seal_block(forwards_block); - Some((backwards_block, op.clone())) - } else { - None - }; - - // Helper closure to test if the length in `op` is larger than `chunk`, - // and if so do a single chunk. Else this goes to the final block with - // the final operation. - let has_chunk_branch = - |builder: &mut FunctionBuilder<'_>, op: &_, chunk_block, last_chunk_block| { - let len = match *op { - BulkOp::MemoryCopy { len, .. } | BulkOp::MemoryFill { len, .. } => len, - }; - let has_chunk = builder.ins().icmp(IntCC::UnsignedGreaterThan, len, chunk); - match *op { - BulkOp::MemoryCopy { dst, src, len, .. } => { - builder.ins().brif( - has_chunk, - chunk_block, - &[dst.into(), src.into(), len.into()], - last_chunk_block, - &[dst.into(), src.into(), len.into()], - ); - } - BulkOp::MemoryFill { dst, len, .. } => { - builder.ins().brif( - has_chunk, - chunk_block, - &[dst.into(), len.into()], - last_chunk_block, - &[dst.into(), len.into()], - ); - } - } - }; - - let append_block_params = |builder: &mut FunctionBuilder<'_>, block, op: &mut _| match op { - BulkOp::MemoryCopy { dst, src, len, .. } => { - *dst = builder.append_block_param(block, pointer_type); - *src = builder.append_block_param(block, pointer_type); - *len = builder.append_block_param(block, pointer_type); - } - BulkOp::MemoryFill { dst, len, .. } => { - *dst = builder.append_block_param(block, pointer_type); - *len = builder.append_block_param(block, pointer_type); - } - }; - - // Forwards copy: dispatch to the per-chunk loop or the final iteration - // if there's no chunks. - has_chunk_branch(builder, &op, chunk_block, last_chunk_block); - - // Forwards copy: In the block with per-chunk copies, each operation - // performs `chunk` length of bytes and then decrements the current - // length by `chunk`. Afterwards a condition tests if we do another - // chunk or break out for the final chunk. - builder.switch_to_block(chunk_block); - append_block_params(builder, chunk_block, &mut op); - let op_len = match &mut op { - BulkOp::MemoryCopy { len, .. } | BulkOp::MemoryFill { len, .. } => len, - }; - let remaining_len = *op_len; - *op_len = chunk; - raw_call(self, builder, &op); - match &mut op { + // provided `op`. + match op { BulkOp::MemoryCopy { dst, src, len, .. } => { - *dst = builder.ins().iadd(*dst, chunk); - *src = builder.ins().iadd(*src, chunk); - *len = builder.ins().isub(remaining_len, chunk); + let memory_copy = self.builtin_functions.memory_copy(&mut builder.func); + builder.ins().call(memory_copy, &[vmctx, dst, src, len]); } - BulkOp::MemoryFill { len, dst, .. } => { - *dst = builder.ins().iadd(*dst, chunk); - *len = builder.ins().isub(remaining_len, chunk); + BulkOp::MemoryFill { dst, val, len } => { + let memory_fill = self.builtin_functions.memory_fill(&mut builder.func); + builder.ins().call(memory_fill, &[vmctx, dst, val, len]); } - }; - has_chunk_branch(builder, &op, chunk_block, last_chunk_block); - builder.seal_block(chunk_block); - - // Backwards copy: similar to the above but with adjustments on where - // increments/decrements happen. Notably: - // - // * Initial src/end are the final byte address - // * Each chunk starts out by decrementing src/end as opposed to above - // where the increment happens at the end. - // * The final block performs the final decrement before jumping to the - // shared `last_chunk_block` between the forwards/backwards paths. - if let Some((backwards_block, mut op)) = backwards_block { - // Setup `dst=dst+len` and `src=src+len`, then see if we have a - // chunk. - builder.switch_to_block(backwards_block); - builder.seal_block(backwards_block); - let backwards_chunk_block = builder.create_block(); - let backwards_last_chunk_block = builder.create_block(); - let BulkOp::MemoryCopy { dst, src, len, .. } = &mut op else { - unreachable!() - }; - *dst = builder.ins().iadd(*dst, *len); - *src = builder.ins().iadd(*src, *len); - has_chunk_branch( - builder, - &op, - backwards_chunk_block, - backwards_last_chunk_block, - ); - - // Execute the per-chunk backwards copy, adjusting pointers before - // the copy itself. - builder.switch_to_block(backwards_chunk_block); - append_block_params(builder, backwards_chunk_block, &mut op); - let BulkOp::MemoryCopy { dst, src, len, .. } = &mut op else { - unreachable!() - }; - let remaining_len = *len; - *len = chunk; - *dst = builder.ins().isub(*dst, chunk); - *src = builder.ins().isub(*src, chunk); - raw_call(self, builder, &op); - let BulkOp::MemoryCopy { len, .. } = &mut op else { - unreachable!() - }; - *len = builder.ins().isub(remaining_len, chunk); - has_chunk_branch( - builder, - &op, - backwards_chunk_block, - backwards_last_chunk_block, - ); - builder.seal_block(backwards_chunk_block); - - // Final backwards chunk: adjust the dst/src to be their true base - // pointers and then delegate to `last_chunk_block` for the actual - // memcpy. - builder.switch_to_block(backwards_last_chunk_block); - builder.seal_block(backwards_last_chunk_block); - append_block_params(builder, backwards_last_chunk_block, &mut op); - let BulkOp::MemoryCopy { dst, src, len, .. } = &mut op else { - unreachable!() - }; - *dst = builder.ins().isub(*dst, *len); - *src = builder.ins().isub(*src, *len); - builder.ins().jump( - last_chunk_block, - &[(*dst).into(), (*src).into(), (*len).into()], - ); } - - // In the final block we know that the length of the operation is less - // than `chunk`. - builder.switch_to_block(last_chunk_block); - builder.seal_block(last_chunk_block); - append_block_params(builder, last_chunk_block, &mut op); - raw_call(self, builder, &op); + Ok(()) } /// Emits a generic "fill" of `entity` from `dst` for `len` elements, @@ -4021,6 +3715,9 @@ impl FuncEnvironment<'_> { /// `dst` and `len` values must be typed appropriately for `entity`. This /// will perform a bounds-check before actually executing the operation and /// then afterwards will perform the operation. + /// + /// Callers must invoke `pre_translate_bulk_op` before calling this method + /// to properly account for fuel/epoch checks for this bulk operation. fn translate_entity_fill( &mut self, builder: &mut FunctionBuilder<'_>, @@ -4028,7 +3725,6 @@ impl FuncEnvironment<'_> { dst: ir::Value, val: ir::Value, len: ir::Value, - cost_per_unit: u8, ) -> WasmResult<()> { let entity = entity.into(); let idx_type = entity.index_type(self); @@ -4042,37 +3738,22 @@ impl FuncEnvironment<'_> { self.unchecked_cast_wasm_addr_to_native_addr(&mut builder.cursor(), len, idx_type); match entity { - CheckedEntity::Memory(_) => { - self.raw_bulk_memory_operation( - builder, - BulkOp::MemoryFill { - dst: raw_dst_addr, - val, - len: len_ptr, - fuel: BulkFuel { - cost_per_unit, - bytes_per_unit: 1, - }, - }, - ); - } - CheckedEntity::Table { .. } | CheckedEntity::Array { .. } => { - self.emit_raw_array_or_table_fill( - builder, - entity, - raw_dst_addr, + CheckedEntity::Memory(_) => self.raw_bulk_memory_operation( + builder, + BulkOp::MemoryFill { + dst: raw_dst_addr, val, - len_ptr, - cost_per_unit, - )?; + len: len_ptr, + }, + ), + CheckedEntity::Table { .. } | CheckedEntity::Array { .. } => { + self.emit_raw_array_or_table_fill(builder, entity, raw_dst_addr, val, len_ptr) } // Not allowed to be written to in wasm. CheckedEntity::Data { .. } | CheckedEntity::Elem(_) | CheckedEntity::RuntimeData(_) => { unreachable!() } } - - Ok(()) } /// Performs a manual element-by-element fill of `entity`, starting at @@ -4096,7 +3777,6 @@ impl FuncEnvironment<'_> { dst_elem_addr: ir::Value, value: ir::Value, copy_len: ir::Value, - cost_per_element: u8, ) -> WasmResult<()> { let pointer_ty = self.pointer_type(); @@ -4116,19 +3796,14 @@ impl FuncEnvironment<'_> { if entity.allows_memset(self) && let Some(value) = self.fill_value_as_memset(builder, elem_ty, value) { - self.raw_bulk_memory_operation( + return self.raw_bulk_memory_operation( builder, BulkOp::MemoryFill { dst: dst_elem_addr, val: value, len: copy_byte_len, - fuel: BulkFuel { - cost_per_unit: cost_per_element, - bytes_per_unit: u8::try_from(elem_size).unwrap(), - }, }, ); - return Ok(()); } // Funcref values are intern'd when stored on the GC heap, and the @@ -4169,12 +3844,6 @@ impl FuncEnvironment<'_> { builder.insert_block_after(loop_block, current_block); builder.insert_block_after(continue_block, loop_block); - // Before entering the loop below flush our fuel counters to ensure - // that previous instructions' fuel isn't counted once-per-iteration. - if self.tunables.consume_fuel { - self.fuel_increment_var(builder); - } - // Current block: test to see if this is actually an empty copy. If it // is then skip over the entire loop, otherwise enter the loop and // perform the first ieration. @@ -4192,11 +3861,6 @@ impl FuncEnvironment<'_> { // by the element size, then see if we turn again or exit. builder.switch_to_block(loop_block); let elem_addr = builder.append_block_param(loop_block, pointer_ty); - // Consume the configured cost for this element before writing it. - if self.tunables.consume_fuel { - self.fuel_consumed += i64::from(cost_per_element); - } - self.translate_loop_header(builder)?; match entity { CheckedEntity::Table { table, initialized } => { assert!(!is_pre_interned_funcref); @@ -4312,7 +3976,8 @@ impl FuncEnvironment<'_> { len: ir::Value, ) -> WasmResult<()> { let cost = self.tunables.operator_cost.variable().memory_fill_per_byte; - self.translate_entity_fill(builder, memory_index, dst, val, len, cost) + self.pre_translate_bulk_op(builder, len, cost)?; + self.translate_entity_fill(builder, memory_index, dst, val, len) } pub fn translate_memory_init( @@ -4326,6 +3991,7 @@ impl FuncEnvironment<'_> { ) -> WasmResult<()> { let seg_index = DataIndex::from_u32(seg_index); let cost = self.tunables.operator_cost.variable().memory_init_per_byte; + self.pre_translate_bulk_op(builder, len, cost)?; self.translate_entity_copy( builder, memory_index, @@ -4336,7 +4002,6 @@ impl FuncEnvironment<'_> { dst, src, len, - cost, ) } @@ -4379,6 +4044,9 @@ impl FuncEnvironment<'_> { /// of the copy. Both `dst` and `src` have types appropriate to index their /// respective entities, and `len` has a type that's the smaller of the two /// index types. + /// + /// Callers must invoke `pre_translate_bulk_op` before calling this method + /// to properly account for fuel/epoch checks for this bulk operation. fn translate_entity_copy( &mut self, builder: &mut FunctionBuilder<'_>, @@ -4387,7 +4055,6 @@ impl FuncEnvironment<'_> { dst: ir::Value, src: ir::Value, len: ir::Value, - cost_per_unit: u8, ) -> WasmResult<()> { let dst_entity = dst_entity.into(); let src_entity = src_entity.into(); @@ -4449,13 +4116,8 @@ impl FuncEnvironment<'_> { const_len: const_count, src_entity, dst_entity, - fuel: BulkFuel { - cost_per_unit, - bytes_per_unit: 1, - }, }, - ); - Ok(()) + ) } // Tables/arrays are sometimes a memcpy, sometimes a per-element @@ -4470,7 +4132,6 @@ impl FuncEnvironment<'_> { len_ptr, src, const_count, - cost_per_unit, ), // Cannot copy into a data or element segment in wasm. @@ -4686,6 +4347,7 @@ impl FuncEnvironment<'_> { .operator_cost .variable() .table_copy_per_element; + self.pre_translate_bulk_op(builder, len, cost)?; self.translate_entity_copy( builder, CheckedEntity::Table { @@ -4699,7 +4361,6 @@ impl FuncEnvironment<'_> { dst, src, len, - cost, ) } @@ -4729,7 +4390,6 @@ impl FuncEnvironment<'_> { copy_len: ir::Value, src_index: ir::Value, const_count: Option, - cost_per_element: u8, ) -> WasmResult<()> { let pointer_type = self.pointer_type(); assert_eq!(builder.func.dfg.value_type(dst_elem_addr), pointer_type); @@ -4808,7 +4468,7 @@ impl FuncEnvironment<'_> { // parameters (or expand it inline; see `raw_bulk_memory_operation`). if !type_forbids_memcpy && dst_element_size == src_element_size { let const_len = const_count.and_then(|c| c.checked_mul(u64::from(dst_element_size))); - self.raw_bulk_memory_operation( + return self.raw_bulk_memory_operation( builder, BulkOp::MemoryCopy { dst: dst_elem_addr, @@ -4817,13 +4477,8 @@ impl FuncEnvironment<'_> { const_len, src_entity, dst_entity, - fuel: BulkFuel { - cost_per_unit: cost_per_element, - bytes_per_unit: u8::try_from(dst_element_size).unwrap(), - }, }, ); - return Ok(()); } // For other copies, this is a per-element loop. Use the helper to @@ -4837,7 +4492,6 @@ impl FuncEnvironment<'_> { src_elem_addr, copy_len, src_index, - cost_per_element, &|this, builder, dst, src, src_index| { let write_ty = dst_entity.storage_type(this); let val = match src_entity { @@ -5044,7 +4698,6 @@ impl FuncEnvironment<'_> { src_elem_addr: ir::Value, copy_len: ir::Value, src_index: ir::Value, - cost_per_element: u8, copy_one: &dyn Fn( &mut Self, &mut FunctionBuilder<'_>, @@ -5096,13 +4749,6 @@ impl FuncEnvironment<'_> { builder.insert_block_after(backwards_block, forward_block); builder.insert_block_after(done_block, backwards_block); - // Update our local fuel counter, if enabled, before entering the loops - // below. This zeros out `self.fuel_consumed` so we don't consume - // previous fuel on each iteration of the loop. - if self.tunables.consume_fuel { - self.fuel_increment_var(builder); - } - // Terminate `current_block` by testing to see if we're copying any // elements at all. builder @@ -5192,11 +4838,6 @@ impl FuncEnvironment<'_> { let src_cur = builder.append_block_param(forward_block, self.pointer_type()); let src_index = builder.append_block_param(forward_block, src_index_ty); let forward_keepalives = append_keepalive_params(builder, forward_block); - // Consume the configured cost for this element before copying it. - if self.tunables.consume_fuel { - self.fuel_consumed += i64::from(cost_per_element); - } - self.translate_loop_header(builder)?; copy_one(self, builder, dst_cur, src_cur, src_index)?; let dst_next = builder .ins() @@ -5222,10 +4863,6 @@ impl FuncEnvironment<'_> { let src_cur = builder.append_block_param(backwards_block, self.pointer_type()); let src_index = builder.append_block_param(backwards_block, src_index_ty); let backward_keepalives = append_keepalive_params(builder, backwards_block); - if self.tunables.consume_fuel { - self.fuel_consumed += i64::from(cost_per_element); - } - self.translate_loop_header(builder)?; let dst_cur = { let size = builder .ins() @@ -5281,6 +4918,7 @@ impl FuncEnvironment<'_> { .operator_cost .variable() .table_init_per_element; + self.pre_translate_bulk_op(builder, len, cost)?; self.translate_entity_copy( builder, CheckedEntity::Table { @@ -5291,7 +4929,6 @@ impl FuncEnvironment<'_> { dst, src, len, - cost, ) } @@ -5355,6 +4992,78 @@ impl FuncEnvironment<'_> { Ok(builder.ins().ireduce(ir::types::I32, ret)) } + /// Translation prefix before bulk operations such as `memory.copy`. + /// + /// Takes a dynamic value `units` for the size of the operation as well as + /// a `cost_per_unit` configured for this operation. If fuel is enabled + /// this fuel will be consumed, and if epochs are enabled then an epoch + /// check happens. If neither epochs nor fuel are enabled this is a noop. + fn pre_translate_bulk_op( + &mut self, + builder: &mut FunctionBuilder, + units: ir::Value, + cost_per_unit: u8, + ) -> WasmResult<()> { + let const_units = + Self::value_as_const_int(builder, units).map(|c| i64::try_from(c).unwrap_or(i64::MAX)); + + if self.tunables.consume_fuel && cost_per_unit > 0 { + match const_units { + // Fold constant costs directly into internal state. + Some(units) => { + self.fuel_consumed = self + .fuel_consumed + .saturating_add(units.saturating_mul(i64::from(cost_per_unit))) + } + + None => { + // Note that fuel is always a 64-bit counter. + // + // Also note that the cost is clamped to `i64::MAX` to + // prevent fuel counter overflows since `cost` is otherwise + // an untrusted value. + let units_clamped64 = match builder.func.dfg.value_type(units) { + ir::types::I32 => { + let units64 = builder.ins().uextend(ir::types::I64, units); + builder.ins().imul_imm_u(units64, i64::from(cost_per_unit)) + } + ir::types::I64 => { + let fuel = builder.ins().imul_imm_u(units, i64::from(cost_per_unit)); + let max = builder.ins().iconst(ir::types::I64, i64::MAX); + let max_units = builder + .ins() + .iconst(I64, i64::MAX / i64::from(cost_per_unit)); + let saturate = + builder + .ins() + .icmp(IntCC::UnsignedGreaterThan, units, max_units); + builder.ins().select(saturate, max, fuel) + } + _ => unreachable!(), + }; + self.fuel_increment_var(builder); + let fuel = builder.use_var(self.fuel_var); + let fuel = builder.ins().iadd(fuel, units_clamped64); + builder.def_var(self.fuel_var, fuel); + } + } + } + + // Skip explicit fuel/epoch checks for operations which are + // subjectively, and statically, considered cheap. + const SMALL_BULK_OP_COST: i64 = 128; + if let Some(units) = const_units + && let Some(cost) = units.checked_mul(i64::from(cost_per_unit)) + && cost <= SMALL_BULK_OP_COST + { + return Ok(()); + } + + // This isn't a loop header but for fuel/epoch purposes it's the same + // thing. + self.translate_loop_header(builder) + } + pub fn translate_loop_header(&mut self, builder: &mut FunctionBuilder) -> WasmResult<()> { // Additionally if enabled check how much fuel we have remaining to see // if we've run out by this point. @@ -6195,6 +5904,7 @@ impl FuncEnvironment<'_> { .operator_cost .variable() .table_fill_per_element; + self.pre_translate_bulk_op(builder, len, cost)?; self.translate_entity_fill( builder, CheckedEntity::Table { @@ -6204,7 +5914,6 @@ impl FuncEnvironment<'_> { dst, val, len, - cost, ) } @@ -6240,7 +5949,7 @@ impl FuncEnvironment<'_> { .operator_cost .variable() .table_init_per_element; - self.consume_variable_fuel(builder, segment_len, cost); + self.pre_translate_bulk_op(builder, segment_len, cost)?; // Re-use the `table.set` translation for making this a simple function // to define. That re-executes the bounds check which is a bit @@ -6313,7 +6022,8 @@ impl FuncEnvironment<'_> { let len = self.load_runtime_data_length(builder, data); let start = builder.ins().iconst(I32, 0); let cost = self.tunables.operator_cost.variable().memory_init_per_byte; - self.translate_entity_copy(builder, memory, data, offset, start, len, cost)?; + self.pre_translate_bulk_op(builder, len, cost)?; + self.translate_entity_copy(builder, memory, data, offset, start, len)?; // Finalize control-flow for the `MemorySegmentOffset::Static` case // above. @@ -6483,7 +6193,6 @@ enum BulkOp { const_len: Option, src_entity: CheckedEntity, dst_entity: CheckedEntity, - fuel: BulkFuel, }, /// A `memory.fill` operation, setting all bytes of `dst` to `val`. @@ -6496,31 +6205,9 @@ enum BulkOp { dst: ir::Value, val: ir::Value, len: ir::Value, - fuel: BulkFuel, }, } -impl BulkOp { - fn len(&self) -> ir::Value { - match self { - BulkOp::MemoryCopy { len, .. } | BulkOp::MemoryFill { len, .. } => *len, - } - } - - fn fuel(&self) -> BulkFuel { - match self { - BulkOp::MemoryCopy { fuel, .. } | BulkOp::MemoryFill { fuel, .. } => *fuel, - } - } -} - -#[derive(Copy, Clone)] -struct BulkFuel { - cost_per_unit: u8, - /// Number of copied bytes represented by one billable unit. - bytes_per_unit: u8, -} - /// A list of entities which can participate in various kinds of bulk operations /// in wasm. /// diff --git a/crates/cranelift/src/func_environ/gc.rs b/crates/cranelift/src/func_environ/gc.rs index ce984ce7aec8..22c576924029 100644 --- a/crates/cranelift/src/func_environ/gc.rs +++ b/crates/cranelift/src/func_environ/gc.rs @@ -771,9 +771,15 @@ pub fn translate_array_new( array_type_index: TypeIndex, elem: ir::Value, len: ir::Value, - cost_per_element: u8, ) -> WasmResult { log::trace!("translate_array_new({array_type_index:?}, {elem:?}, {len:?})"); + let cost = func_env + .tunables + .operator_cost + .variable() + .array_new_per_element; + func_env.pre_translate_bulk_op(builder, len, cost)?; + let result = gc_compiler(func_env)?.alloc_uninit_array(func_env, builder, array_type_index, len)?; let zero = builder.ins().iconst(ir::types::I32, 0); @@ -788,7 +794,6 @@ pub fn translate_array_new( zero, elem, len, - cost_per_element, )?; log::trace!("translate_array_new(..) -> {result:?}"); Ok(result) @@ -799,9 +804,14 @@ pub fn translate_array_new_default( builder: &mut FunctionBuilder, array_type_index: TypeIndex, len: ir::Value, - cost_per_element: u8, ) -> WasmResult { log::trace!("translate_array_new_default({array_type_index:?}, {len:?})"); + let cost = func_env + .tunables + .operator_cost + .variable() + .array_new_default_per_element; + func_env.pre_translate_bulk_op(builder, len, cost)?; let interned_ty = func_env.module.types[array_type_index].unwrap_module_type_index(); let array_ty = func_env.types.unwrap_array(interned_ty)?; @@ -820,7 +830,6 @@ pub fn translate_array_new_default( zero, elem, len, - cost_per_element, )?; Ok(result) } @@ -1724,8 +1733,10 @@ pub fn translate_array_new_entity( entity: CheckedEntity, entity_offset: ir::Value, len: ir::Value, - cost_per_element: u8, + cost_per_unit: u8, ) -> WasmResult { + env.pre_translate_bulk_op(builder, len, cost_per_unit)?; + // Before actually allocating this array first do a bounds-check on the // passive entity itself. let interned_type_index = env.module.types[array_type_index].unwrap_module_type_index(); @@ -1744,7 +1755,6 @@ pub fn translate_array_new_entity( dst, entity_offset, len, - cost_per_element, )?; Ok(array) diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs index 1ceb91a4ac56..17edbcbd9da3 100644 --- a/crates/wasmtime/src/config.rs +++ b/crates/wasmtime/src/config.rs @@ -690,10 +690,6 @@ impl Config { /// signal handler), then we can ensure that all async code will /// yield to the executor within a bounded time. /// - /// The deadline check cannot be avoided by malicious wasm code. It is safe - /// to use epoch deadlines to limit the execution time of untrusted - /// code. - /// /// The [`Store`](crate::Store) tracks the deadline, and controls /// what happens when the deadline is reached during /// execution. Several behaviors are possible: @@ -739,6 +735,27 @@ impl Config { /// computation and have the desired effect of cancelling a blocking /// operation when a timeout expires. /// + /// ## Limitations with malicious guests + /// + /// Epochs are designed to handle malicious WebAssembly guests -- the + /// deadline check cannot be avoided by WebAssembly code. It is safe to use + /// epoch deadlines to limit the execution time of untrusted code. + /// + /// Note, though, that a current limitation to this is that + /// bulk-data-transfer instructions, such as `memory.copy`, only check the + /// epoch once at the start of the operation. These operations can take a + /// variable amount of time to complete based on how many bytes are being + /// copied. This means that the maximal time slice a guest might take is + /// the maximum of the epoch interval and the largest + /// memory-copy-style-instruction executed. The size of a copy is bounded + /// on the size of linear memory or GC heap size. In the limit, however, a + /// guest using a 64-bit linear memory with a 128GiB size could issue a + /// 128GiB `memory.copy` which would have no preemption within the + /// instruction itself. Hosts which need strict time limits for guests right + /// now are recommended to ensure that the store's allocated heap size + /// (linear memory + GC heap) are bounded with a + /// [`ResourceLimiter`](crate::ResourceLimiter). + /// /// ## When to use fuel vs. epochs /// /// In general, epoch-based interruption results in faster diff --git a/crates/wasmtime/src/runtime/store.rs b/crates/wasmtime/src/runtime/store.rs index 4959c741d918..49804c46e888 100644 --- a/crates/wasmtime/src/runtime/store.rs +++ b/crates/wasmtime/src/runtime/store.rs @@ -1041,6 +1041,10 @@ impl Store { /// The `interval` parameter indicates how much fuel should be /// consumed between yields of an async future. When fuel runs out wasm will trap. /// + /// For limitations related to consumption of fuel and when yield points are + /// injected, see the discussion in + /// [`Config::epoch_interruption`](crate::Config::epoch_interruption). + /// /// # Error /// /// This method will error if fuel is not enabled or `interval` is diff --git a/tests/all/epoch_interruption.rs b/tests/all/epoch_interruption.rs index 6a1c1ff7bb92..1d2b4664f3dc 100644 --- a/tests/all/epoch_interruption.rs +++ b/tests/all/epoch_interruption.rs @@ -496,3 +496,130 @@ async fn drop_future_on_epoch_yield(config: &mut Config) -> Result<()> { assert_eq!(true, alive_flag.load(Ordering::Acquire)); Ok(()) } + +#[test] +fn memory_grow_in_epoch_callback() -> Result<()> { + let mut config = Config::new(); + config.epoch_interruption(true); + config.memory_reservation(0); + config.memory_reservation_for_growth(0); + config.memory_guard_size(0); + config.memory_may_move(true); + config.memory_init_cow(false); + let engine = Engine::new(&config)?; + let module = Module::new( + &engine, + r#" + (module + (memory (export "mem") 1) + (func (export "go") + (memory.fill (i32.const 0) (i32.const 0x41) (i32.const 65536)))) + "#, + )?; + + let mut store: Store> = Store::new(&engine, None); + store.set_epoch_deadline(1); + store.epoch_deadline_callback(move |mut cx| { + if let Some(mem) = *cx.data() { + mem.grow(&mut cx, 5)?; + } + Ok(UpdateDeadline::Continue(0)) + }); + + let instance = Instance::new(&mut store, &module, &[])?; + let mem = instance.get_memory(&mut store, "mem").unwrap(); + *store.data_mut() = Some(mem); + engine.increment_epoch(); + + instance + .get_typed_func::<(), ()>(&mut store, "go")? + .call(&mut store, ())?; + + let data = mem.data(&store); + assert_eq!(data[0], 0x41); + assert_eq!(data[65535], 0x41); + Ok(()) +} + +#[test] +fn table_grow_in_epoch_callback() -> Result<()> { + let mut config = Config::new(); + config.epoch_interruption(true); + let engine = Engine::new(&config)?; + let module = Module::new( + &engine, + r#" + (module + (table $t (export "t") 1 funcref) + (func (export "go") + (table.fill $t (i32.const 0) (ref.null func) (i32.const 1)))) + "#, + )?; + + let mut store: Store> = Store::new(&engine, None); + store.set_epoch_deadline(1); + store.epoch_deadline_callback(move |mut cx| { + if let Some(t) = *cx.data() { + t.grow(&mut cx, 5, Ref::Func(None))?; + } + Ok(UpdateDeadline::Continue(0)) + }); + + let instance = Instance::new(&mut store, &module, &[])?; + let t = instance.get_table(&mut store, "t").unwrap(); + *store.data_mut() = Some(t); + engine.increment_epoch(); + + instance + .get_typed_func::<(), ()>(&mut store, "go")? + .call(&mut store, ())?; + Ok(()) +} + +#[test] +fn gc_during_epoch_callback() -> Result<()> { + let mut config = Config::new(); + config.epoch_interruption(true); + let engine = Engine::new(&config)?; + let module = Module::new( + &engine, + r#" + (module + (type $box (struct (field i32))) + (type $arr (array (mut (ref null $box)))) + (global $sink (mut (ref null $arr)) (ref.null $arr)) + (func $mk (param $n i32) (result (ref $arr)) + (array.new_default $arr (local.get $n))) + (func (export "run") (param $n i32) (result i32) + (local $i i32) + (block $done + (loop $l + (br_if $done (i32.ge_u (local.get $i) (i32.const 40))) + (array.fill $arr (call $mk (local.get $n)) (i32.const 0) + (struct.new $box (i32.const 7)) (local.get $n)) + (global.set $sink (call $mk (i32.const 8))) + (local.set $i (i32.add (local.get $i) (i32.const 1))) + (br $l) + ) + ) + (i32.mul (local.get $n) (i32.const 7)))) + "#, + )?; + + let mut store = Store::new(&engine, ()); + store.set_epoch_deadline(1); + store.epoch_deadline_callback(|mut caller| { + caller.gc(None)?; + Ok(UpdateDeadline::Continue(0)) + }); + engine.increment_epoch(); + + let instance = Instance::new(&mut store, &module, &[])?; + let run = instance.get_typed_func::(&mut store, "run")?; + let n = 200; + for i in 0..5 { + let got = run.call(&mut store, n)?; + assert_eq!(got, 7 * n, "iteration {i} read back {got}"); + } + Ok(()) +} diff --git a/tests/all/fuel.rs b/tests/all/fuel.rs index 445a6c1cbd4b..d90ee5c97209 100644 --- a/tests/all/fuel.rs +++ b/tests/all/fuel.rs @@ -966,6 +966,88 @@ fn table64_variable_operator_cost_saturates(config: &mut Config) -> Result<()> { // i64::MAX * 2 must saturate at i64::MAX rather than wrap to -2. let error = grow.call(&mut store, i64::MAX).unwrap_err(); assert_eq!(error.downcast::().unwrap(), Trap::OutOfFuel); + Ok(()) +} + +#[test] +#[cfg_attr(miri, ignore)] +fn huge_table64_grow_cannot_mint_fuel() -> Result<()> { + huge_table64_grow_cannot_mint_fuel_impl( + r#" + (module + (table $t i64 0 0x10000 (ref null func)) + (func (export "run") (param $delta i64) + (loop $l + (drop (table.grow $t (ref.null func) (local.get $delta))) + (br $l)))) + "#, + ) +} +#[test] +#[cfg_attr(miri, ignore)] +fn huge_table64_grow_cannot_mint_fuel_const() -> Result<()> { + huge_table64_grow_cannot_mint_fuel_impl( + r#" + (module + (table $t i64 0 0x10000 (ref null func)) + (func (export "run") (param $delta i64) + (loop $l + (drop (table.grow $t (ref.null func) (i64.const -500))) + (br $l)))) + "#, + ) +} + +fn huge_table64_grow_cannot_mint_fuel_impl(wat: &str) -> Result<()> { + let mut config = Config::new(); + config.consume_fuel(true); + let engine = Engine::new(&config)?; + let module = Module::new(&engine, wat)?; + + let mut store = Store::new(&engine, ()); + store.set_fuel(100_000)?; + let instance = Instance::new(&mut store, &module, &[])?; + let run = instance.get_typed_func::(&mut store, "run")?; + + let trap = run.call(&mut store, -500).unwrap_err().downcast::()?; + assert_eq!(trap, Trap::OutOfFuel); + assert_eq!(store.get_fuel()?, 0); + Ok(()) +} + +#[test] +fn fuel_around_table_grow() -> Result<()> { + let mut config = Config::new(); + config.consume_fuel(true); + let engine = Engine::new(&config)?; + let module = Module::new( + &engine, + r#" + (module + (type $ft (func)) + (func $f (type $ft)) + (table $t 1 10000000 (ref $ft) (ref.func $f)) + (func (export "grow") (result i32) + (table.grow $t (ref.func $f) (i32.const 9999999))) + (func (export "call") (param i32) + (call_indirect $t (type $ft) (local.get 0)))) + "#, + )?; + + let mut store = Store::new(&engine, ()); + store.set_fuel(2)?; + let instance = Instance::new(&mut store, &module, &[])?; + let grow = instance.get_typed_func::<(), i32>(&mut store, "grow")?; + let trap = grow.call(&mut store, ()).unwrap_err().downcast::()?; + assert_eq!(trap, Trap::OutOfFuel); + + store.set_fuel(u64::MAX)?; + let call = instance.get_typed_func::(&mut store, "call")?; + let trap = call + .call(&mut store, 9999999) + .unwrap_err() + .downcast::()?; + assert_eq!(trap, Trap::TableOutOfBounds); Ok(()) } diff --git a/tests/all/gc.rs b/tests/all/gc.rs index 3058855e3335..79a049da8afd 100644 --- a/tests/all/gc.rs +++ b/tests/all/gc.rs @@ -3788,3 +3788,107 @@ fn initial_size_larger_than_reservation() -> Result<()> { Ok(()) } + +#[test] +#[cfg_attr(miri, ignore)] +fn array_fill_i64_gc_during_epoch() -> Result<()> { + gc_during_epoch( + r#" + (module + (type $arr (array (mut i64))) + (type $box (struct (field i32))) + (func (export "run") (param $n i32) (result i32) + (local $a (ref null $arr)) (local $i i32) (local $s i32) + (local.set $a (array.new_default $arr (local.get $n))) + ;; Keep the collector busy so it has something to move. + (drop (struct.new $box (i32.const 1))) + (array.fill $arr (local.get $a) (i32.const 0) (i64.const 7) (local.get $n)) + (block $done (loop $l + (br_if $done (i32.ge_u (local.get $i) (local.get $n))) + (local.set $s (i32.add (local.get $s) + (i32.wrap_i64 (array.get $arr (local.get $a) (local.get $i))))) + (local.set $i (i32.add (local.get $i) (i32.const 1))) + (br $l))) + (local.get $s))) + "#, + ) +} + +#[test] +#[cfg_attr(miri, ignore)] +fn array_new_gc_during_epoch() -> Result<()> { + gc_during_epoch( + r#" + (module + (type $box (struct (field i32))) + (type $arr (array (mut (ref null $box)))) + (func (export "run") (param $n i32) (result i32) + (local $a (ref null $arr)) (local $i i32) (local $s i32) + (local.set $a (array.new $arr (struct.new $box (i32.const 7)) (local.get $n))) + (block $done (loop $l + (br_if $done (i32.ge_u (local.get $i) (local.get $n))) + (local.set $s (i32.add (local.get $s) + (struct.get $box 0 (ref.as_non_null + (array.get $arr (local.get $a) (local.get $i)))))) + (local.set $i (i32.add (local.get $i) (i32.const 1))) + (br $l))) + (local.get $s))) + "#, + ) +} + +#[test] +#[cfg_attr(miri, ignore)] +fn array_copy_gc_during_epoch() -> Result<()> { + gc_during_epoch( + r#" + (module + (type $box (struct (field i32))) + (type $arr (array (mut (ref null $box)))) + (func (export "run") (param $n i32) (result i32) + (local $a (ref null $arr)) (local $b (ref null $arr)) + (local $i i32) (local $s i32) + (local.set $a (array.new_default $arr (local.get $n))) + (local.set $b (array.new_default $arr (local.get $n))) + (array.fill $arr (local.get $b) (i32.const 0) + (struct.new $box (i32.const 7)) (local.get $n)) + (array.copy $arr $arr (local.get $a) (i32.const 0) + (local.get $b) (i32.const 0) (local.get $n)) + (block $done (loop $l + (br_if $done (i32.ge_u (local.get $i) (local.get $n))) + (local.set $s (i32.add (local.get $s) + (struct.get $box 0 (ref.as_non_null + (array.get $arr (local.get $a) (local.get $i)))))) + (local.set $i (i32.add (local.get $i) (i32.const 1))) + (br $l))) + (local.get $s))) + "#, + ) +} + +fn gc_during_epoch(wat: &str) -> Result<()> { + let mut config = Config::new(); + config.epoch_interruption(true); + let engine = Engine::new(&config)?; + let module = Module::new(&engine, wat)?; + + let mut store = Store::new(&engine, ()); + store.set_epoch_deadline(1); + store.epoch_deadline_callback(|mut caller| { + caller.gc(None)?; + Ok(UpdateDeadline::Continue(0)) + }); + engine.increment_epoch(); + + let instance = Instance::new(&mut store, &module, &[])?; + let f = instance.get_typed_func::(&mut store, "run")?; + + let n = 100; + for i in 0..5 { + match f.call(&mut store, n) { + Ok(got) => assert_eq!(got, 7 * n, "iteration {i} read back {got}"), + Err(e) => panic!("iteration {i} failed: {e:?}"), + } + } + Ok(()) +} diff --git a/tests/disas/gc/array-copy-with-fuel.wat b/tests/disas/gc/array-copy-with-fuel.wat index aeab1b17ebd3..8371c0eed235 100644 --- a/tests/disas/gc/array-copy-with-fuel.wat +++ b/tests/disas/gc/array-copy-with-fuel.wat @@ -28,10 +28,10 @@ ;; stack_limit = gv2 ;; ;; block0(v0: i64, v1: i64, v2: i32, v3: i32, v4: i32, v5: i32, v6: i32): -;; v181 = stack_addr.i64 ss0 -;; store notrap aligned region6 v2, v181 -;; v182 = stack_addr.i64 ss1 -;; store notrap aligned region7 v4, v182 +;; v162 = stack_addr.i64 ss0 +;; store notrap aligned region6 v2, v162 +;; v163 = stack_addr.i64 ss1 +;; store notrap aligned region7 v4, v163 ;; @0020 v7 = load.i64 notrap aligned readonly can_move region0 v0+8 ;; @0020 v8 = load.i64 notrap aligned region2 v7 ;; @0020 v9 = iconst.i64 1 @@ -41,131 +41,108 @@ ;; @0020 brif v12, block2, block3(v10) ;; ;; block2: -;; v191 = iadd.i64 v8, v9 ; v9 = 1 -;; @0020 store notrap aligned region2 v191, v7 +;; v172 = iadd.i64 v8, v9 ; v9 = 1 +;; @0020 store notrap aligned region2 v172, v7 ;; @0020 v14 = call fn0(v0), stack_map=[i32 @ ss0+0, i32 @ ss1+0] ;; @0020 v16 = load.i64 notrap aligned region2 v7 ;; @0020 jump block3(v16) ;; -;; block3(v89: i64): -;; v180 = load.i32 notrap aligned region6 v181 -;; @002b trapz v180, user16 -;; @002b v24 = load.i64 notrap aligned readonly can_move region3 v7+32 -;; @002b v22 = uextend.i64 v180 -;; @002b v25 = iadd v24, v22 -;; @002b v26 = iconst.i64 16 -;; @002b v27 = iadd v25, v26 ; v26 = 16 -;; @002b v28 = load.i32 user2 readonly region5 v27 -;; @002b v30 = uextend.i64 v3 -;; @002b v31 = uextend.i64 v6 -;; @002b v34 = iadd v30, v31 -;; @002b v29 = uextend.i64 v28 -;; @002b v35 = icmp ugt v34, v29 -;; @002b trapnz v35, user17 -;; v174 = load.i32 notrap aligned region7 v182 -;; @002b trapz v174, user16 -;; @002b v46 = uextend.i64 v174 -;; @002b v49 = iadd v24, v46 -;; @002b v51 = iadd v49, v26 ; v26 = 16 -;; @002b v52 = load.i32 user2 readonly region5 v51 -;; @002b v54 = uextend.i64 v5 -;; @002b v58 = iadd v54, v31 -;; @002b v53 = uextend.i64 v52 -;; @002b v59 = icmp ugt v58, v53 -;; @002b trapnz v59, user17 -;; @002b v78 = load.i64 notrap aligned region4 v7+40 -;; @002b v40 = iconst.i64 20 -;; @002b v41 = iadd v25, v40 ; v40 = 20 -;; v184 = iconst.i64 2 -;; v185 = ishl v30, v184 ; v184 = 2 -;; @002b v45 = iadd v41, v185 -;; v189 = ishl v31, v184 ; v184 = 2 -;; @002b v80 = uadd_overflow_trap v45, v189, user2 -;; @002b v79 = iadd v24, v78 -;; @002b v81 = icmp ugt v80, v79 -;; @002b trapnz v81, user2 -;; @002b v65 = iadd v49, v40 ; v40 = 20 -;; v187 = ishl v54, v184 ; v184 = 2 -;; @002b v69 = iadd v65, v187 -;; @002b v87 = uadd_overflow_trap v69, v189, user2 -;; @002b v88 = icmp ugt v87, v79 -;; @002b trapnz v88, user2 -;; @002b v90 = iconst.i64 6 -;; @002b v91 = iadd v89, v90 ; v90 = 6 -;; @002b brif.i32 v6, block4, block7(v91) +;; block3(v25: i64): +;; @002b v26 = iconst.i64 6 +;; @002b v27 = iadd v25, v26 ; v26 = 6 +;; @002b v22 = uextend.i64 v6 +;; @002b v28 = iadd v27, v22 +;; v173 = iconst.i64 0 +;; v174 = icmp sge v28, v173 ; v173 = 0 +;; @002b brif v174, block4, block5(v28) ;; ;; block4: -;; v162 = load.i32 notrap aligned region6 v181 -;; v164 = load.i32 notrap aligned region7 v182 -;; @002b v92 = icmp.i64 ult v45, v69 -;; v192 = iadd.i64 v89, v90 ; v90 = 6 -;; @002b v97 = iadd.i64 v45, v189 -;; @002b v98 = iadd.i64 v69, v189 -;; @002b v100 = iadd.i32 v5, v6 -;; @002b v43 = iconst.i64 4 -;; @002b v141 = iconst.i32 1 -;; @002b brif v92, block5(v45, v69, v5, v162, v164, v192), block6(v97, v98, v100, v162, v164, v192) +;; @002b store.i64 notrap aligned region2 v28, v7 +;; @002b v32 = call fn0(v0), stack_map=[i32 @ ss0+0, i32 @ ss1+0] +;; @002b v34 = load.i64 notrap aligned region2 v7 +;; @002b jump block5(v34) ;; -;; block5(v101: i64, v102: i64, v103: i32, v104: i32, v105: i32, v106: i64): -;; store notrap aligned region6 v104, v181 -;; store notrap aligned region7 v105, v182 -;; v202 = iconst.i64 1 -;; v203 = iadd v106, v202 ; v202 = 1 -;; v204 = iconst.i64 0 -;; v205 = icmp sge v203, v204 ; v204 = 0 -;; @002b brif v205, block8, block9(v203) +;; block5(v139: i64): +;; v161 = load.i32 notrap aligned region6 v162 +;; @002b trapz v161, user16 +;; @002b v37 = load.i64 notrap aligned readonly can_move region3 v7+32 +;; @002b v35 = uextend.i64 v161 +;; @002b v38 = iadd v37, v35 +;; @002b v39 = iconst.i64 16 +;; @002b v40 = iadd v38, v39 ; v39 = 16 +;; @002b v41 = load.i32 user2 readonly region5 v40 +;; @002b v43 = uextend.i64 v3 +;; @002b v47 = iadd v43, v22 +;; @002b v42 = uextend.i64 v41 +;; @002b v48 = icmp ugt v47, v42 +;; @002b trapnz v48, user17 +;; v155 = load.i32 notrap aligned region7 v163 +;; @002b trapz v155, user16 +;; @002b v59 = uextend.i64 v155 +;; @002b v62 = iadd v37, v59 +;; @002b v64 = iadd v62, v39 ; v39 = 16 +;; @002b v65 = load.i32 user2 readonly region5 v64 +;; @002b v67 = uextend.i64 v5 +;; @002b v71 = iadd v67, v22 +;; @002b v66 = uextend.i64 v65 +;; @002b v72 = icmp ugt v71, v66 +;; @002b trapnz v72, user17 +;; @002b v91 = load.i64 notrap aligned region4 v7+40 +;; @002b v53 = iconst.i64 20 +;; @002b v54 = iadd v38, v53 ; v53 = 20 +;; v165 = iconst.i64 2 +;; v166 = ishl v43, v165 ; v165 = 2 +;; @002b v58 = iadd v54, v166 +;; v170 = ishl.i64 v22, v165 ; v165 = 2 +;; @002b v93 = uadd_overflow_trap v58, v170, user2 +;; @002b v92 = iadd v37, v91 +;; @002b v94 = icmp ugt v93, v92 +;; @002b trapnz v94, user2 +;; @002b v78 = iadd v62, v53 ; v53 = 20 +;; v168 = ishl v67, v165 ; v165 = 2 +;; @002b v82 = iadd v78, v168 +;; @002b v100 = uadd_overflow_trap v82, v170, user2 +;; @002b v101 = icmp ugt v100, v92 +;; @002b trapnz v101, user2 +;; @002b brif.i32 v6, block6, block9 ;; -;; block6(v123: i64, v124: i64, v125: i32, v126: i32, v127: i32, v128: i64): -;; store notrap aligned region7 v126, v182 -;; store notrap aligned region6 v127, v181 -;; v193 = iconst.i64 1 -;; v194 = iadd v128, v193 ; v193 = 1 -;; v195 = iconst.i64 0 -;; v196 = icmp sge v194, v195 ; v195 = 0 -;; @002b brif v196, block10, block11(v194) +;; block6: +;; v143 = load.i32 notrap aligned region6 v162 +;; v145 = load.i32 notrap aligned region7 v163 +;; @002b v102 = icmp.i64 ult v58, v82 +;; @002b v107 = iadd.i64 v58, v170 +;; @002b v108 = iadd.i64 v82, v170 +;; @002b v110 = iadd.i32 v5, v6 +;; @002b v56 = iconst.i64 4 +;; @002b v133 = iconst.i32 1 +;; @002b brif v102, block7(v58, v82, v5), block8(v107, v108, v110) ;; -;; block7(v148: i64): -;; @002f jump block1 -;; -;; block8: -;; @002b store.i64 notrap aligned region2 v203, v7 -;; @002b v112 = call fn0(v0), stack_map=[i32 @ ss0+0, i32 @ ss1+0] -;; @002b v114 = load.i64 notrap aligned region2 v7 -;; @002b jump block9(v114) +;; block7(v111: i64, v112: i64, v113: i32): +;; @002b v116 = load.i32 user2 little region5 v112 +;; @002b store user2 little region5 v116, v111 +;; v180 = iconst.i64 4 +;; v181 = iadd v112, v180 ; v180 = 4 +;; @002b v123 = icmp eq v181, v108 +;; v182 = iadd v111, v180 ; v180 = 4 +;; v183 = iconst.i32 1 +;; v184 = iadd v113, v183 ; v183 = 1 +;; @002b brif v123, block9, block7(v182, v181, v184) ;; -;; block9(v145: i64): -;; @002b v115 = load.i32 user2 little region5 v102 -;; @002b store user2 little region5 v115, v101 -;; v150 = load.i32 notrap aligned region6 v181 -;; v152 = load.i32 notrap aligned region7 v182 -;; v206 = iconst.i64 4 -;; v207 = iadd.i64 v102, v206 ; v206 = 4 -;; @002b v122 = icmp eq v207, v98 -;; v208 = iadd.i64 v101, v206 ; v206 = 4 -;; v209 = iconst.i32 1 -;; v210 = iadd.i32 v103, v209 ; v209 = 1 -;; @002b brif v122, block7(v145), block5(v208, v207, v210, v150, v152, v145) +;; block8(v124: i64, v125: i64, v126: i32): +;; v175 = iconst.i64 4 +;; v176 = isub v125, v175 ; v175 = 4 +;; @002b v135 = load.i32 user2 little region5 v176 +;; v177 = isub v124, v175 ; v175 = 4 +;; @002b store user2 little region5 v135, v177 +;; @002b v136 = icmp eq v176, v82 +;; v178 = iconst.i32 1 +;; v179 = isub v126, v178 ; v178 = 1 +;; @002b brif v136, block9, block8(v177, v176, v179) ;; -;; block10: -;; @002b store.i64 notrap aligned region2 v194, v7 -;; @002b v134 = call fn0(v0), stack_map=[i32 @ ss1+0, i32 @ ss0+0] -;; @002b v136 = load.i64 notrap aligned region2 v7 -;; @002b jump block11(v136) -;; -;; block11(v146: i64): -;; v197 = iconst.i64 4 -;; v198 = isub.i64 v124, v197 ; v197 = 4 -;; @002b v143 = load.i32 user2 little region5 v198 -;; v199 = isub.i64 v123, v197 ; v197 = 4 -;; @002b store user2 little region5 v143, v199 -;; v156 = load.i32 notrap aligned region7 v182 -;; v158 = load.i32 notrap aligned region6 v181 -;; @002b v144 = icmp eq v198, v69 -;; v200 = iconst.i32 1 -;; v201 = isub.i32 v125, v200 ; v200 = 1 -;; @002b brif v144, block7(v146), block6(v199, v198, v201, v156, v158, v146) +;; block9: +;; @002f jump block1 ;; ;; block1: -;; @002f store.i64 notrap aligned region2 v148, v7 +;; @002f store.i64 notrap aligned region2 v139, v7 ;; @002f return ;; } diff --git a/tests/disas/memory-copy-epochs.wat b/tests/disas/memory-copy-epochs.wat index f712972b8774..d6113471a161 100644 --- a/tests/disas/memory-copy-epochs.wat +++ b/tests/disas/memory-copy-epochs.wat @@ -37,101 +37,35 @@ ;; @001e v10 = call fn0(v0) ;; @001e jump block2(v10) ;; -;; block2(v59: i64): -;; @0025 v14 = load.i64 notrap aligned region6 v0+64 -;; @0025 v15 = uextend.i64 v2 -;; @0025 v16 = uextend.i64 v4 -;; @0025 v19 = iadd v15, v16 -;; @0025 v20 = icmp ugt v19, v14 -;; @0025 trapnz v20, heap_oob -;; @0025 v27 = uextend.i64 v3 -;; @0025 v31 = iadd v27, v16 -;; @0025 v32 = icmp ugt v31, v14 -;; @0025 trapnz v32, heap_oob -;; @0025 v21 = load.i64 notrap aligned readonly can_move region5 v0+56 -;; @0025 v37 = iadd v21, v27 -;; @0025 v25 = iadd v21, v15 -;; @0025 v40 = icmp ugt v37, v25 -;; @0025 brif v40, block6, block7 -;; -;; block4(v42: i64, v43: i64, v44: i64, v47: i64): -;; @0025 v46 = load.i64 notrap aligned region3 v5 -;; @0025 v48 = icmp uge v46, v47 -;; @0025 brif v48, block9, block8(v47) -;; -;; block5(v86: i64, v87: i64, v88: i64, v92: i64): -;; @0025 v91 = load.i64 notrap aligned region3 v5 -;; @0025 v94 = icmp uge v91, v92 -;; @0025 brif v94, block17, block16 -;; -;; block6: -;; v108 = iconst.i64 0x0800_0000 -;; v109 = icmp.i64 ugt v16, v108 ; v108 = 0x0800_0000 -;; @0025 brif v109, block4(v25, v37, v16, v59), block5(v25, v37, v16, v59) -;; -;; block9 cold: -;; @0025 v50 = load.i64 notrap aligned region4 v7+8 -;; @0025 v51 = icmp.i64 uge v46, v50 -;; @0025 brif v51, block10, block8(v50) -;; -;; block10 cold: -;; @0025 v52 = call fn0(v0) -;; @0025 jump block8(v52) -;; -;; block8(v60: i64): -;; @0025 call fn1(v0, v42, v43, v108) ; v108 = 0x0800_0000 -;; @0025 v55 = isub.i64 v44, v108 ; v108 = 0x0800_0000 -;; @0025 v56 = icmp ugt v55, v108 ; v108 = 0x0800_0000 -;; @0025 v53 = iadd.i64 v42, v108 ; v108 = 0x0800_0000 -;; @0025 v54 = iadd.i64 v43, v108 ; v108 = 0x0800_0000 -;; @0025 brif v56, block4(v53, v54, v55, v60), block5(v53, v54, v55, v60) -;; -;; block7: -;; @0025 v39 = iconst.i64 0x0800_0000 -;; @0025 v63 = icmp.i64 ugt v16, v39 ; v39 = 0x0800_0000 -;; @0025 v61 = iadd.i64 v25, v16 -;; @0025 v62 = iadd.i64 v37, v16 -;; @0025 brif v63, block11(v61, v62, v16, v59), block12(v61, v62, v16, v59) -;; -;; block11(v64: i64, v65: i64, v66: i64, v71: i64): -;; @0025 v70 = load.i64 notrap aligned region3 v5 -;; @0025 v72 = icmp uge v70, v71 -;; @0025 brif v72, block14, block13(v71) -;; -;; block14 cold: -;; @0025 v74 = load.i64 notrap aligned region4 v7+8 -;; @0025 v75 = icmp.i64 uge v70, v74 -;; @0025 brif v75, block15, block13(v74) -;; -;; block15 cold: -;; @0025 v76 = call fn0(v0) -;; @0025 jump block13(v76) -;; -;; block13(v80: i64): -;; v103 = iconst.i64 0x0800_0000 -;; v104 = isub.i64 v64, v103 ; v103 = 0x0800_0000 -;; v105 = isub.i64 v65, v103 ; v103 = 0x0800_0000 -;; @0025 call fn1(v0, v104, v105, v103) ; v103 = 0x0800_0000 -;; v106 = isub.i64 v66, v103 ; v103 = 0x0800_0000 -;; v107 = icmp ugt v106, v103 ; v103 = 0x0800_0000 -;; @0025 brif v107, block11(v104, v105, v106, v80), block12(v104, v105, v106, v80) -;; -;; block12(v81: i64, v82: i64, v83: i64, v93: i64): -;; @0025 v84 = isub v81, v83 -;; @0025 v85 = isub v82, v83 -;; @0025 jump block5(v84, v85, v83, v93) -;; -;; block17 cold: -;; @0025 v96 = load.i64 notrap aligned region4 v7+8 -;; @0025 v97 = icmp.i64 uge v91, v96 -;; @0025 brif v97, block18, block16 -;; -;; block18 cold: -;; @0025 v98 = call fn0(v0) -;; @0025 jump block16 -;; -;; block16: -;; @0025 call fn1(v0, v86, v87, v88) +;; block2(v16: i64): +;; @0025 v15 = load.i64 notrap aligned region3 v5 +;; @0025 v17 = icmp uge v15, v16 +;; @0025 brif v17, block5, block4 +;; +;; block5 cold: +;; @0025 v19 = load.i64 notrap aligned region4 v7+8 +;; @0025 v20 = icmp.i64 uge v15, v19 +;; @0025 brif v20, block6, block4 +;; +;; block6 cold: +;; @0025 v21 = call fn0(v0) +;; @0025 jump block4 +;; +;; block4: +;; @0025 v22 = load.i64 notrap aligned region6 v0+64 +;; @0025 v23 = uextend.i64 v2 +;; @0025 v24 = uextend.i64 v4 +;; @0025 v27 = iadd v23, v24 +;; @0025 v28 = icmp ugt v27, v22 +;; @0025 trapnz v28, heap_oob +;; @0025 v35 = uextend.i64 v3 +;; @0025 v39 = iadd v35, v24 +;; @0025 v40 = icmp ugt v39, v22 +;; @0025 trapnz v40, heap_oob +;; @0025 v29 = load.i64 notrap aligned readonly can_move region5 v0+56 +;; @0025 v33 = iadd v29, v23 +;; @0025 v45 = iadd v29, v35 +;; @0025 call fn1(v0, v33, v45, v24) ;; @0029 jump block1 ;; ;; block1: diff --git a/tests/disas/memory-copy-fuel-const-len.wat b/tests/disas/memory-copy-fuel-const-len.wat new file mode 100644 index 000000000000..aee7af900c1b --- /dev/null +++ b/tests/disas/memory-copy-fuel-const-len.wat @@ -0,0 +1,184 @@ +;;! target = 'x86_64' +;;! test = 'optimize' +;;! flags = '-Wfuel=100' + +(module + (memory 1) + (func $copy_16 (param i32 i32) + (memory.copy (local.get 0) (local.get 1) (i32.const 16)) + ) + (func $fill_128 (param i32) + (memory.fill (local.get 0) (i32.const 0) (i32.const 128)) + ) + (func $fill_4096 (param i32) + (memory.fill (local.get 0) (i32.const 0) (i32.const 4096)) + ) +) +;; function u0:0(i64 vmctx, i64, i32, i32) tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 67108888 "VMStoreContext+0x18" +;; region2 = 67108864 "VMStoreContext+0x0" +;; region3 = 603979776 "VMMemoryDefinition+0x0" +;; region4 = 603979784 "VMMemoryDefinition+0x8" +;; region5 = 201326592 "DefinedMemory(StaticModuleIndex(0), DefinedMemoryIndex(0))" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 +;; gv2 = load.i64 notrap aligned region1 gv1+24 +;; sig0 = (i64 vmctx) -> i8 tail +;; fn0 = colocated u805306368:12 sig0 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64, v2: i32, v3: i32): +;; @0023 v4 = load.i64 notrap aligned readonly can_move region0 v0+8 +;; @0023 v5 = load.i64 notrap aligned region2 v4 +;; @0023 v6 = iconst.i64 1 +;; @0023 v7 = iadd v5, v6 ; v6 = 1 +;; @0023 v8 = iconst.i64 0 +;; @0023 v9 = icmp sge v7, v8 ; v8 = 0 +;; @0023 brif v9, block2, block3(v7) +;; +;; block2: +;; v58 = iadd.i64 v5, v6 ; v6 = 1 +;; @0023 store notrap aligned region2 v58, v4 +;; @0023 v11 = call fn0(v0) +;; @0023 v13 = load.i64 notrap aligned region2 v4 +;; @0023 jump block3(v13) +;; +;; block3(v43: i64): +;; @002a v17 = load.i64 notrap aligned region4 v0+64 +;; @002a v18 = uextend.i64 v2 +;; v47 = iconst.i64 16 +;; @002a v22 = iadd v18, v47 ; v47 = 16 +;; @002a v23 = icmp ugt v22, v17 +;; @002a trapnz v23, heap_oob +;; @002a v30 = uextend.i64 v3 +;; @002a v34 = iadd v30, v47 ; v47 = 16 +;; @002a v35 = icmp ugt v34, v17 +;; @002a trapnz v35, heap_oob +;; @002a v24 = load.i64 notrap aligned readonly can_move region3 v0+56 +;; @002a v40 = iadd v24, v30 +;; @002a v42 = load.i8x16 notrap aligned little region5 v40 +;; @002a v28 = iadd v24, v18 +;; @002a store notrap aligned little region5 v42, v28 +;; @002e jump block1 +;; +;; block1: +;; @002e v44 = iconst.i64 20 +;; @002e v45 = iadd.i64 v43, v44 ; v44 = 20 +;; @002e store notrap aligned region2 v45, v4 +;; @002e return +;; } +;; +;; function u0:1(i64 vmctx, i64, i32) tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 67108888 "VMStoreContext+0x18" +;; region2 = 67108864 "VMStoreContext+0x0" +;; region3 = 603979776 "VMMemoryDefinition+0x0" +;; region4 = 603979784 "VMMemoryDefinition+0x8" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 +;; gv2 = load.i64 notrap aligned region1 gv1+24 +;; sig0 = (i64 vmctx) -> i8 tail +;; sig1 = (i64 vmctx, i64, i32, i64) tail +;; fn0 = colocated u805306368:12 sig0 +;; fn1 = colocated u805306368:2 sig1 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64, v2: i32): +;; @0030 v3 = load.i64 notrap aligned readonly can_move region0 v0+8 +;; @0030 v4 = load.i64 notrap aligned region2 v3 +;; @0030 v5 = iconst.i64 1 +;; @0030 v6 = iadd v4, v5 ; v5 = 1 +;; @0030 v7 = iconst.i64 0 +;; @0030 v8 = icmp sge v6, v7 ; v7 = 0 +;; @0030 brif v8, block2, block3(v6) +;; +;; block2: +;; v43 = iadd.i64 v4, v5 ; v5 = 1 +;; @0030 store notrap aligned region2 v43, v3 +;; @0030 v10 = call fn0(v0) +;; @0030 v12 = load.i64 notrap aligned region2 v3 +;; @0030 jump block3(v12) +;; +;; block3(v29: i64): +;; @0038 v16 = load.i64 notrap aligned region4 v0+64 +;; @0038 v17 = uextend.i64 v2 +;; v33 = iconst.i64 128 +;; @0038 v21 = iadd v17, v33 ; v33 = 128 +;; @0038 v22 = icmp ugt v21, v16 +;; @0038 trapnz v22, heap_oob +;; @0038 v23 = load.i64 notrap aligned readonly can_move region3 v0+56 +;; @0038 v27 = iadd v23, v17 +;; @0033 v14 = iconst.i32 0 +;; @0038 call fn1(v0, v27, v14, v33) ; v14 = 0, v33 = 128 +;; @003b jump block1 +;; +;; block1: +;; @003b v30 = iconst.i64 132 +;; @003b v31 = iadd.i64 v29, v30 ; v30 = 132 +;; @003b store notrap aligned region2 v31, v3 +;; @003b return +;; } +;; +;; function u0:2(i64 vmctx, i64, i32) tail { +;; region0 = 8 "VMContext+0x8" +;; region1 = 67108888 "VMStoreContext+0x18" +;; region2 = 67108864 "VMStoreContext+0x0" +;; region3 = 603979776 "VMMemoryDefinition+0x0" +;; region4 = 603979784 "VMMemoryDefinition+0x8" +;; gv0 = vmctx +;; gv1 = load.i64 notrap aligned readonly can_move region0 gv0+8 +;; gv2 = load.i64 notrap aligned region1 gv1+24 +;; sig0 = (i64 vmctx) -> i8 tail +;; sig1 = (i64 vmctx, i64, i32, i64) tail +;; fn0 = colocated u805306368:12 sig0 +;; fn1 = colocated u805306368:2 sig1 +;; stack_limit = gv2 +;; +;; block0(v0: i64, v1: i64, v2: i32): +;; @003d v3 = load.i64 notrap aligned readonly can_move region0 v0+8 +;; @003d v4 = load.i64 notrap aligned region2 v3 +;; @003d v5 = iconst.i64 1 +;; @003d v6 = iadd v4, v5 ; v5 = 1 +;; @003d v7 = iconst.i64 0 +;; @003d v8 = icmp sge v6, v7 ; v7 = 0 +;; @003d brif v8, block2, block3(v6) +;; +;; block2: +;; v50 = iadd.i64 v4, v5 ; v5 = 1 +;; @003d store notrap aligned region2 v50, v3 +;; @003d v10 = call fn0(v0) +;; @003d v12 = load.i64 notrap aligned region2 v3 +;; @003d jump block3(v12) +;; +;; block3(v16: i64): +;; @0045 v17 = iconst.i64 4100 +;; @0045 v18 = iadd v16, v17 ; v17 = 4100 +;; v51 = iconst.i64 0 +;; v52 = icmp sge v18, v51 ; v51 = 0 +;; @0045 brif v52, block4, block5(v18) +;; +;; block4: +;; v53 = iadd.i64 v16, v17 ; v17 = 4100 +;; @0045 store notrap aligned region2 v53, v3 +;; @0045 v22 = call fn0(v0) +;; @0045 v24 = load.i64 notrap aligned region2 v3 +;; @0045 jump block5(v24) +;; +;; block5(v39: i64): +;; @0045 v25 = load.i64 notrap aligned region4 v0+64 +;; @0045 v26 = uextend.i64 v2 +;; v40 = iconst.i64 4096 +;; @0045 v30 = iadd v26, v40 ; v40 = 4096 +;; @0045 v31 = icmp ugt v30, v25 +;; @0045 trapnz v31, heap_oob +;; @0045 v32 = load.i64 notrap aligned readonly can_move region3 v0+56 +;; @0045 v36 = iadd v32, v26 +;; @0040 v14 = iconst.i32 0 +;; @0045 call fn1(v0, v36, v14, v40) ; v14 = 0, v40 = 4096 +;; @0048 jump block1 +;; +;; block1: +;; @0048 store.i64 notrap aligned region2 v39, v3 +;; @0048 return +;; } diff --git a/tests/disas/memory-copy-fuel.wat b/tests/disas/memory-copy-fuel.wat index 45e72b7043d4..29104ad29719 100644 --- a/tests/disas/memory-copy-fuel.wat +++ b/tests/disas/memory-copy-fuel.wat @@ -33,110 +33,44 @@ ;; @001e brif v10, block2, block3(v8) ;; ;; block2: -;; v106 = iadd.i64 v6, v7 ; v7 = 1 -;; @001e store notrap aligned region2 v106, v5 +;; v61 = iadd.i64 v6, v7 ; v7 = 1 +;; @001e store notrap aligned region2 v61, v5 ;; @001e v12 = call fn0(v0) ;; @001e v14 = load.i64 notrap aligned region2 v5 ;; @001e jump block3(v14) ;; -;; block3(v43: i64): -;; @0025 v18 = load.i64 notrap aligned region4 v0+64 -;; @0025 v19 = uextend.i64 v2 -;; @0025 v20 = uextend.i64 v4 -;; @0025 v23 = iadd v19, v20 -;; @0025 v24 = icmp ugt v23, v18 -;; @0025 trapnz v24, heap_oob -;; @0025 v31 = uextend.i64 v3 -;; @0025 v35 = iadd v31, v20 -;; @0025 v36 = icmp ugt v35, v18 -;; @0025 trapnz v36, heap_oob -;; @0025 v25 = load.i64 notrap aligned readonly can_move region3 v0+56 -;; @0025 v41 = iadd v25, v31 -;; @0025 v29 = iadd v25, v19 -;; @0025 v47 = icmp ugt v41, v29 -;; @0025 brif v47, block6, block7 -;; -;; block4(v49: i64, v50: i64, v51: i64, v52: i64): -;; @0025 v53 = iadd v52, v116 ; v116 = 0x0800_0000 -;; v120 = iconst.i64 0 -;; v121 = icmp sge v53, v120 ; v120 = 0 -;; @0025 brif v121, block8, block9(v53) -;; -;; block5(v89: i64, v90: i64, v91: i64, v92: i64): -;; @0025 v94 = iadd v92, v91 -;; v123 = iconst.i64 0 -;; v124 = icmp sge v94, v123 ; v123 = 0 -;; @0025 brif v124, block14, block15(v94) -;; -;; block6: -;; v116 = iconst.i64 0x0800_0000 -;; v117 = icmp.i64 ugt v20, v116 ; v116 = 0x0800_0000 -;; v118 = iconst.i64 4 -;; v119 = iadd.i64 v43, v118 ; v118 = 4 -;; @0025 brif v117, block4(v29, v41, v20, v119), block5(v29, v41, v20, v119) -;; -;; block8: -;; v122 = iadd.i64 v52, v116 ; v116 = 0x0800_0000 -;; @0025 store notrap aligned region2 v122, v5 -;; @0025 v57 = call fn0(v0) -;; @0025 v59 = load.i64 notrap aligned region2 v5 -;; @0025 jump block9(v59) -;; -;; block9(v64: i64): -;; @0025 call fn1(v0, v49, v50, v116) ; v116 = 0x0800_0000 -;; @0025 v62 = isub.i64 v51, v116 ; v116 = 0x0800_0000 -;; @0025 v63 = icmp ugt v62, v116 ; v116 = 0x0800_0000 -;; @0025 v60 = iadd.i64 v49, v116 ; v116 = 0x0800_0000 -;; @0025 v61 = iadd.i64 v50, v116 ; v116 = 0x0800_0000 -;; @0025 brif v63, block4(v60, v61, v62, v64), block5(v60, v61, v62, v64) -;; -;; block7: -;; @0025 v46 = iconst.i64 0x0800_0000 -;; @0025 v67 = icmp.i64 ugt v20, v46 ; v46 = 0x0800_0000 -;; @0025 v65 = iadd.i64 v29, v20 -;; @0025 v66 = iadd.i64 v41, v20 -;; @0025 v44 = iconst.i64 4 -;; @0025 v45 = iadd.i64 v43, v44 ; v44 = 4 -;; @0025 brif v67, block10(v65, v66, v20, v45), block11(v65, v66, v20, v45) -;; -;; block10(v68: i64, v69: i64, v70: i64, v73: i64): -;; v107 = iconst.i64 0x0800_0000 -;; v108 = iadd v73, v107 ; v107 = 0x0800_0000 -;; v109 = iconst.i64 0 -;; v110 = icmp sge v108, v109 ; v109 = 0 -;; @0025 brif v110, block12, block13(v108) -;; -;; block12: -;; @0025 store.i64 notrap aligned region2 v108, v5 -;; @0025 v78 = call fn0(v0) -;; @0025 v80 = load.i64 notrap aligned region2 v5 -;; @0025 jump block13(v80) -;; -;; block13(v83: i64): -;; v111 = iconst.i64 0x0800_0000 -;; v112 = isub.i64 v68, v111 ; v111 = 0x0800_0000 -;; v113 = isub.i64 v69, v111 ; v111 = 0x0800_0000 -;; @0025 call fn1(v0, v112, v113, v111) ; v111 = 0x0800_0000 -;; v114 = isub.i64 v70, v111 ; v111 = 0x0800_0000 -;; v115 = icmp ugt v114, v111 ; v111 = 0x0800_0000 -;; @0025 brif v115, block10(v112, v113, v114, v83), block11(v112, v113, v114, v83) -;; -;; block11(v84: i64, v85: i64, v86: i64, v93: i64): -;; @0025 v87 = isub v84, v86 -;; @0025 v88 = isub v85, v86 -;; @0025 jump block5(v87, v88, v86, v93) -;; -;; block14: -;; @0025 store.i64 notrap aligned region2 v94, v5 -;; @0025 v98 = call fn0(v0) -;; @0025 v100 = load.i64 notrap aligned region2 v5 -;; @0025 jump block15(v100) -;; -;; block15(v102: i64): -;; @0025 call fn1(v0, v89, v90, v91) +;; block3(v21: i64): +;; @0025 v22 = iconst.i64 4 +;; @0025 v23 = iadd v21, v22 ; v22 = 4 +;; @0025 v18 = uextend.i64 v4 +;; @0025 v24 = iadd v23, v18 +;; v62 = iconst.i64 0 +;; v63 = icmp sge v24, v62 ; v62 = 0 +;; @0025 brif v63, block4, block5(v24) +;; +;; block4: +;; @0025 store.i64 notrap aligned region2 v24, v5 +;; @0025 v28 = call fn0(v0) +;; @0025 v30 = load.i64 notrap aligned region2 v5 +;; @0025 jump block5(v30) +;; +;; block5(v57: i64): +;; @0025 v31 = load.i64 notrap aligned region4 v0+64 +;; @0025 v32 = uextend.i64 v2 +;; @0025 v36 = iadd v32, v18 +;; @0025 v37 = icmp ugt v36, v31 +;; @0025 trapnz v37, heap_oob +;; @0025 v44 = uextend.i64 v3 +;; @0025 v48 = iadd v44, v18 +;; @0025 v49 = icmp ugt v48, v31 +;; @0025 trapnz v49, heap_oob +;; @0025 v38 = load.i64 notrap aligned readonly can_move region3 v0+56 +;; @0025 v42 = iadd v38, v32 +;; @0025 v54 = iadd v38, v44 +;; @0025 call fn1(v0, v42, v54, v18) ;; @0029 jump block1 ;; ;; block1: -;; @0029 store.i64 notrap aligned region2 v102, v5 +;; @0029 store.i64 notrap aligned region2 v57, v5 ;; @0029 return ;; }