diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index af864e99fb2c..8fe8bd484bff 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1046,9 +1046,9 @@ jobs: strategy: matrix: crate: - - "wasmtime" + - "wasmtime --features pulley" - "wasmtime-cli" - - "wasmtime-environ" + - "wasmtime-environ --all-features" - "pulley-interpreter --all-features" needs: determine if: needs.determine.outputs.test-miri && github.repository == 'bytecodealliance/wasmtime' diff --git a/cranelift/codegen/src/isa/pulley_shared/inst/args.rs b/cranelift/codegen/src/isa/pulley_shared/inst/args.rs index e70dfbaf2249..0d6dc6161104 100644 --- a/cranelift/codegen/src/isa/pulley_shared/inst/args.rs +++ b/cranelift/codegen/src/isa/pulley_shared/inst/args.rs @@ -180,7 +180,9 @@ impl Amode { + frame_layout.outgoing_args_size; i64::from(sp_offset) - offset } - StackAMode::Slot(offset) => *offset, + StackAMode::Slot(offset) => { + offset + i64::from(state.frame_layout().outgoing_args_size) + } StackAMode::OutgoingArg(offset) => *offset, }, } diff --git a/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs b/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs index 6131cee2c461..42095c0d6e99 100644 --- a/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs +++ b/cranelift/codegen/src/isa/pulley_shared/inst/emit.rs @@ -549,9 +549,15 @@ fn pulley_emit

( *start_offset = sink.cur_offset(); } - Inst::PushFrame => enc::push_frame(sink), + Inst::PushFrame => { + sink.add_trap(ir::TrapCode::STACK_OVERFLOW); + enc::push_frame(sink); + } Inst::PopFrame => enc::pop_frame(sink), - Inst::StackAlloc32 { amt } => enc::stack_alloc32(sink, *amt), + Inst::StackAlloc32 { amt } => { + sink.add_trap(ir::TrapCode::STACK_OVERFLOW); + enc::stack_alloc32(sink, *amt); + } Inst::StackFree32 { amt } => enc::stack_free32(sink, *amt), Inst::Zext8 { dst, src } => enc::zext8(sink, dst, src), diff --git a/cranelift/codegen/src/isa/pulley_shared/lower.isle b/cranelift/codegen/src/isa/pulley_shared/lower.isle index 00de3b70838c..6f6a4fff5edc 100644 --- a/cranelift/codegen/src/isa/pulley_shared/lower.isle +++ b/cranelift/codegen/src/isa/pulley_shared/lower.isle @@ -59,7 +59,7 @@ ;;;; Rules for `trapz` and `trapnz` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; -(rule (lower (trapz a @ (value_type (fits_in_64 ty)) code)) +(rule (lower (trapz a @ (value_type (ty_32_or_64 ty)) code)) (let ((zero Reg (pulley_xconst8 0))) (side_effect (pulley_trap_if (IntCC.Equal) (ty_to_operand_size ty) @@ -67,7 +67,7 @@ zero code)))) -(rule (lower (trapnz a @ (value_type (fits_in_64 ty)) code)) +(rule (lower (trapnz a @ (value_type (ty_32_or_64 ty)) code)) (let ((zero Reg (pulley_xconst8 0))) (side_effect (pulley_trap_if (IntCC.NotEqual) (ty_to_operand_size ty) @@ -77,14 +77,14 @@ ;; Fold `(trap[n]z (icmp ...))` together. -(rule 1 (lower (trapz (icmp cc a b @ (value_type (fits_in_64 ty))) code)) +(rule 1 (lower (trapz (icmp cc a b @ (value_type (ty_32_or_64 ty))) code)) (side_effect (pulley_trap_if (intcc_complement cc) (ty_to_operand_size ty) a b code))) -(rule 1 (lower (trapnz (icmp cc a b @ (value_type (fits_in_64 ty))) code)) +(rule 1 (lower (trapnz (icmp cc a b @ (value_type (ty_32_or_64 ty))) code)) (side_effect (pulley_trap_if cc (ty_to_operand_size ty) a diff --git a/crates/cranelift/src/compiler.rs b/crates/cranelift/src/compiler.rs index cbe2bfa9e7d2..ea87148ca01a 100644 --- a/crates/cranelift/src/compiler.rs +++ b/crates/cranelift/src/compiler.rs @@ -30,7 +30,7 @@ use wasmtime_environ::{ AddressMapSection, BuiltinFunctionIndex, CacheStore, CompileError, DefinedFuncIndex, FlagValue, FunctionBodyData, FunctionLoc, HostCall, ModuleTranslation, ModuleTypesBuilder, PtrSize, RelocationTarget, StackMapInformation, StaticModuleIndex, TrapEncodingBuilder, TrapSentinel, - Tunables, VMOffsets, WasmFuncType, WasmFunctionInfo, WasmValType, + TripleExt, Tunables, VMOffsets, WasmFuncType, WasmFunctionInfo, WasmValType, }; #[cfg(feature = "component-model")] @@ -152,12 +152,7 @@ impl Compiler { // `call` instruction where the name is `colocated: false`. This will // force a pulley-specific relocation to get emitted in addition to // using the `call_indirect_host` instruction. - let is_pulley = match self.isa.triple().architecture { - target_lexicon::Architecture::Pulley32 => true, - target_lexicon::Architecture::Pulley64 => true, - _ => false, - }; - if is_pulley { + if self.isa.triple().is_pulley() { let mut new_signature = signature.clone(); new_signature .params @@ -246,7 +241,7 @@ impl wasmtime_environ::Compiler for Compiler { // abort for the whole program since the runtime limits configured by // the embedder should cause wasm to trap before it reaches that // (ensuring the host has enough space as well for its functionality). - if !func_env.is_pulley() { + if !isa.triple().is_pulley() { let vmctx = context .func .create_global_value(ir::GlobalValueData::VMContext); diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index 53db72ca394c..b4cb0f879b19 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -22,9 +22,9 @@ use wasmparser::{Operator, WasmFeatures}; use wasmtime_environ::{ BuiltinFunctionIndex, DataIndex, ElemIndex, EngineOrModuleTypeIndex, FuncIndex, GlobalIndex, IndexType, Memory, MemoryIndex, Module, ModuleInternedTypeIndex, ModuleTranslation, - ModuleTypesBuilder, PtrSize, Table, TableIndex, Tunables, TypeConvert, TypeIndex, VMOffsets, - WasmCompositeInnerType, WasmFuncType, WasmHeapTopType, WasmHeapType, WasmRefType, WasmResult, - WasmValType, + ModuleTypesBuilder, PtrSize, Table, TableIndex, TripleExt, Tunables, TypeConvert, TypeIndex, + VMOffsets, WasmCompositeInnerType, WasmFuncType, WasmHeapTopType, WasmHeapType, WasmRefType, + WasmResult, WasmValType, }; use wasmtime_environ::{FUNCREF_INIT_BIT, FUNCREF_MASK}; @@ -1187,16 +1187,6 @@ impl<'module_environment> FuncEnvironment<'module_environment> { i32::from(self.offsets.ptr.vm_func_ref_type_index()), ) } - - /// Returns whether the current compilation target is for the Pulley - /// interpreter. - pub fn is_pulley(&self) -> bool { - match self.isa.triple().architecture { - target_lexicon::Architecture::Pulley32 => true, - target_lexicon::Architecture::Pulley64 => true, - _ => false, - } - } } struct Call<'a, 'func, 'module_env> { @@ -3419,7 +3409,7 @@ impl FuncEnvironment<'_> { /// being targetted since the Pulley runtime doesn't catch segfaults for /// itself. pub fn clif_memory_traps_enabled(&self) -> bool { - self.tunables.signals_based_traps && !self.is_pulley() + self.tunables.signals_based_traps && !self.isa.triple().is_pulley() } /// Returns whether it's acceptable to have CLIF instructions natively trap, @@ -3429,7 +3419,7 @@ impl FuncEnvironment<'_> { /// unconditionally since Pulley doesn't use hardware-based traps in its /// runtime. pub fn clif_instruction_traps_enabled(&self) -> bool { - self.tunables.signals_based_traps || self.is_pulley() + self.tunables.signals_based_traps || self.isa.triple().is_pulley() } } diff --git a/crates/cranelift/src/obj.rs b/crates/cranelift/src/obj.rs index a55c08b88441..e64be78367a3 100644 --- a/crates/cranelift/src/obj.rs +++ b/crates/cranelift/src/obj.rs @@ -22,11 +22,11 @@ use cranelift_control::ControlPlane; use gimli::write::{Address, EhFrame, EndianVec, FrameTable, Writer}; use gimli::RunTimeEndian; use object::write::{Object, SectionId, StandardSegment, Symbol, SymbolId, SymbolSection}; -use object::{Architecture, SectionKind, SymbolFlags, SymbolKind, SymbolScope}; +use object::{Architecture, SectionFlags, SectionKind, SymbolFlags, SymbolKind, SymbolScope}; use std::collections::HashMap; use std::ops::Range; -use wasmtime_environ::obj::LibCall; -use wasmtime_environ::{Compiler, Unsigned}; +use wasmtime_environ::obj::{self, LibCall}; +use wasmtime_environ::{Compiler, TripleExt, Unsigned}; const TEXT_SECTION_NAME: &[u8] = b".text"; @@ -83,6 +83,18 @@ impl<'a> ModuleTextBuilder<'a> { SectionKind::Text, ); + // If this target is Pulley then flag the text section as not needing the + // executable bit in virtual memory which means that the runtime won't + // try to call `Mmap::make_exectuable`, which makes Pulley more + // portable. + if compiler.triple().is_pulley() { + let section = obj.section_mut(text_section); + assert!(matches!(section.flags, SectionFlags::None)); + section.flags = SectionFlags::Elf { + sh_flags: obj::SH_WASMTIME_NOT_EXECUTED, + }; + } + Self { compiler, obj, diff --git a/crates/environ/src/compile/mod.rs b/crates/environ/src/compile/mod.rs index 642e20c2debe..db4bf9548518 100644 --- a/crates/environ/src/compile/mod.rs +++ b/crates/environ/src/compile/mod.rs @@ -338,6 +338,9 @@ pub trait Compiler: Send + Sync { // 64 KB is the maximal page size (i.e. memory translation granule size) // supported by the architecture and is used on some platforms. (_, Architecture::Aarch64(..)) => 0x10000, + // Conservatively assume the max-of-all-supported-hosts for pulley + // and round up to 64k. + (_, Architecture::Pulley32 | Architecture::Pulley64) => 0x10000, _ => 0x1000, } } diff --git a/crates/environ/src/compile/module_artifacts.rs b/crates/environ/src/compile/module_artifacts.rs index bc1d22634c5b..7d9d132eddd6 100644 --- a/crates/environ/src/compile/module_artifacts.rs +++ b/crates/environ/src/compile/module_artifacts.rs @@ -28,9 +28,6 @@ pub struct ObjectBuilder<'a> { /// will go. data: SectionId, - /// The target triple for this compilation. - triple: target_lexicon::Triple, - /// The section identifier for function name information, or otherwise where /// the `name` custom section of wasm is copied into. /// @@ -46,11 +43,7 @@ pub struct ObjectBuilder<'a> { impl<'a> ObjectBuilder<'a> { /// Creates a new builder for the `obj` specified. - pub fn new( - mut obj: Object<'a>, - tunables: &'a Tunables, - triple: target_lexicon::Triple, - ) -> ObjectBuilder<'a> { + pub fn new(mut obj: Object<'a>, tunables: &'a Tunables) -> ObjectBuilder<'a> { let data = obj.add_section( obj.segment_name(StandardSegment::Data).to_vec(), obj::ELF_WASM_DATA.as_bytes().to_vec(), @@ -60,7 +53,6 @@ impl<'a> ObjectBuilder<'a> { obj, tunables, data, - triple, names: None, dwarf: None, } @@ -225,12 +217,6 @@ impl<'a> ObjectBuilder<'a> { self.push_debuginfo(&mut dwarf, &debuginfo); } - let is_pulley = matches!( - self.triple.architecture, - target_lexicon::Architecture::Pulley32 | target_lexicon::Architecture::Pulley64 - ); - assert!(!is_pulley || wasm_to_array_trampolines.is_empty()); - Ok(CompiledModuleInfo { module, funcs, @@ -240,7 +226,6 @@ impl<'a> ObjectBuilder<'a> { has_unparsed_debuginfo, code_section_offset: debuginfo.wasm_file.code_section_offset, has_wasm_debuginfo: self.tunables.parse_wasm_debuginfo, - is_pulley, dwarf, }, }) diff --git a/crates/environ/src/component/info.rs b/crates/environ/src/component/info.rs index e1b6f2edf58b..8c5442a6d243 100644 --- a/crates/environ/src/component/info.rs +++ b/crates/environ/src/component/info.rs @@ -452,19 +452,30 @@ pub struct CanonicalOptions { } /// Possible encodings of strings within the component model. -// -// Note that the `repr(u8)` is load-bearing here since this is used in an -// `extern "C" fn()` function argument which is called from cranelift-compiled -// code so we must know the representation of this. #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Hash)] #[allow(missing_docs, reason = "self-describing variants")] -#[repr(u8)] pub enum StringEncoding { Utf8, Utf16, CompactUtf16, } +impl StringEncoding { + /// Decodes the `u8` provided back into a `StringEncoding`, if it's valid. + pub fn from_u8(val: u8) -> Option { + if val == StringEncoding::Utf8 as u8 { + return Some(StringEncoding::Utf8); + } + if val == StringEncoding::Utf16 as u8 { + return Some(StringEncoding::Utf16); + } + if val == StringEncoding::CompactUtf16 as u8 { + return Some(StringEncoding::CompactUtf16); + } + None + } +} + /// Possible transcoding operations that must be provided by the host. /// /// Note that each transcoding operation may have a unique signature depending diff --git a/crates/environ/src/ext.rs b/crates/environ/src/ext.rs new file mode 100644 index 000000000000..8311c4789a96 --- /dev/null +++ b/crates/environ/src/ext.rs @@ -0,0 +1,18 @@ +use target_lexicon::{Architecture, Triple}; + +/// Extension methods for `target_lexicon::Triple`. +pub trait TripleExt { + /// Helper for returning whether this target is for pulley, wasmtime's + /// interpreter. + fn is_pulley(&self) -> bool; +} + +impl TripleExt for Triple { + fn is_pulley(&self) -> bool { + match self.architecture { + Architecture::Pulley32 => true, + Architecture::Pulley64 => true, + _ => false, + } + } +} diff --git a/crates/environ/src/lib.rs b/crates/environ/src/lib.rs index 555e5586678a..91c887b55a52 100644 --- a/crates/environ/src/lib.rs +++ b/crates/environ/src/lib.rs @@ -19,6 +19,7 @@ mod address_map; mod builtin; mod demangling; mod error; +mod ext; mod gc; mod hostcall; mod module; @@ -33,6 +34,7 @@ mod tunables; mod types; mod vmoffsets; +pub use self::ext::*; pub use crate::address_map::*; pub use crate::builtin::*; pub use crate::demangling::*; diff --git a/crates/environ/src/module_artifacts.rs b/crates/environ/src/module_artifacts.rs index 6707da994ebe..1d51f1215cdb 100644 --- a/crates/environ/src/module_artifacts.rs +++ b/crates/environ/src/module_artifacts.rs @@ -107,10 +107,6 @@ pub struct Metadata { /// weren't found in the original wasm module itself. pub has_wasm_debuginfo: bool, - /// Whether this artifact contains Pulley bytecode (instead of machine code) - /// or not. - pub is_pulley: bool, - /// Dwarf sections and the offsets at which they're stored in the /// ELF_WASMTIME_DWARF pub dwarf: Vec<(u8, Range)>, diff --git a/crates/fuzzing/src/generators/config.rs b/crates/fuzzing/src/generators/config.rs index fd7180853e16..8fbdb737aa2b 100644 --- a/crates/fuzzing/src/generators/config.rs +++ b/crates/fuzzing/src/generators/config.rs @@ -233,7 +233,7 @@ impl Config { InstanceAllocationStrategy::Pooling(_) ), compiler: match self.wasmtime.compiler_strategy { - CompilerStrategy::Cranelift => wasmtime_wast_util::Compiler::Cranelift, + CompilerStrategy::Cranelift => wasmtime_wast_util::Compiler::CraneliftNative, CompilerStrategy::Winch => wasmtime_wast_util::Compiler::Winch, }, } diff --git a/crates/misc/component-test-util/src/lib.rs b/crates/misc/component-test-util/src/lib.rs index bc7993d69cd8..eff4a4299e5c 100644 --- a/crates/misc/component-test-util/src/lib.rs +++ b/crates/misc/component-test-util/src/lib.rs @@ -132,16 +132,23 @@ forward_impls! { /// Helper method to apply `wast_config` to `config`. pub fn apply_wast_config(config: &mut Config, wast_config: &wasmtime_wast_util::WastConfig) { + use wasmtime_wast_util::{Collector, Compiler}; + config.strategy(match wast_config.compiler { - wasmtime_wast_util::Compiler::Cranelift => wasmtime::Strategy::Cranelift, - wasmtime_wast_util::Compiler::Winch => wasmtime::Strategy::Winch, + Compiler::CraneliftNative | Compiler::CraneliftPulley => wasmtime::Strategy::Cranelift, + Compiler::Winch => wasmtime::Strategy::Winch, }); - config.collector(match wast_config.collector { - wasmtime_wast_util::Collector::Auto => wasmtime::Collector::Auto, - wasmtime_wast_util::Collector::Null => wasmtime::Collector::Null, - wasmtime_wast_util::Collector::DeferredReferenceCounting => { - wasmtime::Collector::DeferredReferenceCounting + if let Compiler::CraneliftPulley = wast_config.compiler { + if cfg!(target_pointer_width = "32") { + config.target("pulley32").unwrap(); + } else { + config.target("pulley64").unwrap(); } + } + config.collector(match wast_config.collector { + Collector::Auto => wasmtime::Collector::Auto, + Collector::Null => wasmtime::Collector::Null, + Collector::DeferredReferenceCounting => wasmtime::Collector::DeferredReferenceCounting, }); } diff --git a/crates/test-macros/src/lib.rs b/crates/test-macros/src/lib.rs index 7c125bbc653c..5a9c8c89a12c 100644 --- a/crates/test-macros/src/lib.rs +++ b/crates/test-macros/src/lib.rs @@ -55,7 +55,7 @@ impl TestConfig { self.strategies.retain(|s| *s != Compiler::Winch); Ok(()) } else if meta.path.is_ident("Cranelift") { - self.strategies.retain(|s| *s != Compiler::Cranelift); + self.strategies.retain(|s| *s != Compiler::CraneliftNative); Ok(()) } else { Err(meta.error("Unknown strategy")) @@ -97,7 +97,7 @@ impl TestConfig { impl Default for TestConfig { fn default() -> Self { Self { - strategies: vec![Compiler::Cranelift, Compiler::Winch], + strategies: vec![Compiler::CraneliftNative, Compiler::Winch], flags: Default::default(), test_attribute: None, } diff --git a/crates/wasmtime/src/compile.rs b/crates/wasmtime/src/compile.rs index 86339b44dba3..1222fb9607c4 100644 --- a/crates/wasmtime/src/compile.rs +++ b/crates/wasmtime/src/compile.rs @@ -80,8 +80,7 @@ pub(crate) fn build_artifacts( .context("failed to parse WebAssembly module")?; let functions = mem::take(&mut translation.function_body_inputs); - let compile_inputs = - CompileInputs::for_module(engine.compiler().triple(), &types, &translation, functions); + let compile_inputs = CompileInputs::for_module(&types, &translation, functions); let unlinked_compile_outputs = compile_inputs.compile(engine)?; let (compiled_funcs, function_indices) = unlinked_compile_outputs.pre_link(); @@ -324,9 +323,6 @@ struct CompileOutput { /// The collection of things we need to compile for a Wasm module or component. #[derive(Default)] struct CompileInputs<'a> { - // Whether or not we need to compile wasm-to-native trampolines. - need_wasm_to_array_trampolines: bool, - inputs: Vec>, } @@ -337,18 +333,11 @@ impl<'a> CompileInputs<'a> { /// Create the `CompileInputs` for a core Wasm module. fn for_module( - triple: &target_lexicon::Triple, types: &'a ModuleTypesBuilder, translation: &'a ModuleTranslation<'a>, functions: PrimaryMap>, ) -> Self { - let mut ret = CompileInputs { - need_wasm_to_array_trampolines: !matches!( - triple.architecture, - target_lexicon::Architecture::Pulley32 | target_lexicon::Architecture::Pulley64 - ), - inputs: vec![], - }; + let mut ret = CompileInputs { inputs: vec![] }; let module_index = StaticModuleIndex::from_u32(0); ret.collect_inputs_in_translations(types, [(module_index, translation, functions)]); @@ -370,14 +359,7 @@ impl<'a> CompileInputs<'a> { ), >, ) -> Self { - let triple = engine.compiler().triple(); - let mut ret = CompileInputs { - need_wasm_to_array_trampolines: !matches!( - triple.architecture, - target_lexicon::Architecture::Pulley32 | target_lexicon::Architecture::Pulley64 - ), - inputs: vec![], - }; + let mut ret = CompileInputs { inputs: vec![] }; ret.collect_inputs_in_translations(types.module_types_builder(), module_translations); let tunables = engine.tunables(); @@ -518,28 +500,25 @@ impl<'a> CompileInputs<'a> { } } - if self.need_wasm_to_array_trampolines { - let mut trampoline_types_seen = HashSet::new(); - for (_func_type_index, trampoline_type_index) in types.trampoline_types() { - let is_new = trampoline_types_seen.insert(trampoline_type_index); - if !is_new { - continue; - } - let trampoline_func_ty = types[trampoline_type_index].unwrap_func(); - self.push_input(move |compiler| { - let trampoline = - compiler.compile_wasm_to_array_trampoline(trampoline_func_ty)?; - Ok(CompileOutput { - key: CompileKey::wasm_to_array_trampoline(trampoline_type_index), - symbol: format!( - "signatures[{}]::wasm_to_array_trampoline", - trampoline_type_index.as_u32() - ), - function: CompiledFunction::Function(trampoline), - info: None, - }) - }); + let mut trampoline_types_seen = HashSet::new(); + for (_func_type_index, trampoline_type_index) in types.trampoline_types() { + let is_new = trampoline_types_seen.insert(trampoline_type_index); + if !is_new { + continue; } + let trampoline_func_ty = types[trampoline_type_index].unwrap_func(); + self.push_input(move |compiler| { + let trampoline = compiler.compile_wasm_to_array_trampoline(trampoline_func_ty)?; + Ok(CompileOutput { + key: CompileKey::wasm_to_array_trampoline(trampoline_type_index), + symbol: format!( + "signatures[{}]::wasm_to_array_trampoline", + trampoline_type_index.as_u32() + ), + function: CompiledFunction::Function(trampoline), + info: None, + }) + }); } } @@ -738,8 +717,7 @@ impl FunctionIndices { )?; } - let mut obj = - wasmtime_environ::ObjectBuilder::new(obj, tunables, compiler.triple().clone()); + let mut obj = wasmtime_environ::ObjectBuilder::new(obj, tunables); let mut artifacts = Artifacts::default(); // Remove this as it's not needed by anything below and we'll debug @@ -831,29 +809,23 @@ impl FunctionIndices { }) .collect(); - let wasm_to_array_trampolines = match engine.compiler().triple().architecture { - target_lexicon::Architecture::Pulley32 - | target_lexicon::Architecture::Pulley64 => vec![], - _ => { - let unique_and_sorted_trampoline_sigs = translation - .module - .types - .iter() - .map(|(_, ty)| *ty) - .filter(|idx| types[*idx].is_func()) - .map(|idx| types.trampoline_type(idx)) - .collect::>(); - unique_and_sorted_trampoline_sigs - .iter() - .map(|idx| { - let trampoline = types.trampoline_type(*idx); - let key = CompileKey::wasm_to_array_trampoline(trampoline); - let compiled = wasm_to_array_trampolines[&key]; - (*idx, symbol_ids_and_locs[compiled.unwrap_function()].1) - }) - .collect() - } - }; + let unique_and_sorted_trampoline_sigs = translation + .module + .types + .iter() + .map(|(_, ty)| *ty) + .filter(|idx| types[*idx].is_func()) + .map(|idx| types.trampoline_type(idx)) + .collect::>(); + let wasm_to_array_trampolines = unique_and_sorted_trampoline_sigs + .iter() + .map(|idx| { + let trampoline = types.trampoline_type(*idx); + let key = CompileKey::wasm_to_array_trampoline(trampoline); + let compiled = wasm_to_array_trampolines[&key]; + (*idx, symbol_ids_and_locs[compiled.unwrap_function()].1) + }) + .collect(); obj.append(translation, funcs, wasm_to_array_trampolines) }) diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs index f20ef72aec4c..9aeaa1406caf 100644 --- a/crates/wasmtime/src/config.rs +++ b/crates/wasmtime/src/config.rs @@ -1927,35 +1927,17 @@ impl Config { #[cfg(any(feature = "cranelift", feature = "winch"))] match self.compiler_config.strategy { None | Some(Strategy::Cranelift) => match self.compiler_target().architecture { - // Pulley doesn't support most of wasm at this time and there's - // lots of panicking bits and pieces within the backend. This - // doesn't fully cover all panicking cases but it's at least a - // starting place to have a ratchet. As the pulley backend is - // developed this'll get filtered down over time. + // Pulley is just starting and most errors are because of + // unsupported lowerings which is a first-class error. Some + // errors are panics though due to unimplemented bits in ABI + // code and those causes are listed here. target_lexicon::Architecture::Pulley32 | target_lexicon::Architecture::Pulley64 => { - WasmFeatures::SATURATING_FLOAT_TO_INT - | WasmFeatures::SIGN_EXTENSION - | WasmFeatures::REFERENCE_TYPES - | WasmFeatures::MULTI_VALUE - | WasmFeatures::BULK_MEMORY - | WasmFeatures::SIMD + WasmFeatures::SIMD | WasmFeatures::RELAXED_SIMD - | WasmFeatures::THREADS - | WasmFeatures::SHARED_EVERYTHING_THREADS | WasmFeatures::TAIL_CALL | WasmFeatures::FLOATS - | WasmFeatures::MULTI_MEMORY - | WasmFeatures::EXCEPTIONS | WasmFeatures::MEMORY64 - | WasmFeatures::EXTENDED_CONST - | WasmFeatures::FUNCTION_REFERENCES - | WasmFeatures::MEMORY_CONTROL - | WasmFeatures::GC - | WasmFeatures::CUSTOM_PAGE_SIZES - | WasmFeatures::LEGACY_EXCEPTIONS | WasmFeatures::GC_TYPES - | WasmFeatures::STACK_SWITCHING - | WasmFeatures::WIDE_ARITHMETIC } // Other Cranelift backends are either 100% missing or complete diff --git a/crates/wasmtime/src/engine.rs b/crates/wasmtime/src/engine.rs index 535bdda9e669..8062ef934911 100644 --- a/crates/wasmtime/src/engine.rs +++ b/crates/wasmtime/src/engine.rs @@ -255,7 +255,8 @@ impl Engine { fn _check_compatible_with_native_host(&self) -> Result<(), String> { #[cfg(any(feature = "cranelift", feature = "winch"))] { - use target_lexicon::{Architecture, PointerWidth, Triple}; + use target_lexicon::Triple; + use wasmtime_environ::TripleExt; let compiler = self.compiler(); @@ -268,16 +269,18 @@ impl Engine { return true; } - // Otherwise if there's a mismatch the only allowed - // configuration at this time is that any target can run Pulley, - // Wasmtime's interpreter. This only works though if the - // pointer-width of pulley matches the pointer-width of the - // host, so check that here. - match host.pointer_width() { - Ok(PointerWidth::U32) => target.architecture == Architecture::Pulley32, - Ok(PointerWidth::U64) => target.architecture == Architecture::Pulley64, - _ => false, + // If there's a mismatch and the target is a compatible pulley + // target, then that's also ok to run. + if cfg!(feature = "pulley") + && target.is_pulley() + && target.pointer_width() == host.pointer_width() + && target.endianness() == host.endianness() + { + return true; } + + // ... otherwise everything else is considered not a match. + false }; if !target_matches_host() { diff --git a/crates/wasmtime/src/runtime/component/func/host.rs b/crates/wasmtime/src/runtime/component/func/host.rs index d55ac2ce5237..1461554ad8a9 100644 --- a/crates/wasmtime/src/runtime/component/func/host.rs +++ b/crates/wasmtime/src/runtime/component/func/host.rs @@ -41,11 +41,11 @@ impl HostFunc { extern "C" fn entrypoint( cx: *mut VMOpaqueContext, data: *mut u8, - ty: TypeFuncIndex, - flags: InstanceFlags, + ty: u32, + flags: *mut u8, memory: *mut VMMemoryDefinition, realloc: *mut VMFuncRef, - string_encoding: StringEncoding, + string_encoding: u8, storage: *mut MaybeUninit, storage_len: usize, ) -> bool @@ -61,11 +61,11 @@ impl HostFunc { instance, types, store, - ty, - flags, + TypeFuncIndex::from_u32(ty), + InstanceFlags::from_raw(flags), memory, realloc, - string_encoding, + StringEncoding::from_u8(string_encoding).unwrap(), core::slice::from_raw_parts_mut(storage, storage_len), |store, args| (*data)(store, args), ) @@ -424,11 +424,11 @@ fn validate_inbounds_dynamic(abi: &CanonicalAbiInfo, memory: &[u8], ptr: &ValRaw extern "C" fn dynamic_entrypoint( cx: *mut VMOpaqueContext, data: *mut u8, - ty: TypeFuncIndex, - flags: InstanceFlags, + ty: u32, + flags: *mut u8, memory: *mut VMMemoryDefinition, realloc: *mut VMFuncRef, - string_encoding: StringEncoding, + string_encoding: u8, storage: *mut MaybeUninit, storage_len: usize, ) -> bool @@ -442,11 +442,11 @@ where instance, types, store, - ty, - flags, + TypeFuncIndex::from_u32(ty), + InstanceFlags::from_raw(flags), memory, realloc, - string_encoding, + StringEncoding::from_u8(string_encoding).unwrap(), core::slice::from_raw_parts_mut(storage, storage_len), |store, params, results| (*data)(store, params, results), ) diff --git a/crates/wasmtime/src/runtime/func.rs b/crates/wasmtime/src/runtime/func.rs index 2a8748de2615..ce335651641e 100644 --- a/crates/wasmtime/src/runtime/func.rs +++ b/crates/wasmtime/src/runtime/func.rs @@ -1,7 +1,7 @@ use crate::prelude::*; use crate::runtime::vm::{ - ExportFunction, SendSyncPtr, StoreBox, VMArrayCallHostFuncContext, VMContext, VMFuncRef, - VMFunctionImport, VMOpaqueContext, + ExportFunction, InterpreterRef, SendSyncPtr, StoreBox, VMArrayCallHostFuncContext, VMContext, + VMFuncRef, VMFunctionImport, VMOpaqueContext, }; use crate::runtime::Uninhabited; use crate::store::{AutoAssertNoGc, StoreData, StoreOpaque, Stored}; @@ -1068,10 +1068,12 @@ impl Func { func_ref: NonNull, params_and_returns: *mut [ValRaw], ) -> Result<()> { - invoke_wasm_and_catch_traps(store, |caller| { - func_ref - .as_ref() - .array_call(caller.cast::(), params_and_returns) + invoke_wasm_and_catch_traps(store, |caller, vm| { + func_ref.as_ref().array_call( + vm, + VMOpaqueContext::from_vmcontext(caller), + params_and_returns, + ) }) } @@ -1592,7 +1594,7 @@ impl Func { /// can pass to the called wasm function, if desired. pub(crate) fn invoke_wasm_and_catch_traps( store: &mut StoreContextMut<'_, T>, - closure: impl FnMut(*mut VMContext) -> bool, + closure: impl FnMut(*mut VMContext, Option>) -> bool, ) -> Result<()> { unsafe { let exit = enter_wasm(store); diff --git a/crates/wasmtime/src/runtime/func/typed.rs b/crates/wasmtime/src/runtime/func/typed.rs index 5d233b8a1945..26f5e2dfdb2f 100644 --- a/crates/wasmtime/src/runtime/func/typed.rs +++ b/crates/wasmtime/src/runtime/func/typed.rs @@ -211,7 +211,7 @@ where // the memory go away, so the size matters here for performance. let mut captures = (func, storage); - let result = invoke_wasm_and_catch_traps(store, |caller| { + let result = invoke_wasm_and_catch_traps(store, |caller, vm| { let (func_ref, storage) = &mut captures; let storage_len = mem::size_of_val::>(storage) / mem::size_of::(); let storage: *mut Storage<_, _> = storage; @@ -219,7 +219,7 @@ where let storage = core::ptr::slice_from_raw_parts_mut(storage, storage_len); func_ref .as_ref() - .array_call(VMOpaqueContext::from_vmcontext(caller), storage) + .array_call(vm, VMOpaqueContext::from_vmcontext(caller), storage) }); let (_, storage) = captures; diff --git a/crates/wasmtime/src/runtime/instance.rs b/crates/wasmtime/src/runtime/instance.rs index ab8e12d16d01..5e3117fe4383 100644 --- a/crates/wasmtime/src/runtime/instance.rs +++ b/crates/wasmtime/src/runtime/instance.rs @@ -361,10 +361,12 @@ impl Instance { let f = instance.get_exported_func(start); let caller_vmctx = instance.vmctx(); unsafe { - super::func::invoke_wasm_and_catch_traps(store, |_default_caller| { - f.func_ref - .as_ref() - .array_call(VMOpaqueContext::from_vmcontext(caller_vmctx), &mut []) + super::func::invoke_wasm_and_catch_traps(store, |_default_caller, vm| { + f.func_ref.as_ref().array_call( + vm, + VMOpaqueContext::from_vmcontext(caller_vmctx), + &mut [], + ) })?; } Ok(()) diff --git a/crates/wasmtime/src/runtime/module/registry.rs b/crates/wasmtime/src/runtime/module/registry.rs index 194ff9206aa9..84d0822c31d7 100644 --- a/crates/wasmtime/src/runtime/module/registry.rs +++ b/crates/wasmtime/src/runtime/module/registry.rs @@ -256,7 +256,6 @@ type GlobalRegistry = BTreeMap)>; /// Find which registered region of code contains the given program counter, and /// what offset that PC is within that module's code. -#[cfg(all(feature = "signals-based-traps", not(miri)))] pub fn lookup_code(pc: usize) -> Option<(Arc, usize)> { let all_modules = global_code().read(); let (_end, (start, module)) = all_modules.range(pc..).next()?; diff --git a/crates/wasmtime/src/runtime/store.rs b/crates/wasmtime/src/runtime/store.rs index 7651c7900a88..cbae0ccd64ce 100644 --- a/crates/wasmtime/src/runtime/store.rs +++ b/crates/wasmtime/src/runtime/store.rs @@ -84,8 +84,8 @@ use crate::prelude::*; use crate::runtime::vm::mpk::{self, ProtectionKey, ProtectionMask}; use crate::runtime::vm::{ Backtrace, ExportGlobal, GcRootsList, GcStore, InstanceAllocationRequest, InstanceAllocator, - InstanceHandle, ModuleRuntimeInfo, OnDemandInstanceAllocator, SignalHandler, StoreBox, - StorePtr, VMContext, VMFuncRef, VMGcRef, VMRuntimeLimits, + InstanceHandle, Interpreter, InterpreterRef, ModuleRuntimeInfo, OnDemandInstanceAllocator, + SignalHandler, StoreBox, StorePtr, VMContext, VMFuncRef, VMGcRef, VMRuntimeLimits, }; use crate::trampoline::VMHostGlobalContext; use crate::type_registry::RegisteredType; @@ -103,6 +103,7 @@ use core::ops::{Deref, DerefMut, Range}; use core::pin::Pin; use core::ptr; use core::task::{Context, Poll}; +use wasmtime_environ::TripleExt; mod context; pub use self::context::*; @@ -387,6 +388,11 @@ pub struct StoreOpaque { component_calls: crate::runtime::vm::component::CallContexts, #[cfg(feature = "component-model")] host_resource_data: crate::component::HostResourceData, + + /// State related to the Pulley interpreter if that's enabled and configured + /// for this store's `Engine`. This is `None` if pulley was disabled at + /// compile time or if it's not being used by the `Engine`. + interpreter: Option, } #[cfg(feature = "async")] @@ -575,6 +581,11 @@ impl Store { component_calls: Default::default(), #[cfg(feature = "component-model")] host_resource_data: Default::default(), + interpreter: if cfg!(feature = "pulley") && engine.target().is_pulley() { + Some(Interpreter::new()) + } else { + None + }, }, limiter: None, call_hook: None, @@ -1972,7 +1983,6 @@ impl StoreOpaque { /// with spectre mitigations enabled since the hardware fault address is /// always zero in these situations which means that the trapping context /// doesn't have enough information to report the fault address. - #[cfg(all(feature = "signals-based-traps", not(miri)))] pub(crate) fn wasm_fault( &self, pc: usize, @@ -2133,6 +2143,11 @@ at https://bytecodealliance.org/security. } } } + + pub(crate) fn interpreter(&mut self) -> Option> { + let i = self.interpreter.as_mut()?; + Some(i.as_interpreter_ref()) + } } impl StoreContextMut<'_, T> { diff --git a/crates/wasmtime/src/runtime/trap.rs b/crates/wasmtime/src/runtime/trap.rs index 358a96519abb..2b95ac6c3110 100644 --- a/crates/wasmtime/src/runtime/trap.rs +++ b/crates/wasmtime/src/runtime/trap.rs @@ -93,7 +93,6 @@ pub(crate) fn from_runtime_box( // otherwise the information about what the wasm was doing when the // error was generated would be lost. crate::runtime::vm::TrapReason::User(error) => (error, None), - #[cfg(all(feature = "signals-based-traps", not(miri)))] crate::runtime::vm::TrapReason::Jit { pc, faulting_addr, diff --git a/crates/wasmtime/src/runtime/vm.rs b/crates/wasmtime/src/runtime/vm.rs index 8a8db87e169c..4abefb6da3c5 100644 --- a/crates/wasmtime/src/runtime/vm.rs +++ b/crates/wasmtime/src/runtime/vm.rs @@ -45,6 +45,13 @@ pub mod debug_builtins; pub mod libcalls; pub mod mpk; +#[cfg(feature = "pulley")] +pub(crate) mod interpreter; +#[cfg(not(feature = "pulley"))] +pub(crate) mod interpreter_disabled; +#[cfg(not(feature = "pulley"))] +pub(crate) use interpreter_disabled as interpreter; + #[cfg(feature = "debug-builtins")] pub use wasmtime_jit_debug::gdb_jit_int::GdbJitImageRegistration; @@ -63,6 +70,7 @@ pub use crate::runtime::vm::instance::{ InstanceLimits, PoolConcurrencyLimitError, PoolingInstanceAllocator, PoolingInstanceAllocatorConfig, }; +pub use crate::runtime::vm::interpreter::*; pub use crate::runtime::vm::memory::{ Memory, RuntimeLinearMemory, RuntimeMemoryCreator, SharedMemory, }; diff --git a/crates/wasmtime/src/runtime/vm/component.rs b/crates/wasmtime/src/runtime/vm/component.rs index 263045b57091..813416e134b0 100644 --- a/crates/wasmtime/src/runtime/vm/component.rs +++ b/crates/wasmtime/src/runtime/vm/component.rs @@ -109,11 +109,11 @@ pub struct ComponentInstance { pub type VMLoweringCallee = extern "C" fn( vmctx: *mut VMOpaqueContext, data: *mut u8, - ty: TypeFuncIndex, - flags: InstanceFlags, + ty: u32, + flags: *mut u8, opt_memory: *mut VMMemoryDefinition, opt_realloc: *mut VMFuncRef, - string_encoding: StringEncoding, + string_encoding: u8, args_and_results: *mut mem::MaybeUninit, nargs_and_results: usize, ) -> bool; @@ -802,6 +802,16 @@ pub struct InstanceFlags(SendSyncPtr); #[allow(missing_docs)] impl InstanceFlags { + /// Wraps the given pointer as an `InstanceFlags` + /// + /// # Unsafety + /// + /// This is a raw pointer argument which needs to be valid for the lifetime + /// that `InstanceFlags` is used. + pub unsafe fn from_raw(ptr: *mut u8) -> InstanceFlags { + InstanceFlags(SendSyncPtr::new(NonNull::new(ptr.cast()).unwrap())) + } + #[inline] pub unsafe fn may_leave(&self) -> bool { *(*self.as_raw()).as_i32() & FLAG_MAY_LEAVE != 0 diff --git a/crates/wasmtime/src/runtime/vm/interpreter.rs b/crates/wasmtime/src/runtime/vm/interpreter.rs new file mode 100644 index 000000000000..c12c21e74efa --- /dev/null +++ b/crates/wasmtime/src/runtime/vm/interpreter.rs @@ -0,0 +1,324 @@ +use crate::prelude::*; +use crate::runtime::vm::vmcontext::VMArrayCallNative; +use crate::runtime::vm::{tls, TrapRegisters, TrapTest, VMContext, VMOpaqueContext}; +use crate::ValRaw; +use core::ptr::NonNull; +use pulley_interpreter::interp::{DoneReason, RegType, Val, Vm, XRegVal}; +use pulley_interpreter::{Reg, XReg}; +use wasmtime_environ::{BuiltinFunctionIndex, HostCall}; + +/// Interpreter state stored within a `Store`. +#[repr(transparent)] +pub struct Interpreter { + /// Pulley VM state, stored behind a `Box` to make the storage in + /// `Store` only pointer-sized (that way if you enable pulley but don't + /// use it it's low-overhead). + pulley: Box, +} + +impl Interpreter { + /// Creates a new interpreter ready to interpret code. + pub fn new() -> Interpreter { + Interpreter { + pulley: Box::new(Vm::new()), + } + } + + /// Returns the `InterpreterRef` structure which can be used to actually + /// execute interpreted code. + pub fn as_interpreter_ref(&mut self) -> InterpreterRef<'_> { + InterpreterRef(&mut self.pulley) + } +} + +/// Wrapper around `&mut pulley_interpreter::Vm` to enable compiling this to a +/// zero-sized structure when pulley is disabled at compile time. +#[repr(transparent)] +pub struct InterpreterRef<'a>(&'a mut Vm); + +#[derive(Clone, Copy)] +struct Setjmp { + sp: *mut u8, + fp: *mut u8, + lr: *mut u8, +} + +impl InterpreterRef<'_> { + /// Invokes interpreted code. + /// + /// The `bytecode` pointer should previously have been produced by Cranelift + /// and `callee` / `caller` / `args_and_results` are normal array-call + /// arguments being passed around. + pub unsafe fn call( + mut self, + mut bytecode: NonNull, + callee: *mut VMOpaqueContext, + caller: *mut VMOpaqueContext, + args_and_results: *mut [ValRaw], + ) -> bool { + // Initialize argument registers with the ABI arguments. + let args = [ + XRegVal::new_ptr(callee).into(), + XRegVal::new_ptr(caller).into(), + XRegVal::new_ptr(args_and_results.cast::()).into(), + XRegVal::new_u64(args_and_results.len() as u64).into(), + ]; + self.0.call_start(&args); + + // Fake a "poor man's setjmp" for now by saving some critical context to + // get restored when a trap happens. This pseudo-implements the stack + // unwinding necessary for a trap. + // + // See more comments in `trap` below about how this isn't actually + // correct as it's not saving all callee-save state. + let setjmp = Setjmp { + sp: self.0[XReg::sp].get_ptr(), + fp: self.0[XReg::fp].get_ptr(), + lr: self.0[XReg::lr].get_ptr(), + }; + + // Run the interpreter as much as possible until it finishes, and then + // handle each finish condition differently. + let ret = loop { + match self.0.call_run(bytecode) { + // If the VM returned entirely then read the return value and + // return that (it indicates whether a trap happened or not. + DoneReason::ReturnToHost(()) => { + match self.0.call_end([RegType::XReg]).next().unwrap() { + #[allow( + clippy::cast_possible_truncation, + reason = "intentionally reading the lower bits only" + )] + Val::XReg(xreg) => break (xreg.get_u32() as u8) != 0, + _ => unreachable!(), + } + } + // If the VM wants to call out to the host then dispatch that + // here based on `sig`. Once that returns we can resume + // execution at `resume`. + DoneReason::CallIndirectHost { id, resume } => { + self.call_indirect_host(id); + bytecode = resume; + } + // If the VM trapped then process that here and return `false`. + DoneReason::Trap(pc) => { + self.trap(pc, setjmp); + break false; + } + } + }; + + debug_assert!(self.0[XReg::sp].get_ptr() == setjmp.sp); + debug_assert!(self.0[XReg::fp].get_ptr() == setjmp.fp); + debug_assert!(self.0[XReg::lr].get_ptr() == setjmp.lr); + ret + } + + /// Handles an interpreter trap. This will initialize the trap state stored + /// in TLS via the `test_if_trap` helper below by reading the pc/fp of the + /// interpreter and seeing if that's a valid opcode to trap at. + fn trap(&mut self, pc: NonNull, setjmp: Setjmp) { + let result = tls::with(|s| { + let s = s.unwrap(); + s.test_if_trap( + TrapRegisters { + pc: pc.as_ptr() as usize, + fp: self.0[XReg::fp].get_ptr::() as usize, + }, + None, + |_| false, + ) + }); + + match result { + // This shouldn't be possible, so this is a fatal error if it + // happens. + TrapTest::NotWasm => panic!("pulley trap at {pc:?} without trap code registered"), + + // Not possible with our closure above returning `false`. + TrapTest::HandledByEmbedder => unreachable!(), + + // Trap was handled, yay! We don't use `jmp_buf`. + TrapTest::Trap { jmp_buf: _ } => {} + } + + // Perform a "longjmp" by restoring the "setjmp" context saved when this + // started. + // + // FIXME: this is not restoring callee-save state. For example if + // there's more than one Pulley activation on the stack that means that + // the previous one is expecting the callee (the host) to preserve all + // callee-save registers. That's not restored here which means with + // multiple activations we're effectively corrupting callee-save + // registers. + // + // One fix for this is to possibly update the `SystemV` ABI on pulley to + // have no callee-saved registers and make everything caller-saved. That + // would force all trampolines to save all state which is basically + // what we want as they'll naturally restore state if we later return to + // them. + let Setjmp { sp, fp, lr } = setjmp; + self.0[XReg::sp].set_ptr(sp); + self.0[XReg::fp].set_ptr(fp); + self.0[XReg::lr].set_ptr(lr); + } + + /// Handles the `call_indirect_host` instruction, dispatching the `sig` + /// number here which corresponds to `wasmtime_environ::HostCall`. + #[allow( + clippy::cast_possible_truncation, + clippy::cast_sign_loss, + reason = "macro-generated code" + )] + unsafe fn call_indirect_host(&mut self, id: u8) { + let id = u32::from(id); + let fnptr = self.0[XReg::x0].get_ptr::(); + let mut arg_reg = 1; + + /// Helper macro to invoke a builtin. + /// + /// Used as: + /// + /// `call(@builtin(ty1, ty2, ...) -> retty)` - invoke a core or + /// component builtin with the macro-defined signature. + /// + /// `call(@host Ty(ty1, ty2, ...) -> retty)` - invoke a host function + /// with the type `Ty`. The other types in the macro are checked by + /// rustc to match the actual `Ty` definition in Rust. + macro_rules! call { + (@builtin($($param:ident),*) $(-> $result:ident)?) => {{ + type T = unsafe extern "C" fn($(call!(@ty $param)),*) $(-> call!(@ty $result))?; + call!(@host T($($param),*) $(-> $result)?); + }}; + (@host $ty:ident($($param:ident),*) $(-> $result:ident)?) => {{ + // Convert the pointer from pulley to a native function pointer. + union GetNative { + fnptr: *mut u8, + host: $ty, + } + let host = GetNative { fnptr }.host; + + // Decode each argument according to this macro, pulling + // arguments from successive registers. + let ret = host($({ + let reg = XReg::new(arg_reg).unwrap(); + arg_reg += 1; + call!(@get $param reg) + }),*); + let _ = arg_reg; // silence last dead arg_reg increment warning + + // Store the return value, if one is here, in x0. + $( + let dst = XReg::x0; + call!(@set $result dst ret); + )? + let _ = ret; // silence warning if no return value + + // Return from the outer `call_indirect_host` host function as + // it's been processed. + return; + }}; + + // Conversion from macro-defined types to Rust host types. + (@ty bool) => (bool); + (@ty u8) => (u8); + (@ty u32) => (u32); + (@ty i32) => (i32); + (@ty u64) => (u64); + (@ty i64) => (i64); + (@ty vmctx) => (*mut VMContext); + (@ty pointer) => (*mut u8); + (@ty ptr_u8) => (*mut u8); + (@ty ptr_u16) => (*mut u16); + (@ty ptr_size) => (*mut usize); + (@ty size) => (usize); + + // Conversion from a pulley register value to the macro-defined + // type. + (@get u8 $reg:ident) => (self.0[$reg].get_i32() as u8); + (@get u32 $reg:ident) => (self.0[$reg].get_u32()); + (@get i32 $reg:ident) => (self.0[$reg].get_i32()); + (@get i64 $reg:ident) => (self.0[$reg].get_i64()); + (@get vmctx $reg:ident) => (self.0[$reg].get_ptr()); + (@get pointer $reg:ident) => (self.0[$reg].get_ptr()); + (@get ptr $reg:ident) => (self.0[$reg].get_ptr()); + (@get ptr_u8 $reg:ident) => (self.0[$reg].get_ptr()); + (@get ptr_u16 $reg:ident) => (self.0[$reg].get_ptr()); + (@get ptr_size $reg:ident) => (self.0[$reg].get_ptr()); + (@get size $reg:ident) => (self.0[$reg].get_ptr::() as usize); + + // Conversion from a Rust value back into a macro-defined type, + // stored in a pulley register. + (@set bool $reg:ident $val:ident) => (self.0[$reg].set_i32(i32::from($val))); + (@set i32 $reg:ident $val:ident) => (self.0[$reg].set_i32($val)); + (@set u64 $reg:ident $val:ident) => (self.0[$reg].set_u64($val)); + (@set i64 $reg:ident $val:ident) => (self.0[$reg].set_i64($val)); + (@set pointer $reg:ident $val:ident) => (self.0[$reg].set_ptr($val)); + (@set size $reg:ident $val:ident) => (self.0[$reg].set_ptr($val as *mut u8)); + } + + // With the helper macro above structure this into: + // + // foreach [core, component] + // * dispatch the call-the-host function pointer type + // * dispatch all builtins by their index. + // + // The hope is that this is relatively easy for LLVM to optimize since + // it's a bunch of: + // + // if id == 0 { ...; return; } + // if id == 1 { ...; return; } + // if id == 2 { ...; return; } + // ... + // + + if id == const { HostCall::ArrayCall.index() } { + call!(@host VMArrayCallNative(ptr, ptr, ptr, size) -> bool); + } + + macro_rules! core { + ( + $( + $( #[cfg($attr:meta)] )? + $name:ident($($pname:ident: $param:ident ),* ) $(-> $result:ident)?; + )* + ) => { + $( + $( #[cfg($attr)] )? + if id == const { HostCall::Builtin(BuiltinFunctionIndex::$name()).index() } { + call!(@builtin($($param),*) $(-> $result)?); + } + )* + } + } + wasmtime_environ::foreach_builtin_function!(core); + + #[cfg(feature = "component-model")] + { + use crate::runtime::vm::component::VMLoweringCallee; + use wasmtime_environ::component::ComponentBuiltinFunctionIndex; + + if id == const { HostCall::ComponentLowerImport.index() } { + call!(@host VMLoweringCallee(ptr, ptr, u32, ptr, ptr, ptr, u8, ptr, size) -> bool); + } + + macro_rules! component { + ( + $( + $name:ident($($pname:ident: $param:ident ),* ) $(-> $result:ident)?; + )* + ) => { + $( + if id == const { HostCall::ComponentBuiltin(ComponentBuiltinFunctionIndex::$name()).index() } { + call!(@builtin($($param),*) $(-> $result)?); + } + )* + } + } + wasmtime_environ::foreach_builtin_component_function!(component); + } + + // if we got this far then something has gone seriously wrong. + unreachable!() + } +} diff --git a/crates/wasmtime/src/runtime/vm/interpreter_disabled.rs b/crates/wasmtime/src/runtime/vm/interpreter_disabled.rs new file mode 100644 index 000000000000..d410c2cb1bbc --- /dev/null +++ b/crates/wasmtime/src/runtime/vm/interpreter_disabled.rs @@ -0,0 +1,49 @@ +//! Stubs for when pulley is disabled at compile time. +//! +//! Note that this is structured so that these structures are all zero-sized and +//! `Option` is also zero-sized so there should be no runtime cost for +//! having these structures plumbed around. + +use crate::runtime::vm::VMOpaqueContext; +use crate::runtime::Uninhabited; +use crate::ValRaw; +use core::marker; +use core::mem; +use core::ptr::NonNull; + +pub struct Interpreter { + empty: Uninhabited, +} + +const _: () = assert!(mem::size_of::() == 0); +const _: () = assert!(mem::size_of::>() == 0); + +impl Interpreter { + pub fn new() -> Interpreter { + unreachable!() + } + + pub fn as_interpreter_ref(&mut self) -> InterpreterRef<'_> { + match self.empty {} + } +} + +pub struct InterpreterRef<'a> { + empty: Uninhabited, + _marker: marker::PhantomData<&'a mut Interpreter>, +} + +const _: () = assert!(mem::size_of::>() == 0); +const _: () = assert!(mem::size_of::>>() == 0); + +impl InterpreterRef<'_> { + pub unsafe fn call( + self, + _bytecode: NonNull, + _callee: *mut VMOpaqueContext, + _caller: *mut VMOpaqueContext, + _args_and_results: *mut [ValRaw], + ) -> bool { + match self.empty {} + } +} diff --git a/crates/wasmtime/src/runtime/vm/traphandlers.rs b/crates/wasmtime/src/runtime/vm/traphandlers.rs index 5eb3128dd5fd..c5fb8d608157 100644 --- a/crates/wasmtime/src/runtime/vm/traphandlers.rs +++ b/crates/wasmtime/src/runtime/vm/traphandlers.rs @@ -16,9 +16,10 @@ mod signals; pub use self::signals::*; use crate::prelude::*; +use crate::runtime::module::lookup_code; use crate::runtime::store::StoreOpaque; use crate::runtime::vm::sys::traphandlers; -use crate::runtime::vm::{Instance, VMContext, VMRuntimeLimits}; +use crate::runtime::vm::{Instance, InterpreterRef, VMContext, VMRuntimeLimits}; use crate::{StoreContextMut, WasmBacktrace}; use core::cell::Cell; use core::ops::Range; @@ -30,6 +31,26 @@ pub use self::tls::{tls_eager_initialize, AsyncWasmCallState, PreviousAsyncWasmC pub use traphandlers::SignalHandler; +pub(crate) struct TrapRegisters { + pub pc: usize, + pub fp: usize, +} + +/// Return value from `test_if_trap`. +pub(crate) enum TrapTest { + /// Not a wasm trap, need to delegate to whatever process handler is next. + NotWasm, + /// This trap was handled by the embedder via custom embedding APIs. + #[cfg_attr(miri, expect(dead_code, reason = "using #[cfg] too unergonomic"))] + HandledByEmbedder, + /// This is a wasm trap, it needs to be handled. + #[cfg_attr(miri, expect(dead_code, reason = "using #[cfg] too unergonomic"))] + Trap { + /// How to longjmp back to the original wasm frame. + jmp_buf: *const u8, + }, +} + fn lazy_per_thread_init() { traphandlers::lazy_per_thread_init(); } @@ -283,7 +304,6 @@ pub enum TrapReason { User(Error), /// A trap raised from Cranelift-generated code. - #[cfg(all(feature = "signals-based-traps", not(miri)))] Jit { /// The program counter where this trap originated. /// @@ -334,17 +354,41 @@ pub unsafe fn catch_traps( mut closure: F, ) -> Result<(), Box> where - F: FnMut(*mut VMContext) -> bool, + F: FnMut(*mut VMContext, Option>) -> bool, { let caller = store.0.default_caller(); + let result = CallThreadState::new(store.0, caller).with(|cx| match store.0.interpreter() { + // In interpreted mode directly invoke the host closure since we won't + // be using host-based `setjmp`/`longjmp` as that's not going to save + // the context we want. + Some(r) => { + cx.jmp_buf + .set(CallThreadState::JMP_BUF_INTERPRETER_SENTINEL); + closure(caller, Some(r)) + } - let result = CallThreadState::new(store.0, caller).with(|cx| { - traphandlers::wasmtime_setjmp( + // In native mode, however, defer to C to do the `setjmp` since Rust + // doesn't understand `setjmp`. + // + // Note that here we pass a function pointer to C to catch longjmp + // within, here it's `call_closure`, and that passes `None` for the + // interpreter since this branch is only ever taken if the interpreter + // isn't present. + None => traphandlers::wasmtime_setjmp( cx.jmp_buf.as_ptr(), - call_closure::, + { + extern "C" fn call_closure(payload: *mut u8, caller: *mut VMContext) -> bool + where + F: FnMut(*mut VMContext, Option>) -> bool, + { + unsafe { (*(payload as *mut F))(caller, None) } + } + + call_closure:: + }, &mut closure as *mut F as *mut u8, caller, - ) + ), }); return match result { @@ -357,13 +401,6 @@ where #[cfg(all(feature = "std", panic = "unwind"))] Err((UnwindReason::Panic(panic), _, _)) => std::panic::resume_unwind(panic), }; - - extern "C" fn call_closure(payload: *mut u8, caller: *mut VMContext) -> bool - where - F: FnMut(*mut VMContext) -> bool, - { - unsafe { (*(payload as *mut F))(caller) } - } } // Module to hide visibility of the `CallThreadState::prev` field and force @@ -416,6 +453,8 @@ mod call_thread_state { } impl CallThreadState { + pub const JMP_BUF_INTERPRETER_SENTINEL: *mut u8 = 1 as *mut u8; + #[inline] pub(super) fn new(store: &mut StoreOpaque, caller: *mut VMContext) -> CallThreadState { let limits = unsafe { *Instance::from_vmctx(caller, |i| i.runtime_limits()) }; @@ -580,6 +619,91 @@ impl CallThreadState { Some(this) }) } + + /// Trap handler using our thread-local state. + /// + /// * `regs` - some special program registers at the time that the trap + /// happened, for example `pc`. + /// * `faulting_addr` - the system-provided address that the a fault, if + /// any, happened at. This is used when debug-asserting that all segfaults + /// are known to live within a `Store` in a valid range. + /// * `call_handler` - a closure used to invoke the platform-specific + /// signal handler for each instance, if available. + /// + /// Attempts to handle the trap if it's a wasm trap. Returns a `TrapTest` + /// which indicates what this could be, such as: + /// + /// * `TrapTest::NotWasm` - not a wasm fault, this should get forwarded to + /// the next platform-specific fault handler. + /// * `TrapTest::HandledByEmbedder` - the embedder `call_handler` handled + /// this signal, nothing else to do. + /// * `TrapTest::Trap` - this is a wasm trap an the stack needs to be + /// unwound now. + pub(crate) fn test_if_trap( + &self, + regs: TrapRegisters, + faulting_addr: Option, + call_handler: impl Fn(&SignalHandler) -> bool, + ) -> TrapTest { + // If we haven't even started to handle traps yet, bail out. + if self.jmp_buf.get().is_null() { + return TrapTest::NotWasm; + } + + // First up see if any instance registered has a custom trap handler, + // in which case run them all. If anything handles the trap then we + // return that the trap was handled. + let _ = &call_handler; + #[cfg(all(feature = "signals-based-traps", not(miri)))] + if let Some(handler) = self.signal_handler { + if unsafe { call_handler(&*handler) } { + return TrapTest::HandledByEmbedder; + } + } + + // If this fault wasn't in wasm code, then it's not our problem + let Some((code, text_offset)) = lookup_code(regs.pc) else { + return TrapTest::NotWasm; + }; + + // If the fault was at a location that was not marked as potentially + // trapping, then that's a bug in Cranelift/Winch/etc. Don't try to + // catch the trap and pretend this isn't wasm so the program likely + // aborts. + let Some(trap) = code.lookup_trap_code(text_offset) else { + return TrapTest::NotWasm; + }; + + // If all that passed then this is indeed a wasm trap, so return the + // `jmp_buf` passed to `wasmtime_longjmp` to resume. + self.set_jit_trap(regs, faulting_addr, trap); + TrapTest::Trap { + jmp_buf: self.take_jmp_buf(), + } + } + + pub(crate) fn take_jmp_buf(&self) -> *const u8 { + self.jmp_buf.replace(ptr::null()) + } + + pub(crate) fn set_jit_trap( + &self, + TrapRegisters { pc, fp, .. }: TrapRegisters, + faulting_addr: Option, + trap: wasmtime_environ::Trap, + ) { + let backtrace = self.capture_backtrace(self.limits, Some((pc, fp))); + let coredump = self.capture_coredump(self.limits, Some((pc, fp))); + self.unwind.set(Some(( + UnwindReason::Trap(TrapReason::Jit { + pc, + faulting_addr, + trap, + }), + backtrace, + coredump, + ))) + } } // A private inner module for managing the TLS state that we require across diff --git a/crates/wasmtime/src/runtime/vm/traphandlers/signals.rs b/crates/wasmtime/src/runtime/vm/traphandlers/signals.rs index 1a9e8da94d0e..ac5d6a3409a7 100644 --- a/crates/wasmtime/src/runtime/vm/traphandlers/signals.rs +++ b/crates/wasmtime/src/runtime/vm/traphandlers/signals.rs @@ -6,30 +6,8 @@ //! thise module serves as a shared entrypoint for initialization entrypoints //! (`init_traps`) and testing if a trapping opcode is wasm (`test_if_trap`). -use crate::runtime::module::lookup_code; use crate::sync::RwLock; use crate::vm::sys::traphandlers::TrapHandler; -use crate::vm::traphandlers::{CallThreadState, SignalHandler, TrapReason, UnwindReason}; -use core::ptr; - -pub(crate) struct TrapRegisters { - pub pc: usize, - pub fp: usize, -} - -/// Return value from `test_if_trap`. -pub(crate) enum TrapTest { - /// Not a wasm trap, need to delegate to whatever process handler is next. - NotWasm, - /// This trap was handled by the embedder via custom embedding APIs. - HandledByEmbedder, - /// This is a wasm trap, it needs to be handled. - #[cfg_attr(miri, allow(dead_code))] - Trap { - /// How to longjmp back to the original wasm frame. - jmp_buf: *const u8, - }, -} /// Platform-specific trap-handler state. /// @@ -84,81 +62,3 @@ pub unsafe fn deinit_traps() { let mut lock = TRAP_HANDLER.write(); let _ = lock.take(); } - -impl CallThreadState { - /// Trap handler using our thread-local state. - /// - /// * `pc` - the program counter the trap happened at - /// * `call_handler` - a closure used to invoke the platform-specific - /// signal handler for each instance, if available. - /// - /// Attempts to handle the trap if it's a wasm trap. Returns a few - /// different things: - /// - /// * null - the trap didn't look like a wasm trap and should continue as a - /// trap - /// * 1 as a pointer - the trap was handled by a custom trap handler on an - /// instance, and the trap handler should quickly return. - /// * a different pointer - a jmp_buf buffer to longjmp to, meaning that - /// the wasm trap was successfully handled. - pub(crate) fn test_if_trap( - &self, - regs: TrapRegisters, - faulting_addr: Option, - call_handler: impl Fn(&SignalHandler) -> bool, - ) -> TrapTest { - // If we haven't even started to handle traps yet, bail out. - if self.jmp_buf.get().is_null() { - return TrapTest::NotWasm; - } - - // First up see if any instance registered has a custom trap handler, - // in which case run them all. If anything handles the trap then we - // return that the trap was handled. - if let Some(handler) = self.signal_handler { - if unsafe { call_handler(&*handler) } { - return TrapTest::HandledByEmbedder; - } - } - - // If this fault wasn't in wasm code, then it's not our problem - let Some((code, text_offset)) = lookup_code(regs.pc) else { - return TrapTest::NotWasm; - }; - - let Some(trap) = code.lookup_trap_code(text_offset) else { - return TrapTest::NotWasm; - }; - - self.set_jit_trap(regs, faulting_addr, trap); - - // If all that passed then this is indeed a wasm trap, so return the - // `jmp_buf` passed to `wasmtime_longjmp` to resume. - TrapTest::Trap { - jmp_buf: self.take_jmp_buf(), - } - } - - pub(crate) fn take_jmp_buf(&self) -> *const u8 { - self.jmp_buf.replace(ptr::null()) - } - - pub(crate) fn set_jit_trap( - &self, - TrapRegisters { pc, fp, .. }: TrapRegisters, - faulting_addr: Option, - trap: wasmtime_environ::Trap, - ) { - let backtrace = self.capture_backtrace(self.limits, Some((pc, fp))); - let coredump = self.capture_coredump(self.limits, Some((pc, fp))); - self.unwind.set(Some(( - UnwindReason::Trap(TrapReason::Jit { - pc, - faulting_addr, - trap, - }), - backtrace, - coredump, - ))) - } -} diff --git a/crates/wasmtime/src/runtime/vm/vmcontext.rs b/crates/wasmtime/src/runtime/vm/vmcontext.rs index 8b14680e22c0..61c78620b1b7 100644 --- a/crates/wasmtime/src/runtime/vm/vmcontext.rs +++ b/crates/wasmtime/src/runtime/vm/vmcontext.rs @@ -5,7 +5,7 @@ mod vm_host_func_context; pub use self::vm_host_func_context::VMArrayCallHostFuncContext; use crate::prelude::*; -use crate::runtime::vm::{GcStore, VMGcRef}; +use crate::runtime::vm::{GcStore, InterpreterRef, VMGcRef}; use crate::store::StoreOpaque; use core::cell::UnsafeCell; use core::ffi::c_void; @@ -720,8 +720,35 @@ impl VMFuncRef { /// # Unsafety /// /// This method is unsafe because it can be called with any pointers. They - /// must all be valid for this wasm function call to proceed. + /// must all be valid for this wasm function call to proceed. For example + /// the `caller` must be valid machine code if `pulley` is `None` or it must + /// be valid bytecode if `pulley` is `Some`. Additionally `args_and_results` + /// must be large enough to handle all the arguments/results for this call. + /// + /// Note that the unsafety invariants to maintain here are not currently + /// exhaustively documented. pub unsafe fn array_call( + &self, + pulley: Option>, + caller: *mut VMOpaqueContext, + args_and_results: *mut [ValRaw], + ) -> bool { + match pulley { + Some(vm) => self.array_call_interpreted(vm, caller, args_and_results), + None => self.array_call_native(caller, args_and_results), + } + } + + unsafe fn array_call_interpreted( + &self, + vm: InterpreterRef<'_>, + caller: *mut VMOpaqueContext, + args_and_results: *mut [ValRaw], + ) -> bool { + vm.call(self.array_call.cast(), self.vmctx, caller, args_and_results) + } + + unsafe fn array_call_native( &self, caller: *mut VMOpaqueContext, args_and_results: *mut [ValRaw], diff --git a/crates/wast-util/src/lib.rs b/crates/wast-util/src/lib.rs index e88c18cb0594..cc82f83e4a24 100644 --- a/crates/wast-util/src/lib.rs +++ b/crates/wast-util/src/lib.rs @@ -200,6 +200,15 @@ macro_rules! define_test_config { pub struct TestConfig { $(pub $option: Option,)* } + + impl TestConfig { + $( + pub fn $option(&self) -> bool { + self.$option.unwrap_or(false) + } + )* + } + } } @@ -229,25 +238,81 @@ pub struct WastConfig { pub collector: Collector, } +/// Different compilers that can be tested in Wasmtime. #[derive(PartialEq, Debug, Copy, Clone)] pub enum Compiler { - Cranelift, + /// Cranelift backend. + /// + /// This tests the Cranelift code generator for native platforms. This + /// notably excludes Pulley since that's listed separately below even though + /// Pulley is a backend of Cranelift. This is only used for native code + /// generation such as x86_64. + CraneliftNative, + + /// Winch backend. + /// + /// This tests the Winch backend for native platforms. Currently Winch + /// primarily supports x86_64. Winch, + + /// Pulley interpreter. + /// + /// This tests the Cranelift pulley backend plus the pulley execution + /// environment of the output bytecode. Note that this is separate from + /// `Cranelift` above to be able to test both on platforms where Cranelift + /// has native codegen support. + CraneliftPulley, } impl Compiler { + /// Returns whether this compiler is known to fail for the provided + /// `TestConfig`. + /// + /// This function will determine if the configuration of the test provided + /// is known to guarantee fail. This effectively tracks the proposal support + /// for each compiler backend/runtime and tests whether `config` enables or + /// disables features that aren't supported. + /// + /// Note that this is closely aligned with + /// `Config::compiler_panicking_wasm_features`. pub fn should_fail(&self, config: &TestConfig) -> bool { match self { - Compiler::Cranelift => {} + // Currently Cranelift supports all wasm proposals that wasmtime + // tests. + Compiler::CraneliftNative => {} + + // Winch doesn't have quite the full breadth of support that + // Cranelift has quite yet. Compiler::Winch => { - // A few proposals that winch has no support for. - if config.gc == Some(true) - || config.threads == Some(true) - || config.tail_call == Some(true) - || config.function_references == Some(true) - || config.gc == Some(true) - || config.relaxed_simd == Some(true) - || config.gc_types == Some(true) + if config.gc() + || config.threads() + || config.tail_call() + || config.function_references() + || config.gc() + || config.relaxed_simd() + || config.gc_types() + { + return true; + } + } + + // Pulley is just getting started, it implements almost no proposals + // yet. + Compiler::CraneliftPulley => { + // Unsupported proposals + if config.memory64() + || config.custom_page_sizes() + || config.multi_memory() + || config.threads() + || config.gc() + || config.function_references() + || config.relaxed_simd() + || config.reference_types() + || config.tail_call() + || config.extended_const() + || config.wide_arithmetic() + || config.simd() + || config.gc_types() { return true; } @@ -256,6 +321,27 @@ impl Compiler { false } + + /// Returns whether this complier configuration supports the current host + /// architecture. + pub fn supports_host(&self) -> bool { + match self { + Compiler::CraneliftNative => { + cfg!(target_arch = "x86_64") + || cfg!(target_arch = "aarch64") + || cfg!(target_arch = "riscv64") + || cfg!(target_arch = "s390x") + } + Compiler::Winch => { + cfg!(target_arch = "x86_64") + } + Compiler::CraneliftPulley => { + // FIXME(#9747) pulley needs more refactoring to support a + // big-endian host. + cfg!(target_endian = "little") + } + } + } } #[derive(PartialEq, Debug, Copy, Clone)] @@ -269,10 +355,7 @@ impl WastTest { /// Returns whether this test exercises the GC types and might want to use /// multiple different garbage collectors. pub fn test_uses_gc_types(&self) -> bool { - self.config - .gc - .or(self.config.function_references) - .unwrap_or(false) + self.config.gc() || self.config.function_references() } /// Returns the optional spec proposal that this test is associated with. @@ -283,8 +366,74 @@ impl WastTest { /// Returns whether this test should fail under the specified extra /// configuration. pub fn should_fail(&self, config: &WastConfig) -> bool { - // Winch only supports x86_64 at this time. - if config.compiler == Compiler::Winch && !cfg!(target_arch = "x86_64") { + if !config.compiler.supports_host() { + return true; + } + + // Some tests are known to fail with the pooling allocator + if config.pooling { + let unsupported = [ + // allocates too much memory for the pooling configuration here + "misc_testsuite/memory64/more-than-4gb.wast", + // shared memories + pooling allocator aren't supported yet + "misc_testsuite/memory-combos.wast", + "misc_testsuite/threads/LB.wast", + "misc_testsuite/threads/LB_atomic.wast", + "misc_testsuite/threads/MP.wast", + "misc_testsuite/threads/MP_atomic.wast", + "misc_testsuite/threads/MP_wait.wast", + "misc_testsuite/threads/SB.wast", + "misc_testsuite/threads/SB_atomic.wast", + "misc_testsuite/threads/atomics_notify.wast", + "misc_testsuite/threads/atomics_wait_address.wast", + "misc_testsuite/threads/wait_notify.wast", + "spec_testsuite/proposals/threads/atomic.wast", + "spec_testsuite/proposals/threads/exports.wast", + "spec_testsuite/proposals/threads/memory.wast", + ]; + + if unsupported.iter().any(|part| self.path.ends_with(part)) { + return true; + } + } + + // Pulley is in a bit of a special state at this time where it supports + // only a subset of the initial MVP of WebAssembly. That means that no + // test technically passes by default but a few do happen to use just + // the right subset of wasm that we can pass it. For now maintain an + // allow-list of tests that are known to pass in Pulley. As tests are + // fixed they should get added to this list. Over time this list will + // instead get inverted to "these tests are known to fail" once Pulley + // implements more proposals. + if config.compiler == Compiler::CraneliftPulley { + let supported = [ + "custom-page-sizes/custom-page-sizes-invalid.wast", + "exception-handling/exports.wast", + "extended-const/data.wast", + "misc_testsuite/component-model/adapter.wast", + "misc_testsuite/component-model/aliasing.wast", + "misc_testsuite/component-model/import.wast", + "misc_testsuite/component-model/instance.wast", + "misc_testsuite/component-model/linking.wast", + "misc_testsuite/component-model/nested.wast", + "misc_testsuite/component-model/types.wast", + "misc_testsuite/elem-ref-null.wast", + "misc_testsuite/elem_drop.wast", + "misc_testsuite/empty.wast", + "misc_testsuite/fib.wast", + "misc_testsuite/func-400-params.wast", + "misc_testsuite/gc/more-rec-groups-than-types.wast", + "misc_testsuite/gc/rec-group-funcs.wast", + "misc_testsuite/rs2wasm-add-func.wast", + "misc_testsuite/stack_overflow.wast", + "misc_testsuite/winch/misc.wast", + "threads/exports.wast", + ]; + + if supported.iter().any(|part| self.path.ends_with(part)) { + return false; + } + return true; } @@ -427,33 +576,6 @@ impl WastTest { } } - // Some tests are known to fail with the pooling allocator - if config.pooling { - let unsupported = [ - // allocates too much memory for the pooling configuration here - "misc_testsuite/memory64/more-than-4gb.wast", - // shared memories + pooling allocator aren't supported yet - "misc_testsuite/memory-combos.wast", - "misc_testsuite/threads/LB.wast", - "misc_testsuite/threads/LB_atomic.wast", - "misc_testsuite/threads/MP.wast", - "misc_testsuite/threads/MP_atomic.wast", - "misc_testsuite/threads/MP_wait.wast", - "misc_testsuite/threads/SB.wast", - "misc_testsuite/threads/SB_atomic.wast", - "misc_testsuite/threads/atomics_notify.wast", - "misc_testsuite/threads/atomics_wait_address.wast", - "misc_testsuite/threads/wait_notify.wast", - "spec_testsuite/proposals/threads/atomic.wast", - "spec_testsuite/proposals/threads/exports.wast", - "spec_testsuite/proposals/threads/memory.wast", - ]; - - if unsupported.iter().any(|part| self.path.ends_with(part)) { - return true; - } - } - false } } diff --git a/pulley/examples/objdump.rs b/pulley/examples/objdump.rs index 7e9c7f8ed154..ad7b57b0dd51 100644 --- a/pulley/examples/objdump.rs +++ b/pulley/examples/objdump.rs @@ -5,7 +5,7 @@ //! cargo run --example objdump -F disas -p pulley-interpreter foo.cwasm use anyhow::{bail, Result}; -use object::{File, Object as _, ObjectSection, ObjectSymbol, SectionKind, SymbolKind}; +use object::{File, Object as _, ObjectSection, ObjectSymbol, SymbolKind}; use pulley_interpreter::decode::Decoder; use pulley_interpreter::disas::Disassembler; @@ -14,7 +14,7 @@ fn main() -> Result<()> { let image = File::parse(&cwasm[..])?; - let text = match image.sections().find(|s| s.kind() == SectionKind::Text) { + let text = match image.sections().find(|s| s.name().ok() == Some(".text")) { Some(section) => section.data()?, None => bail!("no text section"), }; diff --git a/pulley/fuzz/src/interp.rs b/pulley/fuzz/src/interp.rs index 4cf59b973c65..7e434709830a 100644 --- a/pulley/fuzz/src/interp.rs +++ b/pulley/fuzz/src/interp.rs @@ -1,5 +1,5 @@ use pulley_interpreter::{ - interp::Vm, + interp::{DoneReason, Vm}, op::{self, ExtendedOp, Op}, *, }; @@ -30,8 +30,8 @@ pub fn interp(ops: Vec) { let args = &[]; let rets = &[]; match vm.call(NonNull::from(&encoded[0]), args, rets.into_iter().copied()) { - Ok(rets) => assert_eq!(rets.count(), 0), - Err(pc) => { + DoneReason::ReturnToHost(rets) => assert_eq!(rets.count(), 0), + DoneReason::Trap(pc) => { let pc = pc.as_ptr() as usize; let start = &encoded[0] as *const u8 as usize; @@ -47,6 +47,7 @@ pub fn interp(ops: Vec) { assert_eq!(encoded[index + 1], a); assert_eq!(encoded[index + 2], b); } + DoneReason::CallIndirectHost { .. } => unreachable!(), }; } } diff --git a/pulley/src/interp.rs b/pulley/src/interp.rs index 55d718b70a58..d7507a73e5e7 100644 --- a/pulley/src/interp.rs +++ b/pulley/src/interp.rs @@ -4,7 +4,6 @@ use crate::decode::*; use crate::encode::Encode; use crate::imms::*; use crate::regs::*; -use crate::ExtendedOpcode; use alloc::string::ToString; use alloc::{vec, vec::Vec}; use core::fmt; @@ -77,7 +76,28 @@ impl Vm { func: NonNull, args: &[Val], rets: impl IntoIterator + 'a, - ) -> Result + 'a, NonNull> { + ) -> DoneReason + 'a> { + self.call_start(args); + + match self.call_run(func) { + DoneReason::ReturnToHost(()) => DoneReason::ReturnToHost(self.call_end(rets)), + DoneReason::Trap(pc) => DoneReason::Trap(pc), + DoneReason::CallIndirectHost { id, resume } => { + DoneReason::CallIndirectHost { id, resume } + } + } + } + + /// Peforms the initial part of [`Vm::call`] in setting up the `args` + /// provided in registers according to Pulley's ABI. + /// + /// # Unsafety + /// + /// All the same unsafety as `call` and additiionally, you must + /// invoke `call_run` and then `call_end` after calling `call_start`. + /// If you don't want to wrangle these invocations, use `call` instead + /// of `call_{start,run,end}`. + pub unsafe fn call_start<'a>(&'a mut self, args: &[Val]) { // NB: make sure this method stays in sync with // `PulleyMachineDeps::compute_arg_locs`! @@ -101,14 +121,45 @@ impl Vm { }, } } + } - self.run(func)?; + /// Peforms the internal part of [`Vm::call`] where bytecode is actually + /// executed. + /// + /// # Unsafety + /// + /// In addition to all the invariants documented for `call`, you + /// may only invoke `call_run` after invoking `call_start` to + /// initialize this call's arguments. + pub unsafe fn call_run(&mut self, pc: NonNull) -> DoneReason<()> { + self.state.debug_assert_done_reason_none(); + let interpreter = Interpreter { + state: &mut self.state, + pc: UnsafeBytecodeStream::new(pc), + }; + let done = interpreter.run(); + self.state.done_decode(done) + } + + /// Peforms the tail end of [`Vm::call`] by returning the values as + /// determined by `rets` according to Pulley's ABI. + /// + /// # Unsafety + /// + /// In addition to the invariants documented for `call`, this may + /// only be called after `call_run`. + pub unsafe fn call_end<'a>( + &'a mut self, + rets: impl IntoIterator + 'a, + ) -> impl Iterator + 'a { + // NB: make sure this method stays in sync with + // `PulleyMachineDeps::compute_arg_locs`! let mut x_rets = (0..16).map(|x| XReg::new_unchecked(x)); let mut f_rets = (0..16).map(|f| FReg::new_unchecked(f)); let mut v_rets = (0..16).map(|v| VReg::new_unchecked(v)); - Ok(rets.into_iter().map(move |ty| match ty { + rets.into_iter().map(move |ty| match ty { RegType::XReg => match x_rets.next() { Some(reg) => Val::XReg(self.state[reg]), None => todo!("stack slots"), @@ -121,43 +172,7 @@ impl Vm { Some(reg) => Val::VReg(self.state[reg]), None => todo!("stack slots"), }, - })) - } - - unsafe fn run(&mut self, pc: NonNull) -> Result<(), NonNull> { - let interpreter = Interpreter { - state: &mut self.state, - pc: UnsafeBytecodeStream::new(pc), - }; - match interpreter.run() { - Done::ReturnToHost => self.return_to_host(), - Done::Trap(pc) => self.trap(pc), - Done::HostCall => self.host_call(), - } - } - - #[cold] - #[inline(never)] - fn return_to_host(&self) -> Result<(), NonNull> { - Ok(()) - } - - #[cold] - #[inline(never)] - fn trap(&self, pc: NonNull) -> Result<(), NonNull> { - // We are given the VM's PC upon having executed a trap instruction, - // which is actually pointing to the next instruction after the - // trap. Back the PC up to point exactly at the trap. - let trap_pc = unsafe { - NonNull::new_unchecked(pc.as_ptr().byte_sub(ExtendedOpcode::ENCODED_SIZE_OF_TRAP)) - }; - Err(trap_pc) - } - - #[cold] - #[inline(never)] - fn host_call(&self) -> Result<(), NonNull> { - todo!() + }) } } @@ -526,6 +541,7 @@ pub struct MachineState { f_regs: [FRegVal; FReg::RANGE.end as usize], v_regs: [VRegVal; VReg::RANGE.end as usize], stack: Vec, + done_reason: Option>, } unsafe impl Send for MachineState {} @@ -538,6 +554,7 @@ impl fmt::Debug for MachineState { f_regs, v_regs, stack: _, + done_reason: _, } = self; struct RegMap<'a, R>(&'a [R], fn(u8) -> alloc::string::String); @@ -571,6 +588,20 @@ impl fmt::Debug for MachineState { macro_rules! index_reg { ($reg_ty:ty,$value_ty:ty,$field:ident) => { + impl Index<$reg_ty> for Vm { + type Output = $value_ty; + + fn index(&self, reg: $reg_ty) -> &Self::Output { + &self.state[reg] + } + } + + impl IndexMut<$reg_ty> for Vm { + fn index_mut(&mut self, reg: $reg_ty) -> &mut Self::Output { + &mut self.state[reg] + } + } + impl Index<$reg_ty> for MachineState { type Output = $value_ty; @@ -599,6 +630,7 @@ impl MachineState { f_regs: Default::default(), v_regs: Default::default(), stack, + done_reason: None, }; // Take care to construct SP such that we preserve pointer provenance @@ -615,20 +647,72 @@ impl MachineState { } } -/// The reason the interpreter loop terminated. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -enum Done { - /// A `ret` instruction was executed and the call stack was empty. This is - /// how the loop normally ends. - ReturnToHost, +/// Inner private module to prevent creation of the `Done` structure outside of +/// this module. +mod done { + use super::{Interpreter, MachineState}; + use core::ptr::NonNull; + + /// Zero-sized sentinel indicating that pulley execution has halted. + /// + /// The reason for halting is stored in `MachineState`. + #[derive(Copy, Clone, Debug, PartialEq, Eq)] + pub struct Done { + _priv: (), + } + + /// Reason that the pulley interpreter has ceased execution. + pub enum DoneReason { + /// A trap happened at this bytecode instruction. + Trap(NonNull), + /// The `call_indirect_host` instruction was executed. + CallIndirectHost { + /// The payload of `call_indirect_host`. + id: u8, + /// Where to resume execution after the host has finished. + resume: NonNull, + }, + /// Pulley has finished and the provided value is being returned. + ReturnToHost(T), + } + + impl MachineState { + pub(super) fn debug_assert_done_reason_none(&mut self) { + debug_assert!(self.done_reason.is_none()); + } + + pub(super) fn done_decode(&mut self, Done { _priv }: Done) -> DoneReason<()> { + self.done_reason.take().unwrap() + } + } + + impl Interpreter<'_> { + /// Finishes execution by recording `DoneReason::Trap`. + pub fn done_trap(&mut self, pc: NonNull) -> Done { + self.state.done_reason = Some(DoneReason::Trap(pc)); + Done { _priv: () } + } - /// A `trap` instruction was executed at the given PC. - Trap(NonNull), + /// Finishes execution by recording `DoneReason::CallIndirectHost`. + pub fn done_call_indirect_host(&mut self, id: u8) -> Done { + self.state.done_reason = Some(DoneReason::CallIndirectHost { + id, + resume: self.pc.as_ptr(), + }); + Done { _priv: () } + } - #[allow(dead_code)] - HostCall, + /// Finishes execution by recording `DoneReason::ReturnToHost`. + pub fn done_return_to_host(&mut self) -> Done { + self.state.done_reason = Some(DoneReason::ReturnToHost(())); + Done { _priv: () } + } + } } +use done::Done; +pub use done::DoneReason; + struct Interpreter<'a> { state: &'a mut MachineState, pc: UnsafeBytecodeStream, @@ -676,7 +760,7 @@ impl Interpreter<'_> { let sp_raw = sp as usize; let base_raw = self.state.stack.as_ptr() as usize; if sp_raw < base_raw { - return ControlFlow::Break(Done::Trap(pc)); + return ControlFlow::Break(self.done_trap(pc)); } self.set_sp_unchecked(sp); ControlFlow::Continue(()) @@ -731,7 +815,7 @@ impl OpVisitor for Interpreter<'_> { fn ret(&mut self) -> ControlFlow { let lr = self.state[XReg::lr]; if lr == XRegVal::HOST_RETURN_ADDR { - ControlFlow::Break(Done::ReturnToHost) + ControlFlow::Break(self.done_return_to_host()) } else { let return_addr = lr.get_ptr(); self.pc = unsafe { UnsafeBytecodeStream::new(NonNull::new_unchecked(return_addr)) }; @@ -1288,7 +1372,8 @@ impl OpVisitor for Interpreter<'_> { // SAFETY: part of the contract of the interpreter is only dealing with // valid bytecode, so this offset should be safe. self.pc = unsafe { self.pc.offset(idx * 4) }; - let rel = unwrap_uninhabited(PcRelOffset::decode(&mut self.pc)); + let mut tmp = self.pc; + let rel = unwrap_uninhabited(PcRelOffset::decode(&mut tmp)); self.pc_rel_jump(rel, 0) } @@ -1350,11 +1435,11 @@ impl ExtendedOpVisitor for Interpreter<'_> { } fn trap(&mut self) -> ControlFlow { - ControlFlow::Break(Done::Trap(self.pc.as_ptr())) + let trap_pc = self.current_pc::(); + ControlFlow::Break(self.done_trap(trap_pc)) } - fn call_indirect_host(&mut self, sig: u8) -> ControlFlow { - let _ = sig; // TODO: should stash this somewhere - ControlFlow::Break(Done::ReturnToHost) + fn call_indirect_host(&mut self, id: u8) -> ControlFlow { + ControlFlow::Break(self.done_call_indirect_host(id)) } } diff --git a/pulley/src/opcode.rs b/pulley/src/opcode.rs index de1d8420eb04..0ff21217e316 100644 --- a/pulley/src/opcode.rs +++ b/pulley/src/opcode.rs @@ -84,9 +84,6 @@ macro_rules! define_extended_opcode { for_each_extended_op!(define_extended_opcode); impl ExtendedOpcode { - #[cfg_attr(not(feature = "interp"), allow(unused))] - pub(crate) const ENCODED_SIZE_OF_TRAP: usize = 3; - /// Create a new `ExtendedOpcode` from the given bytes. /// /// Returns `None` if `bytes` is not a valid extended opcode. @@ -108,16 +105,3 @@ impl ExtendedOpcode { core::mem::transmute(byte) } } - -#[cfg(all(test, feature = "encode"))] -mod tests { - use super::*; - use alloc::vec::Vec; - - #[test] - fn encoded_size_of_trap() { - let mut buf = Vec::new(); - crate::encode::trap(&mut buf); - assert_eq!(ExtendedOpcode::ENCODED_SIZE_OF_TRAP, buf.len()); - } -} diff --git a/pulley/tests/all/interp.rs b/pulley/tests/all/interp.rs index 8015cfed1412..886add4be222 100644 --- a/pulley/tests/all/interp.rs +++ b/pulley/tests/all/interp.rs @@ -1,7 +1,10 @@ //! Interpreter tests. use interp::Val; -use pulley_interpreter::{interp::Vm, *}; +use pulley_interpreter::{ + interp::{DoneReason, Vm}, + *, +}; use std::{cell::UnsafeCell, fmt::Debug, ptr::NonNull}; fn encoded(ops: &[Op]) -> Vec { @@ -16,8 +19,11 @@ fn encoded(ops: &[Op]) -> Vec { unsafe fn run(vm: &mut Vm, ops: &[Op]) -> Result<(), NonNull> { let _ = env_logger::try_init(); let ops = encoded(ops); - let _ = vm.call(NonNull::from(&ops[..]).cast(), &[], [])?; - Ok(()) + match vm.call(NonNull::from(&ops[..]).cast(), &[], []) { + DoneReason::ReturnToHost(_) => Ok(()), + DoneReason::Trap(pc) => Err(pc), + DoneReason::CallIndirectHost { .. } => unimplemented!(), + } } unsafe fn assert_one( diff --git a/tests/all/pulley.rs b/tests/all/pulley.rs index f3695283c2d9..ef24819b5590 100644 --- a/tests/all/pulley.rs +++ b/tests/all/pulley.rs @@ -15,10 +15,25 @@ fn pulley_config() -> Config { config } +// Pulley is known to not support big-endian platforms at this time, so assert +// that big-endian platforms do indeed fail and success is only on little-endian +// platforms. When pulley has support for big-endian this will get deleted. +fn assert_result_expected(r: Result) { + match r { + Ok(_) => { + assert!(!cfg!(target_endian = "big")); + } + Err(e) => { + assert!(cfg!(target_endian = "big"), "bad error: {e:?}"); + } + } +} + #[test] fn can_compile_pulley_module() -> Result<()> { let engine = Engine::new(&pulley_config())?; - Module::new(&engine, "(module)")?; + assert_result_expected(Module::new(&engine, "(module)")); + Ok(()) } @@ -61,16 +76,16 @@ fn pulley_wrong_architecture_is_rejected() -> Result<()> { #[cfg(not(miri))] fn can_run_on_cli() -> Result<()> { use crate::cli_tests::run_wasmtime; - run_wasmtime(&[ + assert_result_expected(run_wasmtime(&[ "--target", pulley_target(), "tests/all/cli_tests/empty-module.wat", - ])?; - run_wasmtime(&[ + ])); + assert_result_expected(run_wasmtime(&[ "run", "--target", pulley_target(), "tests/all/cli_tests/empty-module.wat", - ])?; + ])); Ok(()) } diff --git a/tests/disas/pulley/epoch-simple.wat b/tests/disas/pulley/epoch-simple.wat index 6eace319937b..1f503e82902e 100644 --- a/tests/disas/pulley/epoch-simple.wat +++ b/tests/disas/pulley/epoch-simple.wat @@ -15,5 +15,5 @@ ;; br_if x7, 0x8 // target = 0x1b ;; 19: pop_frame ;; ret -;; 1b: call 0xa // target = 0x25 +;; 1b: call 0x83 // target = 0x9e ;; 20: jump 0xfffffffffffffff9 // target = 0x19 diff --git a/tests/wast.rs b/tests/wast.rs index cfbd12a06899..68abc4923726 100644 --- a/tests/wast.rs +++ b/tests/wast.rs @@ -22,7 +22,16 @@ fn main() { // run this test in. for test in tests { let test_uses_gc_types = test.test_uses_gc_types(); - for compiler in [Compiler::Cranelift, Compiler::Winch] { + for compiler in [ + Compiler::CraneliftNative, + Compiler::Winch, + Compiler::CraneliftPulley, + ] { + // Skip compilers that have no support for this host. + if !compiler.supports_host() { + continue; + } + for pooling in [true, false] { let collectors: &[_] = if !pooling && test_uses_gc_types { &[Collector::DeferredReferenceCounting, Collector::Null] @@ -104,12 +113,12 @@ fn run_wast(test: &WastTest, config: WastConfig) -> anyhow::Result<()> { // `crates/wast-util/src/lib.rs` file. let should_fail = test.should_fail(&config); - let multi_memory = test_config.multi_memory.unwrap_or(false); - let test_hogs_memory = test_config.hogs_memory.unwrap_or(false); - let relaxed_simd = test_config.relaxed_simd.unwrap_or(false); + let multi_memory = test_config.multi_memory(); + let test_hogs_memory = test_config.hogs_memory(); + let relaxed_simd = test_config.relaxed_simd(); let is_cranelift = match config.compiler { - Compiler::Cranelift => true, + Compiler::CraneliftNative | Compiler::CraneliftPulley => true, _ => false, };