diff --git a/compiler/rustc_codegen_gcc/src/asm.rs b/compiler/rustc_codegen_gcc/src/asm.rs index a04cd4735f478..cec3ecd6683b9 100644 --- a/compiler/rustc_codegen_gcc/src/asm.rs +++ b/compiler/rustc_codegen_gcc/src/asm.rs @@ -611,10 +611,14 @@ fn reg_to_gcc(reg: InlineAsmRegOrRegClass) -> ConstraintOrRegister { } // They can be retrieved from https://gcc.gnu.org/onlinedocs/gcc/Machine-Constraints.html InlineAsmRegOrRegClass::RegClass(reg) => match reg { - InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => "r", - InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) => "w", - InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => "x", - InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => { + InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) + | InlineAsmRegClass::Arm64EC(AArch64InlineAsmRegClass::reg) => "r", + InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) + | InlineAsmRegClass::Arm64EC(AArch64InlineAsmRegClass::vreg) => "w", + InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) + | InlineAsmRegClass::Arm64EC(AArch64InlineAsmRegClass::vreg_low16) => "x", + InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) + | InlineAsmRegClass::Arm64EC(AArch64InlineAsmRegClass::preg) => { unreachable!("clobber-only") } InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => "r", @@ -698,12 +702,16 @@ fn reg_to_gcc(reg: InlineAsmRegOrRegClass) -> ConstraintOrRegister { /// the type is, as long as it is valid for the constraint code. fn dummy_output_type<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, reg: InlineAsmRegClass) -> Type<'gcc> { match reg { - InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => cx.type_i32(), + InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) + | InlineAsmRegClass::Arm64EC(AArch64InlineAsmRegClass::reg) => cx.type_i32(), InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) - | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => { + | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) + | InlineAsmRegClass::Arm64EC(AArch64InlineAsmRegClass::vreg) + | InlineAsmRegClass::Arm64EC(AArch64InlineAsmRegClass::vreg_low16) => { cx.type_vector(cx.type_i64(), 2) } - InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => { + InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) + | InlineAsmRegClass::Arm64EC(AArch64InlineAsmRegClass::preg) => { unreachable!("clobber-only") } InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => cx.type_i32(), @@ -863,12 +871,16 @@ fn modifier_to_gcc( // The modifiers can be retrieved from // https://gcc.gnu.org/onlinedocs/gcc/Modifiers.html#Modifiers match reg { - InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => modifier, + InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) + | InlineAsmRegClass::Arm64EC(AArch64InlineAsmRegClass::reg) => modifier, InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) - | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => { + | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) + | InlineAsmRegClass::Arm64EC(AArch64InlineAsmRegClass::vreg) + | InlineAsmRegClass::Arm64EC(AArch64InlineAsmRegClass::vreg_low16) => { if modifier == Some('v') { None } else { modifier } } - InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => { + InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) + | InlineAsmRegClass::Arm64EC(AArch64InlineAsmRegClass::preg) => { unreachable!("clobber-only") } InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => None, diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs index 82ca3f519f740..662e3eb9c5362 100644 --- a/compiler/rustc_codegen_llvm/src/asm.rs +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -542,59 +542,107 @@ fn xmm_reg_index(reg: InlineAsmReg) -> Option { /// If the register is an AArch64 integer register then return its index. fn a64_reg_index(reg: InlineAsmReg) -> Option { - use AArch64InlineAsmReg::*; // Unlike `a64_vreg_index`, we can't subtract `x0` to get the u32 because // `x19` and `x29` are missing and the integer constants for the // `x0`..`x30` enum variants don't all match the register number. E.g. the // integer constant for `x18` is 18, but the constant for `x20` is 19. Some(match reg { - InlineAsmReg::AArch64(r) => match r { - x0 => 0, - x1 => 1, - x2 => 2, - x3 => 3, - x4 => 4, - x5 => 5, - x6 => 6, - x7 => 7, - x8 => 8, - x9 => 9, - x10 => 10, - x11 => 11, - x12 => 12, - x13 => 13, - x14 => 14, - x15 => 15, - x16 => 16, - x17 => 17, - x18 => 18, - // x19 is reserved - x20 => 20, - x21 => 21, - x22 => 22, - x23 => 23, - x24 => 24, - x25 => 25, - x26 => 26, - x27 => 27, - x28 => 28, - // x29 is reserved - x30 => 30, - _ => return None, - }, + InlineAsmReg::AArch64(r) => { + use AArch64InlineAsmReg::*; + match r { + x0 => 0, + x1 => 1, + x2 => 2, + x3 => 3, + x4 => 4, + x5 => 5, + x6 => 6, + x7 => 7, + x8 => 8, + x9 => 9, + x10 => 10, + x11 => 11, + x12 => 12, + x13 => 13, + x14 => 14, + x15 => 15, + x16 => 16, + x17 => 17, + x18 => 18, + // x19 is reserved + x20 => 20, + x21 => 21, + x22 => 22, + x23 => 23, + x24 => 24, + x25 => 25, + x26 => 26, + x27 => 27, + x28 => 28, + // x29 is reserved + x30 => 30, + _ => return None, + } + } + InlineAsmReg::Arm64EC(r) => { + use Arm64ECInlineAsmReg::*; + match r { + x0 => 0, + x1 => 1, + x2 => 2, + x3 => 3, + x4 => 4, + x5 => 5, + x6 => 6, + x7 => 7, + x8 => 8, + x9 => 9, + x10 => 10, + x11 => 11, + x12 => 12, + // x13 is reserved + // x14 is reserved + x15 => 15, + x16 => 16, + x17 => 17, + // x18 is reserved + // x19 is reserved + x20 => 20, + x21 => 21, + x22 => 22, + // x23 is reserved + // x24 is reserved + x25 => 25, + x26 => 26, + x27 => 27, + // x28 is reserved + // x29 is reserved + x30 => 30, + _ => return None, + } + } _ => return None, }) } /// If the register is an AArch64 vector register then return its index. fn a64_vreg_index(reg: InlineAsmReg) -> Option { - use AArch64InlineAsmReg::*; match reg { - InlineAsmReg::AArch64(reg) if reg as u32 >= v0 as u32 && reg as u32 <= v31 as u32 => { - Some(reg as u32 - v0 as u32) + InlineAsmReg::AArch64(reg) => { + use AArch64InlineAsmReg::*; + if reg as u32 >= v0 as u32 && reg as u32 <= v31 as u32 { + return Some(reg as u32 - v0 as u32); + } } - _ => None, + InlineAsmReg::Arm64EC(reg) => { + use Arm64ECInlineAsmReg::*; + if reg as u32 >= v0 as u32 && reg as u32 <= v15 as u32 { + return Some(reg as u32 - v0 as u32); + } + } + _ => {} } + None } /// Converts a register class to an LLVM constraint code. @@ -625,7 +673,13 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> // We use i32 as the type for discarded outputs 'w' }; - if class == 'x' && reg == InlineAsmReg::AArch64(AArch64InlineAsmReg::x30) { + if class == 'x' + && matches!( + reg, + InlineAsmReg::AArch64(AArch64InlineAsmReg::x30) + | InlineAsmReg::Arm64EC(Arm64ECInlineAsmReg::x30) + ) + { // LLVM doesn't recognize x30. use lr instead. "{lr}".to_string() } else { @@ -656,10 +710,15 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> // The constraints can be retrieved from // https://llvm.org/docs/LangRef.html#supported-constraint-code-list InlineAsmRegOrRegClass::RegClass(reg) => match reg { - AArch64(AArch64InlineAsmRegClass::reg) => "r", - AArch64(AArch64InlineAsmRegClass::vreg) => "w", - AArch64(AArch64InlineAsmRegClass::vreg_low16) => "x", - AArch64(AArch64InlineAsmRegClass::preg) => unreachable!("clobber-only"), + AArch64(AArch64InlineAsmRegClass::reg) | Arm64EC(AArch64InlineAsmRegClass::reg) => "r", + AArch64(AArch64InlineAsmRegClass::vreg) | Arm64EC(AArch64InlineAsmRegClass::vreg) => { + "w" + } + AArch64(AArch64InlineAsmRegClass::vreg_low16) + | Arm64EC(AArch64InlineAsmRegClass::vreg_low16) => "x", + AArch64(AArch64InlineAsmRegClass::preg) | Arm64EC(AArch64InlineAsmRegClass::preg) => { + unreachable!("clobber-only") + } Arm(ArmInlineAsmRegClass::reg) => "r", Arm(ArmInlineAsmRegClass::sreg) | Arm(ArmInlineAsmRegClass::dreg_low16) @@ -734,15 +793,20 @@ fn modifier_to_llvm( // The modifiers can be retrieved from // https://llvm.org/docs/LangRef.html#asm-template-argument-modifiers match reg { - AArch64(AArch64InlineAsmRegClass::reg) => modifier, - AArch64(AArch64InlineAsmRegClass::vreg) | AArch64(AArch64InlineAsmRegClass::vreg_low16) => { + AArch64(AArch64InlineAsmRegClass::reg) | Arm64EC(AArch64InlineAsmRegClass::reg) => modifier, + AArch64(AArch64InlineAsmRegClass::vreg) + | AArch64(AArch64InlineAsmRegClass::vreg_low16) + | Arm64EC(AArch64InlineAsmRegClass::vreg) + | Arm64EC(AArch64InlineAsmRegClass::vreg_low16) => { if modifier == Some('v') { None } else { modifier } } - AArch64(AArch64InlineAsmRegClass::preg) => unreachable!("clobber-only"), + AArch64(AArch64InlineAsmRegClass::preg) | Arm64EC(AArch64InlineAsmRegClass::preg) => { + unreachable!("clobber-only") + } Arm(ArmInlineAsmRegClass::reg) => None, Arm(ArmInlineAsmRegClass::sreg) | Arm(ArmInlineAsmRegClass::sreg_low16) => None, Arm(ArmInlineAsmRegClass::dreg) @@ -817,11 +881,16 @@ fn modifier_to_llvm( fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &'ll Type { use InlineAsmRegClass::*; match reg { - AArch64(AArch64InlineAsmRegClass::reg) => cx.type_i32(), - AArch64(AArch64InlineAsmRegClass::vreg) | AArch64(AArch64InlineAsmRegClass::vreg_low16) => { - cx.type_vector(cx.type_i64(), 2) + AArch64(AArch64InlineAsmRegClass::reg) | Arm64EC(AArch64InlineAsmRegClass::reg) => { + cx.type_i32() + } + AArch64(AArch64InlineAsmRegClass::vreg) + | AArch64(AArch64InlineAsmRegClass::vreg_low16) + | Arm64EC(AArch64InlineAsmRegClass::vreg) + | Arm64EC(AArch64InlineAsmRegClass::vreg_low16) => cx.type_vector(cx.type_i64(), 2), + AArch64(AArch64InlineAsmRegClass::preg) | Arm64EC(AArch64InlineAsmRegClass::preg) => { + unreachable!("clobber-only") } - AArch64(AArch64InlineAsmRegClass::preg) => unreachable!("clobber-only"), Arm(ArmInlineAsmRegClass::reg) => cx.type_i32(), Arm(ArmInlineAsmRegClass::sreg) | Arm(ArmInlineAsmRegClass::sreg_low16) => cx.type_f32(), Arm(ArmInlineAsmRegClass::dreg) @@ -922,7 +991,10 @@ fn llvm_fixup_input<'ll, 'tcx>( use InlineAsmRegClass::*; let dl = &bx.tcx.data_layout; match (reg, layout.abi) { - (AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => { + ( + AArch64(AArch64InlineAsmRegClass::vreg) | Arm64EC(AArch64InlineAsmRegClass::vreg), + Abi::Scalar(s), + ) => { if let Primitive::Int(Integer::I8, _) = s.primitive() { let vec_ty = bx.cx.type_vector(bx.cx.type_i8(), 8); bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0)) @@ -930,9 +1002,11 @@ fn llvm_fixup_input<'ll, 'tcx>( value } } - (AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) - if s.primitive() != Primitive::Float(Float::F128) => - { + ( + AArch64(AArch64InlineAsmRegClass::vreg_low16) + | Arm64EC(AArch64InlineAsmRegClass::vreg_low16), + Abi::Scalar(s), + ) if s.primitive() != Primitive::Float(Float::F128) => { let elem_ty = llvm_asm_scalar_type(bx.cx, s); let count = 16 / layout.size.bytes(); let vec_ty = bx.cx.type_vector(elem_ty, count); @@ -943,9 +1017,11 @@ fn llvm_fixup_input<'ll, 'tcx>( } bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0)) } - (AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Vector { element, count }) - if layout.size.bytes() == 8 => - { + ( + AArch64(AArch64InlineAsmRegClass::vreg_low16) + | Arm64EC(AArch64InlineAsmRegClass::vreg_low16), + Abi::Vector { element, count }, + ) if layout.size.bytes() == 8 => { let elem_ty = llvm_asm_scalar_type(bx.cx, element); let vec_ty = bx.cx.type_vector(elem_ty, count); let indices: Vec<_> = (0..count * 2).map(|x| bx.const_i32(x as i32)).collect(); @@ -1064,25 +1140,32 @@ fn llvm_fixup_output<'ll, 'tcx>( ) -> &'ll Value { use InlineAsmRegClass::*; match (reg, layout.abi) { - (AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => { + ( + AArch64(AArch64InlineAsmRegClass::vreg) | Arm64EC(AArch64InlineAsmRegClass::vreg), + Abi::Scalar(s), + ) => { if let Primitive::Int(Integer::I8, _) = s.primitive() { bx.extract_element(value, bx.const_i32(0)) } else { value } } - (AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) - if s.primitive() != Primitive::Float(Float::F128) => - { + ( + AArch64(AArch64InlineAsmRegClass::vreg_low16) + | Arm64EC(AArch64InlineAsmRegClass::vreg_low16), + Abi::Scalar(s), + ) if s.primitive() != Primitive::Float(Float::F128) => { value = bx.extract_element(value, bx.const_i32(0)); if let Primitive::Pointer(_) = s.primitive() { value = bx.inttoptr(value, layout.llvm_type(bx.cx)); } value } - (AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Vector { element, count }) - if layout.size.bytes() == 8 => - { + ( + AArch64(AArch64InlineAsmRegClass::vreg_low16) + | Arm64EC(AArch64InlineAsmRegClass::vreg_low16), + Abi::Vector { element, count }, + ) if layout.size.bytes() == 8 => { let elem_ty = llvm_asm_scalar_type(bx.cx, element); let vec_ty = bx.cx.type_vector(elem_ty, count * 2); let indices: Vec<_> = (0..count).map(|x| bx.const_i32(x as i32)).collect(); @@ -1195,23 +1278,30 @@ fn llvm_fixup_output_type<'ll, 'tcx>( ) -> &'ll Type { use InlineAsmRegClass::*; match (reg, layout.abi) { - (AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => { + ( + AArch64(AArch64InlineAsmRegClass::vreg) | Arm64EC(AArch64InlineAsmRegClass::vreg), + Abi::Scalar(s), + ) => { if let Primitive::Int(Integer::I8, _) = s.primitive() { cx.type_vector(cx.type_i8(), 8) } else { layout.llvm_type(cx) } } - (AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) - if s.primitive() != Primitive::Float(Float::F128) => - { + ( + AArch64(AArch64InlineAsmRegClass::vreg_low16) + | Arm64EC(AArch64InlineAsmRegClass::vreg_low16), + Abi::Scalar(s), + ) if s.primitive() != Primitive::Float(Float::F128) => { let elem_ty = llvm_asm_scalar_type(cx, s); let count = 16 / layout.size.bytes(); cx.type_vector(elem_ty, count) } - (AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Vector { element, count }) - if layout.size.bytes() == 8 => - { + ( + AArch64(AArch64InlineAsmRegClass::vreg_low16) + | Arm64EC(AArch64InlineAsmRegClass::vreg_low16), + Abi::Vector { element, count }, + ) if layout.size.bytes() == 8 => { let elem_ty = llvm_asm_scalar_type(cx, element); cx.type_vector(elem_ty, count * 2) } diff --git a/compiler/rustc_target/src/asm/aarch64.rs b/compiler/rustc_target/src/asm/aarch64.rs index 067f5c3a9568b..078066821d78f 100644 --- a/compiler/rustc_target/src/asm/aarch64.rs +++ b/compiler/rustc_target/src/asm/aarch64.rs @@ -88,20 +88,6 @@ fn reserved_x18( } } -fn restricted_for_arm64ec( - arch: InlineAsmArch, - _reloc_model: RelocModel, - _target_features: &FxIndexSet, - _target: &Target, - _is_clobber: bool, -) -> Result<(), &'static str> { - if arch == InlineAsmArch::Arm64EC { - Err("x13, x14, x23, x24, x28, v16-v31, z*, p*, ffr cannot be used for Arm64EC") - } else { - Ok(()) - } -} - def_regs! { AArch64 AArch64InlineAsmReg AArch64InlineAsmRegClass { x0: reg = ["x0", "w0"], @@ -117,8 +103,8 @@ def_regs! { x10: reg = ["x10", "w10"], x11: reg = ["x11", "w11"], x12: reg = ["x12", "w12"], - x13: reg = ["x13", "w13"] % restricted_for_arm64ec, - x14: reg = ["x14", "w14"] % restricted_for_arm64ec, + x13: reg = ["x13", "w13"], + x14: reg = ["x14", "w14"], x15: reg = ["x15", "w15"], x16: reg = ["x16", "w16"], x17: reg = ["x17", "w17"], @@ -126,14 +112,13 @@ def_regs! { x20: reg = ["x20", "w20"], x21: reg = ["x21", "w21"], x22: reg = ["x22", "w22"], - x23: reg = ["x23", "w23"] % restricted_for_arm64ec, - x24: reg = ["x24", "w24"] % restricted_for_arm64ec, + x23: reg = ["x23", "w23"], + x24: reg = ["x24", "w24"], x25: reg = ["x25", "w25"], x26: reg = ["x26", "w26"], x27: reg = ["x27", "w27"], - x28: reg = ["x28", "w28"] % restricted_for_arm64ec, + x28: reg = ["x28", "w28"], x30: reg = ["x30", "w30", "lr", "wlr"], - // FIXME: z* cannot be used for Arm64EC v0: vreg, vreg_low16 = ["v0", "b0", "h0", "s0", "d0", "q0", "z0"], v1: vreg, vreg_low16 = ["v1", "b1", "h1", "s1", "d1", "q1", "z1"], v2: vreg, vreg_low16 = ["v2", "b2", "h2", "s2", "d2", "q2", "z2"], @@ -150,39 +135,39 @@ def_regs! { v13: vreg, vreg_low16 = ["v13", "b13", "h13", "s13", "d13", "q13", "z13"], v14: vreg, vreg_low16 = ["v14", "b14", "h14", "s14", "d14", "q14", "z14"], v15: vreg, vreg_low16 = ["v15", "b15", "h15", "s15", "d15", "q15", "z15"], - v16: vreg = ["v16", "b16", "h16", "s16", "d16", "q16", "z16"] % restricted_for_arm64ec, - v17: vreg = ["v17", "b17", "h17", "s17", "d17", "q17", "z17"] % restricted_for_arm64ec, - v18: vreg = ["v18", "b18", "h18", "s18", "d18", "q18", "z18"] % restricted_for_arm64ec, - v19: vreg = ["v19", "b19", "h19", "s19", "d19", "q19", "z19"] % restricted_for_arm64ec, - v20: vreg = ["v20", "b20", "h20", "s20", "d20", "q20", "z20"] % restricted_for_arm64ec, - v21: vreg = ["v21", "b21", "h21", "s21", "d21", "q21", "z21"] % restricted_for_arm64ec, - v22: vreg = ["v22", "b22", "h22", "s22", "d22", "q22", "z22"] % restricted_for_arm64ec, - v23: vreg = ["v23", "b23", "h23", "s23", "d23", "q23", "z23"] % restricted_for_arm64ec, - v24: vreg = ["v24", "b24", "h24", "s24", "d24", "q24", "z24"] % restricted_for_arm64ec, - v25: vreg = ["v25", "b25", "h25", "s25", "d25", "q25", "z25"] % restricted_for_arm64ec, - v26: vreg = ["v26", "b26", "h26", "s26", "d26", "q26", "z26"] % restricted_for_arm64ec, - v27: vreg = ["v27", "b27", "h27", "s27", "d27", "q27", "z27"] % restricted_for_arm64ec, - v28: vreg = ["v28", "b28", "h28", "s28", "d28", "q28", "z28"] % restricted_for_arm64ec, - v29: vreg = ["v29", "b29", "h29", "s29", "d29", "q29", "z29"] % restricted_for_arm64ec, - v30: vreg = ["v30", "b30", "h30", "s30", "d30", "q30", "z30"] % restricted_for_arm64ec, - v31: vreg = ["v31", "b31", "h31", "s31", "d31", "q31", "z31"] % restricted_for_arm64ec, - p0: preg = ["p0"] % restricted_for_arm64ec, - p1: preg = ["p1"] % restricted_for_arm64ec, - p2: preg = ["p2"] % restricted_for_arm64ec, - p3: preg = ["p3"] % restricted_for_arm64ec, - p4: preg = ["p4"] % restricted_for_arm64ec, - p5: preg = ["p5"] % restricted_for_arm64ec, - p6: preg = ["p6"] % restricted_for_arm64ec, - p7: preg = ["p7"] % restricted_for_arm64ec, - p8: preg = ["p8"] % restricted_for_arm64ec, - p9: preg = ["p9"] % restricted_for_arm64ec, - p10: preg = ["p10"] % restricted_for_arm64ec, - p11: preg = ["p11"] % restricted_for_arm64ec, - p12: preg = ["p12"] % restricted_for_arm64ec, - p13: preg = ["p13"] % restricted_for_arm64ec, - p14: preg = ["p14"] % restricted_for_arm64ec, - p15: preg = ["p15"] % restricted_for_arm64ec, - ffr: preg = ["ffr"] % restricted_for_arm64ec, + v16: vreg = ["v16", "b16", "h16", "s16", "d16", "q16", "z16"], + v17: vreg = ["v17", "b17", "h17", "s17", "d17", "q17", "z17"], + v18: vreg = ["v18", "b18", "h18", "s18", "d18", "q18", "z18"], + v19: vreg = ["v19", "b19", "h19", "s19", "d19", "q19", "z19"], + v20: vreg = ["v20", "b20", "h20", "s20", "d20", "q20", "z20"], + v21: vreg = ["v21", "b21", "h21", "s21", "d21", "q21", "z21"], + v22: vreg = ["v22", "b22", "h22", "s22", "d22", "q22", "z22"], + v23: vreg = ["v23", "b23", "h23", "s23", "d23", "q23", "z23"], + v24: vreg = ["v24", "b24", "h24", "s24", "d24", "q24", "z24"], + v25: vreg = ["v25", "b25", "h25", "s25", "d25", "q25", "z25"], + v26: vreg = ["v26", "b26", "h26", "s26", "d26", "q26", "z26"], + v27: vreg = ["v27", "b27", "h27", "s27", "d27", "q27", "z27"], + v28: vreg = ["v28", "b28", "h28", "s28", "d28", "q28", "z28"], + v29: vreg = ["v29", "b29", "h29", "s29", "d29", "q29", "z29"], + v30: vreg = ["v30", "b30", "h30", "s30", "d30", "q30", "z30"], + v31: vreg = ["v31", "b31", "h31", "s31", "d31", "q31", "z31"], + p0: preg = ["p0"], + p1: preg = ["p1"], + p2: preg = ["p2"], + p3: preg = ["p3"], + p4: preg = ["p4"], + p5: preg = ["p5"], + p6: preg = ["p6"], + p7: preg = ["p7"], + p8: preg = ["p8"], + p9: preg = ["p9"], + p10: preg = ["p10"], + p11: preg = ["p11"], + p12: preg = ["p12"], + p13: preg = ["p13"], + p14: preg = ["p14"], + p15: preg = ["p15"], + ffr: preg = ["ffr"], #error = ["x19", "w19"] => "x19 is used internally by LLVM and cannot be used as an operand for inline asm", #error = ["x29", "w29", "fp", "wfp"] => diff --git a/compiler/rustc_target/src/asm/arm64ec.rs b/compiler/rustc_target/src/asm/arm64ec.rs new file mode 100644 index 0000000000000..f7962f05fd4fa --- /dev/null +++ b/compiler/rustc_target/src/asm/arm64ec.rs @@ -0,0 +1,110 @@ +use std::fmt; + +use rustc_span::Symbol; + +use super::{AArch64InlineAsmRegClass, InlineAsmArch}; + +def_regs! { + Arm64EC Arm64ECInlineAsmReg AArch64InlineAsmRegClass { + x0: reg = ["x0", "w0"], + x1: reg = ["x1", "w1"], + x2: reg = ["x2", "w2"], + x3: reg = ["x3", "w3"], + x4: reg = ["x4", "w4"], + x5: reg = ["x5", "w5"], + x6: reg = ["x6", "w6"], + x7: reg = ["x7", "w7"], + x8: reg = ["x8", "w8"], + x9: reg = ["x9", "w9"], + x10: reg = ["x10", "w10"], + x11: reg = ["x11", "w11"], + x12: reg = ["x12", "w12"], + x15: reg = ["x15", "w15"], + x16: reg = ["x16", "w16"], + x17: reg = ["x17", "w17"], + x20: reg = ["x20", "w20"], + x21: reg = ["x21", "w21"], + x22: reg = ["x22", "w22"], + x25: reg = ["x25", "w25"], + x26: reg = ["x26", "w26"], + x27: reg = ["x27", "w27"], + x30: reg = ["x30", "w30", "lr", "wlr"], + v0: vreg, vreg_low16 = ["v0", "b0", "h0", "s0", "d0", "q0"], + v1: vreg, vreg_low16 = ["v1", "b1", "h1", "s1", "d1", "q1"], + v2: vreg, vreg_low16 = ["v2", "b2", "h2", "s2", "d2", "q2"], + v3: vreg, vreg_low16 = ["v3", "b3", "h3", "s3", "d3", "q3"], + v4: vreg, vreg_low16 = ["v4", "b4", "h4", "s4", "d4", "q4"], + v5: vreg, vreg_low16 = ["v5", "b5", "h5", "s5", "d5", "q5"], + v6: vreg, vreg_low16 = ["v6", "b6", "h6", "s6", "d6", "q6"], + v7: vreg, vreg_low16 = ["v7", "b7", "h7", "s7", "d7", "q7"], + v8: vreg, vreg_low16 = ["v8", "b8", "h8", "s8", "d8", "q8"], + v9: vreg, vreg_low16 = ["v9", "b9", "h9", "s9", "d9", "q9"], + v10: vreg, vreg_low16 = ["v10", "b10", "h10", "s10", "d10", "q10"], + v11: vreg, vreg_low16 = ["v11", "b11", "h11", "s11", "d11", "q11"], + v12: vreg, vreg_low16 = ["v12", "b12", "h12", "s12", "d12", "q12"], + v13: vreg, vreg_low16 = ["v13", "b13", "h13", "s13", "d13", "q13"], + v14: vreg, vreg_low16 = ["v14", "b14", "h14", "s14", "d14", "q14"], + v15: vreg, vreg_low16 = ["v15", "b15", "h15", "s15", "d15", "q15"], + #error = ["x18", "w18"] => + "x18 is a reserved register on this target", + #error = ["x19", "w19"] => + "x19 is used internally by LLVM and cannot be used as an operand for inline asm", + #error = ["x29", "w29", "fp", "wfp"] => + "the frame pointer cannot be used as an operand for inline asm", + #error = ["sp", "wsp"] => + "the stack pointer cannot be used as an operand for inline asm", + #error = ["xzr", "wzr"] => + "the zero register cannot be used as an operand for inline asm", + #error = [ + "x13", "w13", + "x14", "w14", + "x23", "w23", + "x24", "w24", + "x28", "w28", + "v16", "b16", "h16", "s16", "d16", "q16", + "v17", "b17", "h17", "s17", "d17", "q17", + "v18", "b18", "h18", "s18", "d18", "q18", + "v19", "b19", "h19", "s19", "d19", "q19", + "v20", "b20", "h20", "s20", "d20", "q20", + "v21", "b21", "h21", "s21", "d21", "q21", + "v22", "b22", "h22", "s22", "d22", "q22", + "v23", "b23", "h23", "s23", "d23", "q23", + "v24", "b24", "h24", "s24", "d24", "q24", + "v25", "b25", "h25", "s25", "d25", "q25", + "v26", "b26", "h26", "s26", "d26", "q26", + "v27", "b27", "h27", "s27", "d27", "q27", + "v28", "b28", "h28", "s28", "d28", "q28", + "v29", "b29", "h29", "s29", "d29", "q29", + "v30", "b30", "h30", "s30", "d30", "q30", + "v31", "b31", "h31", "s31", "d31", "q31" + ] => + "x13, x14, x23, x24, x28, v16-v31 cannot be used for Arm64EC", + #error = [ + "z0", "z1", "z2", "z3", "z4", "z5", "z6", "z7", "z8", "z9", + "z10", "z11", "z12", "z13", "z14", "z15", "z16", "z17", "z18", "z19", + "z20", "z21", "z22", "z23", "z24", "z25", "z26", "z27", "z28", "z29", + "z30", "z31", + "p0", "p1", "p2", "p3", "p4", "p5", "p6", "p7", "p8", "p9", + "p10", "p11", "p12", "p13", "p14", "p15", + "ffr" + ] => + "SVE cannot be used for Arm64EC", + } +} + +impl Arm64ECInlineAsmReg { + pub fn emit( + self, + out: &mut dyn fmt::Write, + _arch: InlineAsmArch, + modifier: Option, + ) -> fmt::Result { + let (prefix, index) = if (self as u32) < Self::v0 as u32 { + (modifier.unwrap_or('x'), self as u32 - Self::x0 as u32) + } else { + (modifier.unwrap_or('v'), self as u32 - Self::v0 as u32) + }; + assert!(index < 32); + write!(out, "{prefix}{index}") + } +} diff --git a/compiler/rustc_target/src/asm/mod.rs b/compiler/rustc_target/src/asm/mod.rs index ef5143d804be1..d4949bb429b88 100644 --- a/compiler/rustc_target/src/asm/mod.rs +++ b/compiler/rustc_target/src/asm/mod.rs @@ -179,6 +179,7 @@ macro_rules! types { mod aarch64; mod arm; +mod arm64ec; mod avr; mod bpf; mod csky; @@ -197,6 +198,7 @@ mod x86; pub use aarch64::{AArch64InlineAsmReg, AArch64InlineAsmRegClass}; pub use arm::{ArmInlineAsmReg, ArmInlineAsmRegClass}; +pub use arm64ec::Arm64ECInlineAsmReg; pub use avr::{AvrInlineAsmReg, AvrInlineAsmRegClass}; pub use bpf::{BpfInlineAsmReg, BpfInlineAsmRegClass}; pub use csky::{CSKYInlineAsmReg, CSKYInlineAsmRegClass}; @@ -279,6 +281,7 @@ pub enum InlineAsmReg { X86(X86InlineAsmReg), Arm(ArmInlineAsmReg), AArch64(AArch64InlineAsmReg), + Arm64EC(Arm64ECInlineAsmReg), RiscV(RiscVInlineAsmReg), Nvptx(NvptxInlineAsmReg), PowerPC(PowerPCInlineAsmReg), @@ -303,6 +306,7 @@ impl InlineAsmReg { Self::X86(r) => r.name(), Self::Arm(r) => r.name(), Self::AArch64(r) => r.name(), + Self::Arm64EC(r) => r.name(), Self::RiscV(r) => r.name(), Self::PowerPC(r) => r.name(), Self::Hexagon(r) => r.name(), @@ -323,6 +327,7 @@ impl InlineAsmReg { Self::X86(r) => InlineAsmRegClass::X86(r.reg_class()), Self::Arm(r) => InlineAsmRegClass::Arm(r.reg_class()), Self::AArch64(r) => InlineAsmRegClass::AArch64(r.reg_class()), + Self::Arm64EC(r) => InlineAsmRegClass::Arm64EC(r.reg_class()), Self::RiscV(r) => InlineAsmRegClass::RiscV(r.reg_class()), Self::PowerPC(r) => InlineAsmRegClass::PowerPC(r.reg_class()), Self::Hexagon(r) => InlineAsmRegClass::Hexagon(r.reg_class()), @@ -345,9 +350,8 @@ impl InlineAsmReg { Ok(match arch { InlineAsmArch::X86 | InlineAsmArch::X86_64 => Self::X86(X86InlineAsmReg::parse(name)?), InlineAsmArch::Arm => Self::Arm(ArmInlineAsmReg::parse(name)?), - InlineAsmArch::AArch64 | InlineAsmArch::Arm64EC => { - Self::AArch64(AArch64InlineAsmReg::parse(name)?) - } + InlineAsmArch::AArch64 => Self::AArch64(AArch64InlineAsmReg::parse(name)?), + InlineAsmArch::Arm64EC => Self::Arm64EC(Arm64ECInlineAsmReg::parse(name)?), InlineAsmArch::RiscV32 | InlineAsmArch::RiscV64 => { Self::RiscV(RiscVInlineAsmReg::parse(name)?) } @@ -385,6 +389,7 @@ impl InlineAsmReg { Self::X86(r) => r.validate(arch, reloc_model, target_features, target, is_clobber), Self::Arm(r) => r.validate(arch, reloc_model, target_features, target, is_clobber), Self::AArch64(r) => r.validate(arch, reloc_model, target_features, target, is_clobber), + Self::Arm64EC(r) => r.validate(arch, reloc_model, target_features, target, is_clobber), Self::RiscV(r) => r.validate(arch, reloc_model, target_features, target, is_clobber), Self::PowerPC(r) => r.validate(arch, reloc_model, target_features, target, is_clobber), Self::Hexagon(r) => r.validate(arch, reloc_model, target_features, target, is_clobber), @@ -414,6 +419,7 @@ impl InlineAsmReg { Self::X86(r) => r.emit(out, arch, modifier), Self::Arm(r) => r.emit(out, arch, modifier), Self::AArch64(r) => r.emit(out, arch, modifier), + Self::Arm64EC(r) => r.emit(out, arch, modifier), Self::RiscV(r) => r.emit(out, arch, modifier), Self::PowerPC(r) => r.emit(out, arch, modifier), Self::Hexagon(r) => r.emit(out, arch, modifier), @@ -434,6 +440,7 @@ impl InlineAsmReg { Self::X86(r) => r.overlapping_regs(|r| cb(Self::X86(r))), Self::Arm(r) => r.overlapping_regs(|r| cb(Self::Arm(r))), Self::AArch64(_) => cb(self), + Self::Arm64EC(_) => cb(self), Self::RiscV(_) => cb(self), Self::PowerPC(r) => r.overlapping_regs(|r| cb(Self::PowerPC(r))), Self::Hexagon(r) => r.overlapping_regs(|r| cb(Self::Hexagon(r))), @@ -456,6 +463,7 @@ pub enum InlineAsmRegClass { X86(X86InlineAsmRegClass), Arm(ArmInlineAsmRegClass), AArch64(AArch64InlineAsmRegClass), + Arm64EC(AArch64InlineAsmRegClass), RiscV(RiscVInlineAsmRegClass), Nvptx(NvptxInlineAsmRegClass), PowerPC(PowerPCInlineAsmRegClass), @@ -480,6 +488,7 @@ impl InlineAsmRegClass { Self::X86(r) => r.name(), Self::Arm(r) => r.name(), Self::AArch64(r) => r.name(), + Self::Arm64EC(r) => r.name(), Self::RiscV(r) => r.name(), Self::Nvptx(r) => r.name(), Self::PowerPC(r) => r.name(), @@ -506,6 +515,7 @@ impl InlineAsmRegClass { Self::X86(r) => r.suggest_class(arch, ty).map(InlineAsmRegClass::X86), Self::Arm(r) => r.suggest_class(arch, ty).map(InlineAsmRegClass::Arm), Self::AArch64(r) => r.suggest_class(arch, ty).map(InlineAsmRegClass::AArch64), + Self::Arm64EC(r) => r.suggest_class(arch, ty).map(InlineAsmRegClass::Arm64EC), Self::RiscV(r) => r.suggest_class(arch, ty).map(InlineAsmRegClass::RiscV), Self::Nvptx(r) => r.suggest_class(arch, ty).map(InlineAsmRegClass::Nvptx), Self::PowerPC(r) => r.suggest_class(arch, ty).map(InlineAsmRegClass::PowerPC), @@ -535,6 +545,7 @@ impl InlineAsmRegClass { Self::X86(r) => r.suggest_modifier(arch, ty), Self::Arm(r) => r.suggest_modifier(arch, ty), Self::AArch64(r) => r.suggest_modifier(arch, ty), + Self::Arm64EC(r) => r.suggest_modifier(arch, ty), Self::RiscV(r) => r.suggest_modifier(arch, ty), Self::Nvptx(r) => r.suggest_modifier(arch, ty), Self::PowerPC(r) => r.suggest_modifier(arch, ty), @@ -564,6 +575,7 @@ impl InlineAsmRegClass { Self::X86(r) => r.default_modifier(arch), Self::Arm(r) => r.default_modifier(arch), Self::AArch64(r) => r.default_modifier(arch), + Self::Arm64EC(r) => r.default_modifier(arch), Self::RiscV(r) => r.default_modifier(arch), Self::Nvptx(r) => r.default_modifier(arch), Self::PowerPC(r) => r.default_modifier(arch), @@ -592,6 +604,7 @@ impl InlineAsmRegClass { Self::X86(r) => r.supported_types(arch), Self::Arm(r) => r.supported_types(arch), Self::AArch64(r) => r.supported_types(arch), + Self::Arm64EC(r) => r.supported_types(arch), Self::RiscV(r) => r.supported_types(arch), Self::Nvptx(r) => r.supported_types(arch), Self::PowerPC(r) => r.supported_types(arch), @@ -616,9 +629,8 @@ impl InlineAsmRegClass { Self::X86(X86InlineAsmRegClass::parse(name)?) } InlineAsmArch::Arm => Self::Arm(ArmInlineAsmRegClass::parse(name)?), - InlineAsmArch::AArch64 | InlineAsmArch::Arm64EC => { - Self::AArch64(AArch64InlineAsmRegClass::parse(name)?) - } + InlineAsmArch::AArch64 => Self::AArch64(AArch64InlineAsmRegClass::parse(name)?), + InlineAsmArch::Arm64EC => Self::Arm64EC(AArch64InlineAsmRegClass::parse(name)?), InlineAsmArch::RiscV32 | InlineAsmArch::RiscV64 => { Self::RiscV(RiscVInlineAsmRegClass::parse(name)?) } @@ -651,6 +663,7 @@ impl InlineAsmRegClass { Self::X86(r) => r.valid_modifiers(arch), Self::Arm(r) => r.valid_modifiers(arch), Self::AArch64(r) => r.valid_modifiers(arch), + Self::Arm64EC(r) => r.valid_modifiers(arch), Self::RiscV(r) => r.valid_modifiers(arch), Self::Nvptx(r) => r.valid_modifiers(arch), Self::PowerPC(r) => r.valid_modifiers(arch), @@ -803,11 +816,16 @@ pub fn allocatable_registers( arm::fill_reg_map(arch, reloc_model, target_features, target, &mut map); map } - InlineAsmArch::AArch64 | InlineAsmArch::Arm64EC => { + InlineAsmArch::AArch64 => { let mut map = aarch64::regclass_map(); aarch64::fill_reg_map(arch, reloc_model, target_features, target, &mut map); map } + InlineAsmArch::Arm64EC => { + let mut map = aarch64::regclass_map(); + arm64ec::fill_reg_map(arch, reloc_model, target_features, target, &mut map); + map + } InlineAsmArch::RiscV32 | InlineAsmArch::RiscV64 => { let mut map = riscv::regclass_map(); riscv::fill_reg_map(arch, reloc_model, target_features, target, &mut map); @@ -1050,7 +1068,7 @@ impl InlineAsmClobberAbi { } }, InlineAsmClobberAbi::Arm64EC => clobbered_regs! { - AArch64 AArch64InlineAsmReg { + Arm64EC Arm64ECInlineAsmReg { // x13 and x14 cannot be used in Arm64EC. x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x15, diff --git a/tests/ui/asm/aarch64/arm64ec-sve.rs b/tests/ui/asm/aarch64/arm64ec-sve.rs index aaec2d138813f..e163be03ee325 100644 --- a/tests/ui/asm/aarch64/arm64ec-sve.rs +++ b/tests/ui/asm/aarch64/arm64ec-sve.rs @@ -23,11 +23,12 @@ macro_rules! asm { fn f(x: f64) { unsafe { - // FIXME: z* cannot be used for Arm64EC asm!("", out("z0") _, in("z1") x); + //~^ ERROR invalid register `z0`: SVE cannot be used for Arm64EC + //~^^ ERROR invalid register `z1`: SVE cannot be used for Arm64EC asm!("", out("p0") _); - //~^ ERROR cannot use register `p0` + //~^ ERROR invalid register `p0`: SVE cannot be used for Arm64EC asm!("", out("ffr") _); - //~^ ERROR cannot use register `ffr` + //~^ ERROR invalid register `ffr`: SVE cannot be used for Arm64EC } } diff --git a/tests/ui/asm/aarch64/arm64ec-sve.stderr b/tests/ui/asm/aarch64/arm64ec-sve.stderr index c88177d759ecf..acf46a73608d1 100644 --- a/tests/ui/asm/aarch64/arm64ec-sve.stderr +++ b/tests/ui/asm/aarch64/arm64ec-sve.stderr @@ -1,14 +1,26 @@ -error: cannot use register `p0`: x13, x14, x23, x24, x28, v16-v31, z*, p*, ffr cannot be used for Arm64EC - --> $DIR/arm64ec-sve.rs:27:18 +error: invalid register `z0`: SVE cannot be used for Arm64EC + --> $DIR/arm64ec-sve.rs:26:18 | -LL | asm!("", out("p0") _); +LL | asm!("", out("z0") _, in("z1") x); | ^^^^^^^^^^^ -error: cannot use register `ffr`: x13, x14, x23, x24, x28, v16-v31, z*, p*, ffr cannot be used for Arm64EC +error: invalid register `z1`: SVE cannot be used for Arm64EC + --> $DIR/arm64ec-sve.rs:26:31 + | +LL | asm!("", out("z0") _, in("z1") x); + | ^^^^^^^^^^ + +error: invalid register `p0`: SVE cannot be used for Arm64EC --> $DIR/arm64ec-sve.rs:29:18 | +LL | asm!("", out("p0") _); + | ^^^^^^^^^^^ + +error: invalid register `ffr`: SVE cannot be used for Arm64EC + --> $DIR/arm64ec-sve.rs:31:18 + | LL | asm!("", out("ffr") _); | ^^^^^^^^^^^^ -error: aborting due to 2 previous errors +error: aborting due to 4 previous errors