1//===--- arm_sve.td - ARM SVE compiler interface ------------------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This file defines the TableGen definitions from which the ARM SVE header 10// file will be generated. See: 11// 12// https://developer.arm.com/architectures/system-architectures/software-standards/acle 13// 14//===----------------------------------------------------------------------===// 15 16include "arm_sve_sme_incl.td" 17 18//////////////////////////////////////////////////////////////////////////////// 19// Loads 20 21// Load one vector (scalar base) 22def SVLD1 : MInst<"svld1[_{2}]", "dPc", "csilUcUsUiUlhfd", [IsLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_ld1">; 23def SVLD1SB : MInst<"svld1sb_{d}", "dPS", "silUsUiUl", [IsLoad, VerifyRuntimeMode], MemEltTyInt8, "aarch64_sve_ld1">; 24def SVLD1UB : MInst<"svld1ub_{d}", "dPW", "silUsUiUl", [IsLoad, IsZExtReturn, VerifyRuntimeMode], MemEltTyInt8, "aarch64_sve_ld1">; 25def SVLD1SH : MInst<"svld1sh_{d}", "dPT", "ilUiUl", [IsLoad, VerifyRuntimeMode], MemEltTyInt16, "aarch64_sve_ld1">; 26def SVLD1UH : MInst<"svld1uh_{d}", "dPX", "ilUiUl", [IsLoad, IsZExtReturn, VerifyRuntimeMode], MemEltTyInt16, "aarch64_sve_ld1">; 27def SVLD1SW : MInst<"svld1sw_{d}", "dPU", "lUl", [IsLoad, VerifyRuntimeMode], MemEltTyInt32, "aarch64_sve_ld1">; 28def SVLD1UW : MInst<"svld1uw_{d}", "dPY", "lUl", [IsLoad, IsZExtReturn, VerifyRuntimeMode], MemEltTyInt32, "aarch64_sve_ld1">; 29 30let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in { 31 def SVLD1_BF : MInst<"svld1[_{2}]", "dPc", "b", [IsLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_ld1">; 32 def SVLD1_VNUM_BF : MInst<"svld1_vnum[_{2}]", "dPcl", "b", [IsLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_ld1">; 33} 34 35// Load one vector (scalar base, VL displacement) 36def SVLD1_VNUM : MInst<"svld1_vnum[_{2}]", "dPcl", "csilUcUsUiUlhfd", [IsLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_ld1">; 37def SVLD1SB_VNUM : MInst<"svld1sb_vnum_{d}", "dPSl", "silUsUiUl", [IsLoad, VerifyRuntimeMode], MemEltTyInt8, "aarch64_sve_ld1">; 38def SVLD1UB_VNUM : MInst<"svld1ub_vnum_{d}", "dPWl", "silUsUiUl", [IsLoad, IsZExtReturn, VerifyRuntimeMode], MemEltTyInt8, "aarch64_sve_ld1">; 39def SVLD1SH_VNUM : MInst<"svld1sh_vnum_{d}", "dPTl", "ilUiUl", [IsLoad, VerifyRuntimeMode], MemEltTyInt16, "aarch64_sve_ld1">; 40def SVLD1UH_VNUM : MInst<"svld1uh_vnum_{d}", "dPXl", "ilUiUl", [IsLoad, IsZExtReturn, VerifyRuntimeMode], MemEltTyInt16, "aarch64_sve_ld1">; 41def SVLD1SW_VNUM : MInst<"svld1sw_vnum_{d}", "dPUl", "lUl", [IsLoad, VerifyRuntimeMode], MemEltTyInt32, "aarch64_sve_ld1">; 42def SVLD1UW_VNUM : MInst<"svld1uw_vnum_{d}", "dPYl", "lUl", [IsLoad, IsZExtReturn, VerifyRuntimeMode], MemEltTyInt32, "aarch64_sve_ld1">; 43 44let SVETargetGuard = "sve", SMETargetGuard = InvalidMode in { 45// Load one vector (vector base) 46def SVLD1_GATHER_BASES_U : MInst<"svld1_gather[_{2}base]_{d}", "dPu", "ilUiUlfd", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ld1_gather_scalar_offset">; 47def SVLD1SB_GATHER_BASES_U : MInst<"svld1sb_gather[_{2}base]_{d}", "dPu", "ilUiUl", [IsGatherLoad], MemEltTyInt8, "aarch64_sve_ld1_gather_scalar_offset">; 48def SVLD1UB_GATHER_BASES_U : MInst<"svld1ub_gather[_{2}base]_{d}", "dPu", "ilUiUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ld1_gather_scalar_offset">; 49def SVLD1SH_GATHER_BASES_U : MInst<"svld1sh_gather[_{2}base]_{d}", "dPu", "ilUiUl", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ld1_gather_scalar_offset">; 50def SVLD1UH_GATHER_BASES_U : MInst<"svld1uh_gather[_{2}base]_{d}", "dPu", "ilUiUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ld1_gather_scalar_offset">; 51def SVLD1SW_GATHER_BASES_U : MInst<"svld1sw_gather[_{2}base]_{d}", "dPu", "lUl", [IsGatherLoad], MemEltTyInt32, "aarch64_sve_ld1_gather_scalar_offset">; 52def SVLD1UW_GATHER_BASES_U : MInst<"svld1uw_gather[_{2}base]_{d}", "dPu", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ld1_gather_scalar_offset">; 53 54// Load one vector (scalar base, signed vector offset in bytes) 55def SVLD1_GATHER_64B_OFFSETS_S : MInst<"svld1_gather_[{3}]offset[_{d}]", "dPcx", "lUld", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ld1_gather">; 56def SVLD1SB_GATHER_64B_OFFSETS_S : MInst<"svld1sb_gather_[{3}]offset_{d}", "dPSx", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt8, "aarch64_sve_ld1_gather">; 57def SVLD1UB_GATHER_64B_OFFSETS_S : MInst<"svld1ub_gather_[{3}]offset_{d}", "dPWx", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ld1_gather">; 58def SVLD1SH_GATHER_64B_OFFSETS_S : MInst<"svld1sh_gather_[{3}]offset_{d}", "dPTx", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt16, "aarch64_sve_ld1_gather">; 59def SVLD1UH_GATHER_64B_OFFSETS_S : MInst<"svld1uh_gather_[{3}]offset_{d}", "dPXx", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ld1_gather">; 60def SVLD1SW_GATHER_64B_OFFSETS_S : MInst<"svld1sw_gather_[{3}]offset_{d}", "dPUx", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt32, "aarch64_sve_ld1_gather">; 61def SVLD1UW_GATHER_64B_OFFSETS_S : MInst<"svld1uw_gather_[{3}]offset_{d}", "dPYx", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ld1_gather">; 62 63def SVLD1_GATHER_32B_OFFSETS_S : MInst<"svld1_gather_[{3}]offset[_{d}]", "dPcx", "iUif", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ld1_gather_sxtw">; 64def SVLD1SB_GATHER_32B_OFFSETS_S : MInst<"svld1sb_gather_[{3}]offset_{d}", "dPSx", "iUi", [IsGatherLoad, IsByteIndexed], MemEltTyInt8, "aarch64_sve_ld1_gather_sxtw">; 65def SVLD1UB_GATHER_32B_OFFSETS_S : MInst<"svld1ub_gather_[{3}]offset_{d}", "dPWx", "iUi", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ld1_gather_sxtw">; 66def SVLD1SH_GATHER_32B_OFFSETS_S : MInst<"svld1sh_gather_[{3}]offset_{d}", "dPTx", "iUi", [IsGatherLoad, IsByteIndexed], MemEltTyInt16, "aarch64_sve_ld1_gather_sxtw">; 67def SVLD1UH_GATHER_32B_OFFSETS_S : MInst<"svld1uh_gather_[{3}]offset_{d}", "dPXx", "iUi", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ld1_gather_sxtw">; 68 69// Load one vector (scalar base, unsigned vector offset in bytes) 70def SVLD1_GATHER_64B_OFFSETS_U : MInst<"svld1_gather_[{3}]offset[_{d}]", "dPcu", "lUld", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ld1_gather">; 71def SVLD1SB_GATHER_64B_OFFSETS_U : MInst<"svld1sb_gather_[{3}]offset_{d}", "dPSu", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt8, "aarch64_sve_ld1_gather">; 72def SVLD1UB_GATHER_64B_OFFSETS_U : MInst<"svld1ub_gather_[{3}]offset_{d}", "dPWu", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ld1_gather">; 73def SVLD1SH_GATHER_64B_OFFSETS_U : MInst<"svld1sh_gather_[{3}]offset_{d}", "dPTu", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt16, "aarch64_sve_ld1_gather">; 74def SVLD1UH_GATHER_64B_OFFSETS_U : MInst<"svld1uh_gather_[{3}]offset_{d}", "dPXu", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ld1_gather">; 75def SVLD1SW_GATHER_64B_OFFSETS_U : MInst<"svld1sw_gather_[{3}]offset_{d}", "dPUu", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt32, "aarch64_sve_ld1_gather">; 76def SVLD1UW_GATHER_64B_OFFSETS_U : MInst<"svld1uw_gather_[{3}]offset_{d}", "dPYu", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ld1_gather">; 77 78def SVLD1_GATHER_32B_OFFSETS_U : MInst<"svld1_gather_[{3}]offset[_{d}]", "dPcu", "iUif", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ld1_gather_uxtw">; 79def SVLD1SB_GATHER_32B_OFFSETS_U : MInst<"svld1sb_gather_[{3}]offset_{d}", "dPSu", "iUi", [IsGatherLoad, IsByteIndexed], MemEltTyInt8, "aarch64_sve_ld1_gather_uxtw">; 80def SVLD1UB_GATHER_32B_OFFSETS_U : MInst<"svld1ub_gather_[{3}]offset_{d}", "dPWu", "iUi", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ld1_gather_uxtw">; 81def SVLD1SH_GATHER_32B_OFFSETS_U : MInst<"svld1sh_gather_[{3}]offset_{d}", "dPTu", "iUi", [IsGatherLoad, IsByteIndexed], MemEltTyInt16, "aarch64_sve_ld1_gather_uxtw">; 82def SVLD1UH_GATHER_32B_OFFSETS_U : MInst<"svld1uh_gather_[{3}]offset_{d}", "dPXu", "iUi", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ld1_gather_uxtw">; 83 84// Load one vector (vector base, signed scalar offset in bytes) 85def SVLD1_GATHER_OFFSET_S : MInst<"svld1_gather[_{2}base]_offset_{d}", "dPul", "ilUiUlfd", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ld1_gather_scalar_offset">; 86def SVLD1SB_GATHER_OFFSET_S : MInst<"svld1sb_gather[_{2}base]_offset_{d}", "dPul", "ilUiUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt8, "aarch64_sve_ld1_gather_scalar_offset">; 87def SVLD1UB_GATHER_OFFSET_S : MInst<"svld1ub_gather[_{2}base]_offset_{d}", "dPul", "ilUiUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ld1_gather_scalar_offset">; 88def SVLD1SH_GATHER_OFFSET_S : MInst<"svld1sh_gather[_{2}base]_offset_{d}", "dPul", "ilUiUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt16, "aarch64_sve_ld1_gather_scalar_offset">; 89def SVLD1UH_GATHER_OFFSET_S : MInst<"svld1uh_gather[_{2}base]_offset_{d}", "dPul", "ilUiUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ld1_gather_scalar_offset">; 90def SVLD1SW_GATHER_OFFSET_S : MInst<"svld1sw_gather[_{2}base]_offset_{d}", "dPul", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt32, "aarch64_sve_ld1_gather_scalar_offset">; 91def SVLD1UW_GATHER_OFFSET_S : MInst<"svld1uw_gather[_{2}base]_offset_{d}", "dPul", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ld1_gather_scalar_offset">; 92 93// Load one vector (scalar base, signed vector index) 94def SVLD1_GATHER_64B_INDICES_S : MInst<"svld1_gather_[{3}]index[_{d}]", "dPcx", "lUld", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ld1_gather_index">; 95def SVLD1SH_GATHER_64B_INDICES_S : MInst<"svld1sh_gather_[{3}]index_{d}", "dPTx", "lUl", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ld1_gather_index">; 96def SVLD1UH_GATHER_64B_INDICES_S : MInst<"svld1uh_gather_[{3}]index_{d}", "dPXx", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ld1_gather_index">; 97def SVLD1SW_GATHER_64B_INDICES_S : MInst<"svld1sw_gather_[{3}]index_{d}", "dPUx", "lUl", [IsGatherLoad], MemEltTyInt32, "aarch64_sve_ld1_gather_index">; 98def SVLD1UW_GATHER_64B_INDICES_S : MInst<"svld1uw_gather_[{3}]index_{d}", "dPYx", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ld1_gather_index">; 99 100def SVLD1_GATHER_32B_INDICES_S : MInst<"svld1_gather_[{3}]index[_{d}]", "dPcx", "iUif", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ld1_gather_sxtw_index">; 101def SVLD1SH_GATHER_32B_INDICES_S : MInst<"svld1sh_gather_[{3}]index_{d}", "dPTx", "iUi", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ld1_gather_sxtw_index">; 102def SVLD1UH_GATHER_32B_INDICES_S : MInst<"svld1uh_gather_[{3}]index_{d}", "dPXx", "iUi", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ld1_gather_sxtw_index">; 103 104// Load one vector (scalar base, unsigned vector index) 105def SVLD1_GATHER_64B_INDICES_U : MInst<"svld1_gather_[{3}]index[_{d}]", "dPcu", "lUld", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ld1_gather_index">; 106def SVLD1SH_GATHER_64B_INDICES_U : MInst<"svld1sh_gather_[{3}]index_{d}", "dPTu", "lUl", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ld1_gather_index">; 107def SVLD1UH_GATHER_64B_INDICES_U : MInst<"svld1uh_gather_[{3}]index_{d}", "dPXu", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ld1_gather_index">; 108def SVLD1SW_GATHER_64B_INDICES_U : MInst<"svld1sw_gather_[{3}]index_{d}", "dPUu", "lUl", [IsGatherLoad], MemEltTyInt32, "aarch64_sve_ld1_gather_index">; 109def SVLD1UW_GATHER_64B_INDICES_U : MInst<"svld1uw_gather_[{3}]index_{d}", "dPYu", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ld1_gather_index">; 110 111def SVLD1_GATHER_32B_INDICES_U : MInst<"svld1_gather_[{3}]index[_{d}]", "dPcu", "iUif", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ld1_gather_uxtw_index">; 112def SVLD1SH_GATHER_32B_INDICES_U : MInst<"svld1sh_gather_[{3}]index_{d}", "dPTu", "iUi", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ld1_gather_uxtw_index">; 113def SVLD1UH_GATHER_32B_INDICES_U : MInst<"svld1uh_gather_[{3}]index_{d}", "dPXu", "iUi", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ld1_gather_uxtw_index">; 114 115// Load one vector (vector base, signed scalar index) 116def SVLD1_GATHER_INDEX_S : MInst<"svld1_gather[_{2}base]_index_{d}", "dPul", "ilUiUlfd", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ld1_gather_scalar_offset">; 117def SVLD1SH_GATHER_INDEX_S : MInst<"svld1sh_gather[_{2}base]_index_{d}", "dPul", "ilUiUl", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ld1_gather_scalar_offset">; 118def SVLD1UH_GATHER_INDEX_S : MInst<"svld1uh_gather[_{2}base]_index_{d}", "dPul", "ilUiUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ld1_gather_scalar_offset">; 119def SVLD1SW_GATHER_INDEX_S : MInst<"svld1sw_gather[_{2}base]_index_{d}", "dPul", "lUl", [IsGatherLoad], MemEltTyInt32, "aarch64_sve_ld1_gather_scalar_offset">; 120def SVLD1UW_GATHER_INDEX_S : MInst<"svld1uw_gather[_{2}base]_index_{d}", "dPul", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ld1_gather_scalar_offset">; 121 122 123// First-faulting load one vector (scalar base) 124def SVLDFF1 : MInst<"svldff1[_{2}]", "dPc", "csilUcUsUiUlhfd", [IsLoad], MemEltTyDefault, "aarch64_sve_ldff1">; 125def SVLDFF1SB : MInst<"svldff1sb_{d}", "dPS", "silUsUiUl", [IsLoad], MemEltTyInt8, "aarch64_sve_ldff1">; 126def SVLDFF1UB : MInst<"svldff1ub_{d}", "dPW", "silUsUiUl", [IsLoad, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ldff1">; 127def SVLDFF1SH : MInst<"svldff1sh_{d}", "dPT", "ilUiUl", [IsLoad], MemEltTyInt16, "aarch64_sve_ldff1">; 128def SVLDFF1UH : MInst<"svldff1uh_{d}", "dPX", "ilUiUl", [IsLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldff1">; 129def SVLDFF1SW : MInst<"svldff1sw_{d}", "dPU", "lUl", [IsLoad], MemEltTyInt32, "aarch64_sve_ldff1">; 130def SVLDFF1UW : MInst<"svldff1uw_{d}", "dPY", "lUl", [IsLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldff1">; 131 132// First-faulting load one vector (scalar base, VL displacement) 133def SVLDFF1_VNUM : MInst<"svldff1_vnum[_{2}]", "dPcl", "csilUcUsUiUlhfd", [IsLoad], MemEltTyDefault, "aarch64_sve_ldff1">; 134def SVLDFF1SB_VNUM : MInst<"svldff1sb_vnum_{d}", "dPSl", "silUsUiUl", [IsLoad], MemEltTyInt8, "aarch64_sve_ldff1">; 135def SVLDFF1UB_VNUM : MInst<"svldff1ub_vnum_{d}", "dPWl", "silUsUiUl", [IsLoad, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ldff1">; 136def SVLDFF1SH_VNUM : MInst<"svldff1sh_vnum_{d}", "dPTl", "ilUiUl", [IsLoad], MemEltTyInt16, "aarch64_sve_ldff1">; 137def SVLDFF1UH_VNUM : MInst<"svldff1uh_vnum_{d}", "dPXl", "ilUiUl", [IsLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldff1">; 138def SVLDFF1SW_VNUM : MInst<"svldff1sw_vnum_{d}", "dPUl", "lUl", [IsLoad], MemEltTyInt32, "aarch64_sve_ldff1">; 139def SVLDFF1UW_VNUM : MInst<"svldff1uw_vnum_{d}", "dPYl", "lUl", [IsLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldff1">; 140} // let SVETargetGuard = "sve", SMETargetGuard = InvalidMode 141 142let SVETargetGuard = "sve,bf16", SMETargetGuard = InvalidMode in { 143 def SVLDFF1_BF : MInst<"svldff1[_{2}]", "dPc", "b", [IsLoad], MemEltTyDefault, "aarch64_sve_ldff1">; 144 def SVLDFF1_VNUM_BF : MInst<"svldff1_vnum[_{2}]", "dPcl", "b", [IsLoad], MemEltTyDefault, "aarch64_sve_ldff1">; 145} 146 147let SVETargetGuard = "sve", SMETargetGuard = InvalidMode in { 148// First-faulting load one vector (vector base) 149def SVLDFF1_GATHER_BASES_U : MInst<"svldff1_gather[_{2}base]_{d}", "dPu", "ilUiUlfd", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ldff1_gather_scalar_offset">; 150def SVLDFF1SB_GATHER_BASES_U : MInst<"svldff1sb_gather[_{2}base]_{d}", "dPu", "ilUiUl", [IsGatherLoad], MemEltTyInt8, "aarch64_sve_ldff1_gather_scalar_offset">; 151def SVLDFF1UB_GATHER_BASES_U : MInst<"svldff1ub_gather[_{2}base]_{d}", "dPu", "ilUiUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ldff1_gather_scalar_offset">; 152def SVLDFF1SH_GATHER_BASES_U : MInst<"svldff1sh_gather[_{2}base]_{d}", "dPu", "ilUiUl", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ldff1_gather_scalar_offset">; 153def SVLDFF1UH_GATHER_BASES_U : MInst<"svldff1uh_gather[_{2}base]_{d}", "dPu", "ilUiUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldff1_gather_scalar_offset">; 154def SVLDFF1SW_GATHER_BASES_U : MInst<"svldff1sw_gather[_{2}base]_{d}", "dPu", "lUl", [IsGatherLoad], MemEltTyInt32, "aarch64_sve_ldff1_gather_scalar_offset">; 155def SVLDFF1UW_GATHER_BASES_U : MInst<"svldff1uw_gather[_{2}base]_{d}", "dPu", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldff1_gather_scalar_offset">; 156 157// First-faulting load one vector (scalar base, signed vector offset in bytes) 158def SVLDFF1_GATHER_64B_OFFSETS_S : MInst<"svldff1_gather_[{3}]offset[_{d}]", "dPcx", "lUld", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ldff1_gather">; 159def SVLDFF1SB_GATHER_64B_OFFSETS_S : MInst<"svldff1sb_gather_[{3}]offset_{d}", "dPSx", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt8, "aarch64_sve_ldff1_gather">; 160def SVLDFF1UB_GATHER_64B_OFFSETS_S : MInst<"svldff1ub_gather_[{3}]offset_{d}", "dPWx", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ldff1_gather">; 161def SVLDFF1SH_GATHER_64B_OFFSETS_S : MInst<"svldff1sh_gather_[{3}]offset_{d}", "dPTx", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt16, "aarch64_sve_ldff1_gather">; 162def SVLDFF1UH_GATHER_64B_OFFSETS_S : MInst<"svldff1uh_gather_[{3}]offset_{d}", "dPXx", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldff1_gather">; 163def SVLDFF1SW_GATHER_64B_OFFSETS_S : MInst<"svldff1sw_gather_[{3}]offset_{d}", "dPUx", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt32, "aarch64_sve_ldff1_gather">; 164def SVLDFF1UW_GATHER_64B_OFFSETS_S : MInst<"svldff1uw_gather_[{3}]offset_{d}", "dPYx", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldff1_gather">; 165 166def SVLDFF1_GATHER_32B_OFFSETS_S : MInst<"svldff1_gather_[{3}]offset[_{d}]", "dPcx", "iUif", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ldff1_gather_sxtw">; 167def SVLDFF1SB_GATHER_32B_OFFSETS_S : MInst<"svldff1sb_gather_[{3}]offset_{d}", "dPSx", "iUi", [IsGatherLoad, IsByteIndexed], MemEltTyInt8, "aarch64_sve_ldff1_gather_sxtw">; 168def SVLDFF1UB_GATHER_32B_OFFSETS_S : MInst<"svldff1ub_gather_[{3}]offset_{d}", "dPWx", "iUi", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ldff1_gather_sxtw">; 169def SVLDFF1SH_GATHER_32B_OFFSETS_S : MInst<"svldff1sh_gather_[{3}]offset_{d}", "dPTx", "iUi", [IsGatherLoad, IsByteIndexed], MemEltTyInt16, "aarch64_sve_ldff1_gather_sxtw">; 170def SVLDFF1UH_GATHER_32B_OFFSETS_S : MInst<"svldff1uh_gather_[{3}]offset_{d}", "dPXx", "iUi", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldff1_gather_sxtw">; 171 172// First-faulting load one vector (scalar base, unsigned vector offset in bytes) 173def SVLDFF1_GATHER_64B_OFFSETS_U : MInst<"svldff1_gather_[{3}]offset[_{d}]", "dPcu", "lUld", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ldff1_gather">; 174def SVLDFF1SB_GATHER_64B_OFFSETS_U : MInst<"svldff1sb_gather_[{3}]offset_{d}", "dPSu", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt8, "aarch64_sve_ldff1_gather">; 175def SVLDFF1UB_GATHER_64B_OFFSETS_U : MInst<"svldff1ub_gather_[{3}]offset_{d}", "dPWu", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ldff1_gather">; 176def SVLDFF1SH_GATHER_64B_OFFSETS_U : MInst<"svldff1sh_gather_[{3}]offset_{d}", "dPTu", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt16, "aarch64_sve_ldff1_gather">; 177def SVLDFF1UH_GATHER_64B_OFFSETS_U : MInst<"svldff1uh_gather_[{3}]offset_{d}", "dPXu", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldff1_gather">; 178def SVLDFF1SW_GATHER_64B_OFFSETS_U : MInst<"svldff1sw_gather_[{3}]offset_{d}", "dPUu", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt32, "aarch64_sve_ldff1_gather">; 179def SVLDFF1UW_GATHER_64B_OFFSETS_U : MInst<"svldff1uw_gather_[{3}]offset_{d}", "dPYu", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldff1_gather">; 180 181def SVLDFF1_GATHER_32B_OFFSETS_U : MInst<"svldff1_gather_[{3}]offset[_{d}]", "dPcu", "iUif", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ldff1_gather_uxtw">; 182def SVLDFF1SB_GATHER_32B_OFFSETS_U : MInst<"svldff1sb_gather_[{3}]offset_{d}", "dPSu", "iUi", [IsGatherLoad, IsByteIndexed], MemEltTyInt8, "aarch64_sve_ldff1_gather_uxtw">; 183def SVLDFF1UB_GATHER_32B_OFFSETS_U : MInst<"svldff1ub_gather_[{3}]offset_{d}", "dPWu", "iUi", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ldff1_gather_uxtw">; 184def SVLDFF1SH_GATHER_32B_OFFSETS_U : MInst<"svldff1sh_gather_[{3}]offset_{d}", "dPTu", "iUi", [IsGatherLoad, IsByteIndexed], MemEltTyInt16, "aarch64_sve_ldff1_gather_uxtw">; 185def SVLDFF1UH_GATHER_32B_OFFSETS_U : MInst<"svldff1uh_gather_[{3}]offset_{d}", "dPXu", "iUi", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldff1_gather_uxtw">; 186 187// First-faulting load one vector (vector base, signed scalar offset in bytes) 188def SVLDFF1_GATHER_OFFSET_S : MInst<"svldff1_gather[_{2}base]_offset_{d}", "dPul", "ilUiUlfd", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ldff1_gather_scalar_offset">; 189def SVLDFF1SB_GATHER_OFFSET_S : MInst<"svldff1sb_gather[_{2}base]_offset_{d}", "dPul", "ilUiUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt8, "aarch64_sve_ldff1_gather_scalar_offset">; 190def SVLDFF1UB_GATHER_OFFSET_S : MInst<"svldff1ub_gather[_{2}base]_offset_{d}", "dPul", "ilUiUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ldff1_gather_scalar_offset">; 191def SVLDFF1SH_GATHER_OFFSET_S : MInst<"svldff1sh_gather[_{2}base]_offset_{d}", "dPul", "ilUiUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt16, "aarch64_sve_ldff1_gather_scalar_offset">; 192def SVLDFF1UH_GATHER_OFFSET_S : MInst<"svldff1uh_gather[_{2}base]_offset_{d}", "dPul", "ilUiUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldff1_gather_scalar_offset">; 193def SVLDFF1SW_GATHER_OFFSET_S : MInst<"svldff1sw_gather[_{2}base]_offset_{d}", "dPul", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt32, "aarch64_sve_ldff1_gather_scalar_offset">; 194def SVLDFF1UW_GATHER_OFFSET_S : MInst<"svldff1uw_gather[_{2}base]_offset_{d}", "dPul", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldff1_gather_scalar_offset">; 195 196// First-faulting load one vector (scalar base, signed vector index) 197def SVLDFF1_GATHER_64B_INDICES_S : MInst<"svldff1_gather_[{3}]index[_{d}]", "dPcx", "lUld", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ldff1_gather_index">; 198def SVLDFF1SH_GATHER_64B_INDICES_S : MInst<"svldff1sh_gather_[{3}]index_{d}", "dPTx", "lUl", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ldff1_gather_index">; 199def SVLDFF1UH_GATHER_64B_INDICES_S : MInst<"svldff1uh_gather_[{3}]index_{d}", "dPXx", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldff1_gather_index">; 200def SVLDFF1SW_GATHER_64B_INDICES_S : MInst<"svldff1sw_gather_[{3}]index_{d}", "dPUx", "lUl", [IsGatherLoad], MemEltTyInt32, "aarch64_sve_ldff1_gather_index">; 201def SVLDFF1UW_GATHER_64B_INDICES_S : MInst<"svldff1uw_gather_[{3}]index_{d}", "dPYx", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldff1_gather_index">; 202 203def SVLDFF1_GATHER_32B_INDICES_S : MInst<"svldff1_gather_[{3}]index[_{d}]", "dPcx", "iUif", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ldff1_gather_sxtw_index">; 204def SVLDFF1SH_GATHER_32B_INDICES_S : MInst<"svldff1sh_gather_[{3}]index_{d}", "dPTx", "iUi", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ldff1_gather_sxtw_index">; 205def SVLDFF1UH_GATHER_32B_INDICES_S : MInst<"svldff1uh_gather_[{3}]index_{d}", "dPXx", "iUi", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldff1_gather_sxtw_index">; 206 207// First-faulting load one vector (scalar base, unsigned vector index) 208def SVLDFF1_GATHER_64B_INDICES_U : MInst<"svldff1_gather_[{3}]index[_{d}]", "dPcu", "lUld", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ldff1_gather_index">; 209def SVLDFF1SH_GATHER_64B_INDICES_U : MInst<"svldff1sh_gather_[{3}]index_{d}", "dPTu", "lUl", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ldff1_gather_index">; 210def SVLDFF1UH_GATHER_64B_INDICES_U : MInst<"svldff1uh_gather_[{3}]index_{d}", "dPXu", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldff1_gather_index">; 211def SVLDFF1SW_GATHER_64B_INDICES_U : MInst<"svldff1sw_gather_[{3}]index_{d}", "dPUu", "lUl", [IsGatherLoad], MemEltTyInt32, "aarch64_sve_ldff1_gather_index">; 212def SVLDFF1UW_GATHER_64B_INDICES_U : MInst<"svldff1uw_gather_[{3}]index_{d}", "dPYu", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldff1_gather_index">; 213 214def SVLDFF1_GATHER_32B_INDICES_U : MInst<"svldff1_gather_[{3}]index[_{d}]", "dPcu", "iUif", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ldff1_gather_uxtw_index">; 215def SVLDFF1SH_GATHER_32B_INDICES_U : MInst<"svldff1sh_gather_[{3}]index_{d}", "dPTu", "iUi", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ldff1_gather_uxtw_index">; 216def SVLDFF1UH_GATHER_32B_INDICES_U : MInst<"svldff1uh_gather_[{3}]index_{d}", "dPXu", "iUi", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldff1_gather_uxtw_index">; 217 218// First-faulting load one vector (vector base, signed scalar index) 219def SVLDFF1_GATHER_INDEX_S : MInst<"svldff1_gather[_{2}base]_index_{d}", "dPul", "ilUiUlfd", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ldff1_gather_scalar_offset">; 220def SVLDFF1SH_GATHER_INDEX_S : MInst<"svldff1sh_gather[_{2}base]_index_{d}", "dPul", "ilUiUl", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ldff1_gather_scalar_offset">; 221def SVLDFF1UH_GATHER_INDEX_S : MInst<"svldff1uh_gather[_{2}base]_index_{d}", "dPul", "ilUiUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldff1_gather_scalar_offset">; 222def SVLDFF1SW_GATHER_INDEX_S : MInst<"svldff1sw_gather[_{2}base]_index_{d}", "dPul", "lUl", [IsGatherLoad], MemEltTyInt32, "aarch64_sve_ldff1_gather_scalar_offset">; 223def SVLDFF1UW_GATHER_INDEX_S : MInst<"svldff1uw_gather[_{2}base]_index_{d}", "dPul", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldff1_gather_scalar_offset">; 224 225// Non-faulting load one vector (scalar base) 226def SVLDNF1 : MInst<"svldnf1[_{2}]", "dPc", "csilUcUsUiUlhfd", [IsLoad], MemEltTyDefault, "aarch64_sve_ldnf1">; 227def SVLDNF1SB : MInst<"svldnf1sb_{d}", "dPS", "silUsUiUl", [IsLoad], MemEltTyInt8, "aarch64_sve_ldnf1">; 228def SVLDNF1UB : MInst<"svldnf1ub_{d}", "dPW", "silUsUiUl", [IsLoad, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ldnf1">; 229def SVLDNF1SH : MInst<"svldnf1sh_{d}", "dPT", "ilUiUl", [IsLoad], MemEltTyInt16, "aarch64_sve_ldnf1">; 230def SVLDNF1UH : MInst<"svldnf1uh_{d}", "dPX", "ilUiUl", [IsLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldnf1">; 231def SVLDNF1SW : MInst<"svldnf1sw_{d}", "dPU", "lUl", [IsLoad], MemEltTyInt32, "aarch64_sve_ldnf1">; 232def SVLDNF1UW : MInst<"svldnf1uw_{d}", "dPY", "lUl", [IsLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldnf1">; 233 234// Non-faulting load one vector (scalar base, VL displacement) 235def SVLDNF1_VNUM : MInst<"svldnf1_vnum[_{2}]", "dPcl", "csilUcUsUiUlhfd", [IsLoad], MemEltTyDefault, "aarch64_sve_ldnf1">; 236def SVLDNF1SB_VNUM : MInst<"svldnf1sb_vnum_{d}", "dPSl", "silUsUiUl", [IsLoad], MemEltTyInt8, "aarch64_sve_ldnf1">; 237def SVLDNF1UB_VNUM : MInst<"svldnf1ub_vnum_{d}", "dPWl", "silUsUiUl", [IsLoad, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ldnf1">; 238def SVLDNF1SH_VNUM : MInst<"svldnf1sh_vnum_{d}", "dPTl", "ilUiUl", [IsLoad], MemEltTyInt16, "aarch64_sve_ldnf1">; 239def SVLDNF1UH_VNUM : MInst<"svldnf1uh_vnum_{d}", "dPXl", "ilUiUl", [IsLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldnf1">; 240def SVLDNF1SW_VNUM : MInst<"svldnf1sw_vnum_{d}", "dPUl", "lUl", [IsLoad], MemEltTyInt32, "aarch64_sve_ldnf1">; 241def SVLDNF1UW_VNUM : MInst<"svldnf1uw_vnum_{d}", "dPYl", "lUl", [IsLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldnf1">; 242} // let SVETargetGuard = "sve", SMETargetGuard = InvalidMode 243 244let SVETargetGuard = "sve,bf16", SMETargetGuard = InvalidMode in { 245 def SVLDNF1_BF : MInst<"svldnf1[_{2}]", "dPc", "b", [IsLoad], MemEltTyDefault, "aarch64_sve_ldnf1">; 246 def SVLDNF1_VNUM_BF : MInst<"svldnf1_vnum[_{2}]", "dPcl", "b", [IsLoad], MemEltTyDefault, "aarch64_sve_ldnf1">; 247} 248 249// Load one vector, unextended load, non-temporal (scalar base) 250def SVLDNT1 : MInst<"svldnt1[_{2}]", "dPc", "csilUcUsUiUlhfd", [IsLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_ldnt1">; 251 252// Load one vector, unextended load, non-temporal (scalar base, VL displacement) 253def SVLDNT1_VNUM : MInst<"svldnt1_vnum[_{2}]", "dPcl", "csilUcUsUiUlhfd", [IsLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_ldnt1">; 254 255let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in { 256 def SVLDNT1_BF : MInst<"svldnt1[_{2}]", "dPc", "b", [IsLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_ldnt1">; 257 def SVLDNT1_VNUM_BF : MInst<"svldnt1_vnum[_{2}]", "dPcl", "b", [IsLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_ldnt1">; 258} 259 260// Load one quadword and replicate (scalar base) 261def SVLD1RQ : SInst<"svld1rq[_{2}]", "dPc", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_ld1rq", [VerifyRuntimeMode]>; 262 263let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in { 264 def SVLD1RQ_BF : SInst<"svld1rq[_{2}]", "dPc", "b", MergeNone, "aarch64_sve_ld1rq", [VerifyRuntimeMode]>; 265} 266 267multiclass StructLoad<string name, string proto, string i> { 268 def : SInst<name, proto, "csilUcUsUiUlhfd", MergeNone, i, [IsStructLoad, VerifyRuntimeMode]>; 269 let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in { 270 def: SInst<name, proto, "b", MergeNone, i, [IsStructLoad, VerifyRuntimeMode]>; 271 } 272} 273 274// Load N-element structure into N vectors (scalar base) 275defm SVLD2 : StructLoad<"svld2[_{2}]", "2Pc", "aarch64_sve_ld2_sret">; 276defm SVLD3 : StructLoad<"svld3[_{2}]", "3Pc", "aarch64_sve_ld3_sret">; 277defm SVLD4 : StructLoad<"svld4[_{2}]", "4Pc", "aarch64_sve_ld4_sret">; 278 279// Load N-element structure into N vectors (scalar base, VL displacement) 280defm SVLD2_VNUM : StructLoad<"svld2_vnum[_{2}]", "2Pcl", "aarch64_sve_ld2_sret">; 281defm SVLD3_VNUM : StructLoad<"svld3_vnum[_{2}]", "3Pcl", "aarch64_sve_ld3_sret">; 282defm SVLD4_VNUM : StructLoad<"svld4_vnum[_{2}]", "4Pcl", "aarch64_sve_ld4_sret">; 283 284// Load one octoword and replicate (scalar base) 285let SVETargetGuard = "sve,f64mm", SMETargetGuard = InvalidMode in { 286 def SVLD1RO : SInst<"svld1ro[_{2}]", "dPc", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_ld1ro">; 287} 288let SVETargetGuard = "sve,f64mm,bf16", SMETargetGuard = InvalidMode in { 289 def SVLD1RO_BF16 : SInst<"svld1ro[_{2}]", "dPc", "b", MergeNone, "aarch64_sve_ld1ro">; 290} 291 292let SVETargetGuard = "sve,bf16", SMETargetGuard = InvalidMode in { 293 def SVBFMMLA : SInst<"svbfmmla[_{0}]", "MMdd", "b", MergeNone, "aarch64_sve_bfmmla", [IsOverloadNone]>; 294} 295 296let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in { 297 def SVBFDOT : SInst<"svbfdot[_{0}]", "MMdd", "b", MergeNone, "aarch64_sve_bfdot", [IsOverloadNone, VerifyRuntimeMode]>; 298 def SVBFMLALB : SInst<"svbfmlalb[_{0}]", "MMdd", "b", MergeNone, "aarch64_sve_bfmlalb", [IsOverloadNone, VerifyRuntimeMode]>; 299 def SVBFMLALT : SInst<"svbfmlalt[_{0}]", "MMdd", "b", MergeNone, "aarch64_sve_bfmlalt", [IsOverloadNone, VerifyRuntimeMode]>; 300 def SVBFDOT_N : SInst<"svbfdot[_n_{0}]", "MMda", "b", MergeNone, "aarch64_sve_bfdot", [IsOverloadNone, VerifyRuntimeMode]>; 301 def SVBFMLALB_N : SInst<"svbfmlalb[_n_{0}]", "MMda", "b", MergeNone, "aarch64_sve_bfmlalb", [IsOverloadNone, VerifyRuntimeMode]>; 302 def SVBFMLALT_N : SInst<"svbfmlalt[_n_{0}]", "MMda", "b", MergeNone, "aarch64_sve_bfmlalt", [IsOverloadNone, VerifyRuntimeMode]>; 303 def SVBFDOT_LANE : SInst<"svbfdot_lane[_{0}]", "MMddi", "b", MergeNone, "aarch64_sve_bfdot_lane_v2", [IsOverloadNone, VerifyRuntimeMode], [ImmCheck<3, ImmCheck0_3>]>; 304 def SVBFMLALB_LANE : SInst<"svbfmlalb_lane[_{0}]", "MMddi", "b", MergeNone, "aarch64_sve_bfmlalb_lane_v2", [IsOverloadNone, VerifyRuntimeMode], [ImmCheck<3, ImmCheck0_7>]>; 305 def SVBFMLALT_LANE : SInst<"svbfmlalt_lane[_{0}]", "MMddi", "b", MergeNone, "aarch64_sve_bfmlalt_lane_v2", [IsOverloadNone, VerifyRuntimeMode], [ImmCheck<3, ImmCheck0_7>]>; 306} // let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" 307 308let SVETargetGuard = "sve2p1", SMETargetGuard = InvalidMode in { 309 // Contiguous zero-extending load to quadword (single vector). 310 def SVLD1UWQ : MInst<"svld1uwq[_{d}]", "dPc", "iUif", [IsLoad], MemEltTyInt32, "aarch64_sve_ld1uwq">; 311 def SVLD1UWQ_VNUM : MInst<"svld1uwq_vnum[_{d}]", "dPcl", "iUif", [IsLoad], MemEltTyInt32, "aarch64_sve_ld1uwq">; 312 313 def SVLD1UDQ : MInst<"svld1udq[_{d}]", "dPc", "lUld", [IsLoad], MemEltTyInt64, "aarch64_sve_ld1udq">; 314 def SVLD1UDQ_VNUM : MInst<"svld1udq_vnum[_{d}]", "dPcl", "lUld", [IsLoad], MemEltTyInt64, "aarch64_sve_ld1udq">; 315 316 // Load one vector (vector base + scalar offset) 317 def SVLD1Q_GATHER_U64BASE_OFFSET : MInst<"svld1q_gather[_{2}base]_offset_{d}", "dPgl", "cUcsUsiUilUlfhdb", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ld1q_gather_scalar_offset">; 318 def SVLD1Q_GATHER_U64BASE : MInst<"svld1q_gather[_{2}base]_{d}", "dPg", "cUcsUsiUilUlfhdb", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ld1q_gather_scalar_offset">; 319 320 // Load one vector (scalar base + vector offset) 321 def SVLD1Q_GATHER_U64OFFSET : MInst<"svld1q_gather_[{3}]offset[_{d}]", "dPcg", "cUcsUsiUilUlfhdb", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ld1q_gather_vector_offset">; 322 323 // Load N-element structure into N vectors (scalar base) 324 defm SVLD2Q : StructLoad<"svld2q[_{2}]", "2Pc", "aarch64_sve_ld2q_sret">; 325 defm SVLD3Q : StructLoad<"svld3q[_{2}]", "3Pc", "aarch64_sve_ld3q_sret">; 326 defm SVLD4Q : StructLoad<"svld4q[_{2}]", "4Pc", "aarch64_sve_ld4q_sret">; 327 328 // Load N-element structure into N vectors (scalar base, VL displacement) 329 defm SVLD2Q_VNUM : StructLoad<"svld2q_vnum[_{2}]", "2Pcl", "aarch64_sve_ld2q_sret">; 330 defm SVLD3Q_VNUM : StructLoad<"svld3q_vnum[_{2}]", "3Pcl", "aarch64_sve_ld3q_sret">; 331 defm SVLD4Q_VNUM : StructLoad<"svld4q_vnum[_{2}]", "4Pcl", "aarch64_sve_ld4q_sret">; 332 333 // Load quadwords (scalar base + vector index) 334 def SVLD1Q_GATHER_INDICES_U : MInst<"svld1q_gather_[{3}]index[_{d}]", "dPcg", "sUsiUilUlbhfd", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ld1q_gather_index">; 335 336 // Load quadwords (vector base + scalar index) 337 def SVLD1Q_GATHER_INDEX_S : MInst<"svld1q_gather[_{2}base]_index_{d}", "dPgl", "sUsiUilUlbhfd", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ld1q_gather_scalar_offset">; 338} 339 340//////////////////////////////////////////////////////////////////////////////// 341// Stores 342 343// Store one vector (scalar base) 344def SVST1 : MInst<"svst1[_{d}]", "vPpd", "csilUcUsUiUlhfd", [IsStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_st1">; 345def SVST1B_S : MInst<"svst1b[_{d}]", "vPAd", "sil", [IsStore, VerifyRuntimeMode], MemEltTyInt8, "aarch64_sve_st1">; 346def SVST1B_U : MInst<"svst1b[_{d}]", "vPEd", "UsUiUl", [IsStore, VerifyRuntimeMode], MemEltTyInt8, "aarch64_sve_st1">; 347def SVST1H_S : MInst<"svst1h[_{d}]", "vPBd", "il", [IsStore, VerifyRuntimeMode], MemEltTyInt16, "aarch64_sve_st1">; 348def SVST1H_U : MInst<"svst1h[_{d}]", "vPFd", "UiUl", [IsStore, VerifyRuntimeMode], MemEltTyInt16, "aarch64_sve_st1">; 349def SVST1W_S : MInst<"svst1w[_{d}]", "vPCd", "l", [IsStore, VerifyRuntimeMode], MemEltTyInt32, "aarch64_sve_st1">; 350def SVST1W_U : MInst<"svst1w[_{d}]", "vPGd", "Ul", [IsStore, VerifyRuntimeMode], MemEltTyInt32, "aarch64_sve_st1">; 351 352// Store one vector (scalar base, VL displacement) 353def SVST1_VNUM : MInst<"svst1_vnum[_{d}]", "vPpld", "csilUcUsUiUlhfd", [IsStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_st1">; 354def SVST1B_VNUM_S : MInst<"svst1b_vnum[_{d}]", "vPAld", "sil", [IsStore, VerifyRuntimeMode], MemEltTyInt8, "aarch64_sve_st1">; 355def SVST1B_VNUM_U : MInst<"svst1b_vnum[_{d}]", "vPEld", "UsUiUl", [IsStore, VerifyRuntimeMode], MemEltTyInt8, "aarch64_sve_st1">; 356def SVST1H_VNUM_S : MInst<"svst1h_vnum[_{d}]", "vPBld", "il", [IsStore, VerifyRuntimeMode], MemEltTyInt16, "aarch64_sve_st1">; 357def SVST1H_VNUM_U : MInst<"svst1h_vnum[_{d}]", "vPFld", "UiUl", [IsStore, VerifyRuntimeMode], MemEltTyInt16, "aarch64_sve_st1">; 358def SVST1W_VNUM_S : MInst<"svst1w_vnum[_{d}]", "vPCld", "l", [IsStore, VerifyRuntimeMode], MemEltTyInt32, "aarch64_sve_st1">; 359def SVST1W_VNUM_U : MInst<"svst1w_vnum[_{d}]", "vPGld", "Ul", [IsStore, VerifyRuntimeMode], MemEltTyInt32, "aarch64_sve_st1">; 360 361let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in { 362 def SVST1_BF : MInst<"svst1[_{d}]", "vPpd", "b", [IsStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_st1">; 363 def SVST1_VNUM_BF : MInst<"svst1_vnum[_{d}]", "vPpld", "b", [IsStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_st1">; 364} 365 366let SVETargetGuard = "sve", SMETargetGuard = InvalidMode in { 367// Store one vector (vector base) 368def SVST1_SCATTER_BASES_U : MInst<"svst1_scatter[_{2}base_{d}]", "vPud", "ilUiUlfd", [IsScatterStore], MemEltTyDefault, "aarch64_sve_st1_scatter_scalar_offset">; 369def SVST1B_SCATTER_BASES_U : MInst<"svst1b_scatter[_{2}base_{d}]", "vPud", "ilUiUl", [IsScatterStore], MemEltTyInt8, "aarch64_sve_st1_scatter_scalar_offset">; 370def SVST1H_SCATTER_BASES_U : MInst<"svst1h_scatter[_{2}base_{d}]", "vPud", "ilUiUl", [IsScatterStore], MemEltTyInt16, "aarch64_sve_st1_scatter_scalar_offset">; 371def SVST1W_SCATTER_BASES_U : MInst<"svst1w_scatter[_{2}base_{d}]", "vPud", "lUl", [IsScatterStore], MemEltTyInt32, "aarch64_sve_st1_scatter_scalar_offset">; 372 373// Store one vector (scalar base, signed vector offset in bytes) 374def SVST1_SCATTER_64B_OFFSETS_S : MInst<"svst1_scatter_[{3}]offset[_{d}]", "vPpxd", "lUld", [IsScatterStore, IsByteIndexed], MemEltTyDefault, "aarch64_sve_st1_scatter">; 375def SVST1B_SCATTER_64B_OFFSETS_SS : MInst<"svst1b_scatter_[{3}]offset[_{d}]", "vPAxd", "l", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_st1_scatter">; 376def SVST1B_SCATTER_64B_OFFSETS_SU : MInst<"svst1b_scatter_[{3}]offset[_{d}]", "vPExd", "Ul", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_st1_scatter">; 377def SVST1H_SCATTER_64B_OFFSETS_SS : MInst<"svst1h_scatter_[{3}]offset[_{d}]", "vPBxd", "l", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_st1_scatter">; 378def SVST1H_SCATTER_64B_OFFSETS_SU : MInst<"svst1h_scatter_[{3}]offset[_{d}]", "vPFxd", "Ul", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_st1_scatter">; 379def SVST1W_SCATTER_64B_OFFSETS_SS : MInst<"svst1w_scatter_[{3}]offset[_{d}]", "vPCxd", "l", [IsScatterStore, IsByteIndexed], MemEltTyInt32, "aarch64_sve_st1_scatter">; 380def SVST1W_SCATTER_64B_OFFSETS_SU : MInst<"svst1w_scatter_[{3}]offset[_{d}]", "vPGxd", "Ul", [IsScatterStore, IsByteIndexed], MemEltTyInt32, "aarch64_sve_st1_scatter">; 381 382def SVST1_SCATTER_32B_OFFSETS_S : MInst<"svst1_scatter_[{3}]offset[_{d}]", "vPpxd", "iUif", [IsScatterStore, IsByteIndexed], MemEltTyDefault, "aarch64_sve_st1_scatter_sxtw">; 383def SVST1B_SCATTER_32B_OFFSETS_SS : MInst<"svst1b_scatter_[{3}]offset[_{d}]", "vPAxd", "i", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_st1_scatter_sxtw">; 384def SVST1B_SCATTER_32B_OFFSETS_SU : MInst<"svst1b_scatter_[{3}]offset[_{d}]", "vPExd", "Ui", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_st1_scatter_sxtw">; 385def SVST1H_SCATTER_32B_OFFSETS_SS : MInst<"svst1h_scatter_[{3}]offset[_{d}]", "vPBxd", "i", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_st1_scatter_sxtw">; 386def SVST1H_SCATTER_32B_OFFSETS_SU : MInst<"svst1h_scatter_[{3}]offset[_{d}]", "vPFxd", "Ui", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_st1_scatter_sxtw">; 387 388// Store one vector (scalar base, unsigned vector offset in bytes) 389def SVST1_SCATTER_64B_OFFSETS_U : MInst<"svst1_scatter_[{3}]offset[_{d}]", "vPpud", "lUld", [IsScatterStore, IsByteIndexed], MemEltTyDefault, "aarch64_sve_st1_scatter">; 390def SVST1B_SCATTER_64B_OFFSETS_US : MInst<"svst1b_scatter_[{3}]offset[_{d}]", "vPAud", "l", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_st1_scatter">; 391def SVST1B_SCATTER_64B_OFFSETS_UU : MInst<"svst1b_scatter_[{3}]offset[_{d}]", "vPEud", "Ul", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_st1_scatter">; 392def SVST1H_SCATTER_64B_OFFSETS_US : MInst<"svst1h_scatter_[{3}]offset[_{d}]", "vPBud", "l", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_st1_scatter">; 393def SVST1H_SCATTER_64B_OFFSETS_UU : MInst<"svst1h_scatter_[{3}]offset[_{d}]", "vPFud", "Ul", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_st1_scatter">; 394def SVST1W_SCATTER_64B_OFFSETS_US : MInst<"svst1w_scatter_[{3}]offset[_{d}]", "vPCud", "l", [IsScatterStore, IsByteIndexed], MemEltTyInt32, "aarch64_sve_st1_scatter">; 395def SVST1W_SCATTER_64B_OFFSETS_UU : MInst<"svst1w_scatter_[{3}]offset[_{d}]", "vPGud", "Ul", [IsScatterStore, IsByteIndexed], MemEltTyInt32, "aarch64_sve_st1_scatter">; 396 397def SVST1_SCATTER_32B_OFFSETS_U : MInst<"svst1_scatter_[{3}]offset[_{d}]", "vPpud", "iUif", [IsScatterStore, IsByteIndexed], MemEltTyDefault, "aarch64_sve_st1_scatter_uxtw">; 398def SVST1B_SCATTER_32B_OFFSETS_US : MInst<"svst1b_scatter_[{3}]offset[_{d}]", "vPAud", "i", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_st1_scatter_uxtw">; 399def SVST1B_SCATTER_32B_OFFSETS_UU : MInst<"svst1b_scatter_[{3}]offset[_{d}]", "vPEud", "Ui", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_st1_scatter_uxtw">; 400def SVST1H_SCATTER_32B_OFFSETS_US : MInst<"svst1h_scatter_[{3}]offset[_{d}]", "vPBud", "i", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_st1_scatter_uxtw">; 401def SVST1H_SCATTER_32B_OFFSETS_UU : MInst<"svst1h_scatter_[{3}]offset[_{d}]", "vPFud", "Ui", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_st1_scatter_uxtw">; 402 403// Store one vector (vector base, signed scalar offset in bytes) 404def SVST1_SCATTER_OFFSET_S : MInst<"svst1_scatter[_{2}base]_offset[_{d}]", "vPuld", "ilUiUlfd", [IsScatterStore, IsByteIndexed], MemEltTyDefault, "aarch64_sve_st1_scatter_scalar_offset">; 405def SVST1B_SCATTER_OFFSET_S : MInst<"svst1b_scatter[_{2}base]_offset[_{d}]", "vPuld", "ilUiUl", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_st1_scatter_scalar_offset">; 406def SVST1H_SCATTER_OFFSET_S : MInst<"svst1h_scatter[_{2}base]_offset[_{d}]", "vPuld", "ilUiUl", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_st1_scatter_scalar_offset">; 407def SVST1W_SCATTER_OFFSET_S : MInst<"svst1w_scatter[_{2}base]_offset[_{d}]", "vPuld", "lUl", [IsScatterStore, IsByteIndexed], MemEltTyInt32, "aarch64_sve_st1_scatter_scalar_offset">; 408 409// Store one vector (scalar base, signed vector index) 410def SVST1_SCATTER_64B_INDICES_S : MInst<"svst1_scatter_[{3}]index[_{d}]", "vPpxd", "lUld", [IsScatterStore], MemEltTyDefault, "aarch64_sve_st1_scatter_index">; 411def SVST1H_SCATTER_64B_INDICES_SS : MInst<"svst1h_scatter_[{3}]index[_{d}]", "vPBxd", "l", [IsScatterStore], MemEltTyInt16, "aarch64_sve_st1_scatter_index">; 412def SVST1H_SCATTER_64B_INDICES_SU : MInst<"svst1h_scatter_[{3}]index[_{d}]", "vPFxd", "Ul", [IsScatterStore], MemEltTyInt16, "aarch64_sve_st1_scatter_index">; 413def SVST1W_SCATTER_64B_INDICES_SS : MInst<"svst1w_scatter_[{3}]index[_{d}]", "vPCxd", "l", [IsScatterStore], MemEltTyInt32, "aarch64_sve_st1_scatter_index">; 414def SVST1W_SCATTER_64B_INDICES_SU : MInst<"svst1w_scatter_[{3}]index[_{d}]", "vPGxd", "Ul", [IsScatterStore], MemEltTyInt32, "aarch64_sve_st1_scatter_index">; 415 416def SVST1_SCATTER_32B_INDICES_S : MInst<"svst1_scatter_[{3}]index[_{d}]", "vPpxd", "iUif", [IsScatterStore], MemEltTyDefault, "aarch64_sve_st1_scatter_sxtw_index">; 417def SVST1H_SCATTER_32B_INDICES_SS : MInst<"svst1h_scatter_[{3}]index[_{d}]", "vPBxd", "i", [IsScatterStore], MemEltTyInt16, "aarch64_sve_st1_scatter_sxtw_index">; 418def SVST1H_SCATTER_32B_INDICES_SU : MInst<"svst1h_scatter_[{3}]index[_{d}]", "vPFxd", "Ui", [IsScatterStore], MemEltTyInt16, "aarch64_sve_st1_scatter_sxtw_index">; 419 420// Store one vector (scalar base, unsigned vector index) 421def SVST1_SCATTER_64B_INDICES_U : MInst<"svst1_scatter_[{3}]index[_{d}]", "vPpud", "lUld", [IsScatterStore], MemEltTyDefault, "aarch64_sve_st1_scatter_index">; 422def SVST1H_SCATTER_64B_INDICES_US : MInst<"svst1h_scatter_[{3}]index[_{d}]", "vPBud", "l", [IsScatterStore], MemEltTyInt16, "aarch64_sve_st1_scatter_index">; 423def SVST1H_SCATTER_64B_INDICES_UU : MInst<"svst1h_scatter_[{3}]index[_{d}]", "vPFud", "Ul", [IsScatterStore], MemEltTyInt16, "aarch64_sve_st1_scatter_index">; 424def SVST1W_SCATTER_64B_INDICES_US : MInst<"svst1w_scatter_[{3}]index[_{d}]", "vPCud", "l", [IsScatterStore], MemEltTyInt32, "aarch64_sve_st1_scatter_index">; 425def SVST1W_SCATTER_64B_INDICES_UU : MInst<"svst1w_scatter_[{3}]index[_{d}]", "vPGud", "Ul", [IsScatterStore], MemEltTyInt32, "aarch64_sve_st1_scatter_index">; 426 427def SVST1_SCATTER_32B_INDICES_U : MInst<"svst1_scatter_[{3}]index[_{d}]", "vPpud", "iUif", [IsScatterStore], MemEltTyDefault, "aarch64_sve_st1_scatter_uxtw_index">; 428def SVST1H_SCATTER_32B_INDICES_US : MInst<"svst1h_scatter_[{3}]index[_{d}]", "vPBud", "i", [IsScatterStore], MemEltTyInt16, "aarch64_sve_st1_scatter_uxtw_index">; 429def SVST1H_SCATTER_32B_INDICES_UU : MInst<"svst1h_scatter_[{3}]index[_{d}]", "vPFud", "Ui", [IsScatterStore], MemEltTyInt16, "aarch64_sve_st1_scatter_uxtw_index">; 430 431// Store one vector (vector base, signed scalar index) 432def SVST1_SCATTER_INDEX_S : MInst<"svst1_scatter[_{2}base]_index[_{d}]", "vPuld", "ilUiUlfd", [IsScatterStore], MemEltTyDefault, "aarch64_sve_st1_scatter_scalar_offset">; 433def SVST1H_SCATTER_INDEX_S : MInst<"svst1h_scatter[_{2}base]_index[_{d}]", "vPuld", "ilUiUl", [IsScatterStore], MemEltTyInt16, "aarch64_sve_st1_scatter_scalar_offset">; 434def SVST1W_SCATTER_INDEX_S : MInst<"svst1w_scatter[_{2}base]_index[_{d}]", "vPuld", "lUl", [IsScatterStore], MemEltTyInt32, "aarch64_sve_st1_scatter_scalar_offset">; 435} // let SVETargetGuard = "sve" 436 437multiclass StructStore<string name, string proto, string i> { 438 def : SInst<name, proto, "csilUcUsUiUlhfd", MergeNone, i, [IsStructStore, VerifyRuntimeMode]>; 439 let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in { 440 def: SInst<name, proto, "b", MergeNone, i, [IsStructStore, VerifyRuntimeMode]>; 441 } 442} 443// Store N vectors into N-element structure (scalar base) 444defm SVST2 : StructStore<"svst2[_{d}]", "vPp2", "aarch64_sve_st2">; 445defm SVST3 : StructStore<"svst3[_{d}]", "vPp3", "aarch64_sve_st3">; 446defm SVST4 : StructStore<"svst4[_{d}]", "vPp4", "aarch64_sve_st4">; 447 448// Store N vectors into N-element structure (scalar base, VL displacement) 449defm SVST2_VNUM : StructStore<"svst2_vnum[_{d}]", "vPpl2", "aarch64_sve_st2">; 450defm SVST3_VNUM : StructStore<"svst3_vnum[_{d}]", "vPpl3", "aarch64_sve_st3">; 451defm SVST4_VNUM : StructStore<"svst4_vnum[_{d}]", "vPpl4", "aarch64_sve_st4">; 452 453// Store one vector, with no truncation, non-temporal (scalar base) 454def SVSTNT1 : MInst<"svstnt1[_{d}]", "vPpd", "csilUcUsUiUlhfd", [IsStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_stnt1">; 455 456// Store one vector, with no truncation, non-temporal (scalar base, VL displacement) 457def SVSTNT1_VNUM : MInst<"svstnt1_vnum[_{d}]", "vPpld", "csilUcUsUiUlhfd", [IsStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_stnt1">; 458 459let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in { 460 def SVSTNT1_BF : MInst<"svstnt1[_{d}]", "vPpd", "b", [IsStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_stnt1">; 461 def SVSTNT1_VNUM_BF : MInst<"svstnt1_vnum[_{d}]", "vPpld", "b", [IsStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_stnt1">; 462} 463 464let SVETargetGuard = "sve2p1", SMETargetGuard = InvalidMode in { 465 // Contiguous truncating store from quadword (single vector). 466 def SVST1UWQ : MInst<"svst1wq[_{d}]", "vPcd", "iUif", [IsStore], MemEltTyInt32, "aarch64_sve_st1wq">; 467 def SVST1UWQ_VNUM : MInst<"svst1wq_vnum[_{d}]", "vPcld", "iUif", [IsStore], MemEltTyInt32, "aarch64_sve_st1wq">; 468 469 def SVST1UDQ : MInst<"svst1dq[_{d}]", "vPcd", "lUld", [IsStore], MemEltTyInt64, "aarch64_sve_st1dq">; 470 def SVST1UDQ_VNUM : MInst<"svst1dq_vnum[_{d}]", "vPcld", "lUld", [IsStore], MemEltTyInt64, "aarch64_sve_st1dq">; 471 472 // Store one vector (vector base + scalar offset) 473 def SVST1Q_SCATTER_U64BASE_OFFSET : MInst<"svst1q_scatter[_{2}base]_offset[_{d}]", "vPgld", "cUcsUsiUilUlfhdb", [IsScatterStore, IsByteIndexed], MemEltTyDefault, "aarch64_sve_st1q_scatter_scalar_offset">; 474 def SVST1Q_SCATTER_U64BASE : MInst<"svst1q_scatter[_{2}base][_{d}]", "vPgd", "cUcsUsiUilUlfhdb", [IsScatterStore, IsByteIndexed], MemEltTyDefault, "aarch64_sve_st1q_scatter_scalar_offset">; 475 476 // Store one vector (scalar base + vector offset) 477 def SVST1Q_SCATTER_U64OFFSET : MInst<"svst1q_scatter_[{3}]offset[_{d}]", "vPpgd", "cUcsUsiUilUlfhdb", [IsScatterStore, IsByteIndexed], MemEltTyDefault, "aarch64_sve_st1q_scatter_vector_offset">; 478 479 // Store N vectors into N-element structure (scalar base) 480 defm SVST2Q : StructStore<"svst2q[_{d}]", "vPc2", "aarch64_sve_st2q">; 481 defm SVST3Q : StructStore<"svst3q[_{d}]", "vPc3", "aarch64_sve_st3q">; 482 defm SVST4Q : StructStore<"svst4q[_{d}]", "vPc4", "aarch64_sve_st4q">; 483 484 // Store N vectors into N-element structure (scalar base, VL displacement) 485 defm SVST2Q_VNUM : StructStore<"svst2q_vnum[_{d}]", "vPcl2", "aarch64_sve_st2q">; 486 defm SVST3Q_VNUM : StructStore<"svst3q_vnum[_{d}]", "vPcl3", "aarch64_sve_st3q">; 487 defm SVST4Q_VNUM : StructStore<"svst4q_vnum[_{d}]", "vPcl4", "aarch64_sve_st4q">; 488 489 // Scatter store quadwords (scalar base + vector index) 490 def SVST1Q_SCATTER_INDICES_U : MInst<"svst1q_scatter_[{3}]index[_{d}]", "vPpgd", "sUsiUilUlbhfd", [IsScatterStore], MemEltTyDefault, "aarch64_sve_st1q_scatter_index">; 491 492 // Scatter store quadwords (vector base + scalar index) 493 def SVST1Q_SCATTER_INDEX_S : MInst<"svst1q_scatter[_{2}base]_index[_{d}]", "vPgld", "sUsiUilUlbhfd", [IsScatterStore], MemEltTyDefault, "aarch64_sve_st1q_scatter_scalar_offset">; 494} 495 496//////////////////////////////////////////////////////////////////////////////// 497// Prefetches 498 499// Prefetch (Scalar base) 500def SVPRFB : MInst<"svprfb", "vPQJ", "c", [IsPrefetch, VerifyRuntimeMode], MemEltTyInt8, "aarch64_sve_prf">; 501def SVPRFH : MInst<"svprfh", "vPQJ", "s", [IsPrefetch, VerifyRuntimeMode], MemEltTyInt16, "aarch64_sve_prf">; 502def SVPRFW : MInst<"svprfw", "vPQJ", "i", [IsPrefetch, VerifyRuntimeMode], MemEltTyInt32, "aarch64_sve_prf">; 503def SVPRFD : MInst<"svprfd", "vPQJ", "l", [IsPrefetch, VerifyRuntimeMode], MemEltTyInt64, "aarch64_sve_prf">; 504 505// Prefetch (Scalar base, VL displacement) 506def SVPRFB_VNUM : MInst<"svprfb_vnum", "vPQlJ", "c", [IsPrefetch, VerifyRuntimeMode], MemEltTyInt8, "aarch64_sve_prf">; 507def SVPRFH_VNUM : MInst<"svprfh_vnum", "vPQlJ", "s", [IsPrefetch, VerifyRuntimeMode], MemEltTyInt16, "aarch64_sve_prf">; 508def SVPRFW_VNUM : MInst<"svprfw_vnum", "vPQlJ", "i", [IsPrefetch, VerifyRuntimeMode], MemEltTyInt32, "aarch64_sve_prf">; 509def SVPRFD_VNUM : MInst<"svprfd_vnum", "vPQlJ", "l", [IsPrefetch, VerifyRuntimeMode], MemEltTyInt64, "aarch64_sve_prf">; 510 511let SVETargetGuard = "sve", SMETargetGuard = InvalidMode in { 512// Prefetch (Vector bases) 513def SVPRFB_GATHER_BASES : MInst<"svprfb_gather[_{2}base]", "vPdJ", "UiUl", [IsGatherPrefetch], MemEltTyInt8, "aarch64_sve_prfb_gather_scalar_offset">; 514def SVPRFH_GATHER_BASES : MInst<"svprfh_gather[_{2}base]", "vPdJ", "UiUl", [IsGatherPrefetch], MemEltTyInt16, "aarch64_sve_prfh_gather_scalar_offset">; 515def SVPRFW_GATHER_BASES : MInst<"svprfw_gather[_{2}base]", "vPdJ", "UiUl", [IsGatherPrefetch], MemEltTyInt32, "aarch64_sve_prfw_gather_scalar_offset">; 516def SVPRFD_GATHER_BASES : MInst<"svprfd_gather[_{2}base]", "vPdJ", "UiUl", [IsGatherPrefetch], MemEltTyInt64, "aarch64_sve_prfd_gather_scalar_offset">; 517 518// Prefetch (Scalar base, Vector offsets) 519def SVPRFB_GATHER_32B_OFFSETS_S : MInst<"svprfb_gather_[{3}]offset", "vPQdJ", "i", [IsGatherPrefetch], MemEltTyInt8, "aarch64_sve_prfb_gather_sxtw_index">; 520def SVPRFH_GATHER_32B_OFFSETS_S : MInst<"svprfh_gather_[{3}]index", "vPQdJ", "i", [IsGatherPrefetch], MemEltTyInt16, "aarch64_sve_prfh_gather_sxtw_index">; 521def SVPRFW_GATHER_32B_OFFSETS_S : MInst<"svprfw_gather_[{3}]index", "vPQdJ", "i", [IsGatherPrefetch], MemEltTyInt32, "aarch64_sve_prfw_gather_sxtw_index">; 522def SVPRFD_GATHER_32B_OFFSETS_S : MInst<"svprfd_gather_[{3}]index", "vPQdJ", "i", [IsGatherPrefetch], MemEltTyInt64, "aarch64_sve_prfd_gather_sxtw_index">; 523 524def SVPRFB_GATHER_64B_OFFSETS_S : MInst<"svprfb_gather_[{3}]offset", "vPQdJ", "l", [IsGatherPrefetch], MemEltTyInt8, "aarch64_sve_prfb_gather_index">; 525def SVPRFH_GATHER_64B_OFFSETS_S : MInst<"svprfh_gather_[{3}]index", "vPQdJ", "l", [IsGatherPrefetch], MemEltTyInt16, "aarch64_sve_prfh_gather_index">; 526def SVPRFW_GATHER_64B_OFFSETS_S : MInst<"svprfw_gather_[{3}]index", "vPQdJ", "l", [IsGatherPrefetch], MemEltTyInt32, "aarch64_sve_prfw_gather_index">; 527def SVPRFD_GATHER_64B_OFFSETS_S : MInst<"svprfd_gather_[{3}]index", "vPQdJ", "l", [IsGatherPrefetch], MemEltTyInt64, "aarch64_sve_prfd_gather_index">; 528 529def SVPRFB_GATHER_32B_OFFSETS_U : MInst<"svprfb_gather_[{3}]offset", "vPQdJ", "Ui", [IsGatherPrefetch], MemEltTyInt8, "aarch64_sve_prfb_gather_uxtw_index">; 530def SVPRFH_GATHER_32B_OFFSETS_U : MInst<"svprfh_gather_[{3}]index", "vPQdJ", "Ui", [IsGatherPrefetch], MemEltTyInt16, "aarch64_sve_prfh_gather_uxtw_index">; 531def SVPRFW_GATHER_32B_OFFSETS_U : MInst<"svprfw_gather_[{3}]index", "vPQdJ", "Ui", [IsGatherPrefetch], MemEltTyInt32, "aarch64_sve_prfw_gather_uxtw_index">; 532def SVPRFD_GATHER_32B_OFFSETS_U : MInst<"svprfd_gather_[{3}]index", "vPQdJ", "Ui", [IsGatherPrefetch], MemEltTyInt64, "aarch64_sve_prfd_gather_uxtw_index">; 533 534def SVPRFB_GATHER_64B_OFFSETS_U : MInst<"svprfb_gather_[{3}]offset", "vPQdJ", "Ul", [IsGatherPrefetch], MemEltTyInt8, "aarch64_sve_prfb_gather_index">; 535def SVPRFH_GATHER_64B_OFFSETS_U : MInst<"svprfh_gather_[{3}]index", "vPQdJ", "Ul", [IsGatherPrefetch], MemEltTyInt16, "aarch64_sve_prfh_gather_index">; 536def SVPRFW_GATHER_64B_OFFSETS_U : MInst<"svprfw_gather_[{3}]index", "vPQdJ", "Ul", [IsGatherPrefetch], MemEltTyInt32, "aarch64_sve_prfw_gather_index">; 537def SVPRFD_GATHER_64B_OFFSETS_U : MInst<"svprfd_gather_[{3}]index", "vPQdJ", "Ul", [IsGatherPrefetch], MemEltTyInt64, "aarch64_sve_prfd_gather_index">; 538 539// Prefetch (Vector bases, scalar offset) 540def SVPRFB_GATHER_BASES_OFFSET : MInst<"svprfb_gather[_{2}base]_offset", "vPdlJ", "UiUl", [IsGatherPrefetch], MemEltTyInt8, "aarch64_sve_prfb_gather_scalar_offset">; 541def SVPRFH_GATHER_BASES_OFFSET : MInst<"svprfh_gather[_{2}base]_index", "vPdlJ", "UiUl", [IsGatherPrefetch], MemEltTyInt16, "aarch64_sve_prfh_gather_scalar_offset">; 542def SVPRFW_GATHER_BASES_OFFSET : MInst<"svprfw_gather[_{2}base]_index", "vPdlJ", "UiUl", [IsGatherPrefetch], MemEltTyInt32, "aarch64_sve_prfw_gather_scalar_offset">; 543def SVPRFD_GATHER_BASES_OFFSET : MInst<"svprfd_gather[_{2}base]_index", "vPdlJ", "UiUl", [IsGatherPrefetch], MemEltTyInt64, "aarch64_sve_prfd_gather_scalar_offset">; 544} // let SVETargetGuard = "sve" 545 546//////////////////////////////////////////////////////////////////////////////// 547// Address calculations 548 549let SVETargetGuard = "sve", SMETargetGuard = InvalidMode in { 550def SVADRB : SInst<"svadrb[_{0}base]_[{2}]offset", "uud", "ilUiUl", MergeNone, "aarch64_sve_adrb">; 551def SVADRH : SInst<"svadrh[_{0}base]_[{2}]index", "uud", "ilUiUl", MergeNone, "aarch64_sve_adrh">; 552def SVADRW : SInst<"svadrw[_{0}base]_[{2}]index", "uud", "ilUiUl", MergeNone, "aarch64_sve_adrw">; 553def SVADRD : SInst<"svadrd[_{0}base]_[{2}]index", "uud", "ilUiUl", MergeNone, "aarch64_sve_adrd">; 554} // let SVETargetGuard = "sve" 555 556//////////////////////////////////////////////////////////////////////////////// 557// Scalar to vector 558 559def SVDUPQ_8 : SInst<"svdupq[_n]_{d}", "dssssssssssssssss", "cUc", MergeNone, "", [VerifyRuntimeMode]>; 560def SVDUPQ_16 : SInst<"svdupq[_n]_{d}", "dssssssss", "sUsh", MergeNone, "", [VerifyRuntimeMode]>; 561let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in { 562 def SVDUPQ_BF16 : SInst<"svdupq[_n]_{d}", "dssssssss", "b", MergeNone, "", [VerifyRuntimeMode]>; 563} 564def SVDUPQ_32 : SInst<"svdupq[_n]_{d}", "dssss", "iUif", MergeNone, "", [VerifyRuntimeMode]>; 565def SVDUPQ_64 : SInst<"svdupq[_n]_{d}", "dss", "lUld", MergeNone, "", [VerifyRuntimeMode]>; 566 567multiclass svdup_base<string n, string p, MergeType mt, string i> { 568 def NAME : SInst<n, p, "csilUcUsUiUlhfd", mt, i, [VerifyRuntimeMode]>; 569 let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in { 570 def _BF16: SInst<n, p, "b", mt, i, [VerifyRuntimeMode]>; 571 } 572} 573 574defm SVDUP : svdup_base<"svdup[_n]_{d}", "ds", MergeNone, "aarch64_sve_dup_x">; 575defm SVDUP_M : svdup_base<"svdup[_n]_{d}", "ddPs", MergeOp1, "aarch64_sve_dup">; 576defm SVDUP_X : svdup_base<"svdup[_n]_{d}", "dPs", MergeAnyExp, "aarch64_sve_dup">; 577defm SVDUP_Z : svdup_base<"svdup[_n]_{d}", "dPs", MergeZeroExp, "aarch64_sve_dup">; 578 579def SVINDEX : SInst<"svindex_{d}", "dss", "csilUcUsUiUl", MergeNone, "aarch64_sve_index", [VerifyRuntimeMode]>; 580 581// Integer arithmetic 582 583multiclass SInstZPZ<string name, string types, string intrinsic> { 584 def _M : SInst<name # "[_{d}]", "ddPd", types, MergeOp1, intrinsic, [VerifyRuntimeMode]>; 585 def _X : SInst<name # "[_{d}]", "dPd", types, MergeAnyExp, intrinsic, [VerifyRuntimeMode]>; 586 def _Z : SInst<name # "[_{d}]", "dPd", types, MergeZeroExp, intrinsic, [VerifyRuntimeMode]>; 587} 588 589defm SVABS : SInstZPZ<"svabs", "csil", "aarch64_sve_abs">; 590defm SVNEG : SInstZPZ<"svneg", "csil", "aarch64_sve_neg">; 591 592//------------------------------------------------------------------------------ 593 594multiclass SInstZPZZ<string name, string types, string m_intrinsic, string x_intrinsic, list<FlagType> flags=[]> { 595 def _M : SInst<name # "[_{d}]", "dPdd", types, MergeOp1, m_intrinsic, !listconcat(flags, [VerifyRuntimeMode])>; 596 def _X : SInst<name # "[_{d}]", "dPdd", types, MergeAny, x_intrinsic, !listconcat(flags, [VerifyRuntimeMode])>; 597 def _Z : SInst<name # "[_{d}]", "dPdd", types, MergeZero, m_intrinsic, !listconcat(flags, [VerifyRuntimeMode])>; 598 599 def _N_M : SInst<name # "[_n_{d}]", "dPda", types, MergeOp1, m_intrinsic, !listconcat(flags, [VerifyRuntimeMode])>; 600 def _N_X : SInst<name # "[_n_{d}]", "dPda", types, MergeAny, x_intrinsic, !listconcat(flags, [VerifyRuntimeMode])>; 601 def _N_Z : SInst<name # "[_n_{d}]", "dPda", types, MergeZero, m_intrinsic, !listconcat(flags, [VerifyRuntimeMode])>; 602} 603 604defm SVABD_S : SInstZPZZ<"svabd", "csil", "aarch64_sve_sabd", "aarch64_sve_sabd_u">; 605defm SVABD_U : SInstZPZZ<"svabd", "UcUsUiUl", "aarch64_sve_uabd", "aarch64_sve_uabd_u">; 606defm SVADD : SInstZPZZ<"svadd", "csilUcUsUiUl", "aarch64_sve_add", "aarch64_sve_add_u">; 607defm SVDIV_S : SInstZPZZ<"svdiv", "il", "aarch64_sve_sdiv", "aarch64_sve_sdiv_u">; 608defm SVDIV_U : SInstZPZZ<"svdiv", "UiUl", "aarch64_sve_udiv", "aarch64_sve_udiv_u">; 609defm SVDIVR_S : SInstZPZZ<"svdivr", "il", "aarch64_sve_sdivr", "aarch64_sve_sdiv_u", [ReverseMergeAnyBinOp]>; 610defm SVDIVR_U : SInstZPZZ<"svdivr", "UiUl", "aarch64_sve_udivr", "aarch64_sve_udiv_u", [ReverseMergeAnyBinOp]>; 611defm SVMAX_S : SInstZPZZ<"svmax", "csil", "aarch64_sve_smax", "aarch64_sve_smax_u">; 612defm SVMAX_U : SInstZPZZ<"svmax", "UcUsUiUl", "aarch64_sve_umax", "aarch64_sve_umax_u">; 613defm SVMIN_S : SInstZPZZ<"svmin", "csil", "aarch64_sve_smin", "aarch64_sve_smin_u">; 614defm SVMIN_U : SInstZPZZ<"svmin", "UcUsUiUl", "aarch64_sve_umin", "aarch64_sve_umin_u">; 615defm SVMUL : SInstZPZZ<"svmul", "csilUcUsUiUl", "aarch64_sve_mul", "aarch64_sve_mul_u">; 616defm SVMULH_S : SInstZPZZ<"svmulh", "csil", "aarch64_sve_smulh", "aarch64_sve_smulh_u">; 617defm SVMULH_U : SInstZPZZ<"svmulh", "UcUsUiUl", "aarch64_sve_umulh", "aarch64_sve_umulh_u">; 618defm SVSUB : SInstZPZZ<"svsub", "csilUcUsUiUl", "aarch64_sve_sub", "aarch64_sve_sub_u">; 619defm SVSUBR : SInstZPZZ<"svsubr", "csilUcUsUiUl", "aarch64_sve_subr", "aarch64_sve_sub_u", [ReverseMergeAnyBinOp]>; 620 621//------------------------------------------------------------------------------ 622 623multiclass SInstZPZZZ<string name, string types, string m_intrinsic, string x_intrinsic, list<FlagType> flags=[]> { 624 def _M : SInst<name # "[_{d}]", "dPddd", types, MergeOp1, m_intrinsic, flags>; 625 def _X : SInst<name # "[_{d}]", "dPddd", types, MergeAny, x_intrinsic, flags>; 626 def _Z : SInst<name # "[_{d}]", "dPddd", types, MergeZero, m_intrinsic, flags>; 627 628 def _N_M : SInst<name # "[_n_{d}]", "dPdda", types, MergeOp1, m_intrinsic, flags>; 629 def _N_X : SInst<name # "[_n_{d}]", "dPdda", types, MergeAny, x_intrinsic, flags>; 630 def _N_Z : SInst<name # "[_n_{d}]", "dPdda", types, MergeZero, m_intrinsic, flags>; 631} 632 633defm SVMAD : SInstZPZZZ<"svmad", "csilUcUsUiUl", "aarch64_sve_mad", "aarch64_sve_mla_u", [ReverseMergeAnyAccOp, VerifyRuntimeMode]>; 634defm SVMLA : SInstZPZZZ<"svmla", "csilUcUsUiUl", "aarch64_sve_mla", "aarch64_sve_mla_u", [VerifyRuntimeMode]>; 635defm SVMLS : SInstZPZZZ<"svmls", "csilUcUsUiUl", "aarch64_sve_mls", "aarch64_sve_mls_u", [VerifyRuntimeMode]>; 636defm SVMSB : SInstZPZZZ<"svmsb", "csilUcUsUiUl", "aarch64_sve_msb", "aarch64_sve_mls_u", [ReverseMergeAnyAccOp, VerifyRuntimeMode]>; 637 638//------------------------------------------------------------------------------ 639 640def SVDOT_S : SInst<"svdot[_{0}]", "ddqq", "il", MergeNone, "aarch64_sve_sdot", [VerifyRuntimeMode]>; 641def SVDOT_U : SInst<"svdot[_{0}]", "ddqq", "UiUl", MergeNone, "aarch64_sve_udot", [VerifyRuntimeMode]>; 642def SVQADD_S : SInst<"svqadd[_{d}]", "ddd", "csil", MergeNone, "aarch64_sve_sqadd_x", [VerifyRuntimeMode]>; 643def SVQADD_U : SInst<"svqadd[_{d}]", "ddd", "UcUsUiUl", MergeNone, "aarch64_sve_uqadd_x", [VerifyRuntimeMode]>; 644def SVQSUB_S : SInst<"svqsub[_{d}]", "ddd", "csil", MergeNone, "aarch64_sve_sqsub_x", [VerifyRuntimeMode]>; 645def SVQSUB_U : SInst<"svqsub[_{d}]", "ddd", "UcUsUiUl", MergeNone, "aarch64_sve_uqsub_x", [VerifyRuntimeMode]>; 646 647def SVDOT_N_S : SInst<"svdot[_n_{0}]", "ddqr", "il", MergeNone, "aarch64_sve_sdot", [VerifyRuntimeMode]>; 648def SVDOT_N_U : SInst<"svdot[_n_{0}]", "ddqr", "UiUl", MergeNone, "aarch64_sve_udot", [VerifyRuntimeMode]>; 649def SVQADD_N_S : SInst<"svqadd[_n_{d}]", "dda", "csil", MergeNone, "aarch64_sve_sqadd_x", [VerifyRuntimeMode]>; 650def SVQADD_N_U : SInst<"svqadd[_n_{d}]", "dda", "UcUsUiUl", MergeNone, "aarch64_sve_uqadd_x", [VerifyRuntimeMode]>; 651def SVQSUB_N_S : SInst<"svqsub[_n_{d}]", "dda", "csil", MergeNone, "aarch64_sve_sqsub_x", [VerifyRuntimeMode]>; 652def SVQSUB_N_U : SInst<"svqsub[_n_{d}]", "dda", "UcUsUiUl", MergeNone, "aarch64_sve_uqsub_x", [VerifyRuntimeMode]>; 653 654def SVDOT_LANE_S : SInst<"svdot_lane[_{d}]", "ddqqi", "il", MergeNone, "aarch64_sve_sdot_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndexDot, 2>]>; 655def SVDOT_LANE_U : SInst<"svdot_lane[_{d}]", "ddqqi", "UiUl", MergeNone, "aarch64_sve_udot_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndexDot, 2>]>; 656 657//////////////////////////////////////////////////////////////////////////////// 658// Logical operations 659 660defm SVAND : SInstZPZZ<"svand", "csilUcUsUiUl", "aarch64_sve_and", "aarch64_sve_and_u">; 661defm SVBIC : SInstZPZZ<"svbic", "csilUcUsUiUl", "aarch64_sve_bic", "aarch64_sve_bic_u">; 662defm SVEOR : SInstZPZZ<"sveor", "csilUcUsUiUl", "aarch64_sve_eor", "aarch64_sve_eor_u">; 663defm SVORR : SInstZPZZ<"svorr", "csilUcUsUiUl", "aarch64_sve_orr", "aarch64_sve_orr_u">; 664 665defm SVCNOT : SInstZPZ<"svcnot", "csilUcUsUiUl", "aarch64_sve_cnot">; 666defm SVNOT : SInstZPZ<"svnot", "csilUcUsUiUl", "aarch64_sve_not">; 667 668//////////////////////////////////////////////////////////////////////////////// 669// Shifts 670 671multiclass SInst_SHIFT<string name, string intrinsic, string ts, string wide_ts> { 672 def _M : SInst<name # "[_{d}]", "dPdu", ts, MergeOp1, intrinsic, [VerifyRuntimeMode]>; 673 def _X : SInst<name # "[_{d}]", "dPdu", ts, MergeAny, intrinsic # _u, [VerifyRuntimeMode]>; 674 def _Z : SInst<name # "[_{d}]", "dPdu", ts, MergeZero, intrinsic, [VerifyRuntimeMode]>; 675 676 def _N_M : SInst<name # "[_n_{d}]", "dPdL", ts, MergeOp1, intrinsic, [VerifyRuntimeMode]>; 677 def _N_X : SInst<name # "[_n_{d}]", "dPdL", ts, MergeAny, intrinsic # _u, [VerifyRuntimeMode]>; 678 def _N_Z : SInst<name # "[_n_{d}]", "dPdL", ts, MergeZero, intrinsic, [VerifyRuntimeMode]>; 679 680 def _WIDE_M : SInst<name # _wide # "[_{d}]", "dPdg", wide_ts, MergeOp1, intrinsic # _wide, [VerifyRuntimeMode]>; 681 def _WIDE_X : SInst<name # _wide # "[_{d}]", "dPdg", wide_ts, MergeAny, intrinsic # _wide, [VerifyRuntimeMode]>; 682 def _WIDE_Z : SInst<name # _wide # "[_{d}]", "dPdg", wide_ts, MergeZero, intrinsic # _wide, [VerifyRuntimeMode]>; 683 684 def _WIDE_N_M : SInst<name # _wide # "[_n_{d}]", "dPdf", wide_ts, MergeOp1, intrinsic # _wide, [VerifyRuntimeMode]>; 685 def _WIDE_N_X : SInst<name # _wide # "[_n_{d}]", "dPdf", wide_ts, MergeAny, intrinsic # _wide, [VerifyRuntimeMode]>; 686 def _WIDE_N_Z : SInst<name # _wide # "[_n_{d}]", "dPdf", wide_ts, MergeZero, intrinsic # _wide, [VerifyRuntimeMode]>; 687} 688 689defm SVASR : SInst_SHIFT<"svasr", "aarch64_sve_asr", "csil", "csi">; 690defm SVLSL : SInst_SHIFT<"svlsl", "aarch64_sve_lsl", "csilUcUsUiUl", "csiUcUsUi">; 691defm SVLSR : SInst_SHIFT<"svlsr", "aarch64_sve_lsr", "UcUsUiUl", "UcUsUi">; 692 693def SVASRD_M : SInst<"svasrd[_n_{d}]", "dPdi", "csil", MergeOp1, "aarch64_sve_asrd", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftRight, 1>]>; 694def SVASRD_X : SInst<"svasrd[_n_{d}]", "dPdi", "csil", MergeAny, "aarch64_sve_asrd", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftRight, 1>]>; 695def SVASRD_Z : SInst<"svasrd[_n_{d}]", "dPdi", "csil", MergeZero, "aarch64_sve_asrd", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftRight, 1>]>; 696 697def SVINSR : SInst<"svinsr[_n_{d}]", "dds", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_insr", [VerifyRuntimeMode]>; 698let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in { 699 def SVINSR_BF16 : SInst<"svinsr[_n_{d}]", "dds", "b", MergeNone, "aarch64_sve_insr", [VerifyRuntimeMode]>; 700} 701 702//////////////////////////////////////////////////////////////////////////////// 703// Integer reductions 704 705def SVADDV_S : SInst<"svaddv[_{d}]", "lPd", "csil", MergeNone, "aarch64_sve_saddv", [VerifyRuntimeMode]>; 706def SVADDV_U : SInst<"svaddv[_{d}]", "nPd", "UcUsUiUl", MergeNone, "aarch64_sve_uaddv", [VerifyRuntimeMode]>; 707def SVANDV : SInst<"svandv[_{d}]", "sPd", "csilUcUsUiUl", MergeNone, "aarch64_sve_andv", [VerifyRuntimeMode]>; 708def SVEORV : SInst<"sveorv[_{d}]", "sPd", "csilUcUsUiUl", MergeNone, "aarch64_sve_eorv", [VerifyRuntimeMode]>; 709def SVMAXV_S : SInst<"svmaxv[_{d}]", "sPd", "csil", MergeNone, "aarch64_sve_smaxv", [VerifyRuntimeMode]>; 710def SVMAXV_U : SInst<"svmaxv[_{d}]", "sPd", "UcUsUiUl", MergeNone, "aarch64_sve_umaxv", [VerifyRuntimeMode]>; 711def SVMINV_S : SInst<"svminv[_{d}]", "sPd", "csil", MergeNone, "aarch64_sve_sminv", [VerifyRuntimeMode]>; 712def SVMINV_U : SInst<"svminv[_{d}]", "sPd", "UcUsUiUl", MergeNone, "aarch64_sve_uminv", [VerifyRuntimeMode]>; 713def SVORV : SInst<"svorv[_{d}]", "sPd", "csilUcUsUiUl", MergeNone, "aarch64_sve_orv", [VerifyRuntimeMode]>; 714 715//////////////////////////////////////////////////////////////////////////////// 716// Integer comparisons 717 718def SVCMPEQ : SInst<"svcmpeq[_{d}]", "PPdd", "csilUcUsUiUl", MergeNone, "aarch64_sve_cmpeq", [VerifyRuntimeMode]>; 719def SVCMPNE : SInst<"svcmpne[_{d}]", "PPdd", "csilUcUsUiUl", MergeNone, "aarch64_sve_cmpne", [VerifyRuntimeMode]>; 720def SVCMPGE : SInst<"svcmpge[_{d}]", "PPdd", "csil", MergeNone, "aarch64_sve_cmpge", [VerifyRuntimeMode]>; 721def SVCMPGT : SInst<"svcmpgt[_{d}]", "PPdd", "csil", MergeNone, "aarch64_sve_cmpgt", [VerifyRuntimeMode]>; 722def SVCMPLE : SInst<"svcmple[_{d}]", "PPdd", "csil", MergeNone, "aarch64_sve_cmpge", [ReverseCompare, VerifyRuntimeMode]>; 723def SVCMPLT : SInst<"svcmplt[_{d}]", "PPdd", "csil", MergeNone, "aarch64_sve_cmpgt", [ReverseCompare, VerifyRuntimeMode]>; 724def SVCMPHI : SInst<"svcmpgt[_{d}]", "PPdd", "UcUsUiUl", MergeNone, "aarch64_sve_cmphi", [VerifyRuntimeMode]>; 725def SVCMPHS : SInst<"svcmpge[_{d}]", "PPdd", "UcUsUiUl", MergeNone, "aarch64_sve_cmphs", [VerifyRuntimeMode]>; 726def SVCMPLO : SInst<"svcmplt[_{d}]", "PPdd", "UcUsUiUl", MergeNone, "aarch64_sve_cmphi", [ReverseCompare, VerifyRuntimeMode]>; 727def SVCMPLS : SInst<"svcmple[_{d}]", "PPdd", "UcUsUiUl", MergeNone, "aarch64_sve_cmphs", [ReverseCompare, VerifyRuntimeMode]>; 728 729def SVCMPEQ_N : SInst<"svcmpeq[_n_{d}]", "PPda", "csilUcUsUiUl", MergeNone, "aarch64_sve_cmpeq", [VerifyRuntimeMode]>; 730def SVCMPNE_N : SInst<"svcmpne[_n_{d}]", "PPda", "csilUcUsUiUl", MergeNone, "aarch64_sve_cmpne", [VerifyRuntimeMode]>; 731def SVCMPGE_N : SInst<"svcmpge[_n_{d}]", "PPda", "csil", MergeNone, "aarch64_sve_cmpge", [VerifyRuntimeMode]>; 732def SVCMPGT_N : SInst<"svcmpgt[_n_{d}]", "PPda", "csil", MergeNone, "aarch64_sve_cmpgt", [VerifyRuntimeMode]>; 733def SVCMPLE_N : SInst<"svcmple[_n_{d}]", "PPda", "csil", MergeNone, "aarch64_sve_cmpge", [ReverseCompare, VerifyRuntimeMode]>; 734def SVCMPLT_N : SInst<"svcmplt[_n_{d}]", "PPda", "csil", MergeNone, "aarch64_sve_cmpgt", [ReverseCompare, VerifyRuntimeMode]>; 735def SVCMPHS_N : SInst<"svcmpge[_n_{d}]", "PPda", "UcUsUiUl", MergeNone, "aarch64_sve_cmphs", [VerifyRuntimeMode]>; 736def SVCMPHI_N : SInst<"svcmpgt[_n_{d}]", "PPda", "UcUsUiUl", MergeNone, "aarch64_sve_cmphi", [VerifyRuntimeMode]>; 737def SVCMPLS_N : SInst<"svcmple[_n_{d}]", "PPda", "UcUsUiUl", MergeNone, "aarch64_sve_cmphs", [ReverseCompare, VerifyRuntimeMode]>; 738def SVCMPLO_N : SInst<"svcmplt[_n_{d}]", "PPda", "UcUsUiUl", MergeNone, "aarch64_sve_cmphi", [ReverseCompare, VerifyRuntimeMode]>; 739 740def SVCMPEQ_WIDE : SInst<"svcmpeq_wide[_{d}]", "PPdw", "csi", MergeNone, "aarch64_sve_cmpeq_wide", [VerifyRuntimeMode]>; 741def SVCMPNE_WIDE : SInst<"svcmpne_wide[_{d}]", "PPdw", "csi", MergeNone, "aarch64_sve_cmpne_wide", [VerifyRuntimeMode]>; 742def SVCMPGE_WIDE : SInst<"svcmpge_wide[_{d}]", "PPdw", "csi", MergeNone, "aarch64_sve_cmpge_wide", [VerifyRuntimeMode]>; 743def SVCMPGT_WIDE : SInst<"svcmpgt_wide[_{d}]", "PPdw", "csi", MergeNone, "aarch64_sve_cmpgt_wide", [VerifyRuntimeMode]>; 744def SVCMPLE_WIDE : SInst<"svcmple_wide[_{d}]", "PPdw", "csi", MergeNone, "aarch64_sve_cmple_wide", [VerifyRuntimeMode]>; 745def SVCMPLT_WIDE : SInst<"svcmplt_wide[_{d}]", "PPdw", "csi", MergeNone, "aarch64_sve_cmplt_wide", [VerifyRuntimeMode]>; 746def SVCMPHI_WIDE : SInst<"svcmpgt_wide[_{d}]", "PPdw", "UcUsUi", MergeNone, "aarch64_sve_cmphi_wide", [VerifyRuntimeMode]>; 747def SVCMPHS_WIDE : SInst<"svcmpge_wide[_{d}]", "PPdw", "UcUsUi", MergeNone, "aarch64_sve_cmphs_wide", [VerifyRuntimeMode]>; 748def SVCMPLO_WIDE : SInst<"svcmplt_wide[_{d}]", "PPdw", "UcUsUi", MergeNone, "aarch64_sve_cmplo_wide", [VerifyRuntimeMode]>; 749def SVCMPLS_WIDE : SInst<"svcmple_wide[_{d}]", "PPdw", "UcUsUi", MergeNone, "aarch64_sve_cmpls_wide", [VerifyRuntimeMode]>; 750 751def SVCMPEQ_WIDE_N : SInst<"svcmpeq_wide[_n_{d}]", "PPdj", "csi", MergeNone, "aarch64_sve_cmpeq_wide", [VerifyRuntimeMode]>; 752def SVCMPNE_WIDE_N : SInst<"svcmpne_wide[_n_{d}]", "PPdj", "csi", MergeNone, "aarch64_sve_cmpne_wide", [VerifyRuntimeMode]>; 753def SVCMPGE_WIDE_N : SInst<"svcmpge_wide[_n_{d}]", "PPdj", "csi", MergeNone, "aarch64_sve_cmpge_wide", [VerifyRuntimeMode]>; 754def SVCMPGT_WIDE_N : SInst<"svcmpgt_wide[_n_{d}]", "PPdj", "csi", MergeNone, "aarch64_sve_cmpgt_wide", [VerifyRuntimeMode]>; 755def SVCMPLE_WIDE_N : SInst<"svcmple_wide[_n_{d}]", "PPdj", "csi", MergeNone, "aarch64_sve_cmple_wide", [VerifyRuntimeMode]>; 756def SVCMPLT_WIDE_N : SInst<"svcmplt_wide[_n_{d}]", "PPdj", "csi", MergeNone, "aarch64_sve_cmplt_wide", [VerifyRuntimeMode]>; 757def SVCMPHS_WIDE_N : SInst<"svcmpge_wide[_n_{d}]", "PPdj", "UcUsUi", MergeNone, "aarch64_sve_cmphs_wide", [VerifyRuntimeMode]>; 758def SVCMPHI_WIDE_N : SInst<"svcmpgt_wide[_n_{d}]", "PPdj", "UcUsUi", MergeNone, "aarch64_sve_cmphi_wide", [VerifyRuntimeMode]>; 759def SVCMPLO_WIDE_N : SInst<"svcmplt_wide[_n_{d}]", "PPdj", "UcUsUi", MergeNone, "aarch64_sve_cmplo_wide", [VerifyRuntimeMode]>; 760def SVCMPLS_WIDE_N : SInst<"svcmple_wide[_n_{d}]", "PPdj", "UcUsUi", MergeNone, "aarch64_sve_cmpls_wide", [VerifyRuntimeMode]>; 761 762//////////////////////////////////////////////////////////////////////////////// 763// While comparisons 764 765def SVWHILELE_S32 : SInst<"svwhilele_{d}[_{1}]", "Pkk", "PcPsPiPl", MergeNone, "aarch64_sve_whilele", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>; 766def SVWHILELE_S64 : SInst<"svwhilele_{d}[_{1}]", "Pll", "PcPsPiPl", MergeNone, "aarch64_sve_whilele", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>; 767def SVWHILELO_U32 : SInst<"svwhilelt_{d}[_{1}]", "Pmm", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilelo", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>; 768def SVWHILELO_U64 : SInst<"svwhilelt_{d}[_{1}]", "Pnn", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilelo", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>; 769def SVWHILELS_U32 : SInst<"svwhilele_{d}[_{1}]", "Pmm", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilels", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>; 770def SVWHILELS_U64 : SInst<"svwhilele_{d}[_{1}]", "Pnn", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilels", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>; 771def SVWHILELT_S32 : SInst<"svwhilelt_{d}[_{1}]", "Pkk", "PcPsPiPl", MergeNone, "aarch64_sve_whilelt", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>; 772def SVWHILELT_S64 : SInst<"svwhilelt_{d}[_{1}]", "Pll", "PcPsPiPl", MergeNone, "aarch64_sve_whilelt", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>; 773 774//////////////////////////////////////////////////////////////////////////////// 775// Counting bit 776 777multiclass SInstCLS<string name, string types, string intrinsic, list<FlagType> flags=[]> { 778 def _M : SInst<name # "[_{d}]", "uuPd", types, MergeOp1, intrinsic, flags>; 779 def _X : SInst<name # "[_{d}]", "uPd", types, MergeAnyExp, intrinsic, flags>; 780 def _Z : SInst<name # "[_{d}]", "uPd", types, MergeZeroExp, intrinsic, flags>; 781} 782 783defm SVCLS : SInstCLS<"svcls", "csil", "aarch64_sve_cls", [VerifyRuntimeMode]>; 784defm SVCLZ : SInstCLS<"svclz", "csilUcUsUiUl", "aarch64_sve_clz", [VerifyRuntimeMode]>; 785defm SVCNT : SInstCLS<"svcnt", "csilUcUsUiUlhfd", "aarch64_sve_cnt", [VerifyRuntimeMode]>; 786 787let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in { 788 defm SVCNT_BF16 : SInstCLS<"svcnt", "b", "aarch64_sve_cnt", [VerifyRuntimeMode]>; 789} 790 791//////////////////////////////////////////////////////////////////////////////// 792// Conversion 793 794defm SVEXTB_S : SInstZPZ<"svextb", "sil", "aarch64_sve_sxtb">; 795defm SVEXTB_U : SInstZPZ<"svextb", "UsUiUl", "aarch64_sve_uxtb">; 796defm SVEXTH_S : SInstZPZ<"svexth", "il", "aarch64_sve_sxth">; 797defm SVEXTH_U : SInstZPZ<"svexth", "UiUl", "aarch64_sve_uxth">; 798defm SVEXTW_S : SInstZPZ<"svextw", "l", "aarch64_sve_sxtw">; 799defm SVEXTW_U : SInstZPZ<"svextw", "Ul", "aarch64_sve_uxtw">; 800 801//////////////////////////////////////////////////////////////////////////////// 802// Reversal 803 804defm SVRBIT : SInstZPZ<"svrbit", "csilUcUsUiUl", "aarch64_sve_rbit">; 805defm SVREVB : SInstZPZ<"svrevb", "silUsUiUl", "aarch64_sve_revb">; 806defm SVREVH : SInstZPZ<"svrevh", "ilUiUl", "aarch64_sve_revh">; 807defm SVREVW : SInstZPZ<"svrevw", "lUl", "aarch64_sve_revw">; 808 809//////////////////////////////////////////////////////////////////////////////// 810// Floating-point arithmetic 811 812defm SVABS_F : SInstZPZ<"svabs", "hfd", "aarch64_sve_fabs">; 813defm SVNEG_F : SInstZPZ<"svneg", "hfd", "aarch64_sve_fneg">; 814 815defm SVABD_F : SInstZPZZ<"svabd", "hfd", "aarch64_sve_fabd", "aarch64_sve_fabd_u">; 816defm SVADD_F : SInstZPZZ<"svadd", "hfd", "aarch64_sve_fadd", "aarch64_sve_fadd_u">; 817defm SVDIV_F : SInstZPZZ<"svdiv", "hfd", "aarch64_sve_fdiv", "aarch64_sve_fdiv_u">; 818defm SVDIVR_F : SInstZPZZ<"svdivr", "hfd", "aarch64_sve_fdivr", "aarch64_sve_fdiv_u", [ReverseMergeAnyBinOp]>; 819defm SVMAX_F : SInstZPZZ<"svmax", "hfd", "aarch64_sve_fmax", "aarch64_sve_fmax_u">; 820defm SVMAXNM : SInstZPZZ<"svmaxnm","hfd", "aarch64_sve_fmaxnm", "aarch64_sve_fmaxnm_u">; 821defm SVMIN_F : SInstZPZZ<"svmin", "hfd", "aarch64_sve_fmin", "aarch64_sve_fmin_u">; 822defm SVMINNM : SInstZPZZ<"svminnm","hfd", "aarch64_sve_fminnm", "aarch64_sve_fminnm_u">; 823defm SVMUL_F : SInstZPZZ<"svmul", "hfd", "aarch64_sve_fmul", "aarch64_sve_fmul_u">; 824defm SVMULX : SInstZPZZ<"svmulx", "hfd", "aarch64_sve_fmulx", "aarch64_sve_fmulx_u">; 825defm SVSUB_F : SInstZPZZ<"svsub", "hfd", "aarch64_sve_fsub", "aarch64_sve_fsub_u">; 826defm SVSUBR_F : SInstZPZZ<"svsubr", "hfd", "aarch64_sve_fsubr", "aarch64_sve_fsub_u", [ReverseMergeAnyBinOp]>; 827 828defm SVRECPX : SInstZPZ<"svrecpx", "hfd", "aarch64_sve_frecpx">; 829defm SVRINTA : SInstZPZ<"svrinta", "hfd", "aarch64_sve_frinta">; 830defm SVRINTI : SInstZPZ<"svrinti", "hfd", "aarch64_sve_frinti">; 831defm SVRINTM : SInstZPZ<"svrintm", "hfd", "aarch64_sve_frintm">; 832defm SVRINTN : SInstZPZ<"svrintn", "hfd", "aarch64_sve_frintn">; 833defm SVRINTP : SInstZPZ<"svrintp", "hfd", "aarch64_sve_frintp">; 834defm SVRINTX : SInstZPZ<"svrintx", "hfd", "aarch64_sve_frintx">; 835defm SVRINTZ : SInstZPZ<"svrintz", "hfd", "aarch64_sve_frintz">; 836defm SVSQRT : SInstZPZ<"svsqrt", "hfd", "aarch64_sve_fsqrt">; 837 838let SVETargetGuard = "sve", SMETargetGuard = InvalidMode in { 839def SVEXPA : SInst<"svexpa[_{d}]", "du", "hfd", MergeNone, "aarch64_sve_fexpa_x">; 840def SVTMAD : SInst<"svtmad[_{d}]", "dddi", "hfd", MergeNone, "aarch64_sve_ftmad_x", [], [ImmCheck<2, ImmCheck0_7>]>; 841def SVTSMUL : SInst<"svtsmul[_{d}]", "ddu", "hfd", MergeNone, "aarch64_sve_ftsmul_x">; 842def SVTSSEL : SInst<"svtssel[_{d}]", "ddu", "hfd", MergeNone, "aarch64_sve_ftssel_x">; 843} 844 845def SVSCALE_M : SInst<"svscale[_{d}]", "dPdx", "hfd", MergeOp1, "aarch64_sve_fscale", [VerifyRuntimeMode]>; 846def SVSCALE_X : SInst<"svscale[_{d}]", "dPdx", "hfd", MergeAny, "aarch64_sve_fscale", [VerifyRuntimeMode]>; 847def SVSCALE_Z : SInst<"svscale[_{d}]", "dPdx", "hfd", MergeZero, "aarch64_sve_fscale", [VerifyRuntimeMode]>; 848 849def SVSCALE_N_M : SInst<"svscale[_n_{d}]", "dPdK", "hfd", MergeOp1, "aarch64_sve_fscale", [VerifyRuntimeMode]>; 850def SVSCALE_N_X : SInst<"svscale[_n_{d}]", "dPdK", "hfd", MergeAny, "aarch64_sve_fscale", [VerifyRuntimeMode]>; 851def SVSCALE_N_Z : SInst<"svscale[_n_{d}]", "dPdK", "hfd", MergeZero, "aarch64_sve_fscale", [VerifyRuntimeMode]>; 852 853defm SVMAD_F : SInstZPZZZ<"svmad", "hfd", "aarch64_sve_fmad", "aarch64_sve_fmla_u", [VerifyRuntimeMode, ReverseMergeAnyAccOp]>; 854defm SVMLA_F : SInstZPZZZ<"svmla", "hfd", "aarch64_sve_fmla", "aarch64_sve_fmla_u", [VerifyRuntimeMode]>; 855defm SVMLS_F : SInstZPZZZ<"svmls", "hfd", "aarch64_sve_fmls", "aarch64_sve_fmls_u", [VerifyRuntimeMode]>; 856defm SVMSB_F : SInstZPZZZ<"svmsb", "hfd", "aarch64_sve_fmsb", "aarch64_sve_fmls_u", [VerifyRuntimeMode, ReverseMergeAnyAccOp]>; 857defm SVNMAD_F : SInstZPZZZ<"svnmad", "hfd", "aarch64_sve_fnmad", "aarch64_sve_fnmla_u", [VerifyRuntimeMode, ReverseMergeAnyAccOp]>; 858defm SVNMLA_F : SInstZPZZZ<"svnmla", "hfd", "aarch64_sve_fnmla", "aarch64_sve_fnmla_u", [VerifyRuntimeMode]>; 859defm SVNMLS_F : SInstZPZZZ<"svnmls", "hfd", "aarch64_sve_fnmls", "aarch64_sve_fnmls_u", [VerifyRuntimeMode]>; 860defm SVNMSB_F : SInstZPZZZ<"svnmsb", "hfd", "aarch64_sve_fnmsb", "aarch64_sve_fnmls_u", [VerifyRuntimeMode, ReverseMergeAnyAccOp]>; 861 862def SVCADD_M : SInst<"svcadd[_{d}]", "dPddi", "hfd", MergeOp1, "aarch64_sve_fcadd", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckComplexRot90_270>]>; 863def SVCADD_X : SInst<"svcadd[_{d}]", "dPddi", "hfd", MergeAny, "aarch64_sve_fcadd", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckComplexRot90_270>]>; 864def SVCADD_Z : SInst<"svcadd[_{d}]", "dPddi", "hfd", MergeZero, "aarch64_sve_fcadd", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckComplexRot90_270>]>; 865def SVCMLA_M : SInst<"svcmla[_{d}]", "dPdddi", "hfd", MergeOp1, "aarch64_sve_fcmla", [VerifyRuntimeMode], [ImmCheck<4, ImmCheckComplexRotAll90>]>; 866def SVCMLA_X : SInst<"svcmla[_{d}]", "dPdddi", "hfd", MergeAny, "aarch64_sve_fcmla", [VerifyRuntimeMode], [ImmCheck<4, ImmCheckComplexRotAll90>]>; 867def SVCMLA_Z : SInst<"svcmla[_{d}]", "dPdddi", "hfd", MergeZero, "aarch64_sve_fcmla", [VerifyRuntimeMode], [ImmCheck<4, ImmCheckComplexRotAll90>]>; 868 869def SVCMLA_LANE : SInst<"svcmla_lane[_{d}]", "ddddii", "hf", MergeNone, "aarch64_sve_fcmla_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndexCompRotate, 2>, 870 ImmCheck<4, ImmCheckComplexRotAll90>]>; 871def SVMLA_LANE : SInst<"svmla_lane[_{d}]", "ddddi", "hfd", MergeNone, "aarch64_sve_fmla_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 872def SVMLS_LANE : SInst<"svmls_lane[_{d}]", "ddddi", "hfd", MergeNone, "aarch64_sve_fmls_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 873def SVMUL_LANE : SInst<"svmul_lane[_{d}]", "dddi", "hfd", MergeNone, "aarch64_sve_fmul_lane", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; 874 875def SVRECPE : SInst<"svrecpe[_{d}]", "dd", "hfd", MergeNone, "aarch64_sve_frecpe_x", [VerifyRuntimeMode]>; 876def SVRECPS : SInst<"svrecps[_{d}]", "ddd", "hfd", MergeNone, "aarch64_sve_frecps_x", [VerifyRuntimeMode]>; 877def SVRSQRTE : SInst<"svrsqrte[_{d}]", "dd", "hfd", MergeNone, "aarch64_sve_frsqrte_x", [VerifyRuntimeMode]>; 878def SVRSQRTS : SInst<"svrsqrts[_{d}]", "ddd", "hfd", MergeNone, "aarch64_sve_frsqrts_x", [VerifyRuntimeMode]>; 879 880//////////////////////////////////////////////////////////////////////////////// 881// Floating-point reductions 882 883def SVFADDA : SInst<"svadda[_{d}]", "sPsd", "hfd", MergeNone, "aarch64_sve_fadda", [VerifyRuntimeMode]>; 884def SVFADDV : SInst<"svaddv[_{d}]", "sPd", "hfd", MergeNone, "aarch64_sve_faddv", [VerifyRuntimeMode]>; 885def SVFMAXV : SInst<"svmaxv[_{d}]", "sPd", "hfd", MergeNone, "aarch64_sve_fmaxv", [VerifyRuntimeMode]>; 886def SVFMAXNMV : SInst<"svmaxnmv[_{d}]", "sPd", "hfd", MergeNone, "aarch64_sve_fmaxnmv", [VerifyRuntimeMode]>; 887def SVFMINV : SInst<"svminv[_{d}]", "sPd", "hfd", MergeNone, "aarch64_sve_fminv", [VerifyRuntimeMode]>; 888def SVFMINNMV : SInst<"svminnmv[_{d}]", "sPd", "hfd", MergeNone, "aarch64_sve_fminnmv", [VerifyRuntimeMode]>; 889 890//////////////////////////////////////////////////////////////////////////////// 891// Floating-point comparisons 892 893def SVACGE : SInst<"svacge[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_facge", [VerifyRuntimeMode]>; 894def SVACGT : SInst<"svacgt[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_facgt", [VerifyRuntimeMode]>; 895def SVACLE : SInst<"svacle[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_facge", [ReverseCompare, VerifyRuntimeMode]>; 896def SVACLT : SInst<"svaclt[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_facgt", [ReverseCompare, VerifyRuntimeMode]>; 897def SVCMPUO : SInst<"svcmpuo[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpuo", [VerifyRuntimeMode]>; 898 899def SVACGE_N : SInst<"svacge[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_facge", [VerifyRuntimeMode]>; 900def SVACGT_N : SInst<"svacgt[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_facgt", [VerifyRuntimeMode]>; 901def SVACLE_N : SInst<"svacle[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_facge", [ReverseCompare, VerifyRuntimeMode]>; 902def SVACLT_N : SInst<"svaclt[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_facgt", [ReverseCompare, VerifyRuntimeMode]>; 903def SVCMPUO_N : SInst<"svcmpuo[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpuo", [VerifyRuntimeMode]>; 904 905def SVCMPEQ_F : SInst<"svcmpeq[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpeq", [VerifyRuntimeMode]>; 906def SVCMPNE_F : SInst<"svcmpne[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpne", [VerifyRuntimeMode]>; 907def SVCMPGE_F : SInst<"svcmpge[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpge", [VerifyRuntimeMode]>; 908def SVCMPGT_F : SInst<"svcmpgt[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpgt", [VerifyRuntimeMode]>; 909def SVCMPLE_F : SInst<"svcmple[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpge", [ReverseCompare, VerifyRuntimeMode]>; 910def SVCMPLT_F : SInst<"svcmplt[_{d}]", "PPdd", "hfd", MergeNone, "aarch64_sve_fcmpgt", [ReverseCompare, VerifyRuntimeMode]>; 911 912def SVCMPEQ_F_N : SInst<"svcmpeq[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpeq", [VerifyRuntimeMode]>; 913def SVCMPNE_F_N : SInst<"svcmpne[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpne", [VerifyRuntimeMode]>; 914def SVCMPGE_F_N : SInst<"svcmpge[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpge", [VerifyRuntimeMode]>; 915def SVCMPGT_F_N : SInst<"svcmpgt[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpgt", [VerifyRuntimeMode]>; 916def SVCMPLE_F_N : SInst<"svcmple[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpge", [ReverseCompare, VerifyRuntimeMode]>; 917def SVCMPLT_F_N : SInst<"svcmplt[_n_{d}]", "PPda", "hfd", MergeNone, "aarch64_sve_fcmpgt", [ReverseCompare, VerifyRuntimeMode]>; 918 919//////////////////////////////////////////////////////////////////////////////// 920// Floating-point conversions 921 922multiclass SInstCvtMXZ< 923 string name, string m_types, string xz_types, string types, 924 string intrinsic, list<FlagType> flags = [IsOverloadNone]> { 925 def _M : SInst<name, m_types, types, MergeOp1, intrinsic, !listconcat(flags, [VerifyRuntimeMode])>; 926 def _X : SInst<name, xz_types, types, MergeAnyExp, intrinsic, !listconcat(flags, [VerifyRuntimeMode])>; 927 def _Z : SInst<name, xz_types, types, MergeZeroExp, intrinsic, !listconcat(flags, [VerifyRuntimeMode])>; 928} 929 930multiclass SInstCvtMX<string name, string m_types, string xz_types, 931 string types, string intrinsic, 932 list<FlagType> flags = [IsOverloadNone]> { 933 def _M : SInst<name, m_types, types, MergeOp1, intrinsic, !listconcat(flags, [VerifyRuntimeMode])>; 934 def _X : SInst<name, xz_types, types, MergeAnyExp, intrinsic, !listconcat(flags, [VerifyRuntimeMode])>; 935} 936 937// svcvt_s##_f16 938defm SVFCVTZS_S16_F16 : SInstCvtMXZ<"svcvt_s16[_f16]", "ddPO", "dPO", "s", "aarch64_sve_fcvtzs", [IsOverloadCvt]>; 939defm SVFCVTZS_S32_F16 : SInstCvtMXZ<"svcvt_s32[_f16]", "ddPO", "dPO", "i", "aarch64_sve_fcvtzs_i32f16">; 940defm SVFCVTZS_S64_F16 : SInstCvtMXZ<"svcvt_s64[_f16]", "ddPO", "dPO", "l", "aarch64_sve_fcvtzs_i64f16">; 941 942// svcvt_s##_f32 943defm SVFCVTZS_S32_F32 : SInstCvtMXZ<"svcvt_s32[_f32]", "ddPM", "dPM", "i", "aarch64_sve_fcvtzs", [IsOverloadCvt]>; 944defm SVFCVTZS_S64_F32 : SInstCvtMXZ<"svcvt_s64[_f32]", "ddPM", "dPM", "l", "aarch64_sve_fcvtzs_i64f32">; 945 946let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in { 947 defm SVCVT_BF16_F32 : SInstCvtMXZ<"svcvt_bf16[_f32]", "ddPM", "dPM", "b", "aarch64_sve_fcvt_bf16f32">; 948 def SVCVTNT_BF16_F32 : SInst<"svcvtnt_bf16[_f32]", "ddPM", "b", MergeOp1, "aarch64_sve_fcvtnt_bf16f32", [IsOverloadNone, VerifyRuntimeMode]>; 949} 950 951// svcvt_s##_f64 952defm SVFCVTZS_S32_F64 : SInstCvtMXZ<"svcvt_s32[_f64]", "ttPd", "tPd", "d", "aarch64_sve_fcvtzs_i32f64">; 953defm SVFCVTZS_S64_F64 : SInstCvtMXZ<"svcvt_s64[_f64]", "ddPN", "dPN", "l", "aarch64_sve_fcvtzs", [IsOverloadCvt]>; 954 955// svcvt_u##_f16 956defm SVFCVTZU_U16_F16 : SInstCvtMXZ<"svcvt_u16[_f16]", "ddPO", "dPO", "Us", "aarch64_sve_fcvtzu", [IsOverloadCvt]>; 957defm SVFCVTZU_U32_F16 : SInstCvtMXZ<"svcvt_u32[_f16]", "ddPO", "dPO", "Ui", "aarch64_sve_fcvtzu_i32f16">; 958defm SVFCVTZU_U64_F16 : SInstCvtMXZ<"svcvt_u64[_f16]", "ddPO", "dPO", "Ul", "aarch64_sve_fcvtzu_i64f16">; 959 960// svcvt_u##_f32 961defm SVFCVTZU_U32_F32 : SInstCvtMXZ<"svcvt_u32[_f32]", "ddPM", "dPM", "Ui", "aarch64_sve_fcvtzu", [IsOverloadCvt]>; 962defm SVFCVTZU_U64_F32 : SInstCvtMXZ<"svcvt_u64[_f32]", "ddPM", "dPM", "Ul", "aarch64_sve_fcvtzu_i64f32">; 963 964// svcvt_u##_f64 965defm SVFCVTZU_U32_F64 : SInstCvtMXZ<"svcvt_u32[_f64]", "zzPd", "zPd", "d", "aarch64_sve_fcvtzu_i32f64">; 966defm SVFCVTZU_U64_F64 : SInstCvtMXZ<"svcvt_u64[_f64]", "ddPN", "dPN", "Ul", "aarch64_sve_fcvtzu", [IsOverloadCvt]>; 967 968// svcvt_f16_s## 969defm SVFCVTZS_F16_S16 : SInstCvtMXZ<"svcvt_f16[_s16]", "OOPd", "OPd", "s", "aarch64_sve_scvtf", [IsOverloadCvt]>; 970defm SVFCVTZS_F16_S32 : SInstCvtMXZ<"svcvt_f16[_s32]", "OOPd", "OPd", "i", "aarch64_sve_scvtf_f16i32">; 971defm SVFCVTZS_F16_S64 : SInstCvtMXZ<"svcvt_f16[_s64]", "OOPd", "OPd", "l", "aarch64_sve_scvtf_f16i64">; 972 973// svcvt_f32_s## 974defm SVFCVTZS_F32_S32 : SInstCvtMXZ<"svcvt_f32[_s32]", "MMPd", "MPd", "i", "aarch64_sve_scvtf", [IsOverloadCvt]>; 975defm SVFCVTZS_F32_S64 : SInstCvtMXZ<"svcvt_f32[_s64]", "MMPd", "MPd", "l", "aarch64_sve_scvtf_f32i64">; 976 977// svcvt_f64_s## 978defm SVFCVTZS_F64_S32 : SInstCvtMXZ<"svcvt_f64[_s32]", "ddPt", "dPt", "d", "aarch64_sve_scvtf_f64i32">; 979defm SVFCVTZS_F64_S64 : SInstCvtMXZ<"svcvt_f64[_s64]", "NNPd", "NPd", "l", "aarch64_sve_scvtf", [IsOverloadCvt]>; 980 981// svcvt_f16_u## 982defm SVFCVTZU_F16_U16 : SInstCvtMXZ<"svcvt_f16[_u16]", "OOPd", "OPd", "Us", "aarch64_sve_ucvtf", [IsOverloadCvt]>; 983defm SVFCVTZU_F16_U32 : SInstCvtMXZ<"svcvt_f16[_u32]", "OOPd", "OPd", "Ui", "aarch64_sve_ucvtf_f16i32">; 984defm SVFCVTZU_F16_U64 : SInstCvtMXZ<"svcvt_f16[_u64]", "OOPd", "OPd", "Ul", "aarch64_sve_ucvtf_f16i64">; 985 986// svcvt_f32_u## 987defm SVFCVTZU_F32_U32 : SInstCvtMXZ<"svcvt_f32[_u32]", "MMPd", "MPd", "Ui", "aarch64_sve_ucvtf", [IsOverloadCvt]>; 988defm SVFCVTZU_F32_U64 : SInstCvtMXZ<"svcvt_f32[_u64]", "MMPd", "MPd", "Ul", "aarch64_sve_ucvtf_f32i64">; 989 990// svcvt_f64_u## 991defm SVFCVTZU_F64_U32 : SInstCvtMXZ<"svcvt_f64[_u32]", "ddPz", "dPz", "d", "aarch64_sve_ucvtf_f64i32">; 992defm SVFCVTZU_F64_U64 : SInstCvtMXZ<"svcvt_f64[_u64]", "NNPd", "NPd", "Ul", "aarch64_sve_ucvtf", [IsOverloadCvt]>; 993 994// svcvt_f16_f## 995defm SVFCVT_F16_F32 : SInstCvtMXZ<"svcvt_f16[_f32]", "OOPd", "OPd", "f", "aarch64_sve_fcvt_f16f32">; 996defm SVFCVT_F16_F64 : SInstCvtMXZ<"svcvt_f16[_f64]", "OOPd", "OPd", "d", "aarch64_sve_fcvt_f16f64">; 997 998// svcvt_f32_f## 999defm SVFCVT_F32_F16 : SInstCvtMXZ<"svcvt_f32[_f16]", "ddPO", "dPO", "f", "aarch64_sve_fcvt_f32f16">; 1000defm SVFCVT_F32_F64 : SInstCvtMXZ<"svcvt_f32[_f64]", "MMPd", "MPd", "d", "aarch64_sve_fcvt_f32f64">; 1001 1002// svcvt_f64_f## 1003defm SVFCVT_F64_F16 : SInstCvtMXZ<"svcvt_f64[_f16]", "ddPO", "dPO", "d", "aarch64_sve_fcvt_f64f16">; 1004defm SVFCVT_F64_F32 : SInstCvtMXZ<"svcvt_f64[_f32]", "ddPM", "dPM", "d", "aarch64_sve_fcvt_f64f32">; 1005 1006let SVETargetGuard = "sve2", SMETargetGuard = "sme" in { 1007defm SVCVTLT_F32 : SInstCvtMX<"svcvtlt_f32[_f16]", "ddPh", "dPh", "f", "aarch64_sve_fcvtlt_f32f16">; 1008defm SVCVTLT_F64 : SInstCvtMX<"svcvtlt_f64[_f32]", "ddPh", "dPh", "d", "aarch64_sve_fcvtlt_f64f32">; 1009 1010defm SVCVTX_F32 : SInstCvtMXZ<"svcvtx_f32[_f64]", "MMPd", "MPd", "d", "aarch64_sve_fcvtx_f32f64">; 1011 1012def SVCVTNT_F32 : SInst<"svcvtnt_f16[_f32]", "hhPd", "f", MergeOp1, "aarch64_sve_fcvtnt_f16f32", [IsOverloadNone, VerifyRuntimeMode]>; 1013def SVCVTNT_F64 : SInst<"svcvtnt_f32[_f64]", "hhPd", "d", MergeOp1, "aarch64_sve_fcvtnt_f32f64", [IsOverloadNone, VerifyRuntimeMode]>; 1014// SVCVTNT_X : Implemented as macro by SveEmitter.cpp 1015 1016def SVCVTXNT_F32 : SInst<"svcvtxnt_f32[_f64]", "MMPd", "d", MergeOp1, "aarch64_sve_fcvtxnt_f32f64", [IsOverloadNone, VerifyRuntimeMode]>; 1017// SVCVTXNT_X_F32 : Implemented as macro by SveEmitter.cpp 1018 1019} 1020 1021//////////////////////////////////////////////////////////////////////////////// 1022// Permutations and selection 1023 1024multiclass SVEPerm<string name, string proto, string i> { 1025 def : SInst<name, proto, "csilUcUsUiUlhfd", MergeNone, i, [VerifyRuntimeMode]>; 1026 let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in { 1027 def: SInst<name, proto, "b", MergeNone, i, [VerifyRuntimeMode]>; 1028 } 1029} 1030 1031defm SVCLASTA : SVEPerm<"svclasta[_{d}]", "dPdd", "aarch64_sve_clasta">; 1032defm SVCLASTA_N : SVEPerm<"svclasta[_n_{d}]", "sPsd", "aarch64_sve_clasta_n">; 1033defm SVCLASTB : SVEPerm<"svclastb[_{d}]", "dPdd", "aarch64_sve_clastb">; 1034defm SVCLASTB_N : SVEPerm<"svclastb[_n_{d}]", "sPsd", "aarch64_sve_clastb_n">; 1035 1036let SVETargetGuard = "sve", SMETargetGuard = InvalidMode in { 1037def SVCOMPACT : SInst<"svcompact[_{d}]", "dPd", "ilUiUlfd", MergeNone, "aarch64_sve_compact">; 1038} 1039 1040// Note: svdup_lane is implemented using the intrinsic for TBL to represent a 1041// splat of any possible lane. It is upto LLVM to pick a more efficient 1042// instruction such as DUP (indexed) if the lane index fits the range of the 1043// instruction's immediate. 1044def SVDUP_LANE : SInst<"svdup_lane[_{d}]", "ddL", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_tbl", [VerifyRuntimeMode]>; 1045let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in { 1046def SVDUP_LANE_BF16 : 1047 SInst<"svdup_lane[_{d}]", "ddL", "b", MergeNone, "aarch64_sve_tbl", [VerifyRuntimeMode]>; 1048} 1049 1050def SVDUPQ_LANE : SInst<"svdupq_lane[_{d}]", "ddn", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_dupq_lane", [VerifyRuntimeMode]>; 1051let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in { 1052 def SVDUPQ_LANE_BF16 : SInst<"svdupq_lane[_{d}]", "ddn", "b", MergeNone, "aarch64_sve_dupq_lane", [VerifyRuntimeMode]>; 1053} 1054def SVEXT : SInst<"svext[_{d}]", "dddi", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_ext", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckExtract, 1>]>; 1055defm SVLASTA : SVEPerm<"svlasta[_{d}]", "sPd", "aarch64_sve_lasta">; 1056defm SVLASTB : SVEPerm<"svlastb[_{d}]", "sPd", "aarch64_sve_lastb">; 1057def SVREV : SInst<"svrev[_{d}]", "dd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_rev", [VerifyRuntimeMode]>; 1058def SVSEL : SInst<"svsel[_{d}]", "dPdd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_sel", [VerifyRuntimeMode]>; 1059def SVSPLICE : SInst<"svsplice[_{d}]", "dPdd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_splice", [VerifyRuntimeMode]>; 1060def SVTBL : SInst<"svtbl[_{d}]", "ddu", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_tbl", [VerifyRuntimeMode]>; 1061 1062let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in { 1063 def SVTBL_BF16 : SInst<"svtbl[_{d}]", "ddu", "b", MergeNone, "aarch64_sve_tbl", [VerifyRuntimeMode]>; 1064} 1065 1066def SVTRN1 : SInst<"svtrn1[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_trn1", [VerifyRuntimeMode]>; 1067def SVTRN2 : SInst<"svtrn2[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_trn2", [VerifyRuntimeMode]>; 1068def SVUNPKHI_S : SInst<"svunpkhi[_{d}]", "dh", "sil", MergeNone, "aarch64_sve_sunpkhi", [VerifyRuntimeMode]>; 1069def SVUNPKHI_U : SInst<"svunpkhi[_{d}]", "dh", "UsUiUl", MergeNone, "aarch64_sve_uunpkhi", [VerifyRuntimeMode]>; 1070def SVUNPKLO_S : SInst<"svunpklo[_{d}]", "dh", "sil", MergeNone, "aarch64_sve_sunpklo", [VerifyRuntimeMode]>; 1071def SVUNPKLO_U : SInst<"svunpklo[_{d}]", "dh", "UsUiUl", MergeNone, "aarch64_sve_uunpklo", [VerifyRuntimeMode]>; 1072def SVUZP1 : SInst<"svuzp1[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_uzp1", [VerifyRuntimeMode]>; 1073def SVUZP2 : SInst<"svuzp2[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_uzp2", [VerifyRuntimeMode]>; 1074def SVZIP1 : SInst<"svzip1[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_zip1", [VerifyRuntimeMode]>; 1075def SVZIP2 : SInst<"svzip2[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_zip2", [VerifyRuntimeMode]>; 1076 1077let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in { 1078def SVEXT_BF16 : SInst<"svext[_{d}]", "dddi", "b", MergeNone, "aarch64_sve_ext", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckExtract, 1>]>; 1079def SVREV_BF16 : SInst<"svrev[_{d}]", "dd", "b", MergeNone, "aarch64_sve_rev", [VerifyRuntimeMode]>; 1080def SVSEL_BF16 : SInst<"svsel[_{d}]", "dPdd", "b", MergeNone, "aarch64_sve_sel", [VerifyRuntimeMode]>; 1081def SVSPLICE_BF16 : SInst<"svsplice[_{d}]", "dPdd", "b", MergeNone, "aarch64_sve_splice", [VerifyRuntimeMode]>; 1082def SVTRN1_BF16 : SInst<"svtrn1[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_trn1", [VerifyRuntimeMode]>; 1083def SVTRN2_BF16 : SInst<"svtrn2[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_trn2", [VerifyRuntimeMode]>; 1084def SVUZP1_BF16 : SInst<"svuzp1[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_uzp1", [VerifyRuntimeMode]>; 1085def SVUZP2_BF16 : SInst<"svuzp2[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_uzp2", [VerifyRuntimeMode]>; 1086def SVZIP1_BF16 : SInst<"svzip1[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_zip1", [VerifyRuntimeMode]>; 1087def SVZIP2_BF16 : SInst<"svzip2[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_zip2", [VerifyRuntimeMode]>; 1088} 1089 1090def SVREV_B8 : SInst<"svrev_b8", "PP", "Pc", MergeNone, "aarch64_sve_rev", [VerifyRuntimeMode]>; 1091def SVREV_B16 : SInst<"svrev_b16", "PP", "Pc", MergeNone, "aarch64_sve_rev_b16", [IsOverloadNone, VerifyRuntimeMode]>; 1092def SVREV_B32 : SInst<"svrev_b32", "PP", "Pc", MergeNone, "aarch64_sve_rev_b32", [IsOverloadNone, VerifyRuntimeMode]>; 1093def SVREV_B64 : SInst<"svrev_b64", "PP", "Pc", MergeNone, "aarch64_sve_rev_b64", [IsOverloadNone, VerifyRuntimeMode]>; 1094def SVSEL_B : SInst<"svsel[_b]", "PPPP", "Pc", MergeNone, "aarch64_sve_sel", [VerifyRuntimeMode]>; 1095def SVTRN1_B8 : SInst<"svtrn1_b8", "PPP", "Pc", MergeNone, "aarch64_sve_trn1", [VerifyRuntimeMode]>; 1096def SVTRN1_B16 : SInst<"svtrn1_b16", "PPP", "Pc", MergeNone, "aarch64_sve_trn1_b16", [IsOverloadNone, VerifyRuntimeMode]>; 1097def SVTRN1_B32 : SInst<"svtrn1_b32", "PPP", "Pc", MergeNone, "aarch64_sve_trn1_b32", [IsOverloadNone, VerifyRuntimeMode]>; 1098def SVTRN1_B64 : SInst<"svtrn1_b64", "PPP", "Pc", MergeNone, "aarch64_sve_trn1_b64", [IsOverloadNone, VerifyRuntimeMode]>; 1099def SVTRN2_B8 : SInst<"svtrn2_b8", "PPP", "Pc", MergeNone, "aarch64_sve_trn2", [VerifyRuntimeMode]>; 1100def SVTRN2_B16 : SInst<"svtrn2_b16", "PPP", "Pc", MergeNone, "aarch64_sve_trn2_b16", [IsOverloadNone, VerifyRuntimeMode]>; 1101def SVTRN2_B32 : SInst<"svtrn2_b32", "PPP", "Pc", MergeNone, "aarch64_sve_trn2_b32", [IsOverloadNone, VerifyRuntimeMode]>; 1102def SVTRN2_B64 : SInst<"svtrn2_b64", "PPP", "Pc", MergeNone, "aarch64_sve_trn2_b64", [IsOverloadNone, VerifyRuntimeMode]>; 1103def SVPUNPKHI : SInst<"svunpkhi[_b]", "PP", "Pc", MergeNone, "aarch64_sve_punpkhi", [VerifyRuntimeMode]>; 1104def SVPUNPKLO : SInst<"svunpklo[_b]", "PP", "Pc", MergeNone, "aarch64_sve_punpklo", [VerifyRuntimeMode]>; 1105def SVUZP1_B8 : SInst<"svuzp1_b8", "PPP", "Pc", MergeNone, "aarch64_sve_uzp1", [VerifyRuntimeMode]>; 1106def SVUZP1_B16 : SInst<"svuzp1_b16", "PPP", "Pc", MergeNone, "aarch64_sve_uzp1_b16", [IsOverloadNone, VerifyRuntimeMode]>; 1107def SVUZP1_B32 : SInst<"svuzp1_b32", "PPP", "Pc", MergeNone, "aarch64_sve_uzp1_b32", [IsOverloadNone, VerifyRuntimeMode]>; 1108def SVUZP1_B64 : SInst<"svuzp1_b64", "PPP", "Pc", MergeNone, "aarch64_sve_uzp1_b64", [IsOverloadNone, VerifyRuntimeMode]>; 1109def SVUZP2_B8 : SInst<"svuzp2_b8", "PPP", "Pc", MergeNone, "aarch64_sve_uzp2", [VerifyRuntimeMode]>; 1110def SVUZP2_B16 : SInst<"svuzp2_b16", "PPP", "Pc", MergeNone, "aarch64_sve_uzp2_b16", [IsOverloadNone, VerifyRuntimeMode]>; 1111def SVUZP2_B32 : SInst<"svuzp2_b32", "PPP", "Pc", MergeNone, "aarch64_sve_uzp2_b32", [IsOverloadNone, VerifyRuntimeMode]>; 1112def SVUZP2_B64 : SInst<"svuzp2_b64", "PPP", "Pc", MergeNone, "aarch64_sve_uzp2_b64", [IsOverloadNone, VerifyRuntimeMode]>; 1113def SVZIP1_B8 : SInst<"svzip1_b8", "PPP", "Pc", MergeNone, "aarch64_sve_zip1", [VerifyRuntimeMode]>; 1114def SVZIP1_B16 : SInst<"svzip1_b16", "PPP", "Pc", MergeNone, "aarch64_sve_zip1_b16", [IsOverloadNone, VerifyRuntimeMode]>; 1115def SVZIP1_B32 : SInst<"svzip1_b32", "PPP", "Pc", MergeNone, "aarch64_sve_zip1_b32", [IsOverloadNone, VerifyRuntimeMode]>; 1116def SVZIP1_B64 : SInst<"svzip1_b64", "PPP", "Pc", MergeNone, "aarch64_sve_zip1_b64", [IsOverloadNone, VerifyRuntimeMode]>; 1117def SVZIP2_B : SInst<"svzip2_b8", "PPP", "Pc", MergeNone, "aarch64_sve_zip2", [VerifyRuntimeMode]>; 1118def SVZIP2_B16 : SInst<"svzip2_b16", "PPP", "Pc", MergeNone, "aarch64_sve_zip2_b16", [IsOverloadNone, VerifyRuntimeMode]>; 1119def SVZIP2_B32 : SInst<"svzip2_b32", "PPP", "Pc", MergeNone, "aarch64_sve_zip2_b32", [IsOverloadNone, VerifyRuntimeMode]>; 1120def SVZIP2_B64 : SInst<"svzip2_b64", "PPP", "Pc", MergeNone, "aarch64_sve_zip2_b64", [IsOverloadNone, VerifyRuntimeMode]>; 1121 1122//////////////////////////////////////////////////////////////////////////////// 1123// Predicate creation 1124 1125def SVPFALSE : SInst<"svpfalse[_b]", "Pv", "", MergeNone, "", [IsOverloadNone, VerifyRuntimeMode]>; 1126def SVPTRUE_PAT : SInst<"svptrue_pat_{d}", "PI", "PcPsPiPl", MergeNone, "aarch64_sve_ptrue", [VerifyRuntimeMode]>; 1127def SVPTRUE : SInst<"svptrue_{d}", "Pv", "PcPsPiPl", MergeNone, "aarch64_sve_ptrue", [IsAppendSVALL, VerifyRuntimeMode]>; 1128 1129def SVDUPQ_B8 : SInst<"svdupq[_n]_{d}", "Pssssssssssssssss", "Pc", MergeNone, "", [VerifyRuntimeMode]>; 1130def SVDUPQ_B16 : SInst<"svdupq[_n]_{d}", "Pssssssss", "Ps", MergeNone, "", [VerifyRuntimeMode]>; 1131def SVDUPQ_B32 : SInst<"svdupq[_n]_{d}", "Pssss", "Pi", MergeNone, "", [VerifyRuntimeMode]>; 1132def SVDUPQ_B64 : SInst<"svdupq[_n]_{d}", "Pss", "Pl", MergeNone, "", [VerifyRuntimeMode]>; 1133def SVDUP_N_B : SInst<"svdup[_n]_{d}", "Ps", "PcPsPiPl", MergeNone, "", [VerifyRuntimeMode]>; 1134 1135 1136//////////////////////////////////////////////////////////////////////////////// 1137// Predicate operations 1138 1139def SVAND_B_Z : SInst<"svand[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_and_z", [VerifyRuntimeMode]>; 1140def SVBIC_B_Z : SInst<"svbic[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_bic_z", [VerifyRuntimeMode]>; 1141def SVEOR_B_Z : SInst<"sveor[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_eor_z", [VerifyRuntimeMode]>; 1142def SVMOV_B_Z : SInst<"svmov[_b]_z", "PPP", "Pc", MergeNone, "", [VerifyRuntimeMode]>; // Uses custom expansion 1143def SVNAND_B_Z : SInst<"svnand[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_nand_z", [VerifyRuntimeMode]>; 1144def SVNOR_B_Z : SInst<"svnor[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_nor_z", [VerifyRuntimeMode]>; 1145def SVNOT_B_Z : SInst<"svnot[_b]_z", "PPP", "Pc", MergeNone, "", [VerifyRuntimeMode]>; // Uses custom expansion 1146def SVORN_B_Z : SInst<"svorn[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_orn_z", [VerifyRuntimeMode]>; 1147def SVORR_B_Z : SInst<"svorr[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_orr_z", [VerifyRuntimeMode]>; 1148 1149def SVBRKA : SInst<"svbrka[_b]_m", "PPPP", "Pc", MergeNone, "aarch64_sve_brka", [VerifyRuntimeMode]>; 1150def SVBRKA_Z : SInst<"svbrka[_b]_z", "PPP", "Pc", MergeNone, "aarch64_sve_brka_z", [VerifyRuntimeMode]>; 1151def SVBRKB : SInst<"svbrkb[_b]_m", "PPPP", "Pc", MergeNone, "aarch64_sve_brkb", [VerifyRuntimeMode]>; 1152def SVBRKB_Z : SInst<"svbrkb[_b]_z", "PPP", "Pc", MergeNone, "aarch64_sve_brkb_z", [VerifyRuntimeMode]>; 1153def SVBRKN_Z : SInst<"svbrkn[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_brkn_z", [VerifyRuntimeMode]>; 1154def SVBRKPA_Z : SInst<"svbrkpa[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_brkpa_z", [VerifyRuntimeMode]>; 1155def SVBRKPB_Z : SInst<"svbrkpb[_b]_z", "PPPP", "Pc", MergeNone, "aarch64_sve_brkpb_z", [VerifyRuntimeMode]>; 1156 1157def SVPFIRST : SInst<"svpfirst[_b]", "PPP", "Pc", MergeNone, "aarch64_sve_pfirst", [VerifyRuntimeMode]>; 1158def SVPNEXT : SInst<"svpnext_{d}", "PPP", "PcPsPiPl", MergeNone, "aarch64_sve_pnext", [VerifyRuntimeMode]>; 1159 1160//////////////////////////////////////////////////////////////////////////////// 1161// Testing predicates 1162 1163def SVPTEST_ANY : SInst<"svptest_any", "sPP", "Pc", MergeNone, "aarch64_sve_ptest_any", [VerifyRuntimeMode]>; 1164def SVPTEST_FIRST : SInst<"svptest_first", "sPP", "Pc", MergeNone, "aarch64_sve_ptest_first", [VerifyRuntimeMode]>; 1165def SVPTEST_LAST : SInst<"svptest_last", "sPP", "Pc", MergeNone, "aarch64_sve_ptest_last", [VerifyRuntimeMode]>; 1166 1167//////////////////////////////////////////////////////////////////////////////// 1168// FFR manipulation 1169 1170let SVETargetGuard = "sve", SMETargetGuard = InvalidMode in { 1171def SVRDFFR : SInst<"svrdffr", "Pv", "Pc", MergeNone, "", [IsOverloadNone]>; 1172def SVRDFFR_Z : SInst<"svrdffr_z", "PP", "Pc", MergeNone, "", [IsOverloadNone]>; 1173def SVSETFFR : SInst<"svsetffr", "vv", "", MergeNone, "", [IsOverloadNone]>; 1174def SVWRFFR : SInst<"svwrffr", "vP", "Pc", MergeNone, "", [IsOverloadNone]>; 1175} 1176 1177//////////////////////////////////////////////////////////////////////////////// 1178// Counting elements 1179 1180def SVCNTB_PAT : SInst<"svcntb_pat", "nI", "", MergeNone, "aarch64_sve_cntb", [IsOverloadNone, VerifyRuntimeMode]>; 1181def SVCNTH_PAT : SInst<"svcnth_pat", "nI", "", MergeNone, "aarch64_sve_cnth", [IsOverloadNone, VerifyRuntimeMode]>; 1182def SVCNTW_PAT : SInst<"svcntw_pat", "nI", "", MergeNone, "aarch64_sve_cntw", [IsOverloadNone, VerifyRuntimeMode]>; 1183def SVCNTD_PAT : SInst<"svcntd_pat", "nI", "", MergeNone, "aarch64_sve_cntd", [IsOverloadNone, VerifyRuntimeMode]>; 1184 1185def SVCNTB : SInst<"svcntb", "nv", "", MergeNone, "aarch64_sve_cntb", [IsAppendSVALL, IsOverloadNone, VerifyRuntimeMode]>; 1186def SVCNTH : SInst<"svcnth", "nv", "", MergeNone, "aarch64_sve_cnth", [IsAppendSVALL, IsOverloadNone, VerifyRuntimeMode]>; 1187def SVCNTW : SInst<"svcntw", "nv", "", MergeNone, "aarch64_sve_cntw", [IsAppendSVALL, IsOverloadNone, VerifyRuntimeMode]>; 1188def SVCNTD : SInst<"svcntd", "nv", "", MergeNone, "aarch64_sve_cntd", [IsAppendSVALL, IsOverloadNone, VerifyRuntimeMode]>; 1189 1190def SVCNTP : SInst<"svcntp_{d}", "nPP", "PcPsPiPl", MergeNone, "aarch64_sve_cntp", [VerifyRuntimeMode]>; 1191def SVLEN : SInst<"svlen[_{d}]", "nd", "csilUcUsUiUlhfd", MergeNone, "", [VerifyRuntimeMode]>; 1192 1193let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in { 1194def SVLEN_BF16 : SInst<"svlen[_{d}]", "nd", "b", MergeNone, "", [VerifyRuntimeMode]>; 1195} 1196 1197//////////////////////////////////////////////////////////////////////////////// 1198// Saturating scalar arithmetic 1199 1200class sat_type<string u, string t> { string U = u; string T = t; } 1201def SignedByte : sat_type<"", "c">; 1202def SignedHalf : sat_type<"", "s">; 1203def SignedWord : sat_type<"", "i">; 1204def SignedDoubleWord : sat_type<"", "l">; 1205def UnsignedByte : sat_type<"U", "Uc">; 1206def UnsignedHalf : sat_type<"U", "Us">; 1207def UnsignedWord : sat_type<"U", "Ui">; 1208def UnsignedDoubleWord : sat_type<"U", "Ul">; 1209 1210multiclass SInst_SAT1<string name, string intrinsic, sat_type type> { 1211 def _N32 : SInst<name # "_pat[_n_{d}]", "ssIi", type.U # "i", MergeNone, intrinsic # "_n32", [IsOverloadNone, VerifyRuntimeMode], [ImmCheck<2, ImmCheck1_16>]>; 1212 def _N64 : SInst<name # "_pat[_n_{d}]", "ssIi", type.U # "l", MergeNone, intrinsic # "_n64", [IsOverloadNone, VerifyRuntimeMode], [ImmCheck<2, ImmCheck1_16>]>; 1213 def _N32_ALL : SInst<name # "[_n_{d}]", "ssi", type.U # "i", MergeNone, intrinsic # "_n32", [IsOverloadNone, IsInsertOp1SVALL, VerifyRuntimeMode], [ImmCheck<1, ImmCheck1_16>]>; 1214 def _N64_ALL : SInst<name # "[_n_{d}]", "ssi", type.U # "l", MergeNone, intrinsic # "_n64", [IsOverloadNone, IsInsertOp1SVALL, VerifyRuntimeMode], [ImmCheck<1, ImmCheck1_16>]>; 1215} 1216 1217multiclass SInst_SAT2<string name, string intrinsic, sat_type type> { 1218 def "" : SInst<name # "_pat[_{d}]", "ddIi", type.T, MergeNone, intrinsic, [VerifyRuntimeMode], [ImmCheck<2, ImmCheck1_16>]>; 1219 def _ALL : SInst<name # "[_{d}]", "ddi", type.T, MergeNone, intrinsic, [IsInsertOp1SVALL, VerifyRuntimeMode], [ImmCheck<1, ImmCheck1_16>]>; 1220 1221 def _N32 : SInst<name # "_pat[_n_{d}]", "ssIi", type.U # "i", MergeNone, intrinsic # "_n32", [IsOverloadNone, VerifyRuntimeMode], [ImmCheck<2, ImmCheck1_16>]>; 1222 def _N64 : SInst<name # "_pat[_n_{d}]", "ssIi", type.U # "l", MergeNone, intrinsic # "_n64", [IsOverloadNone, VerifyRuntimeMode], [ImmCheck<2, ImmCheck1_16>]>; 1223 def _N32_ALL : SInst<name # "[_n_{d}]", "ssi", type.U # "i", MergeNone, intrinsic # "_n32", [IsOverloadNone, IsInsertOp1SVALL, VerifyRuntimeMode], [ImmCheck<1, ImmCheck1_16>]>; 1224 def _N64_ALL : SInst<name # "[_n_{d}]", "ssi", type.U # "l", MergeNone, intrinsic # "_n64", [IsOverloadNone, IsInsertOp1SVALL, VerifyRuntimeMode], [ImmCheck<1, ImmCheck1_16>]>; 1225} 1226 1227defm SVQDECB_S : SInst_SAT1<"svqdecb", "aarch64_sve_sqdecb", SignedByte>; 1228defm SVQDECB_U : SInst_SAT1<"svqdecb", "aarch64_sve_uqdecb", UnsignedByte>; 1229defm SVQDECH_S : SInst_SAT2<"svqdech", "aarch64_sve_sqdech", SignedHalf>; 1230defm SVQDECH_U : SInst_SAT2<"svqdech", "aarch64_sve_uqdech", UnsignedHalf>; 1231defm SVQDECW_S : SInst_SAT2<"svqdecw", "aarch64_sve_sqdecw", SignedWord>; 1232defm SVQDECW_U : SInst_SAT2<"svqdecw", "aarch64_sve_uqdecw", UnsignedWord>; 1233defm SVQDECD_S : SInst_SAT2<"svqdecd", "aarch64_sve_sqdecd", SignedDoubleWord>; 1234defm SVQDECD_U : SInst_SAT2<"svqdecd", "aarch64_sve_uqdecd", UnsignedDoubleWord>; 1235 1236defm SVQINCB_S : SInst_SAT1<"svqincb", "aarch64_sve_sqincb", SignedByte>; 1237defm SVQINCB_U : SInst_SAT1<"svqincb", "aarch64_sve_uqincb", UnsignedByte>; 1238defm SVQINCH_S : SInst_SAT2<"svqinch", "aarch64_sve_sqinch", SignedHalf>; 1239defm SVQINCH_U : SInst_SAT2<"svqinch", "aarch64_sve_uqinch", UnsignedHalf>; 1240defm SVQINCW_S : SInst_SAT2<"svqincw", "aarch64_sve_sqincw", SignedWord>; 1241defm SVQINCW_U : SInst_SAT2<"svqincw", "aarch64_sve_uqincw", UnsignedWord>; 1242defm SVQINCD_S : SInst_SAT2<"svqincd", "aarch64_sve_sqincd", SignedDoubleWord>; 1243defm SVQINCD_U : SInst_SAT2<"svqincd", "aarch64_sve_uqincd", UnsignedDoubleWord>; 1244 1245def SVQDECP_S : SInst<"svqdecp[_{d}]", "ddP", "sil", MergeNone, "aarch64_sve_sqdecp", [VerifyRuntimeMode]>; 1246def SVQDECP_U : SInst<"svqdecp[_{d}]", "ddP", "UsUiUl", MergeNone, "aarch64_sve_uqdecp", [VerifyRuntimeMode]>; 1247def SVQINCP_S : SInst<"svqincp[_{d}]", "ddP", "sil", MergeNone, "aarch64_sve_sqincp", [VerifyRuntimeMode]>; 1248def SVQINCP_U : SInst<"svqincp[_{d}]", "ddP", "UsUiUl", MergeNone, "aarch64_sve_uqincp", [VerifyRuntimeMode]>; 1249 1250def SVQDECP_N_S32 : SInst<"svqdecp[_n_s32]_{d}", "kkP", "PcPsPiPl", MergeNone, "aarch64_sve_sqdecp_n32", [VerifyRuntimeMode]>; 1251def SVQDECP_N_S64 : SInst<"svqdecp[_n_s64]_{d}", "llP", "PcPsPiPl", MergeNone, "aarch64_sve_sqdecp_n64", [VerifyRuntimeMode]>; 1252def SVQDECP_N_U32 : SInst<"svqdecp[_n_u32]_{d}", "mmP", "PcPsPiPl", MergeNone, "aarch64_sve_uqdecp_n32", [VerifyRuntimeMode]>; 1253def SVQDECP_N_U64 : SInst<"svqdecp[_n_u64]_{d}", "nnP", "PcPsPiPl", MergeNone, "aarch64_sve_uqdecp_n64", [VerifyRuntimeMode]>; 1254def SVQINCP_N_S32 : SInst<"svqincp[_n_s32]_{d}", "kkP", "PcPsPiPl", MergeNone, "aarch64_sve_sqincp_n32", [VerifyRuntimeMode]>; 1255def SVQINCP_N_S64 : SInst<"svqincp[_n_s64]_{d}", "llP", "PcPsPiPl", MergeNone, "aarch64_sve_sqincp_n64", [VerifyRuntimeMode]>; 1256def SVQINCP_N_U32 : SInst<"svqincp[_n_u32]_{d}", "mmP", "PcPsPiPl", MergeNone, "aarch64_sve_uqincp_n32", [VerifyRuntimeMode]>; 1257def SVQINCP_N_U64 : SInst<"svqincp[_n_u64]_{d}", "nnP", "PcPsPiPl", MergeNone, "aarch64_sve_uqincp_n64", [VerifyRuntimeMode]>; 1258 1259let SVETargetGuard = "sve,i8mm", SMETargetGuard = InvalidMode in { 1260def SVMLLA_S32 : SInst<"svmmla[_s32]", "ddqq","i", MergeNone, "aarch64_sve_smmla">; 1261def SVMLLA_U32 : SInst<"svmmla[_u32]", "ddqq","Ui", MergeNone, "aarch64_sve_ummla">; 1262def SVUSMLLA_S32 : SInst<"svusmmla[_s32]", "ddbq","i", MergeNone, "aarch64_sve_usmmla">; 1263} 1264 1265let SVETargetGuard = "sve,i8mm", SMETargetGuard = "sme,i8mm"in { 1266def SVUSDOT_S : SInst<"svusdot[_s32]", "ddbq", "i", MergeNone, "aarch64_sve_usdot", [VerifyRuntimeMode]>; 1267def SVUSDOT_N_S : SInst<"svusdot[_n_s32]", "ddbr", "i", MergeNone, "aarch64_sve_usdot", [VerifyRuntimeMode]>; 1268def SVSUDOT_S : SInst<"svsudot[_s32]", "ddqb", "i", MergeNone, "aarch64_sve_usdot", [ReverseUSDOT, VerifyRuntimeMode]>; 1269def SVSUDOT_N_S : SInst<"svsudot[_n_s32]", "ddq@", "i", MergeNone, "aarch64_sve_usdot", [ReverseUSDOT, VerifyRuntimeMode]>; 1270 1271def SVUSDOT_LANE_S : SInst<"svusdot_lane[_s32]", "ddbqi", "i", MergeNone, "aarch64_sve_usdot_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndexDot, 2>]>; 1272def SVSUDOT_LANE_S : SInst<"svsudot_lane[_s32]", "ddqbi", "i", MergeNone, "aarch64_sve_sudot_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndexDot, 2>]>; 1273} 1274 1275let SVETargetGuard = "sve,f32mm", SMETargetGuard = InvalidMode in { 1276def SVMLLA_F32 : SInst<"svmmla[_f32]", "dddd","f", MergeNone, "aarch64_sve_fmmla">; 1277} 1278 1279let SVETargetGuard = "sve,f64mm", SMETargetGuard = InvalidMode in { 1280def SVMLLA_F64 : SInst<"svmmla[_f64]", "dddd","d", MergeNone, "aarch64_sve_fmmla">; 1281def SVTRN1Q : SInst<"svtrn1q[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_trn1q">; 1282def SVTRN2Q : SInst<"svtrn2q[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_trn2q">; 1283def SVUZP1Q : SInst<"svuzp1q[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_uzp1q">; 1284def SVUZP2Q : SInst<"svuzp2q[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_uzp2q">; 1285def SVZIP1Q : SInst<"svzip1q[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_zip1q">; 1286def SVZIP2Q : SInst<"svzip2q[_{d}]", "ddd", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_zip2q">; 1287} 1288 1289let SVETargetGuard = "sve,bf16,f64mm", SMETargetGuard = InvalidMode in { 1290def SVTRN1Q_BF16 : SInst<"svtrn1q[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_trn1q">; 1291def SVTRN2Q_BF16 : SInst<"svtrn2q[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_trn2q">; 1292def SVUZP1Q_BF16 : SInst<"svuzp1q[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_uzp1q">; 1293def SVUZP2Q_BF16 : SInst<"svuzp2q[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_uzp2q">; 1294def SVZIP1Q_BF16 : SInst<"svzip1q[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_zip1q">; 1295def SVZIP2Q_BF16 : SInst<"svzip2q[_{d}]", "ddd", "b", MergeNone, "aarch64_sve_zip2q">; 1296} 1297 1298//////////////////////////////////////////////////////////////////////////////// 1299// Vector creation 1300def SVUNDEF_1 : SInst<"svundef_{d}", "dv", "csilUcUsUiUlhfd", MergeNone, "", [IsUndef, VerifyRuntimeMode]>; 1301def SVUNDEF_2 : SInst<"svundef2_{d}", "2v", "csilUcUsUiUlhfd", MergeNone, "", [IsUndef, VerifyRuntimeMode]>; 1302def SVUNDEF_3 : SInst<"svundef3_{d}", "3v", "csilUcUsUiUlhfd", MergeNone, "", [IsUndef, VerifyRuntimeMode]>; 1303def SVUNDEF_4 : SInst<"svundef4_{d}", "4v", "csilUcUsUiUlhfd", MergeNone, "", [IsUndef, VerifyRuntimeMode]>; 1304 1305def SVCREATE_2 : SInst<"svcreate2[_{d}]", "2dd", "csilUcUsUiUlhfd", MergeNone, "", [IsTupleCreate, VerifyRuntimeMode]>; 1306def SVCREATE_3 : SInst<"svcreate3[_{d}]", "3ddd", "csilUcUsUiUlhfd", MergeNone, "", [IsTupleCreate, VerifyRuntimeMode]>; 1307def SVCREATE_4 : SInst<"svcreate4[_{d}]", "4dddd", "csilUcUsUiUlhfd", MergeNone, "", [IsTupleCreate, VerifyRuntimeMode]>; 1308 1309let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in { 1310def SVUNDEF_1_BF16 : SInst<"svundef_{d}", "dv", "b", MergeNone, "", [IsUndef, VerifyRuntimeMode]>; 1311def SVUNDEF_2_BF16 : SInst<"svundef2_{d}", "2v", "b", MergeNone, "", [IsUndef, VerifyRuntimeMode]>; 1312def SVUNDEF_3_BF16 : SInst<"svundef3_{d}", "3v", "b", MergeNone, "", [IsUndef, VerifyRuntimeMode]>; 1313def SVUNDEF_4_BF16 : SInst<"svundef4_{d}", "4v", "b", MergeNone, "", [IsUndef, VerifyRuntimeMode]>; 1314 1315def SVCREATE_2_BF16 : SInst<"svcreate2[_{d}]", "2dd", "b", MergeNone, "", [IsTupleCreate, VerifyRuntimeMode]>; 1316def SVCREATE_3_BF16 : SInst<"svcreate3[_{d}]", "3ddd", "b", MergeNone, "", [IsTupleCreate, VerifyRuntimeMode]>; 1317def SVCREATE_4_BF16 : SInst<"svcreate4[_{d}]", "4dddd", "b", MergeNone, "", [IsTupleCreate, VerifyRuntimeMode]>; 1318} 1319 1320let SVETargetGuard = "sve2p1", SMETargetGuard = "sme2" in { 1321 def SVCREATE_2_B : SInst<"svcreate2[_b]", "2dd", "Pc", MergeNone, "", [IsTupleCreate, VerifyRuntimeMode]>; 1322 def SVCREATE_4_B : SInst<"svcreate4[_b]", "4dddd", "Pc", MergeNone, "", [IsTupleCreate, VerifyRuntimeMode]>; 1323} 1324 1325//////////////////////////////////////////////////////////////////////////////// 1326// Vector insertion and extraction 1327def SVGET_2 : SInst<"svget2[_{d}]", "d2i", "csilUcUsUiUlhfd", MergeNone, "", [IsTupleGet, VerifyRuntimeMode], [ImmCheck<1, ImmCheck0_1>]>; 1328def SVGET_3 : SInst<"svget3[_{d}]", "d3i", "csilUcUsUiUlhfd", MergeNone, "", [IsTupleGet, VerifyRuntimeMode], [ImmCheck<1, ImmCheck0_2>]>; 1329def SVGET_4 : SInst<"svget4[_{d}]", "d4i", "csilUcUsUiUlhfd", MergeNone, "", [IsTupleGet, VerifyRuntimeMode], [ImmCheck<1, ImmCheck0_3>]>; 1330 1331def SVSET_2 : SInst<"svset2[_{d}]", "22id", "csilUcUsUiUlhfd", MergeNone, "", [IsTupleSet, VerifyRuntimeMode], [ImmCheck<1, ImmCheck0_1>]>; 1332def SVSET_3 : SInst<"svset3[_{d}]", "33id", "csilUcUsUiUlhfd", MergeNone, "", [IsTupleSet, VerifyRuntimeMode], [ImmCheck<1, ImmCheck0_2>]>; 1333def SVSET_4 : SInst<"svset4[_{d}]", "44id", "csilUcUsUiUlhfd", MergeNone, "", [IsTupleSet, VerifyRuntimeMode], [ImmCheck<1, ImmCheck0_3>]>; 1334 1335let SVETargetGuard = "sve,bf16", SMETargetGuard = "sme,bf16" in { 1336def SVGET_2_BF16 : SInst<"svget2[_{d}]", "d2i", "b", MergeNone, "", [IsTupleGet, VerifyRuntimeMode], [ImmCheck<1, ImmCheck0_1>]>; 1337def SVGET_3_BF16 : SInst<"svget3[_{d}]", "d3i", "b", MergeNone, "", [IsTupleGet, VerifyRuntimeMode], [ImmCheck<1, ImmCheck0_2>]>; 1338def SVGET_4_BF16 : SInst<"svget4[_{d}]", "d4i", "b", MergeNone, "", [IsTupleGet, VerifyRuntimeMode], [ImmCheck<1, ImmCheck0_3>]>; 1339 1340def SVSET_2_BF16 : SInst<"svset2[_{d}]", "22id", "b", MergeNone, "", [IsTupleSet, VerifyRuntimeMode], [ImmCheck<1, ImmCheck0_1>]>; 1341def SVSET_3_BF16 : SInst<"svset3[_{d}]", "33id", "b", MergeNone, "", [IsTupleSet, VerifyRuntimeMode], [ImmCheck<1, ImmCheck0_2>]>; 1342def SVSET_4_BF16 : SInst<"svset4[_{d}]", "44id", "b", MergeNone, "", [IsTupleSet, VerifyRuntimeMode], [ImmCheck<1, ImmCheck0_3>]>; 1343} 1344 1345let SVETargetGuard = "sve2p1", SMETargetGuard = "sme2" in { 1346 def SVGET_2_B : SInst<"svget2[_b]", "d2i", "Pc", MergeNone, "", [IsTupleGet, VerifyRuntimeMode], [ImmCheck<1, ImmCheck0_1>]>; 1347 def SVGET_4_B : SInst<"svget4[_b]", "d4i", "Pc", MergeNone, "", [IsTupleGet, VerifyRuntimeMode], [ImmCheck<1, ImmCheck0_3>]>; 1348 1349 def SVSET_2_B : SInst<"svset2[_b]", "22id", "Pc", MergeNone, "", [IsTupleSet, VerifyRuntimeMode], [ImmCheck<1, ImmCheck0_1>]>; 1350 def SVSET_4_B : SInst<"svset4[_b]", "44id", "Pc", MergeNone, "", [IsTupleSet, VerifyRuntimeMode], [ImmCheck<1, ImmCheck0_3>]>; 1351} 1352 1353let SVETargetGuard = "sve2p1", SMETargetGuard = "sme2" in { 1354 def SVUNDEF_2_B: Inst<"svundef2_b", "2", "Pc", MergeNone, "", [IsUndef, VerifyRuntimeMode], []>; 1355 def SVUNDEF_4_B: Inst<"svundef4_b", "4", "Pc", MergeNone, "", [IsUndef, VerifyRuntimeMode], []>; 1356} 1357//////////////////////////////////////////////////////////////////////////////// 1358// SVE2 WhileGE/GT 1359let SVETargetGuard = "sve2", SMETargetGuard = "sme" in { 1360def SVWHILEGE_S32 : SInst<"svwhilege_{d}[_{1}]", "Pkk", "PcPsPiPl", MergeNone, "aarch64_sve_whilege", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>; 1361def SVWHILEGE_S64 : SInst<"svwhilege_{d}[_{1}]", "Pll", "PcPsPiPl", MergeNone, "aarch64_sve_whilege", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>; 1362def SVWHILEGT_S32 : SInst<"svwhilegt_{d}[_{1}]", "Pkk", "PcPsPiPl", MergeNone, "aarch64_sve_whilegt", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>; 1363def SVWHILEGT_S64 : SInst<"svwhilegt_{d}[_{1}]", "Pll", "PcPsPiPl", MergeNone, "aarch64_sve_whilegt", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>; 1364def SVWHILEHI_U32 : SInst<"svwhilegt_{d}[_{1}]", "Pmm", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilehi", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>; 1365def SVWHILEHI_U64 : SInst<"svwhilegt_{d}[_{1}]", "Pnn", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilehi", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>; 1366def SVWHILEHS_U32 : SInst<"svwhilege_{d}[_{1}]", "Pmm", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilehs", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>; 1367def SVWHILEHS_U64 : SInst<"svwhilege_{d}[_{1}]", "Pnn", "PUcPUsPUiPUl", MergeNone, "aarch64_sve_whilehs", [IsOverloadWhileOrMultiVecCvt, VerifyRuntimeMode]>; 1368} 1369 1370let SVETargetGuard = "sve2p1", SMETargetGuard = "sme2" in { 1371 def SVWHILEGE_S64_X2 : SInst<"svwhilege_{d}[_{1}]_x2", "2ll", "PcPsPiPl", MergeNone, "aarch64_sve_whilege_x2", [VerifyRuntimeMode]>; 1372 def SVWHILEGT_S64_X2 : SInst<"svwhilegt_{d}[_{1}]_x2", "2ll", "PcPsPiPl", MergeNone, "aarch64_sve_whilegt_x2", [VerifyRuntimeMode]>; 1373 def SVWHILEHI_U64_X2 : SInst<"svwhilegt_{d}[_{1}]_x2", "2nn", "PcPsPiPl", MergeNone, "aarch64_sve_whilehi_x2", [VerifyRuntimeMode]>; 1374 def SVWHILEHS_U64_X2 : SInst<"svwhilege_{d}[_{1}]_x2", "2nn", "PcPsPiPl", MergeNone, "aarch64_sve_whilehs_x2", [VerifyRuntimeMode]>; 1375 def SVWHILELE_S64_X2 : SInst<"svwhilele_{d}[_{1}]_x2", "2ll", "PcPsPiPl", MergeNone, "aarch64_sve_whilele_x2", [VerifyRuntimeMode]>; 1376 def SVWHILELT_S64_X2 : SInst<"svwhilelt_{d}[_{1}]_x2", "2ll", "PcPsPiPl", MergeNone, "aarch64_sve_whilelt_x2", [VerifyRuntimeMode]>; 1377 def SVWHILELO_U64_X2 : SInst<"svwhilelt_{d}[_{1}]_x2", "2nn", "PcPsPiPl", MergeNone, "aarch64_sve_whilelo_x2", [VerifyRuntimeMode]>; 1378 def SVWHILELS_U64_X2 : SInst<"svwhilele_{d}[_{1}]_x2", "2nn", "PcPsPiPl", MergeNone, "aarch64_sve_whilels_x2", [VerifyRuntimeMode]>; 1379 1380} 1381 1382//////////////////////////////////////////////////////////////////////////////// 1383// SVE2 - Uniform DSP operations 1384 1385let SVETargetGuard = "sve2", SMETargetGuard = "sme" in { 1386defm SVQADD_S : SInstZPZZ<"svqadd", "csli", "aarch64_sve_sqadd", "aarch64_sve_sqadd">; 1387defm SVQADD_U : SInstZPZZ<"svqadd", "UcUsUiUl", "aarch64_sve_uqadd", "aarch64_sve_uqadd">; 1388defm SVHADD_S : SInstZPZZ<"svhadd", "csli", "aarch64_sve_shadd", "aarch64_sve_shadd">; 1389defm SVHADD_U : SInstZPZZ<"svhadd", "UcUsUiUl", "aarch64_sve_uhadd", "aarch64_sve_uhadd">; 1390defm SVRHADD_S : SInstZPZZ<"svrhadd", "csli", "aarch64_sve_srhadd", "aarch64_sve_srhadd">; 1391defm SVRHADD_U : SInstZPZZ<"svrhadd", "UcUsUiUl", "aarch64_sve_urhadd", "aarch64_sve_urhadd">; 1392 1393defm SVQSUB_S : SInstZPZZ<"svqsub", "csli", "aarch64_sve_sqsub", "aarch64_sve_sqsub_u">; 1394defm SVQSUB_U : SInstZPZZ<"svqsub", "UcUsUiUl", "aarch64_sve_uqsub", "aarch64_sve_uqsub_u">; 1395defm SVQSUBR_S : SInstZPZZ<"svqsubr", "csli", "aarch64_sve_sqsubr", "aarch64_sve_sqsub_u", [ReverseMergeAnyBinOp]>; 1396defm SVQSUBR_U : SInstZPZZ<"svqsubr", "UcUsUiUl", "aarch64_sve_uqsubr", "aarch64_sve_uqsub_u", [ReverseMergeAnyBinOp]>; 1397defm SVHSUB_S : SInstZPZZ<"svhsub", "csli", "aarch64_sve_shsub", "aarch64_sve_shsub">; 1398defm SVHSUB_U : SInstZPZZ<"svhsub", "UcUsUiUl", "aarch64_sve_uhsub", "aarch64_sve_uhsub">; 1399defm SVHSUBR_S : SInstZPZZ<"svhsubr", "csli", "aarch64_sve_shsubr", "aarch64_sve_shsubr">; 1400defm SVHSUBR_U : SInstZPZZ<"svhsubr", "UcUsUiUl", "aarch64_sve_uhsubr", "aarch64_sve_uhsubr">; 1401 1402defm SVQABS : SInstZPZ<"svqabs", "csil", "aarch64_sve_sqabs">; 1403defm SVQNEG : SInstZPZ<"svqneg", "csil", "aarch64_sve_sqneg">; 1404defm SVRECPE : SInstZPZ<"svrecpe", "Ui", "aarch64_sve_urecpe">; 1405defm SVRSQRTE : SInstZPZ<"svrsqrte", "Ui", "aarch64_sve_ursqrte">; 1406} 1407 1408//------------------------------------------------------------------------------ 1409 1410multiclass SInstZPZxZ<string name, string types, string pat_v, string pat_n, string intrinsic, list<FlagType> flags=[]> { 1411 def _M : SInst<name # "[_{d}]", pat_v, types, MergeOp1, intrinsic, flags>; 1412 def _X : SInst<name # "[_{d}]", pat_v, types, MergeAny, intrinsic, flags>; 1413 def _Z : SInst<name # "[_{d}]", pat_v, types, MergeZero, intrinsic, flags>; 1414 1415 def _N_M : SInst<name # "[_n_{d}]", pat_n, types, MergeOp1, intrinsic, flags>; 1416 def _N_X : SInst<name # "[_n_{d}]", pat_n, types, MergeAny, intrinsic, flags>; 1417 def _N_Z : SInst<name # "[_n_{d}]", pat_n, types, MergeZero, intrinsic, flags>; 1418} 1419 1420let SVETargetGuard = "sve2", SMETargetGuard = "sme" in { 1421defm SVQRSHL_S : SInstZPZxZ<"svqrshl", "csil", "dPdx", "dPdK", "aarch64_sve_sqrshl", [VerifyRuntimeMode]>; 1422defm SVQRSHL_U : SInstZPZxZ<"svqrshl", "UcUsUiUl", "dPdx", "dPdK", "aarch64_sve_uqrshl", [VerifyRuntimeMode]>; 1423defm SVQSHL_S : SInstZPZxZ<"svqshl", "csil", "dPdx", "dPdK", "aarch64_sve_sqshl", [VerifyRuntimeMode]>; 1424defm SVQSHL_U : SInstZPZxZ<"svqshl", "UcUsUiUl", "dPdx", "dPdK", "aarch64_sve_uqshl", [VerifyRuntimeMode]>; 1425defm SVRSHL_S : SInstZPZxZ<"svrshl", "csil", "dPdx", "dPdK", "aarch64_sve_srshl", [VerifyRuntimeMode]>; 1426defm SVRSHL_U : SInstZPZxZ<"svrshl", "UcUsUiUl", "dPdx", "dPdK", "aarch64_sve_urshl", [VerifyRuntimeMode]>; 1427defm SVSQADD : SInstZPZxZ<"svsqadd", "UcUsUiUl", "dPdx", "dPdK", "aarch64_sve_usqadd", [VerifyRuntimeMode]>; 1428defm SVUQADD : SInstZPZxZ<"svuqadd", "csil", "dPdu", "dPdL", "aarch64_sve_suqadd", [VerifyRuntimeMode]>; 1429 1430def SVABA_S : SInst<"svaba[_{d}]", "dddd", "csil" , MergeNone, "aarch64_sve_saba", [VerifyRuntimeMode]>; 1431def SVABA_U : SInst<"svaba[_{d}]", "dddd", "UcUsUiUl", MergeNone, "aarch64_sve_uaba", [VerifyRuntimeMode]>; 1432def SVQDMULH : SInst<"svqdmulh[_{d}]", "ddd", "csil", MergeNone, "aarch64_sve_sqdmulh", [VerifyRuntimeMode]>; 1433def SVQRDMULH : SInst<"svqrdmulh[_{d}]", "ddd", "csil", MergeNone, "aarch64_sve_sqrdmulh", [VerifyRuntimeMode]>; 1434def SVQRDMLAH : SInst<"svqrdmlah[_{d}]", "dddd", "csil", MergeNone, "aarch64_sve_sqrdmlah", [VerifyRuntimeMode]>; 1435def SVQRDMLSH : SInst<"svqrdmlsh[_{d}]", "dddd", "csil", MergeNone, "aarch64_sve_sqrdmlsh", [VerifyRuntimeMode]>; 1436 1437def SVABA_S_N : SInst<"svaba[_n_{d}]", "ddda", "csil", MergeNone, "aarch64_sve_saba", [VerifyRuntimeMode]>; 1438def SVABA_U_N : SInst<"svaba[_n_{d}]", "ddda", "UcUsUiUl", MergeNone, "aarch64_sve_uaba", [VerifyRuntimeMode]>; 1439def SVQDMULH_N : SInst<"svqdmulh[_n_{d}]", "dda", "csil", MergeNone, "aarch64_sve_sqdmulh", [VerifyRuntimeMode]>; 1440def SVQRDMULH_N : SInst<"svqrdmulh[_n_{d}]", "dda", "csil", MergeNone, "aarch64_sve_sqrdmulh", [VerifyRuntimeMode]>; 1441def SVQRDMLAH_N : SInst<"svqrdmlah[_n_{d}]", "ddda", "csil", MergeNone, "aarch64_sve_sqrdmlah", [VerifyRuntimeMode]>; 1442def SVQRDMLSH_N : SInst<"svqrdmlsh[_n_{d}]", "ddda", "csil", MergeNone, "aarch64_sve_sqrdmlsh", [VerifyRuntimeMode]>; 1443 1444def SVQDMULH_LANE : SInst<"svqdmulh_lane[_{d}]", "dddi", "sil", MergeNone, "aarch64_sve_sqdmulh_lane", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; 1445def SVQRDMULH_LANE : SInst<"svqrdmulh_lane[_{d}]", "dddi", "sil", MergeNone, "aarch64_sve_sqrdmulh_lane", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; 1446def SVQRDMLAH_LANE : SInst<"svqrdmlah_lane[_{d}]", "ddddi", "sil", MergeNone, "aarch64_sve_sqrdmlah_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 1447def SVQRDMLSH_LANE : SInst<"svqrdmlsh_lane[_{d}]", "ddddi", "sil", MergeNone, "aarch64_sve_sqrdmlsh_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 1448 1449def SVQSHLU_M : SInst<"svqshlu[_n_{d}]", "uPdi", "csil", MergeOp1, "aarch64_sve_sqshlu", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftLeft, 1>]>; 1450def SVQSHLU_X : SInst<"svqshlu[_n_{d}]", "uPdi", "csil", MergeAny, "aarch64_sve_sqshlu", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftLeft, 1>]>; 1451def SVQSHLU_Z : SInst<"svqshlu[_n_{d}]", "uPdi", "csil", MergeZero, "aarch64_sve_sqshlu", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftLeft, 1>]>; 1452def SVRSHR_M_S : SInst<"svrshr[_n_{d}]", "dPdi", "csil", MergeOp1, "aarch64_sve_srshr", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftRight, 1>]>; 1453def SVRSHR_M_U : SInst<"svrshr[_n_{d}]", "dPdi", "UcUsUiUl", MergeOp1, "aarch64_sve_urshr", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftRight, 1>]>; 1454def SVRSHR_X_S : SInst<"svrshr[_n_{d}]", "dPdi", "csil", MergeAny, "aarch64_sve_srshr", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftRight, 1>]>; 1455def SVRSHR_X_U : SInst<"svrshr[_n_{d}]", "dPdi", "UcUsUiUl", MergeAny, "aarch64_sve_urshr", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftRight, 1>]>; 1456def SVRSHR_Z_S : SInst<"svrshr[_n_{d}]", "dPdi", "csil", MergeZero, "aarch64_sve_srshr", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftRight, 1>]>; 1457def SVRSHR_Z_U : SInst<"svrshr[_n_{d}]", "dPdi", "UcUsUiUl", MergeZero, "aarch64_sve_urshr", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftRight, 1>]>; 1458def SVRSRA_S : SInst<"svrsra[_n_{d}]", "dddi", "csil", MergeNone, "aarch64_sve_srsra", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftRight, 1>]>; 1459def SVRSRA_U : SInst<"svrsra[_n_{d}]", "dddi", "UcUsUiUl", MergeNone, "aarch64_sve_ursra", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftRight, 1>]>; 1460def SVSLI : SInst<"svsli[_n_{d}]", "dddi", "csilUcUsUiUl", MergeNone, "aarch64_sve_sli", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftLeft, 1>]>; 1461def SVSRA_S : SInst<"svsra[_n_{d}]", "dddi", "csil", MergeNone, "aarch64_sve_ssra", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftRight, 1>]>; 1462def SVSRA_U : SInst<"svsra[_n_{d}]", "dddi", "UcUsUiUl", MergeNone, "aarch64_sve_usra", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftRight, 1>]>; 1463def SVSRI : SInst<"svsri[_n_{d}]", "dddi", "csilUcUsUiUl", MergeNone, "aarch64_sve_sri", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftRight, 1>]>; 1464} 1465 1466//////////////////////////////////////////////////////////////////////////////// 1467// SVE2 - Non-widening pairwise arithmetic 1468 1469multiclass SInstPairwise<string name, string types, string intrinsic, list<FlagType> flags=[]> { 1470 def _M : SInst<name # "[_{d}]", "dPdd", types, MergeOp1, intrinsic, flags>; 1471 def _X : SInst<name # "[_{d}]", "dPdd", types, MergeAny, intrinsic, flags>; 1472} 1473 1474let SVETargetGuard = "sve2", SMETargetGuard = "sme" in { 1475defm SVADDP : SInstPairwise<"svaddp", "csliUcUsUiUl", "aarch64_sve_addp", [VerifyRuntimeMode]>; 1476defm SVADDP_F : SInstPairwise<"svaddp", "hfd", "aarch64_sve_faddp", [VerifyRuntimeMode]>; 1477defm SVMAXNMP : SInstPairwise<"svmaxnmp", "hfd", "aarch64_sve_fmaxnmp", [VerifyRuntimeMode]>; 1478defm SVMAXP_F : SInstPairwise<"svmaxp", "hfd", "aarch64_sve_fmaxp", [VerifyRuntimeMode]>; 1479defm SVMAXP_S : SInstPairwise<"svmaxp", "csli", "aarch64_sve_smaxp", [VerifyRuntimeMode]>; 1480defm SVMAXP_U : SInstPairwise<"svmaxp", "UcUsUiUl", "aarch64_sve_umaxp", [VerifyRuntimeMode]>; 1481defm SVMINNMP : SInstPairwise<"svminnmp", "hfd", "aarch64_sve_fminnmp", [VerifyRuntimeMode]>; 1482defm SVMINP_F : SInstPairwise<"svminp", "hfd", "aarch64_sve_fminp", [VerifyRuntimeMode]>; 1483defm SVMINP_S : SInstPairwise<"svminp", "csli", "aarch64_sve_sminp", [VerifyRuntimeMode]>; 1484defm SVMINP_U : SInstPairwise<"svminp", "UcUsUiUl", "aarch64_sve_uminp", [VerifyRuntimeMode]>; 1485} 1486 1487//////////////////////////////////////////////////////////////////////////////// 1488// SVE2 - Widening pairwise arithmetic 1489 1490let SVETargetGuard = "sve2", SMETargetGuard = "sme" in { 1491def SVADALP_S_M : SInst<"svadalp[_{d}]", "dPdh", "sil", MergeOp1, "aarch64_sve_sadalp", [VerifyRuntimeMode]>; 1492def SVADALP_S_X : SInst<"svadalp[_{d}]", "dPdh", "sil", MergeAny, "aarch64_sve_sadalp", [VerifyRuntimeMode]>; 1493def SVADALP_S_Z : SInst<"svadalp[_{d}]", "dPdh", "sil", MergeZero, "aarch64_sve_sadalp", [VerifyRuntimeMode]>; 1494 1495def SVADALP_U_M : SInst<"svadalp[_{d}]", "dPdh", "UsUiUl", MergeOp1, "aarch64_sve_uadalp", [VerifyRuntimeMode]>; 1496def SVADALP_U_X : SInst<"svadalp[_{d}]", "dPdh", "UsUiUl", MergeAny, "aarch64_sve_uadalp", [VerifyRuntimeMode]>; 1497def SVADALP_U_Z : SInst<"svadalp[_{d}]", "dPdh", "UsUiUl", MergeZero, "aarch64_sve_uadalp", [VerifyRuntimeMode]>; 1498} 1499 1500//////////////////////////////////////////////////////////////////////////////// 1501// SVE2 - Bitwise ternary logical instructions 1502// 1503 1504let SVETargetGuard = "sve2", SMETargetGuard = "sme" in { 1505def SVBCAX : SInst<"svbcax[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_bcax", [VerifyRuntimeMode]>; 1506def SVBSL : SInst<"svbsl[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_bsl", [VerifyRuntimeMode]>; 1507def SVBSL1N : SInst<"svbsl1n[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_bsl1n", [VerifyRuntimeMode]>; 1508def SVBSL2N : SInst<"svbsl2n[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_bsl2n", [VerifyRuntimeMode]>; 1509def SVEOR3 : SInst<"sveor3[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_eor3", [VerifyRuntimeMode]>; 1510def SVNBSL : SInst<"svnbsl[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_nbsl", [VerifyRuntimeMode]>; 1511 1512def SVBCAX_N : SInst<"svbcax[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_bcax", [VerifyRuntimeMode]>; 1513def SVBSL_N : SInst<"svbsl[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_bsl", [VerifyRuntimeMode]>; 1514def SVBSL1N_N : SInst<"svbsl1n[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_bsl1n", [VerifyRuntimeMode]>; 1515def SVBSL2N_N : SInst<"svbsl2n[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_bsl2n", [VerifyRuntimeMode]>; 1516def SVEOR3_N : SInst<"sveor3[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_eor3", [VerifyRuntimeMode]>; 1517def SVNBSL_N : SInst<"svnbsl[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_nbsl", [VerifyRuntimeMode]>; 1518def SVXAR_N : SInst<"svxar[_n_{d}]", "dddi", "csilUcUsUiUl", MergeNone, "aarch64_sve_xar", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftRight, 1>]>; 1519} 1520 1521//////////////////////////////////////////////////////////////////////////////// 1522// SVE2 - Large integer arithmetic 1523 1524let SVETargetGuard = "sve2", SMETargetGuard = "sme" in { 1525def SVADCLB : SInst<"svadclb[_{d}]", "dddd", "UiUl", MergeNone, "aarch64_sve_adclb", [VerifyRuntimeMode]>; 1526def SVADCLT : SInst<"svadclt[_{d}]", "dddd", "UiUl", MergeNone, "aarch64_sve_adclt", [VerifyRuntimeMode]>; 1527def SVSBCLB : SInst<"svsbclb[_{d}]", "dddd", "UiUl", MergeNone, "aarch64_sve_sbclb", [VerifyRuntimeMode]>; 1528def SVSBCLT : SInst<"svsbclt[_{d}]", "dddd", "UiUl", MergeNone, "aarch64_sve_sbclt", [VerifyRuntimeMode]>; 1529 1530def SVADCLB_N : SInst<"svadclb[_n_{d}]", "ddda", "UiUl", MergeNone, "aarch64_sve_adclb", [VerifyRuntimeMode]>; 1531def SVADCLT_N : SInst<"svadclt[_n_{d}]", "ddda", "UiUl", MergeNone, "aarch64_sve_adclt", [VerifyRuntimeMode]>; 1532def SVSBCLB_N : SInst<"svsbclb[_n_{d}]", "ddda", "UiUl", MergeNone, "aarch64_sve_sbclb", [VerifyRuntimeMode]>; 1533def SVSBCLT_N : SInst<"svsbclt[_n_{d}]", "ddda", "UiUl", MergeNone, "aarch64_sve_sbclt", [VerifyRuntimeMode]>; 1534} 1535 1536//////////////////////////////////////////////////////////////////////////////// 1537// SVE2 - Multiplication by indexed elements 1538 1539let SVETargetGuard = "sve2", SMETargetGuard = "sme" in { 1540def SVMLA_LANE_2 : SInst<"svmla_lane[_{d}]", "ddddi", "silUsUiUl", MergeNone, "aarch64_sve_mla_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 1541def SVMLS_LANE_2 : SInst<"svmls_lane[_{d}]", "ddddi", "silUsUiUl", MergeNone, "aarch64_sve_mls_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 1542def SVMUL_LANE_2 : SInst<"svmul_lane[_{d}]", "dddi", "silUsUiUl", MergeNone, "aarch64_sve_mul_lane", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; 1543} 1544 1545//////////////////////////////////////////////////////////////////////////////// 1546// SVE2 - Uniform complex integer arithmetic 1547let SVETargetGuard = "sve2", SMETargetGuard = "sme" in { 1548def SVCADD : SInst<"svcadd[_{d}]", "dddi", "csilUcUsUiUl", MergeNone, "aarch64_sve_cadd_x", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckComplexRot90_270>]>; 1549def SVSQCADD : SInst<"svqcadd[_{d}]", "dddi", "csil", MergeNone, "aarch64_sve_sqcadd_x", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckComplexRot90_270>]>; 1550def SVCMLA : SInst<"svcmla[_{d}]", "ddddi", "csilUcUsUiUl", MergeNone, "aarch64_sve_cmla_x", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckComplexRotAll90>]>; 1551def SVCMLA_LANE_X : SInst<"svcmla_lane[_{d}]", "ddddii", "siUsUi", MergeNone, "aarch64_sve_cmla_lane_x", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndexCompRotate, 2>, 1552 ImmCheck<4, ImmCheckComplexRotAll90>]>; 1553def SVSQRDCMLAH_X : SInst<"svqrdcmlah[_{d}]", "ddddi", "csil", MergeNone, "aarch64_sve_sqrdcmlah_x", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckComplexRotAll90>]>; 1554def SVSQRDCMLAH_LANE_X : SInst<"svqrdcmlah_lane[_{d}]", "ddddii", "si", MergeNone, "aarch64_sve_sqrdcmlah_lane_x", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndexCompRotate, 2>, 1555 ImmCheck<4, ImmCheckComplexRotAll90>]>; 1556} 1557 1558//////////////////////////////////////////////////////////////////////////////// 1559// SVE2 - Widening DSP operations 1560 1561multiclass SInstWideDSPAcc<string name, string types, string intrinsic> { 1562 def : SInst<name # "[_{d}]", "ddhh", types, MergeNone, intrinsic, [VerifyRuntimeMode]>; 1563 def _N : SInst<name # "[_n_{d}]", "ddhR", types, MergeNone, intrinsic, [VerifyRuntimeMode]>; 1564} 1565 1566multiclass SInstWideDSPLong<string name, string types, string intrinsic> { 1567 def : SInst<name # "[_{d}]", "dhh", types, MergeNone, intrinsic, [VerifyRuntimeMode]>; 1568 def _N : SInst<name # "[_n_{d}]", "dhR", types, MergeNone, intrinsic, [VerifyRuntimeMode]>; 1569} 1570 1571multiclass SInstWideDSPWide<string name, string types, string intrinsic> { 1572 def : SInst<name # "[_{d}]", "ddh", types, MergeNone, intrinsic, [VerifyRuntimeMode]>; 1573 def _N : SInst<name # "[_n_{d}]", "ddR", types, MergeNone, intrinsic, [VerifyRuntimeMode]>; 1574} 1575 1576let SVETargetGuard = "sve2", SMETargetGuard = "sme" in { 1577defm SVABALB_S : SInstWideDSPAcc<"svabalb", "sil", "aarch64_sve_sabalb">; 1578defm SVABALB_U : SInstWideDSPAcc<"svabalb", "UsUiUl", "aarch64_sve_uabalb">; 1579defm SVABALT_S : SInstWideDSPAcc<"svabalt", "sil", "aarch64_sve_sabalt">; 1580defm SVABALT_U : SInstWideDSPAcc<"svabalt", "UsUiUl", "aarch64_sve_uabalt">; 1581defm SVMLALB_S : SInstWideDSPAcc<"svmlalb", "sil", "aarch64_sve_smlalb">; 1582defm SVMLALB_U : SInstWideDSPAcc<"svmlalb", "UsUiUl", "aarch64_sve_umlalb">; 1583defm SVMLALT_S : SInstWideDSPAcc<"svmlalt", "sil", "aarch64_sve_smlalt">; 1584defm SVMLALT_U : SInstWideDSPAcc<"svmlalt", "UsUiUl", "aarch64_sve_umlalt">; 1585defm SVMLSLB_S : SInstWideDSPAcc<"svmlslb", "sil", "aarch64_sve_smlslb">; 1586defm SVMLSLB_U : SInstWideDSPAcc<"svmlslb", "UsUiUl", "aarch64_sve_umlslb">; 1587defm SVMLSLT_S : SInstWideDSPAcc<"svmlslt", "sil", "aarch64_sve_smlslt">; 1588defm SVMLSLT_U : SInstWideDSPAcc<"svmlslt", "UsUiUl", "aarch64_sve_umlslt">; 1589defm SVQDMLALB : SInstWideDSPAcc<"svqdmlalb", "sil", "aarch64_sve_sqdmlalb">; 1590defm SVQDMLALT : SInstWideDSPAcc<"svqdmlalt", "sil", "aarch64_sve_sqdmlalt">; 1591defm SVQDMLSLB : SInstWideDSPAcc<"svqdmlslb", "sil", "aarch64_sve_sqdmlslb">; 1592defm SVQDMLSLT : SInstWideDSPAcc<"svqdmlslt", "sil", "aarch64_sve_sqdmlslt">; 1593 1594defm SVABDLB_S : SInstWideDSPLong<"svabdlb", "sil", "aarch64_sve_sabdlb">; 1595defm SVABDLB_U : SInstWideDSPLong<"svabdlb", "UsUiUl", "aarch64_sve_uabdlb">; 1596defm SVABDLT_S : SInstWideDSPLong<"svabdlt", "sil", "aarch64_sve_sabdlt">; 1597defm SVABDLT_U : SInstWideDSPLong<"svabdlt", "UsUiUl", "aarch64_sve_uabdlt">; 1598defm SVADDLB_S : SInstWideDSPLong<"svaddlb", "sil", "aarch64_sve_saddlb">; 1599defm SVADDLB_U : SInstWideDSPLong<"svaddlb", "UsUiUl", "aarch64_sve_uaddlb">; 1600defm SVADDLT_S : SInstWideDSPLong<"svaddlt", "sil", "aarch64_sve_saddlt">; 1601defm SVADDLT_U : SInstWideDSPLong<"svaddlt", "UsUiUl", "aarch64_sve_uaddlt">; 1602defm SVMULLB_S : SInstWideDSPLong<"svmullb", "sil", "aarch64_sve_smullb">; 1603defm SVMULLB_U : SInstWideDSPLong<"svmullb", "UsUiUl", "aarch64_sve_umullb">; 1604defm SVMULLT_S : SInstWideDSPLong<"svmullt", "sil", "aarch64_sve_smullt">; 1605defm SVMULLT_U : SInstWideDSPLong<"svmullt", "UsUiUl", "aarch64_sve_umullt">; 1606defm SVQDMULLB : SInstWideDSPLong<"svqdmullb", "sil", "aarch64_sve_sqdmullb">; 1607defm SVQDMULLT : SInstWideDSPLong<"svqdmullt", "sil", "aarch64_sve_sqdmullt">; 1608defm SVSUBLB_S : SInstWideDSPLong<"svsublb", "sil", "aarch64_sve_ssublb">; 1609defm SVSUBLB_U : SInstWideDSPLong<"svsublb", "UsUiUl", "aarch64_sve_usublb">; 1610defm SVSUBLT_S : SInstWideDSPLong<"svsublt", "sil", "aarch64_sve_ssublt">; 1611defm SVSUBLT_U : SInstWideDSPLong<"svsublt", "UsUiUl", "aarch64_sve_usublt">; 1612 1613defm SVADDWB_S : SInstWideDSPWide<"svaddwb", "sil", "aarch64_sve_saddwb">; 1614defm SVADDWB_U : SInstWideDSPWide<"svaddwb", "UsUiUl", "aarch64_sve_uaddwb">; 1615defm SVADDWT_S : SInstWideDSPWide<"svaddwt", "sil", "aarch64_sve_saddwt">; 1616defm SVADDWT_U : SInstWideDSPWide<"svaddwt", "UsUiUl", "aarch64_sve_uaddwt">; 1617defm SVSUBWB_S : SInstWideDSPWide<"svsubwb", "sil", "aarch64_sve_ssubwb">; 1618defm SVSUBWB_U : SInstWideDSPWide<"svsubwb", "UsUiUl", "aarch64_sve_usubwb">; 1619defm SVSUBWT_S : SInstWideDSPWide<"svsubwt", "sil", "aarch64_sve_ssubwt">; 1620defm SVSUBWT_U : SInstWideDSPWide<"svsubwt", "UsUiUl", "aarch64_sve_usubwt">; 1621 1622def SVSHLLB_S_N : SInst<"svshllb[_n_{d}]", "dhi", "sil", MergeNone, "aarch64_sve_sshllb", [VerifyRuntimeMode], [ImmCheck<1, ImmCheckShiftLeft, 0>]>; 1623def SVSHLLB_U_N : SInst<"svshllb[_n_{d}]", "dhi", "UsUiUl", MergeNone, "aarch64_sve_ushllb", [VerifyRuntimeMode], [ImmCheck<1, ImmCheckShiftLeft, 0>]>; 1624def SVSHLLT_S_N : SInst<"svshllt[_n_{d}]", "dhi", "sil", MergeNone, "aarch64_sve_sshllt", [VerifyRuntimeMode], [ImmCheck<1, ImmCheckShiftLeft, 0>]>; 1625def SVSHLLT_U_N : SInst<"svshllt[_n_{d}]", "dhi", "UsUiUl", MergeNone, "aarch64_sve_ushllt", [VerifyRuntimeMode], [ImmCheck<1, ImmCheckShiftLeft, 0>]>; 1626 1627def SVMOVLB_S_N : SInst<"svmovlb[_{d}]", "dh", "sil", MergeNone, "", [VerifyRuntimeMode]>; 1628def SVMOVLB_U_N : SInst<"svmovlb[_{d}]", "dh", "UsUiUl", MergeNone, "", [VerifyRuntimeMode]>; 1629def SVMOVLT_S_N : SInst<"svmovlt[_{d}]", "dh", "sil", MergeNone, "", [VerifyRuntimeMode]>; 1630def SVMOVLT_U_N : SInst<"svmovlt[_{d}]", "dh", "UsUiUl", MergeNone, "", [VerifyRuntimeMode]>; 1631 1632def SVMLALB_S_LANE : SInst<"svmlalb_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_smlalb_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 1633def SVMLALB_U_LANE : SInst<"svmlalb_lane[_{d}]", "ddhhi", "UiUl", MergeNone, "aarch64_sve_umlalb_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 1634def SVMLALT_S_LANE : SInst<"svmlalt_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_smlalt_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 1635def SVMLALT_U_LANE : SInst<"svmlalt_lane[_{d}]", "ddhhi", "UiUl", MergeNone, "aarch64_sve_umlalt_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 1636def SVMLSLB_S_LANE : SInst<"svmlslb_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_smlslb_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 1637def SVMLSLB_U_LANE : SInst<"svmlslb_lane[_{d}]", "ddhhi", "UiUl", MergeNone, "aarch64_sve_umlslb_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 1638def SVMLSLT_S_LANE : SInst<"svmlslt_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_smlslt_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 1639def SVMLSLT_U_LANE : SInst<"svmlslt_lane[_{d}]", "ddhhi", "UiUl", MergeNone, "aarch64_sve_umlslt_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 1640def SVMULLB_S_LANE : SInst<"svmullb_lane[_{d}]", "dhhi", "il", MergeNone, "aarch64_sve_smullb_lane", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; 1641def SVMULLB_U_LANE : SInst<"svmullb_lane[_{d}]", "dhhi", "UiUl", MergeNone, "aarch64_sve_umullb_lane", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; 1642def SVMULLT_S_LANE : SInst<"svmullt_lane[_{d}]", "dhhi", "il", MergeNone, "aarch64_sve_smullt_lane", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; 1643def SVMULLT_U_LANE : SInst<"svmullt_lane[_{d}]", "dhhi", "UiUl", MergeNone, "aarch64_sve_umullt_lane", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; 1644def SVQDMLALB_LANE : SInst<"svqdmlalb_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_sqdmlalb_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 1645def SVQDMLALT_LANE : SInst<"svqdmlalt_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_sqdmlalt_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 1646def SVQDMLSLB_LANE : SInst<"svqdmlslb_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_sqdmlslb_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 1647def SVQDMLSLT_LANE : SInst<"svqdmlslt_lane[_{d}]", "ddhhi", "il", MergeNone, "aarch64_sve_sqdmlslt_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 1648def SVQDMULLB_LANE : SInst<"svqdmullb_lane[_{d}]", "dhhi", "il", MergeNone, "aarch64_sve_sqdmullb_lane", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; 1649def SVQDMULLT_LANE : SInst<"svqdmullt_lane[_{d}]", "dhhi", "il", MergeNone, "aarch64_sve_sqdmullt_lane", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; 1650} 1651 1652//////////////////////////////////////////////////////////////////////////////// 1653// SVE2 - Narrowing DSP operations 1654 1655let SVETargetGuard = "sve2", SMETargetGuard = "sme" in { 1656def SVADDHNB : SInst<"svaddhnb[_{d}]", "hdd", "silUsUiUl", MergeNone, "aarch64_sve_addhnb", [VerifyRuntimeMode]>; 1657def SVADDHNT : SInst<"svaddhnt[_{d}]", "hhdd", "silUsUiUl", MergeNone, "aarch64_sve_addhnt", [VerifyRuntimeMode]>; 1658def SVRADDHNB : SInst<"svraddhnb[_{d}]", "hdd", "silUsUiUl", MergeNone, "aarch64_sve_raddhnb", [VerifyRuntimeMode]>; 1659def SVRADDHNT : SInst<"svraddhnt[_{d}]", "hhdd", "silUsUiUl", MergeNone, "aarch64_sve_raddhnt", [VerifyRuntimeMode]>; 1660def SVRSUBHNB : SInst<"svrsubhnb[_{d}]", "hdd", "silUsUiUl", MergeNone, "aarch64_sve_rsubhnb", [VerifyRuntimeMode]>; 1661def SVRSUBHNT : SInst<"svrsubhnt[_{d}]", "hhdd", "silUsUiUl", MergeNone, "aarch64_sve_rsubhnt", [VerifyRuntimeMode]>; 1662def SVSUBHNB : SInst<"svsubhnb[_{d}]", "hdd", "silUsUiUl", MergeNone, "aarch64_sve_subhnb", [VerifyRuntimeMode]>; 1663def SVSUBHNT : SInst<"svsubhnt[_{d}]", "hhdd", "silUsUiUl", MergeNone, "aarch64_sve_subhnt", [VerifyRuntimeMode]>; 1664 1665def SVADDHNB_N : SInst<"svaddhnb[_n_{d}]", "hda", "silUsUiUl", MergeNone, "aarch64_sve_addhnb", [VerifyRuntimeMode]>; 1666def SVADDHNT_N : SInst<"svaddhnt[_n_{d}]", "hhda", "silUsUiUl", MergeNone, "aarch64_sve_addhnt", [VerifyRuntimeMode]>; 1667def SVRADDHNB_N : SInst<"svraddhnb[_n_{d}]", "hda", "silUsUiUl", MergeNone, "aarch64_sve_raddhnb", [VerifyRuntimeMode]>; 1668def SVRADDHNT_N : SInst<"svraddhnt[_n_{d}]", "hhda", "silUsUiUl", MergeNone, "aarch64_sve_raddhnt", [VerifyRuntimeMode]>; 1669def SVRSUBHNB_N : SInst<"svrsubhnb[_n_{d}]", "hda", "silUsUiUl", MergeNone, "aarch64_sve_rsubhnb", [VerifyRuntimeMode]>; 1670def SVRSUBHNT_N : SInst<"svrsubhnt[_n_{d}]", "hhda", "silUsUiUl", MergeNone, "aarch64_sve_rsubhnt", [VerifyRuntimeMode]>; 1671def SVSUBHNB_N : SInst<"svsubhnb[_n_{d}]", "hda", "silUsUiUl", MergeNone, "aarch64_sve_subhnb", [VerifyRuntimeMode]>; 1672def SVSUBHNT_N : SInst<"svsubhnt[_n_{d}]", "hhda", "silUsUiUl", MergeNone, "aarch64_sve_subhnt", [VerifyRuntimeMode]>; 1673 1674def SVSHRNB : SInst<"svshrnb[_n_{d}]", "hdi", "silUsUiUl", MergeNone, "aarch64_sve_shrnb", [VerifyRuntimeMode], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; 1675def SVRSHRNB : SInst<"svrshrnb[_n_{d}]", "hdi", "silUsUiUl", MergeNone, "aarch64_sve_rshrnb", [VerifyRuntimeMode], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; 1676def SVQSHRUNB : SInst<"svqshrunb[_n_{d}]", "edi", "sil", MergeNone, "aarch64_sve_sqshrunb", [VerifyRuntimeMode], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; 1677def SVQRSHRUNB : SInst<"svqrshrunb[_n_{d}]", "edi", "sil", MergeNone, "aarch64_sve_sqrshrunb", [VerifyRuntimeMode], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; 1678def SVQSHRNB_S : SInst<"svqshrnb[_n_{d}]", "hdi", "sil", MergeNone, "aarch64_sve_sqshrnb", [VerifyRuntimeMode], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; 1679def SVQSHRNB_U : SInst<"svqshrnb[_n_{d}]", "hdi", "UsUiUl", MergeNone, "aarch64_sve_uqshrnb", [VerifyRuntimeMode], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; 1680def SVQRSHRNB_S : SInst<"svqrshrnb[_n_{d}]", "hdi", "sil", MergeNone, "aarch64_sve_sqrshrnb", [VerifyRuntimeMode], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; 1681def SVQRSHRNB_U : SInst<"svqrshrnb[_n_{d}]", "hdi", "UsUiUl", MergeNone, "aarch64_sve_uqrshrnb", [VerifyRuntimeMode], [ImmCheck<1, ImmCheckShiftRightNarrow, 0>]>; 1682 1683def SVSHRNT : SInst<"svshrnt[_n_{d}]", "hhdi", "silUsUiUl", MergeNone, "aarch64_sve_shrnt", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; 1684def SVRSHRNT : SInst<"svrshrnt[_n_{d}]", "hhdi", "silUsUiUl", MergeNone, "aarch64_sve_rshrnt", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; 1685def SVQSHRUNT : SInst<"svqshrunt[_n_{d}]", "eedi", "sil", MergeNone, "aarch64_sve_sqshrunt", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; 1686def SVQRSHRUNT : SInst<"svqrshrunt[_n_{d}]", "eedi", "sil", MergeNone, "aarch64_sve_sqrshrunt", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; 1687def SVQSHRNT_S : SInst<"svqshrnt[_n_{d}]", "hhdi", "sil", MergeNone, "aarch64_sve_sqshrnt", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; 1688def SVQSHRNT_U : SInst<"svqshrnt[_n_{d}]", "hhdi", "UsUiUl", MergeNone, "aarch64_sve_uqshrnt", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; 1689def SVQRSHRNT_S : SInst<"svqrshrnt[_n_{d}]", "hhdi", "sil", MergeNone, "aarch64_sve_sqrshrnt", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; 1690def SVQRSHRNT_U : SInst<"svqrshrnt[_n_{d}]", "hhdi", "UsUiUl", MergeNone, "aarch64_sve_uqrshrnt", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckShiftRightNarrow, 1>]>; 1691} 1692//////////////////////////////////////////////////////////////////////////////// 1693// SVE2 - Unary narrowing operations 1694 1695let SVETargetGuard = "sve2", SMETargetGuard = "sme" in { 1696def SVQXTNB_S : SInst<"svqxtnb[_{d}]", "hd", "sil", MergeNone, "aarch64_sve_sqxtnb", [VerifyRuntimeMode]>; 1697def SVQXTNB_U : SInst<"svqxtnb[_{d}]", "hd", "UsUiUl", MergeNone, "aarch64_sve_uqxtnb", [VerifyRuntimeMode]>; 1698def SVQXTUNB_S : SInst<"svqxtunb[_{d}]", "ed", "sil", MergeNone, "aarch64_sve_sqxtunb", [VerifyRuntimeMode]>; 1699 1700def SVQXTNT_S : SInst<"svqxtnt[_{d}]", "hhd", "sil", MergeNone, "aarch64_sve_sqxtnt", [VerifyRuntimeMode]>; 1701def SVQXTNT_U : SInst<"svqxtnt[_{d}]", "hhd", "UsUiUl", MergeNone, "aarch64_sve_uqxtnt", [VerifyRuntimeMode]>; 1702def SVQXTUNT_S : SInst<"svqxtunt[_{d}]", "eed", "sil", MergeNone, "aarch64_sve_sqxtunt", [VerifyRuntimeMode]>; 1703} 1704 1705//////////////////////////////////////////////////////////////////////////////// 1706// SVE2 - Widening complex integer arithmetic 1707 1708let SVETargetGuard = "sve2", SMETargetGuard = "sme" in { 1709defm SVADDLBT : SInstWideDSPLong<"svaddlbt", "sil", "aarch64_sve_saddlbt">; 1710defm SVSUBLBT : SInstWideDSPLong<"svsublbt", "sil", "aarch64_sve_ssublbt">; 1711defm SVSUBLTB : SInstWideDSPLong<"svsubltb", "sil", "aarch64_sve_ssubltb">; 1712 1713defm SVQDMLALBT : SInstWideDSPAcc<"svqdmlalbt", "sil", "aarch64_sve_sqdmlalbt">; 1714defm SVQDMLSLBT : SInstWideDSPAcc<"svqdmlslbt", "sil", "aarch64_sve_sqdmlslbt">; 1715} 1716 1717//////////////////////////////////////////////////////////////////////////////// 1718// SVE2 - Non-temporal gather/scatter 1719let SVETargetGuard = "sve2", SMETargetGuard = InvalidMode in { 1720// Non-temporal gather load one vector (vector base) 1721def SVLDNT1_GATHER_BASES_U : MInst<"svldnt1_gather[_{2}base]_{0}", "dPu", "ilUiUlfd", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ldnt1_gather_scalar_offset">; 1722def SVLDNT1SB_GATHER_BASES_U : MInst<"svldnt1sb_gather[_{2}base]_{0}", "dPu", "ilUiUl", [IsGatherLoad], MemEltTyInt8, "aarch64_sve_ldnt1_gather_scalar_offset">; 1723def SVLDNT1UB_GATHER_BASES_U : MInst<"svldnt1ub_gather[_{2}base]_{0}", "dPu", "ilUiUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ldnt1_gather_scalar_offset">; 1724def SVLDNT1SH_GATHER_BASES_U : MInst<"svldnt1sh_gather[_{2}base]_{0}", "dPu", "ilUiUl", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ldnt1_gather_scalar_offset">; 1725def SVLDNT1UH_GATHER_BASES_U : MInst<"svldnt1uh_gather[_{2}base]_{0}", "dPu", "ilUiUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldnt1_gather_scalar_offset">; 1726def SVLDNT1SW_GATHER_BASES_U : MInst<"svldnt1sw_gather[_{2}base]_{0}", "dPu", "lUl", [IsGatherLoad], MemEltTyInt32, "aarch64_sve_ldnt1_gather_scalar_offset">; 1727def SVLDNT1UW_GATHER_BASES_U : MInst<"svldnt1uw_gather[_{2}base]_{0}", "dPu", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldnt1_gather_scalar_offset">; 1728 1729// Non-temporal gather load one vector (scalar base, signed vector offset in bytes) 1730def SVLDNT1_GATHER_64B_OFFSETS_S : MInst<"svldnt1_gather_[{3}]offset[_{0}]", "dPcx", "lUld", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ldnt1_gather">; 1731def SVLDNT1SB_GATHER_64B_OFFSETS_S : MInst<"svldnt1sb_gather_[{3}]offset_{0}", "dPSx", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt8, "aarch64_sve_ldnt1_gather">; 1732def SVLDNT1UB_GATHER_64B_OFFSETS_S : MInst<"svldnt1ub_gather_[{3}]offset_{0}", "dPWx", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ldnt1_gather">; 1733def SVLDNT1SH_GATHER_64B_OFFSETS_S : MInst<"svldnt1sh_gather_[{3}]offset_{0}", "dPTx", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt16, "aarch64_sve_ldnt1_gather">; 1734def SVLDNT1UH_GATHER_64B_OFFSETS_S : MInst<"svldnt1uh_gather_[{3}]offset_{0}", "dPXx", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldnt1_gather">; 1735def SVLDNT1SW_GATHER_64B_OFFSETS_S : MInst<"svldnt1sw_gather_[{3}]offset_{0}", "dPUx", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt32, "aarch64_sve_ldnt1_gather">; 1736def SVLDNT1UW_GATHER_64B_OFFSETS_S : MInst<"svldnt1uw_gather_[{3}]offset_{0}", "dPYx", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldnt1_gather">; 1737 1738// Non-temporal gather load one vector (scalar base, unsigned vector offset in bytes) 1739def SVLDNT1_GATHER_64B_OFFSETS_U : MInst<"svldnt1_gather_[{3}]offset[_{0}]", "dPcu", "lUld", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ldnt1_gather">; 1740def SVLDNT1SB_GATHER_64B_OFFSETS_U : MInst<"svldnt1sb_gather_[{3}]offset_{0}", "dPSu", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt8, "aarch64_sve_ldnt1_gather">; 1741def SVLDNT1UB_GATHER_64B_OFFSETS_U : MInst<"svldnt1ub_gather_[{3}]offset_{0}", "dPWu", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ldnt1_gather">; 1742def SVLDNT1SH_GATHER_64B_OFFSETS_U : MInst<"svldnt1sh_gather_[{3}]offset_{0}", "dPTu", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt16, "aarch64_sve_ldnt1_gather">; 1743def SVLDNT1UH_GATHER_64B_OFFSETS_U : MInst<"svldnt1uh_gather_[{3}]offset_{0}", "dPXu", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldnt1_gather">; 1744def SVLDNT1SW_GATHER_64B_OFFSETS_U : MInst<"svldnt1sw_gather_[{3}]offset_{0}", "dPUu", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt32, "aarch64_sve_ldnt1_gather">; 1745def SVLDNT1UW_GATHER_64B_OFFSETS_U : MInst<"svldnt1uw_gather_[{3}]offset_{0}", "dPYu", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldnt1_gather">; 1746 1747def SVLDNT1_GATHER_32B_OFFSETS_U : MInst<"svldnt1_gather_[{3}]offset[_{0}]", "dPcu", "iUif", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ldnt1_gather_uxtw">; 1748def SVLDNT1SB_GATHER_32B_OFFSETS_U : MInst<"svldnt1sb_gather_[{3}]offset_{0}", "dPSu", "iUi", [IsGatherLoad, IsByteIndexed], MemEltTyInt8, "aarch64_sve_ldnt1_gather_uxtw">; 1749def SVLDNT1UB_GATHER_32B_OFFSETS_U : MInst<"svldnt1ub_gather_[{3}]offset_{0}", "dPWu", "iUi", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ldnt1_gather_uxtw">; 1750def SVLDNT1SH_GATHER_32B_OFFSETS_U : MInst<"svldnt1sh_gather_[{3}]offset_{0}", "dPTu", "iUi", [IsGatherLoad, IsByteIndexed], MemEltTyInt16, "aarch64_sve_ldnt1_gather_uxtw">; 1751def SVLDNT1UH_GATHER_32B_OFFSETS_U : MInst<"svldnt1uh_gather_[{3}]offset_{0}", "dPXu", "iUi", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldnt1_gather_uxtw">; 1752 1753// Non-temporal gather load one vector (vector base, scalar offset in bytes) 1754def SVLDNT1_GATHER_OFFSET_S : MInst<"svldnt1_gather[_{2}base]_offset_{0}", "dPul", "ilUiUlfd", [IsGatherLoad, IsByteIndexed], MemEltTyDefault, "aarch64_sve_ldnt1_gather_scalar_offset">; 1755def SVLDNT1SB_GATHER_OFFSET_S : MInst<"svldnt1sb_gather[_{2}base]_offset_{0}", "dPul", "ilUiUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt8, "aarch64_sve_ldnt1_gather_scalar_offset">; 1756def SVLDNT1UB_GATHER_OFFSET_S : MInst<"svldnt1ub_gather[_{2}base]_offset_{0}", "dPul", "ilUiUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt8, "aarch64_sve_ldnt1_gather_scalar_offset">; 1757def SVLDNT1SH_GATHER_OFFSET_S : MInst<"svldnt1sh_gather[_{2}base]_offset_{0}", "dPul", "ilUiUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt16, "aarch64_sve_ldnt1_gather_scalar_offset">; 1758def SVLDNT1UH_GATHER_OFFSET_S : MInst<"svldnt1uh_gather[_{2}base]_offset_{0}", "dPul", "ilUiUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldnt1_gather_scalar_offset">; 1759def SVLDNT1SW_GATHER_OFFSET_S : MInst<"svldnt1sw_gather[_{2}base]_offset_{0}", "dPul", "lUl", [IsGatherLoad, IsByteIndexed], MemEltTyInt32, "aarch64_sve_ldnt1_gather_scalar_offset">; 1760def SVLDNT1UW_GATHER_OFFSET_S : MInst<"svldnt1uw_gather[_{2}base]_offset_{0}", "dPul", "lUl", [IsGatherLoad, IsByteIndexed, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldnt1_gather_scalar_offset">; 1761 1762// Non-temporal gather load one vector (scalar base, signed vector index) 1763def SVLDNT1_GATHER_64B_INDICES_S : MInst<"svldnt1_gather_[{3}]index[_{0}]", "dPcx", "lUld", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ldnt1_gather_index">; 1764def SVLDNT1SH_GATHER_64B_INDICES_S : MInst<"svldnt1sh_gather_[{3}]index_{0}", "dPTx", "lUl", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ldnt1_gather_index">; 1765def SVLDNT1UH_GATHER_64B_INDICES_S : MInst<"svldnt1uh_gather_[{3}]index_{0}", "dPXx", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldnt1_gather_index">; 1766def SVLDNT1SW_GATHER_64B_INDICES_S : MInst<"svldnt1sw_gather_[{3}]index_{0}", "dPUx", "lUl", [IsGatherLoad], MemEltTyInt32, "aarch64_sve_ldnt1_gather_index">; 1767def SVLDNT1UW_GATHER_64B_INDICES_S : MInst<"svldnt1uw_gather_[{3}]index_{0}", "dPYx", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldnt1_gather_index">; 1768 1769// Non temporal gather load one vector (scalar base, unsigned vector index) 1770def SVLDNT1_GATHER_64B_INDICES_U : MInst<"svldnt1_gather_[{3}]index[_{0}]", "dPcu", "lUld", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ldnt1_gather_index">; 1771def SVLDNT1SH_GATHER_64B_INDICES_U : MInst<"svldnt1sh_gather_[{3}]index_{0}", "dPTu", "lUl", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ldnt1_gather_index">; 1772def SVLDNT1UH_GATHER_64B_INDICES_U : MInst<"svldnt1uh_gather_[{3}]index_{0}", "dPXu", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldnt1_gather_index">; 1773def SVLDNT1SW_GATHER_64B_INDICES_U : MInst<"svldnt1sw_gather_[{3}]index_{0}", "dPUu", "lUl", [IsGatherLoad], MemEltTyInt32, "aarch64_sve_ldnt1_gather_index">; 1774def SVLDNT1UW_GATHER_64B_INDICES_U : MInst<"svldnt1uw_gather_[{3}]index_{0}", "dPYu", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldnt1_gather_index">; 1775 1776// Non-temporal gather load one vector (vector base, signed scalar index) 1777def SVLDNT1_GATHER_INDEX_S : MInst<"svldnt1_gather[_{2}base]_index_{0}", "dPul", "ilUiUlfd", [IsGatherLoad], MemEltTyDefault, "aarch64_sve_ldnt1_gather_scalar_offset">; 1778def SVLDNT1SH_GATHER_INDEX_S : MInst<"svldnt1sh_gather[_{2}base]_index_{0}", "dPul", "ilUiUl", [IsGatherLoad], MemEltTyInt16, "aarch64_sve_ldnt1_gather_scalar_offset">; 1779def SVLDNT1UH_GATHER_INDEX_S : MInst<"svldnt1uh_gather[_{2}base]_index_{0}", "dPul", "ilUiUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt16, "aarch64_sve_ldnt1_gather_scalar_offset">; 1780def SVLDNT1SW_GATHER_INDEX_S : MInst<"svldnt1sw_gather[_{2}base]_index_{0}", "dPul", "lUl", [IsGatherLoad], MemEltTyInt32, "aarch64_sve_ldnt1_gather_scalar_offset">; 1781def SVLDNT1UW_GATHER_INDEX_S : MInst<"svldnt1uw_gather[_{2}base]_index_{0}", "dPul", "lUl", [IsGatherLoad, IsZExtReturn], MemEltTyInt32, "aarch64_sve_ldnt1_gather_scalar_offset">; 1782 1783// Non-temporal scatter store one vector (vector base) 1784def SVSTNT1_SCATTER_BASES_U : MInst<"svstnt1_scatter[_{2}base_{d}]", "vPud", "ilUiUlfd", [IsScatterStore], MemEltTyDefault, "aarch64_sve_stnt1_scatter_scalar_offset">; 1785def SVSTNT1B_SCATTER_BASES_U : MInst<"svstnt1b_scatter[_{2}base_{d}]", "vPud", "ilUiUl", [IsScatterStore], MemEltTyInt8, "aarch64_sve_stnt1_scatter_scalar_offset">; 1786def SVSTNT1H_SCATTER_BASES_U : MInst<"svstnt1h_scatter[_{2}base_{d}]", "vPud", "ilUiUl", [IsScatterStore], MemEltTyInt16, "aarch64_sve_stnt1_scatter_scalar_offset">; 1787def SVSTNT1W_SCATTER_BASES_U : MInst<"svstnt1w_scatter[_{2}base_{d}]", "vPud", "lUl", [IsScatterStore], MemEltTyInt32, "aarch64_sve_stnt1_scatter_scalar_offset">; 1788 1789// Non-temporal scatter store one vector (scalar base, signed vector offset in bytes) 1790def SVSTNT1_SCATTER_64B_OFFSETS_S : MInst<"svstnt1_scatter_[{3}]offset[_{d}]", "vPpxd", "lUld", [IsScatterStore, IsByteIndexed], MemEltTyDefault, "aarch64_sve_stnt1_scatter">; 1791def SVSTNT1B_SCATTER_64B_OFFSETS_SS : MInst<"svstnt1b_scatter_[{3}]offset[_{d}]", "vPAxd", "l", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_stnt1_scatter">; 1792def SVSTNT1B_SCATTER_64B_OFFSETS_SU : MInst<"svstnt1b_scatter_[{3}]offset[_{d}]", "vPExd", "Ul", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_stnt1_scatter">; 1793def SVSTNT1H_SCATTER_64B_OFFSETS_SS : MInst<"svstnt1h_scatter_[{3}]offset[_{d}]", "vPBxd", "l", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_stnt1_scatter">; 1794def SVSTNT1H_SCATTER_64B_OFFSETS_SU : MInst<"svstnt1h_scatter_[{3}]offset[_{d}]", "vPFxd", "Ul", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_stnt1_scatter">; 1795def SVSTNT1W_SCATTER_64B_OFFSETS_SS : MInst<"svstnt1w_scatter_[{3}]offset[_{d}]", "vPCxd", "l", [IsScatterStore, IsByteIndexed], MemEltTyInt32, "aarch64_sve_stnt1_scatter">; 1796def SVSTNT1W_SCATTER_64B_OFFSETS_SU : MInst<"svstnt1w_scatter_[{3}]offset[_{d}]", "vPGxd", "Ul", [IsScatterStore, IsByteIndexed], MemEltTyInt32, "aarch64_sve_stnt1_scatter">; 1797 1798// Non-temporal scatter store one vector (scalar base, unsigned vector offset in bytes) 1799def SVSTNT1_SCATTER_64B_OFFSETS_U : MInst<"svstnt1_scatter_[{3}]offset[_{d}]", "vPpud", "lUld", [IsScatterStore, IsByteIndexed], MemEltTyDefault, "aarch64_sve_stnt1_scatter">; 1800def SVSTNT1B_SCATTER_64B_OFFSETS_US : MInst<"svstnt1b_scatter_[{3}]offset[_{d}]", "vPAud", "l", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_stnt1_scatter">; 1801def SVSTNT1B_SCATTER_64B_OFFSETS_UU : MInst<"svstnt1b_scatter_[{3}]offset[_{d}]", "vPEud", "Ul", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_stnt1_scatter">; 1802def SVSTNT1H_SCATTER_64B_OFFSETS_US : MInst<"svstnt1h_scatter_[{3}]offset[_{d}]", "vPBud", "l", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_stnt1_scatter">; 1803def SVSTNT1H_SCATTER_64B_OFFSETS_UU : MInst<"svstnt1h_scatter_[{3}]offset[_{d}]", "vPFud", "Ul", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_stnt1_scatter">; 1804def SVSTNT1W_SCATTER_64B_OFFSETS_US : MInst<"svstnt1w_scatter_[{3}]offset[_{d}]", "vPCud", "l", [IsScatterStore, IsByteIndexed], MemEltTyInt32, "aarch64_sve_stnt1_scatter">; 1805def SVSTNT1W_SCATTER_64B_OFFSETS_UU : MInst<"svstnt1w_scatter_[{3}]offset[_{d}]", "vPGud", "Ul", [IsScatterStore, IsByteIndexed], MemEltTyInt32, "aarch64_sve_stnt1_scatter">; 1806 1807def SVSTNT1_SCATTER_32B_OFFSETS_U : MInst<"svstnt1_scatter_[{3}]offset[_{d}]", "vPpud", "iUif", [IsScatterStore, IsByteIndexed], MemEltTyDefault, "aarch64_sve_stnt1_scatter_uxtw">; 1808def SVSTNT1B_SCATTER_32B_OFFSETS_US : MInst<"svstnt1b_scatter_[{3}]offset[_{d}]", "vPAud", "i", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_stnt1_scatter_uxtw">; 1809def SVSTNT1B_SCATTER_32B_OFFSETS_UU : MInst<"svstnt1b_scatter_[{3}]offset[_{d}]", "vPEud", "Ui", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_stnt1_scatter_uxtw">; 1810def SVSTNT1H_SCATTER_32B_OFFSETS_US : MInst<"svstnt1h_scatter_[{3}]offset[_{d}]", "vPBud", "i", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_stnt1_scatter_uxtw">; 1811def SVSTNT1H_SCATTER_32B_OFFSETS_UU : MInst<"svstnt1h_scatter_[{3}]offset[_{d}]", "vPFud", "Ui", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_stnt1_scatter_uxtw">; 1812 1813// Non-temporal scatter store one vector (vector base, scalar offset in bytes) 1814def SVSTNT1_SCATTER_OFFSET_S : MInst<"svstnt1_scatter[_{2}base]_offset[_{d}]", "vPuld", "ilUiUlfd", [IsScatterStore, IsByteIndexed], MemEltTyDefault, "aarch64_sve_stnt1_scatter_scalar_offset">; 1815def SVSTNT1B_SCATTER_OFFSET_S : MInst<"svstnt1b_scatter[_{2}base]_offset[_{d}]", "vPuld", "ilUiUl", [IsScatterStore, IsByteIndexed], MemEltTyInt8, "aarch64_sve_stnt1_scatter_scalar_offset">; 1816def SVSTNT1H_SCATTER_OFFSET_S : MInst<"svstnt1h_scatter[_{2}base]_offset[_{d}]", "vPuld", "ilUiUl", [IsScatterStore, IsByteIndexed], MemEltTyInt16, "aarch64_sve_stnt1_scatter_scalar_offset">; 1817def SVSTNT1W_SCATTER_OFFSET_S : MInst<"svstnt1w_scatter[_{2}base]_offset[_{d}]", "vPuld", "lUl", [IsScatterStore, IsByteIndexed], MemEltTyInt32, "aarch64_sve_stnt1_scatter_scalar_offset">; 1818 1819// Non-temporal scatter store one vector (scalar base, signed vector index) 1820def SVSTNT1_SCATTER_INDICES_S : MInst<"svstnt1_scatter_[{3}]index[_{d}]", "vPpxd", "lUld", [IsScatterStore], MemEltTyDefault, "aarch64_sve_stnt1_scatter_index">; 1821def SVSTNT1H_SCATTER_INDICES_SS : MInst<"svstnt1h_scatter_[{3}]index[_{d}]", "vPBxd", "l", [IsScatterStore], MemEltTyInt16, "aarch64_sve_stnt1_scatter_index">; 1822def SVSTNT1H_SCATTER_INDICES_SU : MInst<"svstnt1h_scatter_[{3}]index[_{d}]", "vPFxd", "Ul", [IsScatterStore], MemEltTyInt16, "aarch64_sve_stnt1_scatter_index">; 1823def SVSTNT1W_SCATTER_INDICES_SS : MInst<"svstnt1w_scatter_[{3}]index[_{d}]", "vPCxd", "l", [IsScatterStore], MemEltTyInt32, "aarch64_sve_stnt1_scatter_index">; 1824def SVSTNT1W_SCATTER_INDICES_SU : MInst<"svstnt1w_scatter_[{3}]index[_{d}]", "vPGxd", "Ul", [IsScatterStore], MemEltTyInt32, "aarch64_sve_stnt1_scatter_index">; 1825 1826// Non-temporal scatter store one vector (scalar base, unsigned vector index) 1827def SVSTNT1_SCATTER_INDICES_U : MInst<"svstnt1_scatter_[{3}]index[_{d}]", "vPpud", "lUld", [IsScatterStore], MemEltTyDefault, "aarch64_sve_stnt1_scatter_index">; 1828def SVSTNT1H_SCATTER_INDICES_US : MInst<"svstnt1h_scatter_[{3}]index[_{d}]", "vPBud", "l", [IsScatterStore], MemEltTyInt16, "aarch64_sve_stnt1_scatter_index">; 1829def SVSTNT1H_SCATTER_INDICES_UU : MInst<"svstnt1h_scatter_[{3}]index[_{d}]", "vPFud", "Ul", [IsScatterStore], MemEltTyInt16, "aarch64_sve_stnt1_scatter_index">; 1830def SVSTNT1W_SCATTER_INDICES_US : MInst<"svstnt1w_scatter_[{3}]index[_{d}]", "vPCud", "l", [IsScatterStore], MemEltTyInt32, "aarch64_sve_stnt1_scatter_index">; 1831def SVSTNT1W_SCATTER_INDICES_UU : MInst<"svstnt1w_scatter_[{3}]index[_{d}]", "vPGud", "Ul", [IsScatterStore], MemEltTyInt32, "aarch64_sve_stnt1_scatter_index">; 1832 1833// Non-temporal scatter store one vector (vector base, signed scalar index) 1834def SVSTNT1_SCATTER_INDEX_S : MInst<"svstnt1_scatter[_{2}base]_index[_{d}]", "vPuld", "ilUiUlfd", [IsScatterStore], MemEltTyDefault, "aarch64_sve_stnt1_scatter_scalar_offset">; 1835def SVSTNT1H_SCATTER_INDEX_S : MInst<"svstnt1h_scatter[_{2}base]_index[_{d}]", "vPuld", "ilUiUl", [IsScatterStore], MemEltTyInt16, "aarch64_sve_stnt1_scatter_scalar_offset">; 1836def SVSTNT1W_SCATTER_INDEX_S : MInst<"svstnt1w_scatter[_{2}base]_index[_{d}]", "vPuld", "lUl", [IsScatterStore], MemEltTyInt32, "aarch64_sve_stnt1_scatter_scalar_offset">; 1837} 1838 1839//////////////////////////////////////////////////////////////////////////////// 1840// SVE2 - Polynomial arithmetic 1841 1842let SVETargetGuard = "sve2", SMETargetGuard = "sme" in { 1843def SVEORBT : SInst<"sveorbt[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_eorbt", [VerifyRuntimeMode]>; 1844def SVEORBT_N : SInst<"sveorbt[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_eorbt", [VerifyRuntimeMode]>; 1845def SVEORTB : SInst<"sveortb[_{d}]", "dddd", "csilUcUsUiUl", MergeNone, "aarch64_sve_eortb", [VerifyRuntimeMode]>; 1846def SVEORTB_N : SInst<"sveortb[_n_{d}]", "ddda", "csilUcUsUiUl", MergeNone, "aarch64_sve_eortb", [VerifyRuntimeMode]>; 1847def SVPMUL : SInst<"svpmul[_{d}]", "ddd", "Uc", MergeNone, "aarch64_sve_pmul", [VerifyRuntimeMode]>; 1848def SVPMUL_N : SInst<"svpmul[_n_{d}]", "dda", "Uc", MergeNone, "aarch64_sve_pmul", [VerifyRuntimeMode]>; 1849def SVPMULLB : SInst<"svpmullb[_{d}]", "dhh", "UsUl", MergeNone, "", [VerifyRuntimeMode]>; 1850def SVPMULLB_N : SInst<"svpmullb[_n_{d}]", "dhR", "UsUl", MergeNone, "", [VerifyRuntimeMode]>; 1851def SVPMULLB_PAIR : SInst<"svpmullb_pair[_{d}]", "ddd", "UcUi", MergeNone, "aarch64_sve_pmullb_pair", [VerifyRuntimeMode]>; 1852def SVPMULLB_PAIR_N : SInst<"svpmullb_pair[_n_{d}]", "dda", "UcUi", MergeNone, "aarch64_sve_pmullb_pair", [VerifyRuntimeMode]>; 1853def SVPMULLT : SInst<"svpmullt[_{d}]", "dhh", "UsUl", MergeNone, "", [VerifyRuntimeMode]>; 1854def SVPMULLT_N : SInst<"svpmullt[_n_{d}]", "dhR", "UsUl", MergeNone, "", [VerifyRuntimeMode]>; 1855def SVPMULLT_PAIR : SInst<"svpmullt_pair[_{d}]", "ddd", "UcUi", MergeNone, "aarch64_sve_pmullt_pair", [VerifyRuntimeMode]>; 1856def SVPMULLT_PAIR_N : SInst<"svpmullt_pair[_n_{d}]", "dda", "UcUi", MergeNone, "aarch64_sve_pmullt_pair", [VerifyRuntimeMode]>; 1857} 1858 1859//////////////////////////////////////////////////////////////////////////////// 1860// SVE2 - Complex integer dot product 1861 1862let SVETargetGuard = "sve2", SMETargetGuard = "sme" in { 1863def SVCDOT : SInst<"svcdot[_{d}]", "ddqqi", "il", MergeNone, "aarch64_sve_cdot", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckComplexRotAll90>]>; 1864def SVCDOT_LANE : SInst<"svcdot_lane[_{d}]", "ddqqii", "il", MergeNone, "aarch64_sve_cdot_lane", [VerifyRuntimeMode], [ImmCheck<4, ImmCheckComplexRotAll90>, 1865 ImmCheck<3, ImmCheckLaneIndexDot, 2>]>; 1866} 1867 1868//////////////////////////////////////////////////////////////////////////////// 1869// SVE2 - Floating-point widening multiply-accumulate 1870 1871let SVETargetGuard = "sve2", SMETargetGuard = "sme" in { 1872def SVMLALB_F : SInst<"svmlalb[_{d}]", "ddhh", "f", MergeNone, "aarch64_sve_fmlalb", [VerifyRuntimeMode]>; 1873def SVMLALB_F_N : SInst<"svmlalb[_n_{d}]", "ddhR", "f", MergeNone, "aarch64_sve_fmlalb", [VerifyRuntimeMode]>; 1874def SVMLALB_F_LANE : SInst<"svmlalb_lane[_{d}]", "ddhhi", "f", MergeNone, "aarch64_sve_fmlalb_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 1875def SVMLALT_F : SInst<"svmlalt[_{d}]", "ddhh", "f", MergeNone, "aarch64_sve_fmlalt", [VerifyRuntimeMode]>; 1876def SVMLALT_F_N : SInst<"svmlalt[_n_{d}]", "ddhR", "f", MergeNone, "aarch64_sve_fmlalt", [VerifyRuntimeMode]>; 1877def SVMLALT_F_LANE : SInst<"svmlalt_lane[_{d}]", "ddhhi", "f", MergeNone, "aarch64_sve_fmlalt_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 1878def SVMLSLB_F : SInst<"svmlslb[_{d}]", "ddhh", "f", MergeNone, "aarch64_sve_fmlslb", [VerifyRuntimeMode]>; 1879def SVMLSLB_F_N : SInst<"svmlslb[_n_{d}]", "ddhR", "f", MergeNone, "aarch64_sve_fmlslb", [VerifyRuntimeMode]>; 1880def SVMLSLB_F_LANE : SInst<"svmlslb_lane[_{d}]", "ddhhi", "f", MergeNone, "aarch64_sve_fmlslb_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 1881def SVMLSLT_F : SInst<"svmlslt[_{d}]", "ddhh", "f", MergeNone, "aarch64_sve_fmlslt", [VerifyRuntimeMode]>; 1882def SVMLSLT_F_N : SInst<"svmlslt[_n_{d}]", "ddhR", "f", MergeNone, "aarch64_sve_fmlslt", [VerifyRuntimeMode]>; 1883def SVMLSLT_F_LANE : SInst<"svmlslt_lane[_{d}]", "ddhhi", "f", MergeNone, "aarch64_sve_fmlslt_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 1884} 1885 1886//////////////////////////////////////////////////////////////////////////////// 1887// SVE2 - Floating-point integer binary logarithm 1888 1889let SVETargetGuard = "sve2", SMETargetGuard = "sme" in { 1890def SVLOGB_M : SInst<"svlogb[_{d}]", "xxPd", "hfd", MergeOp1, "aarch64_sve_flogb", [VerifyRuntimeMode]>; 1891def SVLOGB_X : SInst<"svlogb[_{d}]", "xPd", "hfd", MergeAnyExp, "aarch64_sve_flogb", [VerifyRuntimeMode]>; 1892def SVLOGB_Z : SInst<"svlogb[_{d}]", "xPd", "hfd", MergeZeroExp, "aarch64_sve_flogb", [VerifyRuntimeMode]>; 1893} 1894 1895//////////////////////////////////////////////////////////////////////////////// 1896// SVE2 - Vector Histogram count 1897 1898let SVETargetGuard = "sve2", SMETargetGuard = InvalidMode in { 1899def SVHISTCNT : SInst<"svhistcnt[_{d}]_z", "uPdd", "ilUiUl", MergeNone, "aarch64_sve_histcnt">; 1900def SVHISTSEG : SInst<"svhistseg[_{d}]", "udd", "cUc", MergeNone, "aarch64_sve_histseg">; 1901} 1902 1903//////////////////////////////////////////////////////////////////////////////// 1904// SVE2 - Character match 1905 1906let SVETargetGuard = "sve2", SMETargetGuard = InvalidMode in { 1907def SVMATCH : SInst<"svmatch[_{d}]", "PPdd", "csUcUs", MergeNone, "aarch64_sve_match">; 1908def SVNMATCH : SInst<"svnmatch[_{d}]", "PPdd", "csUcUs", MergeNone, "aarch64_sve_nmatch">; 1909} 1910 1911//////////////////////////////////////////////////////////////////////////////// 1912// SVE2 - Contiguous conflict detection 1913let SVETargetGuard = "sve2", SMETargetGuard = "sme" in { 1914def SVWHILERW_B : SInst<"svwhilerw[_{1}]", "Pcc", "cUc", MergeNone, "aarch64_sve_whilerw_b", [IsOverloadWhileRW, VerifyRuntimeMode]>; 1915def SVWHILERW_H : SInst<"svwhilerw[_{1}]", "Pcc", "sUsh", MergeNone, "aarch64_sve_whilerw_h", [IsOverloadWhileRW, VerifyRuntimeMode]>; 1916def SVWHILERW_S : SInst<"svwhilerw[_{1}]", "Pcc", "iUif", MergeNone, "aarch64_sve_whilerw_s", [IsOverloadWhileRW, VerifyRuntimeMode]>; 1917def SVWHILERW_D : SInst<"svwhilerw[_{1}]", "Pcc", "lUld", MergeNone, "aarch64_sve_whilerw_d", [IsOverloadWhileRW, VerifyRuntimeMode]>; 1918 1919def SVWHILEWR_B : SInst<"svwhilewr[_{1}]", "Pcc", "cUc", MergeNone, "aarch64_sve_whilewr_b", [IsOverloadWhileRW, VerifyRuntimeMode]>; 1920def SVWHILEWR_H : SInst<"svwhilewr[_{1}]", "Pcc", "sUsh", MergeNone, "aarch64_sve_whilewr_h", [IsOverloadWhileRW, VerifyRuntimeMode]>; 1921def SVWHILEWR_S : SInst<"svwhilewr[_{1}]", "Pcc", "iUif", MergeNone, "aarch64_sve_whilewr_s", [IsOverloadWhileRW, VerifyRuntimeMode]>; 1922def SVWHILEWR_D : SInst<"svwhilewr[_{1}]", "Pcc", "lUld", MergeNone, "aarch64_sve_whilewr_d", [IsOverloadWhileRW, VerifyRuntimeMode]>; 1923} 1924 1925let SVETargetGuard = "sve2,bf16", SMETargetGuard = "sme,bf16" in { 1926def SVWHILERW_H_BF16 : SInst<"svwhilerw[_{1}]", "Pcc", "b", MergeNone, "aarch64_sve_whilerw_h", [IsOverloadWhileRW, VerifyRuntimeMode]>; 1927def SVWHILEWR_H_BF16 : SInst<"svwhilewr[_{1}]", "Pcc", "b", MergeNone, "aarch64_sve_whilewr_h", [IsOverloadWhileRW, VerifyRuntimeMode]>; 1928} 1929 1930//////////////////////////////////////////////////////////////////////////////// 1931// SVE2 - Extended table lookup/permute 1932let SVETargetGuard = "sve2", SMETargetGuard = "sme" in { 1933def SVTBL2 : SInst<"svtbl2[_{d}]", "d2u", "csilUcUsUiUlhfd", MergeNone, "", [VerifyRuntimeMode]>; 1934def SVTBX : SInst<"svtbx[_{d}]", "dddu", "csilUcUsUiUlhfd", MergeNone, "aarch64_sve_tbx", [VerifyRuntimeMode]>; 1935} 1936 1937let SVETargetGuard = "sve2,bf16", SMETargetGuard = "sme,bf16" in { 1938def SVTBL2_BF16 : SInst<"svtbl2[_{d}]", "d2u", "b", MergeNone, "", [VerifyRuntimeMode]>; 1939def SVTBX_BF16 : SInst<"svtbx[_{d}]", "dddu", "b", MergeNone, "aarch64_sve_tbx", [VerifyRuntimeMode]>; 1940} 1941 1942//////////////////////////////////////////////////////////////////////////////// 1943// SVE2 - Optional 1944 1945let SVETargetGuard = "sve2-aes", SMETargetGuard = InvalidMode in { 1946def SVAESD : SInst<"svaesd[_{d}]", "ddd", "Uc", MergeNone, "aarch64_sve_aesd", [IsOverloadNone]>; 1947def SVAESIMC : SInst<"svaesimc[_{d}]", "dd", "Uc", MergeNone, "aarch64_sve_aesimc", [IsOverloadNone]>; 1948def SVAESE : SInst<"svaese[_{d}]", "ddd", "Uc", MergeNone, "aarch64_sve_aese", [IsOverloadNone]>; 1949def SVAESMC : SInst<"svaesmc[_{d}]", "dd", "Uc", MergeNone, "aarch64_sve_aesmc", [IsOverloadNone]>; 1950 1951def SVPMULLB_PAIR_U64 : SInst<"svpmullb_pair[_{d}]", "ddd", "Ul", MergeNone, "aarch64_sve_pmullb_pair">; 1952def SVPMULLB_PAIR_N_U64 : SInst<"svpmullb_pair[_n_{d}]", "dda", "Ul", MergeNone, "aarch64_sve_pmullb_pair">; 1953 1954def SVPMULLT_PAIR_U64 : SInst<"svpmullt_pair[_{d}]", "ddd", "Ul", MergeNone, "aarch64_sve_pmullt_pair">; 1955def SVPMULLT_PAIR_N_U64 : SInst<"svpmullt_pair[_n_{d}]", "dda", "Ul", MergeNone, "aarch64_sve_pmullt_pair">; 1956} 1957 1958let SVETargetGuard = "sve2-sha3", SMETargetGuard = InvalidMode in { // FIXME: valid from sme2p1. 1959def SVRAX1 : SInst<"svrax1[_{d}]", "ddd", "lUl", MergeNone, "aarch64_sve_rax1", [IsOverloadNone]>; 1960} 1961 1962let SVETargetGuard = "sve2-sm4", SMETargetGuard = InvalidMode in { 1963def SVSM4E : SInst<"svsm4e[_{d}]", "ddd", "Ui", MergeNone, "aarch64_sve_sm4e", [IsOverloadNone]>; 1964def SVSM4EKEY : SInst<"svsm4ekey[_{d}]", "ddd", "Ui", MergeNone, "aarch64_sve_sm4ekey", [IsOverloadNone]>; 1965} 1966 1967let SVETargetGuard = "sve2-bitperm", SMETargetGuard = InvalidMode in { 1968def SVBDEP : SInst<"svbdep[_{d}]", "ddd", "UcUsUiUl", MergeNone, "aarch64_sve_bdep_x">; 1969def SVBDEP_N : SInst<"svbdep[_n_{d}]", "dda", "UcUsUiUl", MergeNone, "aarch64_sve_bdep_x">; 1970def SVBEXT : SInst<"svbext[_{d}]", "ddd", "UcUsUiUl", MergeNone, "aarch64_sve_bext_x">; 1971def SVBEXT_N : SInst<"svbext[_n_{d}]", "dda", "UcUsUiUl", MergeNone, "aarch64_sve_bext_x">; 1972def SVBGRP : SInst<"svbgrp[_{d}]", "ddd", "UcUsUiUl", MergeNone, "aarch64_sve_bgrp_x">; 1973def SVBGRP_N : SInst<"svbgrp[_n_{d}]", "dda", "UcUsUiUl", MergeNone, "aarch64_sve_bgrp_x">; 1974} 1975 1976let SVETargetGuard = "sve2p1", SMETargetGuard = "sme" in { 1977def SVPSEL_B : SInst<"svpsel_lane_b8", "PPPm", "Pc", MergeNone, "", [VerifyRuntimeMode], []>; 1978def SVPSEL_H : SInst<"svpsel_lane_b16", "PPPm", "Ps", MergeNone, "", [VerifyRuntimeMode], []>; 1979def SVPSEL_S : SInst<"svpsel_lane_b32", "PPPm", "Pi", MergeNone, "", [VerifyRuntimeMode], []>; 1980def SVPSEL_D : SInst<"svpsel_lane_b64", "PPPm", "Pl", MergeNone, "", [VerifyRuntimeMode], []>; 1981} 1982 1983// Standalone sve2.1 builtins 1984let SVETargetGuard = "sve2p1", SMETargetGuard = InvalidMode in { 1985def SVORQV : SInst<"svorqv[_{d}]", "{Pd", "csilUcUsUiUl", MergeNone, "aarch64_sve_orqv", [IsReductionQV]>; 1986def SVEORQV : SInst<"sveorqv[_{d}]", "{Pd", "csilUcUsUiUl", MergeNone, "aarch64_sve_eorqv", [IsReductionQV]>; 1987def SVADDQV : SInst<"svaddqv[_{d}]", "{Pd", "csilUcUsUiUl", MergeNone, "aarch64_sve_addqv", [IsReductionQV]>; 1988def SVANDQV : SInst<"svandqv[_{d}]", "{Pd", "csilUcUsUiUl", MergeNone, "aarch64_sve_andqv", [IsReductionQV]>; 1989def SVSMAXQV : SInst<"svmaxqv[_{d}]", "{Pd", "csil", MergeNone, "aarch64_sve_smaxqv", [IsReductionQV]>; 1990def SVUMAXQV : SInst<"svmaxqv[_{d}]", "{Pd", "UcUsUiUl", MergeNone, "aarch64_sve_umaxqv", [IsReductionQV]>; 1991def SVSMINQV : SInst<"svminqv[_{d}]", "{Pd", "csil", MergeNone, "aarch64_sve_sminqv", [IsReductionQV]>; 1992def SVUMINQV : SInst<"svminqv[_{d}]", "{Pd", "UcUsUiUl", MergeNone, "aarch64_sve_uminqv", [IsReductionQV]>; 1993 1994def SVFADDQV : SInst<"svaddqv[_{d}]", "{Pd", "hfd", MergeNone, "aarch64_sve_faddqv", [IsReductionQV]>; 1995def SVFMAXNMQV : SInst<"svmaxnmqv[_{d}]", "{Pd", "hfd", MergeNone, "aarch64_sve_fmaxnmqv", [IsReductionQV]>; 1996def SVFMINNMQV : SInst<"svminnmqv[_{d}]", "{Pd", "hfd", MergeNone, "aarch64_sve_fminnmqv", [IsReductionQV]>; 1997def SVFMAXQV : SInst<"svmaxqv[_{d}]", "{Pd", "hfd", MergeNone, "aarch64_sve_fmaxqv", [IsReductionQV]>; 1998def SVFMINQV : SInst<"svminqv[_{d}]", "{Pd", "hfd", MergeNone, "aarch64_sve_fminqv", [IsReductionQV]>; 1999} 2000 2001let SVETargetGuard = "sve2p1", SMETargetGuard = "sme2" in { 2002def SVPEXT_SINGLE : SInst<"svpext_lane_{d}", "P}i", "QcQsQiQl", MergeNone, "aarch64_sve_pext", [VerifyRuntimeMode], [ImmCheck<1, ImmCheck0_3>]>; 2003def SVPEXT_X2 : SInst<"svpext_lane_{d}_x2", "2.P}i", "QcQsQiQl", MergeNone, "aarch64_sve_pext_x2", [VerifyRuntimeMode], [ImmCheck<1, ImmCheck0_1>]>; 2004 2005def SVPSEL_COUNT_ALIAS_B : SInst<"svpsel_lane_c8", "}}Pm", "Pc", MergeNone, "", [VerifyRuntimeMode], []>; 2006def SVPSEL_COUNT_ALIAS_H : SInst<"svpsel_lane_c16", "}}Pm", "Ps", MergeNone, "", [VerifyRuntimeMode], []>; 2007def SVPSEL_COUNT_ALIAS_S : SInst<"svpsel_lane_c32", "}}Pm", "Pi", MergeNone, "", [VerifyRuntimeMode], []>; 2008def SVPSEL_COUNT_ALIAS_D : SInst<"svpsel_lane_c64", "}}Pm", "Pl", MergeNone, "", [VerifyRuntimeMode], []>; 2009 2010def SVWHILEGE_COUNT : SInst<"svwhilege_{d}[_{1}]", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilege_{d}", [IsOverloadNone, VerifyRuntimeMode], [ImmCheck<2, ImmCheck2_4_Mul2>]>; 2011def SVWHILEGT_COUNT : SInst<"svwhilegt_{d}[_{1}]", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilegt_{d}", [IsOverloadNone, VerifyRuntimeMode], [ImmCheck<2, ImmCheck2_4_Mul2>]>; 2012def SVWHILELE_COUNT : SInst<"svwhilele_{d}[_{1}]", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilele_{d}", [IsOverloadNone, VerifyRuntimeMode], [ImmCheck<2, ImmCheck2_4_Mul2>]>; 2013def SVWHILELT_COUNT : SInst<"svwhilelt_{d}[_{1}]", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilelt_{d}", [IsOverloadNone, VerifyRuntimeMode], [ImmCheck<2, ImmCheck2_4_Mul2>]>; 2014def SVWHILELO_COUNT : SInst<"svwhilelt_{d}[_{1}]", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilelo_{d}", [IsOverloadNone, VerifyRuntimeMode], [ImmCheck<2, ImmCheck2_4_Mul2>]>; 2015def SVWHILELS_COUNT : SInst<"svwhilele_{d}[_{1}]", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilels_{d}", [IsOverloadNone, VerifyRuntimeMode], [ImmCheck<2, ImmCheck2_4_Mul2>]>; 2016def SVWHILEHI_COUNT : SInst<"svwhilegt_{d}[_{1}]", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilehi_{d}", [IsOverloadNone, VerifyRuntimeMode], [ImmCheck<2, ImmCheck2_4_Mul2>]>; 2017def SVWHILEHS_COUNT : SInst<"svwhilege_{d}[_{1}]", "}nni", "QcQsQiQl", MergeNone, "aarch64_sve_whilehs_{d}", [IsOverloadNone, VerifyRuntimeMode], [ImmCheck<2, ImmCheck2_4_Mul2>]>; 2018} 2019 2020multiclass MultiVecLoad<string i> { 2021 def SV # NAME # B_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "cUc", [IsStructLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; 2022 def SV # NAME # H_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "sUshb", [IsStructLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; 2023 def SV # NAME # W_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "iUif", [IsStructLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; 2024 def SV # NAME # D_X2 : MInst<"sv" # i # "[_{2}]_x2", "2}c", "lUld", [IsStructLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; 2025 def SV # NAME # B_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "cUc", [IsStructLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; 2026 def SV # NAME # H_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "sUshb", [IsStructLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; 2027 def SV # NAME # W_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "iUif", [IsStructLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; 2028 def SV # NAME # D_X4 : MInst<"sv" # i # "[_{2}]_x4", "4}c", "lUld", [IsStructLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; 2029 2030 def SV # NAME # B_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "cUc", [IsStructLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; 2031 def SV # NAME # H_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "sUshb", [IsStructLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; 2032 def SV # NAME # W_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "iUif", [IsStructLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; 2033 def SV # NAME # D_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}]_x2", "2}cl", "lUld", [IsStructLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; 2034 def SV # NAME # B_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "cUc", [IsStructLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; 2035 def SV # NAME # H_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "sUshb", [IsStructLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; 2036 def SV # NAME # W_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "iUif", [IsStructLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; 2037 def SV # NAME # D_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}]_x4", "4}cl", "lUld", [IsStructLoad, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; 2038} 2039 2040let SVETargetGuard = "sve2p1", SMETargetGuard = "sme2" in { 2041 defm LD1 : MultiVecLoad<"ld1">; 2042 defm LDNT1 : MultiVecLoad<"ldnt1">; 2043} 2044 2045multiclass MultiVecStore<string i> { 2046 def SV # NAME # B_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "cUc", [IsStructStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; 2047 def SV # NAME # H_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "sUshb", [IsStructStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; 2048 def SV # NAME # W_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "iUif", [IsStructStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; 2049 def SV # NAME # D_X2 : MInst<"sv" # i # "[_{2}_x2]", "v}p2", "lUld", [IsStructStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; 2050 def SV # NAME # B_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "cUc", [IsStructStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; 2051 def SV # NAME # H_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "sUshb", [IsStructStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; 2052 def SV # NAME # W_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "iUif", [IsStructStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; 2053 def SV # NAME # D_X4 : MInst<"sv" # i # "[_{2}_x4]", "v}p4", "lUld", [IsStructStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; 2054 2055 def SV # NAME # B_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "cUc", [IsStructStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; 2056 def SV # NAME # H_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "sUshb", [IsStructStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; 2057 def SV # NAME # W_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "iUif", [IsStructStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; 2058 def SV # NAME # D_VNUM_X2 : MInst<"sv" # i # "_vnum" # "[_{2}_x2]", "v}pl2", "lUld", [IsStructStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x2">; 2059 def SV # NAME # B_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "cUc", [IsStructStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; 2060 def SV # NAME # H_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "sUshb", [IsStructStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; 2061 def SV # NAME # W_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "iUif", [IsStructStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; 2062 def SV # NAME # D_VNUM_X4 : MInst<"sv" # i # "_vnum" # "[_{2}_x4]", "v}pl4", "lUld", [IsStructStore, VerifyRuntimeMode], MemEltTyDefault, "aarch64_sve_" # i # "_pn_x4">; 2063} 2064 2065let SVETargetGuard = "sve2p1", SMETargetGuard = "sme2" in { 2066 defm ST1 : MultiVecStore<"st1">; 2067 defm STNT1 : MultiVecStore<"stnt1">; 2068} 2069 2070let SVETargetGuard = "sve2p1", SMETargetGuard = "sme2" in { 2071def SVDOT_X2_S : SInst<"svdot[_{d}_{2}]", "ddhh", "i", MergeNone, "aarch64_sve_sdot_x2", [VerifyRuntimeMode], []>; 2072def SVDOT_X2_U : SInst<"svdot[_{d}_{2}]", "ddhh", "Ui", MergeNone, "aarch64_sve_udot_x2", [VerifyRuntimeMode], []>; 2073def SVDOT_X2_F : SInst<"svdot[_{d}_{2}]", "ddhh", "f", MergeNone, "aarch64_sve_fdot_x2", [VerifyRuntimeMode], []>; 2074def SVDOT_LANE_X2_S : SInst<"svdot_lane[_{d}_{2}]", "ddhhi", "i", MergeNone, "aarch64_sve_sdot_lane_x2", [VerifyRuntimeMode], [ImmCheck<3, ImmCheck0_3>]>; 2075def SVDOT_LANE_X2_U : SInst<"svdot_lane[_{d}_{2}]", "ddhhi", "Ui", MergeNone, "aarch64_sve_udot_lane_x2", [VerifyRuntimeMode], [ImmCheck<3, ImmCheck0_3>]>; 2076def SVDOT_LANE_X2_F : SInst<"svdot_lane[_{d}_{2}]", "ddhhi", "f", MergeNone, "aarch64_sve_fdot_lane_x2", [VerifyRuntimeMode], [ImmCheck<3, ImmCheck0_3>]>; 2077} 2078 2079let SVETargetGuard = "sve2p1", SMETargetGuard = "sme" in { 2080def SVSCLAMP : SInst<"svclamp[_{d}]", "dddd", "csil", MergeNone, "aarch64_sve_sclamp", [VerifyRuntimeMode], []>; 2081def SVUCLAMP : SInst<"svclamp[_{d}]", "dddd", "UcUsUiUl", MergeNone, "aarch64_sve_uclamp", [VerifyRuntimeMode], []>; 2082 2083defm SVREVD : SInstZPZ<"svrevd", "csilUcUsUiUlbhfd", "aarch64_sve_revd">; 2084} 2085 2086let SVETargetGuard = "sve2p1", SMETargetGuard = "sme2" in { 2087 def SVPTRUE_COUNT : SInst<"svptrue_{d}", "}v", "QcQsQiQl", MergeNone, "aarch64_sve_ptrue_{d}", [IsOverloadNone, VerifyRuntimeMode], []>; 2088 2089 def SVPFALSE_COUNT_ALIAS : SInst<"svpfalse_c", "}v", "", MergeNone, "", [IsOverloadNone, VerifyRuntimeMode]>; 2090 2091 def SVFCLAMP : SInst<"svclamp[_{d}]", "dddd", "hfd", MergeNone, "aarch64_sve_fclamp", [VerifyRuntimeMode], []>; 2092 def SVCNTP_COUNT : SInst<"svcntp_{d}", "n}i", "QcQsQiQl", MergeNone, "aarch64_sve_cntp_{d}", [IsOverloadNone, VerifyRuntimeMode], [ImmCheck<1, ImmCheck2_4_Mul2>]>; 2093} 2094 2095let SVETargetGuard = "sve2,b16b16", SMETargetGuard = "sme2,b16b16" in { 2096defm SVMUL_BF : SInstZPZZ<"svmul", "b", "aarch64_sve_fmul", "aarch64_sve_fmul_u", [VerifyRuntimeMode]>; 2097defm SVADD_BF : SInstZPZZ<"svadd", "b", "aarch64_sve_fadd", "aarch64_sve_fadd_u", [VerifyRuntimeMode]>; 2098defm SVSUB_BF : SInstZPZZ<"svsub", "b", "aarch64_sve_fsub", "aarch64_sve_fsub_u", [VerifyRuntimeMode]>; 2099defm SVMAXNM_BF : SInstZPZZ<"svmaxnm","b", "aarch64_sve_fmaxnm", "aarch64_sve_fmaxnm_u", [VerifyRuntimeMode]>; 2100defm SVMINNM_BF : SInstZPZZ<"svminnm","b", "aarch64_sve_fminnm", "aarch64_sve_fminnm_u", [VerifyRuntimeMode]>; 2101defm SVMAX_BF : SInstZPZZ<"svmax", "b", "aarch64_sve_fmax", "aarch64_sve_fmax_u", [VerifyRuntimeMode]>; 2102defm SVMIN_BF : SInstZPZZ<"svmin", "b", "aarch64_sve_fmin", "aarch64_sve_fmin_u", [VerifyRuntimeMode]>; 2103defm SVMLA_BF : SInstZPZZZ<"svmla", "b", "aarch64_sve_fmla", "aarch64_sve_fmla_u", [VerifyRuntimeMode]>; 2104defm SVMLS_BF : SInstZPZZZ<"svmls", "b", "aarch64_sve_fmls", "aarch64_sve_fmls_u", [VerifyRuntimeMode]>; 2105def SVMLA_LANE_BF : SInst<"svmla_lane[_{d}]", "ddddi", "b", MergeNone, "aarch64_sve_fmla_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 2106def SVMLS_LANE_BF : SInst<"svmls_lane[_{d}]", "ddddi", "b", MergeNone, "aarch64_sve_fmls_lane", [VerifyRuntimeMode], [ImmCheck<3, ImmCheckLaneIndex, 2>]>; 2107def SVMUL_LANE_BF : SInst<"svmul_lane[_{d}]", "dddi", "b", MergeNone, "aarch64_sve_fmul_lane", [VerifyRuntimeMode], [ImmCheck<2, ImmCheckLaneIndex, 1>]>; 2108def SVFCLAMP_BF : SInst<"svclamp[_{d}]", "dddd", "b", MergeNone, "aarch64_sve_fclamp", [VerifyRuntimeMode], []>; 2109} 2110 2111// SME2 2112 2113// SME intrinsics which operate only on vectors and do not require ZA should be added here, 2114// as they could possibly become SVE instructions in the future. 2115 2116multiclass MinMaxIntr<string i, string zm, string mul, string t> { 2117 def SVS # NAME : SInst<"sv" # i # "[" # zm # "_{d}_" # mul # "]", t, "csil", MergeNone, "aarch64_sve_s" # i # zm # "_" # mul, [IsStreaming], []>; 2118 def SVU # NAME : SInst<"sv" # i # "[" # zm # "_{d}_" # mul # "]", t, "UcUsUiUl", MergeNone, "aarch64_sve_u" # i # zm # "_" # mul, [IsStreaming], []>; 2119 def SVF # NAME : SInst<"sv" # i # "[" # zm # "_{d}_" # mul # "]", t, "hfd", MergeNone, "aarch64_sve_f" # i # zm # "_" # mul, [IsStreaming], []>; 2120} 2121 2122let SVETargetGuard = InvalidMode, SMETargetGuard = "sme2" in { 2123// == SMAX / UMAX / FMAX == 2124 defm MAX_SINGLE_X2 : MinMaxIntr<"max", "_single", "x2", "22d">; 2125 defm MAX_MULTI_X2 : MinMaxIntr<"max", "", "x2", "222">; 2126 defm MAX_SINGLE_X4 : MinMaxIntr<"max", "_single", "x4", "44d">; 2127 defm MAX_MULTI_X4 : MinMaxIntr<"max", "", "x4", "444">; 2128 2129// == SMIN / UMIN / FMIN == 2130 defm MIN_SINGLE_X2 : MinMaxIntr<"min", "_single", "x2", "22d">; 2131 defm MIN_MULTI_X2 : MinMaxIntr<"min", "", "x2", "222">; 2132 defm MIN_SINGLE_X4 : MinMaxIntr<"min", "_single", "x4", "44d">; 2133 defm MIN_MULTI_X4 : MinMaxIntr<"min", "", "x4", "444">; 2134} 2135 2136multiclass SInstMinMaxByVector<string name> { 2137 def NAME # _SINGLE_X2 : SInst<"sv" # name # "nm[_single_{d}_x2]", "22d", "hfd", MergeNone, "aarch64_sve_f" # name # "nm_single_x2", [IsStreaming], []>; 2138 def NAME # _SINGLE_X4 : SInst<"sv" # name # "nm[_single_{d}_x4]", "44d", "hfd", MergeNone, "aarch64_sve_f" # name # "nm_single_x4", [IsStreaming], []>; 2139 2140 def NAME # _X2 : SInst<"sv" # name # "nm[_{d}_x2]", "222", "hfd", MergeNone, "aarch64_sve_f" # name # "nm_x2", [IsStreaming], []>; 2141 def NAME # _X4 : SInst<"sv" # name # "nm[_{d}_x4]", "444", "hfd", MergeNone, "aarch64_sve_f" # name # "nm_x4", [IsStreaming], []>; 2142} 2143 2144let SVETargetGuard = InvalidMode, SMETargetGuard = "sme2" in { 2145// == FMINNM / FMAXNM == 2146 defm SVMINNM : SInstMinMaxByVector<"min">; 2147 defm SVMAXNM : SInstMinMaxByVector<"max">; 2148} 2149 2150let SVETargetGuard = InvalidMode, SMETargetGuard = "sme2" in { 2151 // FRINTA / FRINTM / FRINTN / FRINTP 2152 def SVRINTA_X2 : SInst<"svrinta[_{d}_x2]", "22", "f", MergeNone, "aarch64_sve_frinta_x2", [IsStreaming], []>; 2153 def SVRINTA_X4 : SInst<"svrinta[_{d}_x4]", "44", "f", MergeNone, "aarch64_sve_frinta_x4", [IsStreaming], []>; 2154 2155 def SVRINTM_X2 : SInst<"svrintm[_{d}_x2]", "22", "f", MergeNone, "aarch64_sve_frintm_x2", [IsStreaming], []>; 2156 def SVRINTM_X4 : SInst<"svrintm[_{d}_x4]", "44", "f", MergeNone, "aarch64_sve_frintm_x4", [IsStreaming], []>; 2157 2158 def SVRINTN_X2 : SInst<"svrintn[_{d}_x2]", "22", "f", MergeNone, "aarch64_sve_frintn_x2", [IsStreaming], []>; 2159 def SVRINTN_X4 : SInst<"svrintn[_{d}_x4]", "44", "f", MergeNone, "aarch64_sve_frintn_x4", [IsStreaming], []>; 2160 2161 def SVRINTP_X2 : SInst<"svrintp[_{d}_x2]", "22", "f", MergeNone, "aarch64_sve_frintp_x2", [IsStreaming], []>; 2162 def SVRINTP_X4 : SInst<"svrintp[_{d}_x4]", "44", "f", MergeNone, "aarch64_sve_frintp_x4", [IsStreaming], []>; 2163} 2164 2165let SVETargetGuard = InvalidMode, SMETargetGuard = "sme2" in { 2166 def SVSCLAMP_X2 : SInst<"svclamp[_single_{d}_x2]", "22dd", "csil", MergeNone, "aarch64_sve_sclamp_single_x2", [IsStreaming], []>; 2167 def SVUCLAMP_X2 : SInst<"svclamp[_single_{d}_x2]", "22dd", "UcUsUiUl", MergeNone, "aarch64_sve_uclamp_single_x2", [IsStreaming], []>; 2168 def SVFCLAMP_X2 : SInst<"svclamp[_single_{d}_x2]", "22dd", "hfd", MergeNone, "aarch64_sve_fclamp_single_x2", [IsStreaming], []>; 2169 2170 def SVSCLAMP_X4 : SInst<"svclamp[_single_{d}_x4]", "44dd", "csil", MergeNone, "aarch64_sve_sclamp_single_x4", [IsStreaming], []>; 2171 def SVUCLAMP_X4 : SInst<"svclamp[_single_{d}_x4]", "44dd", "UcUsUiUl", MergeNone, "aarch64_sve_uclamp_single_x4", [IsStreaming], []>; 2172 def SVFCLAMP_X4 : SInst<"svclamp[_single_{d}_x4]", "44dd", "hfd", MergeNone, "aarch64_sve_fclamp_single_x4", [IsStreaming], []>; 2173} 2174 2175multiclass BfSingleMultiVector<string name> { 2176 def NAME # _SINGLE_X2 : SInst<"sv" # name # "[_single_{d}_x2]", "22d", "b", MergeNone, "aarch64_sve_f" # name # "_single_x2", [IsStreaming], []>; 2177 def NAME # _SINGLE_X4 : SInst<"sv" # name # "[_single_{d}_x4]", "44d", "b", MergeNone, "aarch64_sve_f" # name # "_single_x4", [IsStreaming], []>; 2178 2179 def NAME # _X2 : SInst<"sv" # name # "[_{d}_x2]", "222", "b", MergeNone, "aarch64_sve_f" # name # "_x2", [IsStreaming], []>; 2180 def NAME # _X4 : SInst<"sv" # name # "[_{d}_x4]", "444", "b", MergeNone, "aarch64_sve_f" # name # "_x4", [IsStreaming], []>; 2181} 2182 2183let SVETargetGuard = InvalidMode, SMETargetGuard = "sme2,b16b16"in { 2184 def SVBFCLAMP_X2 : SInst<"svclamp[_single_{d}_x2]", "22dd", "b", MergeNone, "aarch64_sve_bfclamp_single_x2", [IsStreaming], []>; 2185 def SVBFCLAMP_X4 : SInst<"svclamp[_single_{d}_x4]", "44dd", "b", MergeNone, "aarch64_sve_bfclamp_single_x4", [IsStreaming], []>; 2186 2187 // bfmin, bfmax (single, multi) 2188 defm SVBFMIN : BfSingleMultiVector<"min">; 2189 defm SVBFMAX : BfSingleMultiVector<"max">; 2190 2191 // bfminnm, bfmaxnm (single, multi) 2192 defm SVBFMINNM : BfSingleMultiVector<"minnm">; 2193 defm SVBFMAXNM : BfSingleMultiVector<"maxnm">; 2194} 2195 2196let SVETargetGuard = InvalidMode, SMETargetGuard = "sme2" in { 2197// == ADD (vectors) == 2198 def SVADD_SINGLE_X2 : SInst<"svadd[_single_{d}_x2]", "22d", "cUcsUsiUilUl", MergeNone, "aarch64_sve_add_single_x2", [IsStreaming], []>; 2199 def SVADD_SINGLE_X4 : SInst<"svadd[_single_{d}_x4]", "44d", "cUcsUsiUilUl", MergeNone, "aarch64_sve_add_single_x4", [IsStreaming], []>; 2200 2201 // 2-way and 4-way selects 2202 def SVSEL_X2 : SInst<"svsel[_{d}_x2]", "2}22", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_sel_x2", [IsStreaming], []>; 2203 def SVSEL_X4 : SInst<"svsel[_{d}_x4]", "4}44", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_sel_x4", [IsStreaming], []>; 2204 2205 // SRSHL / URSHL 2206 def SVSRSHL_SINGLE_X2 : SInst<"svrshl[_single_{d}_x2]", "22d", "csil", MergeNone, "aarch64_sve_srshl_single_x2", [IsStreaming], []>; 2207 def SVURSHL_SINGLE_X2 : SInst<"svrshl[_single_{d}_x2]", "22d", "UcUsUiUl", MergeNone, "aarch64_sve_urshl_single_x2", [IsStreaming], []>; 2208 def SVSRSHL_SINGLE_X4 : SInst<"svrshl[_single_{d}_x4]", "44d", "csil", MergeNone, "aarch64_sve_srshl_single_x4", [IsStreaming], []>; 2209 def SVURSHL_SINGLE_X4 : SInst<"svrshl[_single_{d}_x4]", "44d", "UcUsUiUl", MergeNone, "aarch64_sve_urshl_single_x4", [IsStreaming], []>; 2210 2211 def SVSRSHL_X2 : SInst<"svrshl[_{d}_x2]", "222", "csil", MergeNone, "aarch64_sve_srshl_x2", [IsStreaming], []>; 2212 def SVURSHL_X2 : SInst<"svrshl[_{d}_x2]", "222", "UcUsUiUl", MergeNone, "aarch64_sve_urshl_x2", [IsStreaming], []>; 2213 def SVSRSHL_X4 : SInst<"svrshl[_{d}_x4]", "444", "csil", MergeNone, "aarch64_sve_srshl_x4", [IsStreaming], []>; 2214 def SVURSHL_X4 : SInst<"svrshl[_{d}_x4]", "444", "UcUsUiUl", MergeNone, "aarch64_sve_urshl_x4", [IsStreaming], []>; 2215 2216 def SVQRSHRN_X4 : SInst<"svqrshrn[_n]_{0}[_{d}_x4]", "q4i", "il", MergeNone, "aarch64_sve_sqrshrn_x4", [IsStreaming], [ImmCheck<1, ImmCheckShiftRight, 0>]>; 2217 def SVUQRSHRN_X4 : SInst<"svqrshrn[_n]_{0}[_{d}_x4]", "b4i", "UiUl", MergeNone, "aarch64_sve_uqrshrn_x4", [IsStreaming], [ImmCheck<1, ImmCheckShiftRight, 0>]>; 2218 2219 // SQRSHR / UQRSHR 2220 def SVQRSHR_X2 : SInst<"svqrshr[_n]_{0}[_{d}_x2]", "h2i", "i", MergeNone, "aarch64_sve_sqrshr_x2", [IsStreaming], [ImmCheck<1, ImmCheck1_16>]>; 2221 def SVUQRSHR_X2 : SInst<"svqrshr[_n]_{0}[_{d}_x2]", "e2i", "Ui", MergeNone, "aarch64_sve_uqrshr_x2", [IsStreaming], [ImmCheck<1, ImmCheck1_16>]>; 2222 def SVQRSHR_X4 : SInst<"svqrshr[_n]_{0}[_{d}_x4]", "q4i", "il", MergeNone, "aarch64_sve_sqrshr_x4", [IsStreaming], [ImmCheck<1, ImmCheckShiftRight, 0>]>; 2223 def SVUQRSHR_X4 : SInst<"svqrshr[_n]_{0}[_{d}_x4]", "b4i", "UiUl", MergeNone, "aarch64_sve_uqrshr_x4", [IsStreaming], [ImmCheck<1, ImmCheckShiftRight, 0>]>; 2224 2225 // SQRSHRU 2226 def SVSQRSHRU_X2 : SInst<"svqrshru[_n]_{0}[_{d}_x2]", "e2i", "i", MergeNone, "aarch64_sve_sqrshru_x2", [IsStreaming], [ImmCheck<1, ImmCheck1_16>]>; 2227 def SVSQRSHRU_X4 : SInst<"svqrshru[_n]_{0}[_{d}_x4]", "b4i", "il", MergeNone, "aarch64_sve_sqrshru_x4", [IsStreaming], [ImmCheck<1, ImmCheckShiftRight, 0>]>; 2228 2229 def SVSQRSHRUN_X4 : SInst<"svqrshrun[_n]_{0}[_{d}_x4]", "b4i", "il", MergeNone, "aarch64_sve_sqrshrun_x4", [IsStreaming], [ImmCheck<1, ImmCheckShiftRight, 0>]>; 2230 2231 // SQDMULH 2232 def SVSQDMULH_SINGLE_X2 : SInst<"svqdmulh[_single_{d}_x2]", "22d", "csil", MergeNone, "aarch64_sve_sqdmulh_single_vgx2", [IsStreaming], []>; 2233 def SVSQDMULH_SINGLE_X4 : SInst<"svqdmulh[_single_{d}_x4]", "44d", "csil", MergeNone, "aarch64_sve_sqdmulh_single_vgx4", [IsStreaming], []>; 2234 def SVSQDMULH_X2 : SInst<"svqdmulh[_{d}_x2]", "222", "csil", MergeNone, "aarch64_sve_sqdmulh_vgx2", [IsStreaming], []>; 2235 def SVSQDMULH_X4 : SInst<"svqdmulh[_{d}_x4]", "444", "csil", MergeNone, "aarch64_sve_sqdmulh_vgx4", [IsStreaming], []>; 2236} 2237 2238let SVETargetGuard = "sve2p1", SMETargetGuard = "sme2" in { 2239 def REINTERPRET_SVBOOL_TO_SVCOUNT : Inst<"svreinterpret[_c]", "}P", "Pc", MergeNone, "", [VerifyRuntimeMode], []>; 2240 def REINTERPRET_SVCOUNT_TO_SVBOOL : Inst<"svreinterpret[_b]", "P}", "Pc", MergeNone, "", [VerifyRuntimeMode], []>; 2241 2242 // SQRSHRN / UQRSHRN 2243 def SVQRSHRN_X2 : SInst<"svqrshrn[_n]_{0}[_{d}_x2]", "h2i", "i", MergeNone, "aarch64_sve_sqrshrn_x2", [VerifyRuntimeMode], [ImmCheck<1, ImmCheck1_16>]>; 2244 def SVUQRSHRN_X2 : SInst<"svqrshrn[_n]_{0}[_{d}_x2]", "e2i", "Ui", MergeNone, "aarch64_sve_uqrshrn_x2", [VerifyRuntimeMode], [ImmCheck<1, ImmCheck1_16>]>; 2245 2246 // SQRSHRUN 2247 def SVSQRSHRUN_X2 : SInst<"svqrshrun[_n]_{0}[_{d}_x2]", "e2i", "i", MergeNone, "aarch64_sve_sqrshrun_x2", [VerifyRuntimeMode], [ImmCheck<1, ImmCheck1_16>]>; 2248} 2249 2250let SVETargetGuard = "sve2p1", SMETargetGuard = InvalidMode in { 2251 // ZIPQ1, ZIPQ2, UZPQ1, UZPQ2 2252 def SVZIPQ1 : SInst<"svzipq1[_{d}]", "ddd", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_zipq1", [], []>; 2253 def SVZIPQ2 : SInst<"svzipq2[_{d}]", "ddd", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_zipq2", [], []>; 2254 def SVUZPQ1 : SInst<"svuzpq1[_{d}]", "ddd", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_uzpq1", [], []>; 2255 def SVUZPQ2 : SInst<"svuzpq2[_{d}]", "ddd", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_uzpq2", [], []>; 2256 // TBLQ, TBXQ 2257 def SVTBLQ : SInst<"svtblq[_{d}]", "ddu", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_tblq">; 2258 def SVTBXQ : SInst<"svtbxq[_{d}]", "dddu", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_tbxq">; 2259 // EXTQ 2260 def EXTQ : SInst<"svextq[_{d}]", "dddk", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_extq", [], [ImmCheck<2, ImmCheck0_15>]>; 2261 2262 // PMOV 2263 // Move to Pred 2264 multiclass PMOV_TO_PRED<string name, string types, string intrinsic, list<FlagType> flags=[], ImmCheckType immCh > { 2265 def _LANE : Inst<name # "_lane[_{d}]", "Pdi", types, MergeNone, intrinsic, flags, [ImmCheck<1, immCh>]>; 2266 def _LANE_ZERO : SInst<name # "[_{d}]", "Pd", types, MergeNone, intrinsic # "_zero", flags, []>; 2267 } 2268 defm SVPMOV_B_TO_PRED : PMOV_TO_PRED<"svpmov", "cUc", "aarch64_sve_pmov_to_pred_lane", [], ImmCheck0_0>; 2269 defm SVPMOV_H_TO_PRED : PMOV_TO_PRED<"svpmov", "sUs", "aarch64_sve_pmov_to_pred_lane", [], ImmCheck0_1>; 2270 defm SVPMOV_S_TO_PRED : PMOV_TO_PRED<"svpmov", "iUi", "aarch64_sve_pmov_to_pred_lane", [], ImmCheck0_3>; 2271 defm SVPMOV_D_TO_PRED : PMOV_TO_PRED<"svpmov", "lUl", "aarch64_sve_pmov_to_pred_lane", [], ImmCheck0_7>; 2272 2273 // Move to Vector 2274 multiclass PMOV_TO_VEC<string name, string types, string intrinsic, list<FlagType> flags=[], ImmCheckType immCh > { 2275 def _M : SInst<name # "_lane[_{d}]", "ddPi", types, MergeOp1, intrinsic # "_merging", flags, [ImmCheck<2, immCh>]>; 2276 def _Z : SInst<name # "_{d}_z", "dP", types, MergeNone, intrinsic # "_zeroing", flags, []>; 2277 } 2278 def SVPMOV_TO_VEC_LANE_B : SInst<"svpmov_{d}_z", "dP", "cUc", MergeNone, "aarch64_sve_pmov_to_vector_lane_zeroing", [], []>; 2279 defm SVPMOV_TO_VEC_LANE_H : PMOV_TO_VEC<"svpmov", "sUs", "aarch64_sve_pmov_to_vector_lane", [], ImmCheck1_1>; 2280 defm SVPMOV_TO_VEC_LANE_S : PMOV_TO_VEC<"svpmov", "iUi", "aarch64_sve_pmov_to_vector_lane", [], ImmCheck1_3>; 2281 defm SVPMOV_TO_VEC_LANE_D : PMOV_TO_VEC<"svpmov", "lUl", "aarch64_sve_pmov_to_vector_lane" ,[], ImmCheck1_7>; 2282} 2283 2284let SVETargetGuard = "sve2p1", SMETargetGuard = "sme2p1" in { 2285 // DUPQ 2286 def SVDUP_LANEQ_B : SInst<"svdup_laneq[_{d}]", "ddi", "cUc", MergeNone, "aarch64_sve_dup_laneq", [VerifyRuntimeMode], [ImmCheck<1, ImmCheck0_15>]>; 2287 def SVDUP_LANEQ_H : SInst<"svdup_laneq[_{d}]", "ddi", "sUsh", MergeNone, "aarch64_sve_dup_laneq", [VerifyRuntimeMode], [ImmCheck<1, ImmCheck0_7>]>; 2288 def SVDUP_LANEQ_S : SInst<"svdup_laneq[_{d}]", "ddi", "iUif", MergeNone, "aarch64_sve_dup_laneq", [VerifyRuntimeMode], [ImmCheck<1, ImmCheck0_3>]>; 2289 def SVDUP_LANEQ_D : SInst<"svdup_laneq[_{d}]", "ddi", "lUld", MergeNone, "aarch64_sve_dup_laneq", [VerifyRuntimeMode], [ImmCheck<1, ImmCheck0_1>]>; 2290} 2291 2292let SVETargetGuard = "sve2p1,bf16", SMETargetGuard = "sme2p1,bf16" in { 2293 def SVDUP_LANEQ_BF16 : SInst<"svdup_laneq[_{d}]", "ddi", "b", MergeNone, "aarch64_sve_dup_laneq", [VerifyRuntimeMode], [ImmCheck<1, ImmCheck0_7>]>; 2294} 2295 2296// 2297// Multi-vector convert to/from floating-point. 2298// 2299let SVETargetGuard = InvalidMode, SMETargetGuard = "sme2" in { 2300 def SVCVT_F16_X2 : SInst<"svcvt_f16[_f32_x2]", "e2", "f", MergeNone, "aarch64_sve_fcvt_x2", [IsStreaming],[]>; 2301 def SVCVT_BF16_X2 : SInst<"svcvt_bf16[_f32_x2]", "$2", "f", MergeNone, "aarch64_sve_bfcvt_x2", [IsOverloadNone, IsStreaming],[]>; 2302 2303 def SVCVT_F32_U32_X2 : SInst<"svcvt_{d}[_u32_x2]", "2.d2.u", "f", MergeNone, "aarch64_sve_ucvtf_x2", [IsStreaming, IsOverloadWhileOrMultiVecCvt], []>; 2304 def SVCVT_U32_F32_X2 : SInst<"svcvt_{d}[_f32_x2]", "2.d2.M", "Ui", MergeNone, "aarch64_sve_fcvtzu_x2", [IsStreaming, IsOverloadWhileOrMultiVecCvt], []>; 2305 def SVCVT_F32_S32_X2 : SInst<"svcvt_{d}[_s32_x2]", "2.d2.x", "f", MergeNone, "aarch64_sve_scvtf_x2", [IsStreaming, IsOverloadWhileOrMultiVecCvt], []>; 2306 def SVCVT_S32_F32_X2 : SInst<"svcvt_{d}[_f32_x2]", "2.d2.M", "i", MergeNone, "aarch64_sve_fcvtzs_x2", [IsStreaming, IsOverloadWhileOrMultiVecCvt], []>; 2307 2308 def SVCVT_F32_U32_X4 : SInst<"svcvt_{d}[_u32_x4]", "4.d4.u", "f", MergeNone, "aarch64_sve_ucvtf_x4", [IsStreaming, IsOverloadWhileOrMultiVecCvt], []>; 2309 def SVCVT_U32_F32_X4 : SInst<"svcvt_{d}[_f32_x4]", "4.d4.M", "Ui", MergeNone, "aarch64_sve_fcvtzu_x4", [IsStreaming, IsOverloadWhileOrMultiVecCvt], []>; 2310 def SVCVT_F32_S32_X4 : SInst<"svcvt_{d}[_s32_x4]", "4.d4.x", "f", MergeNone, "aarch64_sve_scvtf_x4", [IsStreaming, IsOverloadWhileOrMultiVecCvt], []>; 2311 def SVCVT_S32_F32_X4 : SInst<"svcvt_{d}[_f32_x4]", "4.d4.M", "i", MergeNone, "aarch64_sve_fcvtzs_x4", [IsStreaming, IsOverloadWhileOrMultiVecCvt], []>; 2312} 2313 2314let SVETargetGuard = InvalidMode, SMETargetGuard = "sme-f16f16" in { 2315 def SVCVT_F32_X2 : SInst<"svcvt_{d}[_f16_x2]", "2h", "f", MergeNone, "aarch64_sve_fcvt_widen_x2", [ IsStreaming],[]>; 2316} 2317 2318// 2319// Multi-vector floating-point convert from single-precision to interleaved half-precision/BFloat16 2320// 2321let SVETargetGuard = InvalidMode, SMETargetGuard = "sme2" in { 2322 def SVCVTN_F16_X2 : SInst<"svcvtn_f16[_f32_x2]", "e2", "f", MergeNone, "aarch64_sve_fcvtn_x2", [IsStreaming],[]>; 2323 def SVCVTN_BF16_X2 : SInst<"svcvtn_bf16[_f32_x2]", "$2", "f", MergeNone, "aarch64_sve_bfcvtn_x2", [IsOverloadNone, IsStreaming],[]>; 2324} 2325 2326// 2327//Multi-vector floating-point convert from half-precision to deinterleaved single-precision. 2328// 2329let SVETargetGuard = InvalidMode, SMETargetGuard = "sme-f16f16" in { 2330 def SVCVTL_F32_X2 : SInst<"svcvtl_f32[_f16_x2]", "2h", "f", MergeNone, "aarch64_sve_fcvtl_widen_x2", [ IsStreaming],[]>; 2331} 2332 2333// 2334// Multi-vector saturating extract narrow 2335// 2336let SVETargetGuard = InvalidMode, SMETargetGuard = "sme2" in { 2337 def SVQCVT_S16_S32_X2 : SInst<"svqcvt_s16[_{d}_x2]", "h2.d", "i", MergeNone, "aarch64_sve_sqcvt_x2", [IsStreaming], []>; 2338 def SVQCVT_U16_U32_X2 : SInst<"svqcvt_u16[_{d}_x2]", "e2.d", "Ui", MergeNone, "aarch64_sve_uqcvt_x2", [IsStreaming], []>; 2339 def SVQCVT_U16_S32_X2 : SInst<"svqcvt_u16[_{d}_x2]", "e2.d", "i", MergeNone, "aarch64_sve_sqcvtu_x2", [IsStreaming], []>; 2340 2341 def SVQCVT_S8_S32_X4 : SInst<"svqcvt_s8[_{d}_x4]", "q4.d", "i", MergeNone, "aarch64_sve_sqcvt_x4", [IsStreaming], []>; 2342 def SVQCVT_U8_U32_X4 : SInst<"svqcvt_u8[_{d}_x4]", "b4.d", "Ui", MergeNone, "aarch64_sve_uqcvt_x4", [IsStreaming], []>; 2343 def SVQCVT_U8_S32_X4 : SInst<"svqcvt_u8[_{d}_x4]", "b4.d", "i", MergeNone, "aarch64_sve_sqcvtu_x4", [IsStreaming], []>; 2344 2345 def SVQCVT_S16_S64_X4 : SInst<"svqcvt_s16[_{d}_x4]", "q4.d", "l", MergeNone, "aarch64_sve_sqcvt_x4", [IsStreaming], []>; 2346 def SVQCVT_U16_U64_X4 : SInst<"svqcvt_u16[_{d}_x4]", "b4.d", "Ul", MergeNone, "aarch64_sve_uqcvt_x4", [IsStreaming], []>; 2347 def SVQCVT_U16_S64_X4 : SInst<"svqcvt_u16[_{d}_x4]", "b4.d", "l", MergeNone, "aarch64_sve_sqcvtu_x4", [IsStreaming], []>; 2348} 2349 2350// 2351// Multi-vector saturating extract narrow and interleave 2352// 2353let SVETargetGuard = "sve2p1", SMETargetGuard = "sme2" in { 2354 def SVQCVTN_S16_S32_X2 : SInst<"svqcvtn_s16[_{d}_x2]", "h2.d", "i", MergeNone, "aarch64_sve_sqcvtn_x2", [VerifyRuntimeMode], []>; 2355 def SVQCVTN_U16_U32_X2 : SInst<"svqcvtn_u16[_{d}_x2]", "e2.d", "Ui", MergeNone, "aarch64_sve_uqcvtn_x2", [VerifyRuntimeMode], []>; 2356 def SVQCVTN_U16_S32_X2 : SInst<"svqcvtn_u16[_{d}_x2]", "e2.d", "i", MergeNone, "aarch64_sve_sqcvtun_x2", [VerifyRuntimeMode], []>; 2357} 2358 2359let SVETargetGuard = InvalidMode, SMETargetGuard = "sme2" in { 2360 def SVQCVTN_S8_S32_X4 : SInst<"svqcvtn_s8[_{d}_x4]", "q4.d", "i", MergeNone, "aarch64_sve_sqcvtn_x4", [IsStreaming], []>; 2361 def SVQCVTN_U8_U32_X4 : SInst<"svqcvtn_u8[_{d}_x4]", "b4.d", "Ui", MergeNone, "aarch64_sve_uqcvtn_x4", [IsStreaming], []>; 2362 def SVQCVTN_U8_S32_X4 : SInst<"svqcvtn_u8[_{d}_x4]", "b4.d", "i", MergeNone, "aarch64_sve_sqcvtun_x4", [IsStreaming], []>; 2363 2364 def SVQCVTN_S16_S64_X4 : SInst<"svqcvtn_s16[_{d}_x4]", "q4.d", "l", MergeNone, "aarch64_sve_sqcvtn_x4", [IsStreaming], []>; 2365 def SVQCVTN_U16_U64_X4 : SInst<"svqcvtn_u16[_{d}_x4]", "b4.d", "Ul", MergeNone, "aarch64_sve_uqcvtn_x4", [IsStreaming], []>; 2366 def SVQCVTN_U16_S64_X4 : SInst<"svqcvtn_u16[_{d}_x4]", "b4.d", "l", MergeNone, "aarch64_sve_sqcvtun_x4", [IsStreaming], []>; 2367} 2368 2369// 2370// Multi-vector zip/unzip 2371// 2372 2373let SVETargetGuard = InvalidMode, SMETargetGuard = "sme2" in { 2374 def SVZIP_X2 : SInst<"svzip[_{d}_x2]", "22", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_zip_x2", [IsStreaming], []>; 2375 def SVZIPQ_X2 : SInst<"svzipq[_{d}_x2]", "22", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_zipq_x2", [IsStreaming], []>; 2376 def SVZIP_X4 : SInst<"svzip[_{d}_x4]", "44", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_zip_x4", [IsStreaming], []>; 2377 def SVZIPQ_X4 : SInst<"svzipq[_{d}_x4]", "44", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_zipq_x4", [IsStreaming], []>; 2378 2379 def SVUZP_X2 : SInst<"svuzp[_{d}_x2]", "22", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_uzp_x2", [IsStreaming], []>; 2380 def SVUZPQ_X2 : SInst<"svuzpq[_{d}_x2]", "22", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_uzpq_x2", [IsStreaming], []>; 2381 def SVUZP_X4 : SInst<"svuzp[_{d}_x4]", "44", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_uzp_x4", [IsStreaming], []>; 2382 def SVUZPQ_X4 : SInst<"svuzpq[_{d}_x4]", "44", "cUcsUsiUilUlbhfd", MergeNone, "aarch64_sve_uzpq_x4", [IsStreaming], []>; 2383} 2384 2385// 2386// Multi-vector unpack 2387// 2388 2389let SVETargetGuard = InvalidMode, SMETargetGuard = "sme2" in { 2390 def SVSUNPK_X2 : SInst<"svunpk_{d}[_{1}_x2]", "2h", "sil", MergeNone, "aarch64_sve_sunpk_x2", [IsStreaming], []>; 2391 def SVUUNPK_X2 : SInst<"svunpk_{d}[_{1}_x2]", "2h", "UsUiUl", MergeNone, "aarch64_sve_uunpk_x2", [IsStreaming], []>; 2392 def SVSUNPK_X4 : SInst<"svunpk_{d}[_{3}_x4]", "42.h", "sil", MergeNone, "aarch64_sve_sunpk_x4", [IsStreaming], []>; 2393 def SVUUNPK_X4 : SInst<"svunpk_{d}[_{3}_x4]", "42.h", "UsUiUl", MergeNone, "aarch64_sve_uunpk_x4", [IsStreaming], []>; 2394} 2395 2396let SVETargetGuard = "sve2p1", SMETargetGuard = "sme2" in { 2397// == BFloat16 multiply-subtract == 2398 def SVBFMLSLB : SInst<"svbfmlslb[_{d}]", "dd$$", "f", MergeNone, "aarch64_sve_bfmlslb", [IsOverloadNone, VerifyRuntimeMode], []>; 2399 def SVBFMLSLT : SInst<"svbfmlslt[_{d}]", "dd$$", "f", MergeNone, "aarch64_sve_bfmlslt", [IsOverloadNone, VerifyRuntimeMode], []>; 2400 2401 def SVBFMLSLB_LANE : SInst<"svbfmlslb_lane[_{d}]", "dd$$i", "f", MergeNone, "aarch64_sve_bfmlslb_lane", [IsOverloadNone, VerifyRuntimeMode], [ImmCheck<3, ImmCheck0_7>]>; 2402 def SVBFMLSLT_LANE : SInst<"svbfmlslt_lane[_{d}]", "dd$$i", "f", MergeNone, "aarch64_sve_bfmlslt_lane", [IsOverloadNone, VerifyRuntimeMode], [ImmCheck<3, ImmCheck0_7>]>; 2403} 2404