Lines Matching +full:ete +full:- +full:0
1 //=- AArch64InstrInfo.td - Describe the AArch64 Instructions -*- tablegen -*-=//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
11 //===----------------------------------------------------------------------===//
14 //===----------------------------------------------------------------------===//
21 def HasV8_0a : Predicate<"Subtarget->hasV8_0aOps()">,
22 AssemblerPredicate<(all_of HasV8_0aOps), "armv8.0a">;
23 def HasV8_1a : Predicate<"Subtarget->hasV8_1aOps()">,
25 def HasV8_2a : Predicate<"Subtarget->hasV8_2aOps()">,
27 def HasV8_3a : Predicate<"Subtarget->hasV8_3aOps()">,
29 def HasV8_4a : Predicate<"Subtarget->hasV8_4aOps()">,
31 def HasV8_5a : Predicate<"Subtarget->hasV8_5aOps()">,
33 def HasV8_6a : Predicate<"Subtarget->hasV8_6aOps()">,
35 def HasV8_7a : Predicate<"Subtarget->hasV8_7aOps()">,
37 def HasV8_8a : Predicate<"Subtarget->hasV8_8aOps()">,
39 def HasV8_9a : Predicate<"Subtarget->hasV8_9aOps()">,
41 def HasV9_0a : Predicate<"Subtarget->hasV9_0aOps()">,
42 AssemblerPredicateWithAll<(all_of HasV9_0aOps), "armv9-a">;
43 def HasV9_1a : Predicate<"Subtarget->hasV9_1aOps()">,
45 def HasV9_2a : Predicate<"Subtarget->hasV9_2aOps()">,
47 def HasV9_3a : Predicate<"Subtarget->hasV9_3aOps()">,
49 def HasV9_4a : Predicate<"Subtarget->hasV9_4aOps()">,
51 def HasV8_0r : Predicate<"Subtarget->hasV8_0rOps()">,
52 AssemblerPredicateWithAll<(all_of HasV8_0rOps), "armv8-r">;
54 def HasEL2VMSA : Predicate<"Subtarget->hasEL2VMSA()">,
57 def HasEL3 : Predicate<"Subtarget->hasEL3()">,
60 def HasVH : Predicate<"Subtarget->hasVH()">,
63 def HasLOR : Predicate<"Subtarget->hasLOR()">,
66 def HasPAuth : Predicate<"Subtarget->hasPAuth()">,
69 def HasPAuthLR : Predicate<"Subtarget->hasPAuthLR()">,
70 AssemblerPredicateWithAll<(all_of FeaturePAuthLR), "pauth-lr">;
72 def HasJS : Predicate<"Subtarget->hasJS()">,
75 def HasCCIDX : Predicate<"Subtarget->hasCCIDX()">,
78 def HasComplxNum : Predicate<"Subtarget->hasComplxNum()">,
81 def HasNV : Predicate<"Subtarget->hasNV()">,
84 def HasMPAM : Predicate<"Subtarget->hasMPAM()">,
87 def HasDIT : Predicate<"Subtarget->hasDIT()">,
90 def HasTRACEV8_4 : Predicate<"Subtarget->hasTRACEV8_4()">,
93 def HasAM : Predicate<"Subtarget->hasAM()">,
96 def HasSEL2 : Predicate<"Subtarget->hasSEL2()">,
99 def HasTLB_RMI : Predicate<"Subtarget->hasTLB_RMI()">,
100 AssemblerPredicateWithAll<(all_of FeatureTLB_RMI), "tlb-rmi">;
102 def HasFlagM : Predicate<"Subtarget->hasFlagM()">,
105 def HasRCPC_IMMO : Predicate<"Subtarget->hasRCPC_IMMO()">,
106 AssemblerPredicateWithAll<(all_of FeatureRCPC_IMMO), "rcpc-immo">;
108 def HasFPARMv8 : Predicate<"Subtarget->hasFPARMv8()">,
109 AssemblerPredicateWithAll<(all_of FeatureFPARMv8), "fp-armv8">;
110 def HasNEON : Predicate<"Subtarget->isNeonAvailable()">,
112 def HasSM4 : Predicate<"Subtarget->hasSM4()">,
114 def HasSHA3 : Predicate<"Subtarget->hasSHA3()">,
116 def HasSHA2 : Predicate<"Subtarget->hasSHA2()">,
118 def HasAES : Predicate<"Subtarget->hasAES()">,
120 def HasDotProd : Predicate<"Subtarget->hasDotProd()">,
122 def HasCRC : Predicate<"Subtarget->hasCRC()">,
124 def HasCSSC : Predicate<"Subtarget->hasCSSC()">,
126 def HasNoCSSC : Predicate<"!Subtarget->hasCSSC()">;
127 def HasLSE : Predicate<"Subtarget->hasLSE()">,
129 def HasNoLSE : Predicate<"!Subtarget->hasLSE()">;
130 def HasRAS : Predicate<"Subtarget->hasRAS()">,
132 def HasRDM : Predicate<"Subtarget->hasRDM()">,
134 def HasFullFP16 : Predicate<"Subtarget->hasFullFP16()">,
136 def HasNoFullFP16 : Predicate<"!Subtarget->hasFullFP16()">;
137 def HasFP16FML : Predicate<"Subtarget->hasFP16FML()">,
139 def HasSPE : Predicate<"Subtarget->hasSPE()">,
141 def HasFuseAES : Predicate<"Subtarget->hasFuseAES()">,
143 "fuse-aes">;
144 def HasSVE : Predicate<"Subtarget->isSVEAvailable()">,
146 def HasSVE2 : Predicate<"Subtarget->isSVEAvailable() && Subtarget->hasSVE2()">,
148 def HasSVE2p1 : Predicate<"Subtarget->isSVEAvailable() && Subtarget->hasSVE2p1()">,
150 def HasSVE2AES : Predicate<"Subtarget->isSVEAvailable() && Subtarget->hasSVE2AES()">,
151 AssemblerPredicateWithAll<(all_of FeatureSVE2AES), "sve2-aes">;
152 def HasSVE2SM4 : Predicate<"Subtarget->isSVEAvailable() && Subtarget->hasSVE2SM4()">,
153 AssemblerPredicateWithAll<(all_of FeatureSVE2SM4), "sve2-sm4">;
154 def HasSVE2SHA3 : Predicate<"Subtarget->isSVEAvailable() && Subtarget->hasSVE2SHA3()">,
155 AssemblerPredicateWithAll<(all_of FeatureSVE2SHA3), "sve2-sha3">;
156 def HasSVE2BitPerm : Predicate<"Subtarget->isSVEAvailable() && Subtarget->hasSVE2BitPerm()">,
157 … AssemblerPredicateWithAll<(all_of FeatureSVE2BitPerm), "sve2-bitperm">;
158 def HasB16B16 : Predicate<"Subtarget->hasB16B16()">,
161 : Predicate<"Subtarget->hasSME()">,
163 def HasSME : Predicate<"Subtarget->isStreaming() && Subtarget->hasSME()">,
165 def HasSMEF64F64 : Predicate<"Subtarget->isStreaming() && Subtarget->hasSMEF64F64()">,
166 AssemblerPredicateWithAll<(all_of FeatureSMEF64F64), "sme-f64f64">;
167 def HasSMEF16F16 : Predicate<"Subtarget->isStreaming() && Subtarget->hasSMEF16F16()">,
168 AssemblerPredicateWithAll<(all_of FeatureSMEF16F16), "sme-f16f16">;
169 def HasSMEFA64 : Predicate<"Subtarget->isStreaming() && Subtarget->hasSMEFA64()">,
170 AssemblerPredicateWithAll<(all_of FeatureSMEFA64), "sme-fa64">;
171 def HasSMEI16I64 : Predicate<"Subtarget->isStreaming() && Subtarget->hasSMEI16I64()">,
172 AssemblerPredicateWithAll<(all_of FeatureSMEI16I64), "sme-i16i64">;
174 : Predicate<"Subtarget->hasSME2()">,
176 def HasSME2 : Predicate<"Subtarget->isStreaming() && Subtarget->hasSME2()">,
178 def HasSME2p1 : Predicate<"Subtarget->isStreaming() && Subtarget->hasSME2p1()">,
180 def HasFP8 : Predicate<"Subtarget->hasFP8()">,
182 def HasFAMINMAX : Predicate<"Subtarget->hasFAMINMAX()">,
184 def HasFP8FMA : Predicate<"Subtarget->hasFP8FMA()">,
186 def HasSSVE_FP8FMA : Predicate<"Subtarget->hasSSVE_FP8FMA() || "
187 "(Subtarget->hasSVE2() && Subtarget->hasFP8FMA())">,
190 "ssve-fp8fma or (sve2 and fp8fma)">;
191 def HasFP8DOT2 : Predicate<"Subtarget->hasFP8DOT2()">,
193 def HasSSVE_FP8DOT2 : Predicate<"Subtarget->hasSSVE_FP8DOT2() || "
194 "(Subtarget->hasSVE2() && Subtarget->hasFP8DOT2())">,
197 "ssve-fp8dot2 or (sve2 and fp8dot2)">;
198 def HasFP8DOT4 : Predicate<"Subtarget->hasFP8DOT4()">,
200 def HasSSVE_FP8DOT4 : Predicate<"Subtarget->hasSSVE_FP8DOT4() || "
201 "(Subtarget->hasSVE2() && Subtarget->hasFP8DOT4())">,
204 "ssve-fp8dot4 or (sve2 and fp8dot4)">;
205 def HasLUT : Predicate<"Subtarget->hasLUT()">,
207 def HasSME_LUTv2 : Predicate<"Subtarget->isStreaming() && Subtarget->hasSME_LUTv2()">,
208 AssemblerPredicateWithAll<(all_of FeatureSME_LUTv2), "sme-lutv2">;
209 def HasSMEF8F16 : Predicate<"Subtarget->isStreaming() && Subtarget->hasSMEF8F16()">,
210 AssemblerPredicateWithAll<(all_of FeatureSMEF8F16), "sme-f8f16">;
211 def HasSMEF8F32 : Predicate<"Subtarget->isStreaming() && Subtarget->hasSMEF8F32()">,
212 AssemblerPredicateWithAll<(all_of FeatureSMEF8F32), "sme-f8f32">;
217 : Predicate<"Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME())">,
221 : Predicate<"Subtarget->hasSVE2() || (Subtarget->isStreaming() && Subtarget->hasSME())">,
225 : Predicate<"Subtarget->hasSVE2() || (Subtarget->isStreaming() && Subtarget->hasSME2())">,
229 : Predicate<"Subtarget->hasSVE2p1() || (Subtarget->isStreaming() && Subtarget->hasSME())">,
232 : Predicate<"Subtarget->hasSVE2p1() || (Subtarget->isStreaming() && Subtarget->hasSME2())">,
235 : Predicate<"Subtarget->hasSVE2p1() || (Subtarget->isStreaming() && Subtarget->hasSME2p1())">,
239 …: Predicate<"Subtarget->isStreaming() && (Subtarget->hasSMEF16F16() || Subtarget->hasSMEF8F16())">,
241 "sme-f16f16 or sme-f8f16">;
246 : Predicate<"Subtarget->hasNEON()">,
248 def HasRCPC : Predicate<"Subtarget->hasRCPC()">,
250 def HasAltNZCV : Predicate<"Subtarget->hasAlternativeNZCV()">,
252 def HasFRInt3264 : Predicate<"Subtarget->hasFRInt3264()">,
254 def HasSB : Predicate<"Subtarget->hasSB()">,
256 def HasPredRes : Predicate<"Subtarget->hasPredRes()">,
258 def HasCCDP : Predicate<"Subtarget->hasCCDP()">,
260 def HasBTI : Predicate<"Subtarget->hasBTI()">,
262 def HasMTE : Predicate<"Subtarget->hasMTE()">,
264 def HasTME : Predicate<"Subtarget->hasTME()">,
266 def HasETE : Predicate<"Subtarget->hasETE()">,
267 AssemblerPredicateWithAll<(all_of FeatureETE), "ete">;
268 def HasTRBE : Predicate<"Subtarget->hasTRBE()">,
270 def HasBF16 : Predicate<"Subtarget->hasBF16()">,
272 def HasNoBF16 : Predicate<"!Subtarget->hasBF16()">;
273 def HasMatMulInt8 : Predicate<"Subtarget->hasMatMulInt8()">,
275 def HasMatMulFP32 : Predicate<"Subtarget->hasMatMulFP32()">,
277 def HasMatMulFP64 : Predicate<"Subtarget->hasMatMulFP64()">,
279 def HasFPAC : Predicate<"Subtarget->hasFPAC())">,
281 def HasXS : Predicate<"Subtarget->hasXS()">,
283 def HasWFxT : Predicate<"Subtarget->hasWFxT()">,
285 def HasLS64 : Predicate<"Subtarget->hasLS64()">,
287 def HasBRBE : Predicate<"Subtarget->hasBRBE()">,
289 def HasSPE_EEF : Predicate<"Subtarget->hasSPE_EEF()">,
290 AssemblerPredicateWithAll<(all_of FeatureSPE_EEF), "spe-eef">;
291 def HasHBC : Predicate<"Subtarget->hasHBC()">,
293 def HasMOPS : Predicate<"Subtarget->hasMOPS()">,
295 def HasCLRBHB : Predicate<"Subtarget->hasCLRBHB()">,
297 def HasSPECRES2 : Predicate<"Subtarget->hasSPECRES2()">,
299 def HasITE : Predicate<"Subtarget->hasITE()">,
301 def HasTHE : Predicate<"Subtarget->hasTHE()">,
303 def HasRCPC3 : Predicate<"Subtarget->hasRCPC3()">,
305 def HasLSE128 : Predicate<"Subtarget->hasLSE128()">,
307 def HasD128 : Predicate<"Subtarget->hasD128()">,
309 def HasCHK : Predicate<"Subtarget->hasCHK()">,
311 def HasGCS : Predicate<"Subtarget->hasGCS()">,
313 def HasCPA : Predicate<"Subtarget->hasCPA()">,
315 def IsLE : Predicate<"Subtarget->isLittleEndian()">;
316 def IsBE : Predicate<"!Subtarget->isLittleEndian()">;
317 def IsWindows : Predicate<"Subtarget->isTargetWindows()">;
319 : Predicate<"Subtarget->useExperimentalZeroingPseudos()">;
321 : Predicate<"Subtarget->useAlternateSExtLoadCVTF32Pattern()">;
327 def UseScalarIncVL : Predicate<"Subtarget->useScalarIncVL()">;
329 def NoUseScalarIncVL : Predicate<"!Subtarget->useScalarIncVL()">;
331 def UseSVEFPLD1R : Predicate<"!Subtarget->noSVEFPLD1R()">;
334 SDTypeProfile<1, 1, [SDTCisSameAs<0, 1>,
338 //===----------------------------------------------------------------------===//
339 // AArch64-specific DAG Nodes.
342 // SDTBinaryArithWithFlagsOut - RES1, FLAGS = op LHS, RHS
344 [SDTCisSameAs<0, 2>,
345 SDTCisSameAs<0, 3>,
346 SDTCisInt<0>, SDTCisVT<1, i32>]>;
348 // SDTBinaryArithWithFlagsIn - RES1, FLAGS = op LHS, RHS, FLAGS
350 [SDTCisSameAs<0, 1>,
351 SDTCisSameAs<0, 2>,
352 SDTCisInt<0>,
355 // SDTBinaryArithWithFlagsInOut - RES1, FLAGS = op LHS, RHS, FLAGS
357 [SDTCisSameAs<0, 2>,
358 SDTCisSameAs<0, 3>,
359 SDTCisInt<0>,
363 def SDT_AArch64Brcond : SDTypeProfile<0, 3,
364 [SDTCisVT<0, OtherVT>, SDTCisVT<1, i32>,
366 def SDT_AArch64cbz : SDTypeProfile<0, 2, [SDTCisInt<0>, SDTCisVT<1, OtherVT>]>;
367 def SDT_AArch64tbz : SDTypeProfile<0, 3, [SDTCisInt<0>, SDTCisInt<1>,
372 [SDTCisSameAs<0, 1>,
373 SDTCisSameAs<0, 2>,
377 [SDTCisVT<0, i32>,
384 [SDTCisVT<0, i32>,
390 def SDT_AArch64FCmp : SDTypeProfile<0, 2,
391 [SDTCisFP<0>,
392 SDTCisSameAs<0, 1>]>;
393 def SDT_AArch64Dup : SDTypeProfile<1, 1, [SDTCisVec<0>]>;
394 def SDT_AArch64DupLane : SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisInt<2>]>;
395 def SDT_AArch64Insr : SDTypeProfile<1, 2, [SDTCisVec<0>]>;
396 def SDT_AArch64Zip : SDTypeProfile<1, 2, [SDTCisVec<0>,
397 SDTCisSameAs<0, 1>,
398 SDTCisSameAs<0, 2>]>;
401 def SDT_AArch64vecimm : SDTypeProfile<1, 3, [SDTCisVec<0>, SDTCisSameAs<0,1>,
403 def SDT_AArch64UnaryVec: SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisSameAs<0,1>]>;
404 def SDT_AArch64ExtVec: SDTypeProfile<1, 3, [SDTCisVec<0>, SDTCisSameAs<0,1>,
405 SDTCisSameAs<0,2>, SDTCisInt<3>]>;
406 def SDT_AArch64vshift : SDTypeProfile<1, 2, [SDTCisSameAs<0,1>, SDTCisInt<2>]>;
407 def SDT_AArch64Dot: SDTypeProfile<1, 3, [SDTCisVec<0>, SDTCisSameAs<0,1>,
410 def SDT_AArch64vshiftinsert : SDTypeProfile<1, 3, [SDTCisVec<0>, SDTCisInt<3>,
411 SDTCisSameAs<0,1>,
412 SDTCisSameAs<0,2>]>;
414 def SDT_AArch64unvec : SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisSameAs<0,1>]>;
417 def SDT_AArch64binvec : SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisSameAs<0,1>,
418 SDTCisSameAs<0,2>]>;
419 def SDT_AArch64trivec : SDTypeProfile<1, 3, [SDTCisVec<0>, SDTCisSameAs<0,1>,
420 SDTCisSameAs<0,2>,
421 SDTCisSameAs<0,3>]>;
422 def SDT_AArch64TCRET : SDTypeProfile<0, 2, [SDTCisPtrTy<0>]>;
423 def SDT_AArch64PREFETCH : SDTypeProfile<0, 2, [SDTCisVT<0, i32>, SDTCisPtrTy<1>]>;
425 def SDT_AArch64ITOF : SDTypeProfile<1, 1, [SDTCisFP<0>, SDTCisSameAs<0,1>]>;
427 def SDT_AArch64TLSDescCall : SDTypeProfile<0, -2, [SDTCisPtrTy<0>,
430 def SDT_AArch64uaddlp : SDTypeProfile<1, 1, [SDTCisVec<0>, SDTCisVec<1>]>;
432 def SDT_AArch64ldp : SDTypeProfile<2, 1, [SDTCisVT<0, i64>, SDTCisSameAs<0, 1>, SDTCisPtrTy<2>]>;
433 def SDT_AArch64ldiapp : SDTypeProfile<2, 1, [SDTCisVT<0, i64>, SDTCisSameAs<0, 1>, SDTCisPtrTy<2>]>;
434 def SDT_AArch64ldnp : SDTypeProfile<2, 1, [SDTCisVT<0, v4i32>, SDTCisSameAs<0, 1>, SDTCisPtrTy<2>]>;
435 def SDT_AArch64stp : SDTypeProfile<0, 3, [SDTCisVT<0, i64>, SDTCisSameAs<0, 1>, SDTCisPtrTy<2>]>;
436 def SDT_AArch64stilp : SDTypeProfile<0, 3, [SDTCisVT<0, i64>, SDTCisSameAs<0, 1>, SDTCisPtrTy<2>]>;
437 def SDT_AArch64stnp : SDTypeProfile<0, 3, [SDTCisVT<0, v4i32>, SDTCisSameAs<0, 1>, SDTCisPtrTy<2>]>;
448 def SDT_AArch64TLSDescCallSeq : SDTypeProfile<0,1,
449 [SDTCisPtrTy<0>]>;
452 [SDTCisVT<0, i64>, SDTCisVT<1, i32>,
457 SDTCisVec<0>, SDTCisSameAs<0, 1>, SDTCisInt<2>
460 // non-extending masked load fragment.
464 return cast<MaskedLoadSDNode>(N)->getExtensionType() == ISD::NON_EXTLOAD &&
465 cast<MaskedLoadSDNode>(N)->isUnindexed() &&
466 !cast<MaskedLoadSDNode>(N)->isNonTemporal();
472 return (cast<MaskedLoadSDNode>(N)->getExtensionType() == ISD::EXTLOAD ||
473 cast<MaskedLoadSDNode>(N)->getExtensionType() == ISD::ZEXTLOAD) &&
474 cast<MaskedLoadSDNode>(N)->isUnindexed();
479 return cast<MaskedLoadSDNode>(N)->getMemoryVT().getScalarType() == MVT::i8;
484 return cast<MaskedLoadSDNode>(N)->getMemoryVT().getScalarType() == MVT::i16;
489 return cast<MaskedLoadSDNode>(N)->getMemoryVT().getScalarType() == MVT::i32;
495 return cast<MaskedLoadSDNode>(N)->getExtensionType() == ISD::SEXTLOAD &&
496 cast<MaskedLoadSDNode>(N)->isUnindexed();
501 return cast<MaskedLoadSDNode>(N)->getMemoryVT().getScalarType() == MVT::i8;
506 return cast<MaskedLoadSDNode>(N)->getMemoryVT().getScalarType() == MVT::i16;
511 return cast<MaskedLoadSDNode>(N)->getMemoryVT().getScalarType() == MVT::i32;
517 return cast<MaskedLoadSDNode>(N)->getExtensionType() == ISD::NON_EXTLOAD &&
518 cast<MaskedLoadSDNode>(N)->isUnindexed() &&
519 cast<MaskedLoadSDNode>(N)->isNonTemporal();
522 // non-truncating masked store fragment.
526 return !cast<MaskedStoreSDNode>(N)->isTruncatingStore() &&
527 cast<MaskedStoreSDNode>(N)->isUnindexed() &&
528 !cast<MaskedStoreSDNode>(N)->isNonTemporal();
534 return cast<MaskedStoreSDNode>(N)->isTruncatingStore() &&
535 cast<MaskedStoreSDNode>(N)->isUnindexed();
540 return cast<MaskedStoreSDNode>(N)->getMemoryVT().getScalarType() == MVT::i8;
545 return cast<MaskedStoreSDNode>(N)->getMemoryVT().getScalarType() == MVT::i16;
550 return cast<MaskedStoreSDNode>(N)->getMemoryVT().getScalarType() == MVT::i32;
556 return !cast<MaskedStoreSDNode>(N)->isTruncatingStore() &&
557 cast<MaskedStoreSDNode>(N)->isUnindexed() &&
558 cast<MaskedStoreSDNode>(N)->isNonTemporal();
567 bool Signed = MGS->isIndexSigned() ||
568 MGS->getIndex().getValueType().getVectorElementType() == MVT::i64;
569 return Signed && MGS->isIndexScaled();
576 bool Signed = MGS->isIndexSigned() ||
577 MGS->getIndex().getValueType().getVectorElementType() == MVT::i64;
578 return Signed && !MGS->isIndexScaled();
585 bool Signed = MGS->isIndexSigned() ||
586 MGS->getIndex().getValueType().getVectorElementType() == MVT::i64;
587 return !Signed && MGS->isIndexScaled();
594 bool Signed = MGS->isIndexSigned() ||
595 MGS->getIndex().getValueType().getVectorElementType() == MVT::i64;
596 return !Signed && !MGS->isIndexScaled();
613 // top16Zero - answer true if the upper 16 bits of $src are 0, false otherwise
615 return SDValue(N,0)->getValueType(0) == MVT::i32 &&
616 CurDAG->MaskedValueIsZero(SDValue(N,0), APInt::getHighBitsSet(32, 16));
619 // top32Zero - answer true if the upper 32 bits of $src are 0, false otherwise
621 return SDValue(N,0)->getValueType(0) == MVT::i64 &&
622 CurDAG->MaskedValueIsZero(SDValue(N,0), APInt::getHighBitsSet(64, 32));
625 // topbitsallzero - Return true if all bits except the lowest bit are known zero
627 return SDValue(N,0)->getValueType(0) == MVT::i32 &&
628 CurDAG->MaskedValueIsZero(SDValue(N,0), APInt::getHighBitsSet(32, 31));
631 return SDValue(N,0)->getValueType(0) == MVT::i64 &&
632 CurDAG->MaskedValueIsZero(SDValue(N,0), APInt::getHighBitsSet(64, 63));
641 SDCallSeqStart<[ SDTCisVT<0, i32>,
645 SDCallSeqEnd<[ SDTCisVT<0, i32>,
649 SDTypeProfile<0, -1, [SDTCisPtrTy<0>]>,
654 SDTypeProfile<0, -1, [SDTCisPtrTy<0>]>,
659 SDTypeProfile<0, -1, [SDTCisPtrTy<0>]>,
664 SDTypeProfile<0, -1, [SDTCisPtrTy<0>]>,
669 SDTypeProfile<0, -1, [SDTCisPtrTy<0>,
677 SDTypeProfile<0, 5, [SDTCisPtrTy<0>,
684 SDTypeProfile<0, -1, [SDTCisPtrTy<0>,
768 return N->getFlags().hasExact();
840 def SDT_AArch64mull : SDTypeProfile<1, 2, [SDTCisInt<0>, SDTCisInt<1>,
912 def SDT_AArch64SETTAG : SDTypeProfile<0, 2, [SDTCisPtrTy<0>, SDTCisPtrTy<1>]>;
919 SDTCisInt<0>, SDTCisInt<1>, SDTCisOpSmallerThanOp<1, 0>
937 SDTypeProfile<0, 1, [SDTCisPtrTy<0>]>,
941 SDTypeProfile<1, 1, [SDTCisVT<0, i64>, SDTCisVT<1, i32>]>,
944 def SD_AArch64rshrnb : SDTypeProfile<1, 2, [SDTCisVec<0>, SDTCisVec<1>, SDTCisInt<2>]>;
951 [SDTCisInt<0>, SDTCisVec<1>]>, []>;
957 if (N->getOpcode() == ISD::ADD)
959 return CurDAG->isADDLike(SDValue(N,0));
968 // Match mul with enough sign-bits. Can be reduced to a smaller mul operand.
970 return CurDAG->ComputeNumSignBits(N->getOperand(0)) > 32 &&
971 CurDAG->ComputeNumSignBits(N->getOperand(1)) > 32;
974 //===----------------------------------------------------------------------===//
976 //===----------------------------------------------------------------------===//
979 // We could compute these on a per-module basis but doing so requires accessing
981 // to that (see post-commit review comments for r301750).
986 def UseSTRQro : Predicate<"!Subtarget->isSTRQroSlow() || shouldOptForSize(MF)">;
988 // Register restrictions for indirect tail-calls:
989 // - If branch target enforcement is enabled, indirect calls must use x16 or
992 // - If PAuthLR is enabled, x16 is used in the epilogue to hold the address
1000 …llX16X17 : Predicate<[{ MF->getInfo<AArch64FunctionInfo>()->branchTargetEnforcement() && !MF->get…
1002 …ilCallX17 : Predicate<[{ MF->getInfo<AArch64FunctionInfo>()->branchTargetEnforcement() && MF->getI…
1003 // BTI off, PAuthLR on: Any non-callee-saved register except x16
1004 …llNotX16 : Predicate<[{ !MF->getInfo<AArch64FunctionInfo>()->branchTargetEnforcement() && MF->getI…
1005 // BTI off, PAuthLR off: Any non-callee-saved register
1006 …lCallAny : Predicate<[{ !MF->getInfo<AArch64FunctionInfo>()->branchTargetEnforcement() && !MF->get…
1008 def SLSBLRMitigation : Predicate<[{ MF->getSubtarget<AArch64Subtarget>().hardenSlsBlr() }]>;
1009 def NoSLSBLRMitigation : Predicate<[{ !MF->getSubtarget<AArch64Subtarget>().hardenSlsBlr() }]>;
1015 …: Predicate<"!MF->getFunction().hasOptNone() || MF->getProperties().hasProperty(MachineFunctionPro…
1022 //===----------------------------------------------------------------------===//
1024 //===----------------------------------------------------------------------===//
1026 //===----------------------------------------------------------------------===//
1043 // stack-clash protection is enabled.
1051 // stack-clash protection is enabled.
1058 // when stack-clash protection is enabled.
1131 // In general these get lowered into a sequence of three 4-byte instructions.
1132 // 32-bit jump table destination is actually only 2 instructions since we can
1133 // use the table itself as a PC-relative base. But optimization occurs after
1148 // A hardened but more expensive version of jump-table dispatch.
1151 // a plain BR) in a single non-attackable sequence.
1154 // mean that the index can be attacker-controlled. To address that, we also do
1156 // jump-table array. When it doesn't, this branches to the first entry.
1160 // to avoid signing jump-table entries and turning them into pointers.
1175 // Space-consuming pseudo to aid testing of placement and reachability
1194 // This gets lowered to a pair of 4-byte instructions.
1198 // This gets lowered to a 4-byte instruction.
1204 //===----------------------------------------------------------------------===//
1206 //===----------------------------------------------------------------------===//
1209 def : InstAlias<"nop", (HINT 0b000)>;
1210 def : InstAlias<"yield",(HINT 0b001)>;
1211 def : InstAlias<"wfe", (HINT 0b010)>;
1212 def : InstAlias<"wfi", (HINT 0b011)>;
1213 def : InstAlias<"sev", (HINT 0b100)>;
1214 def : InstAlias<"sevl", (HINT 0b101)>;
1215 def : InstAlias<"dgh", (HINT 0b110)>;
1216 def : InstAlias<"esb", (HINT 0b10000)>, Requires<[HasRAS]>;
1222 def : InstAlias<"bti", (HINT 32), 0>;
1223 def : InstAlias<"bti $op", (HINT btihint_op:$op), 0>;
1232 def CLREX : CRmSystemI<imm0_15, 0b010, "clrex">;
1234 // NOTE: ideally, this would have mayStore = 0, mayLoad = 0, but we cannot
1237 def DMB : CRmSystemI<barrier_op, 0b101, "dmb",
1240 def DSB : CRmSystemI<barrier_op, 0b100, "dsb",
1243 def ISB : CRmSystemI<barrier_op, 0b110, "isb",
1246 def TSB : CRmSystemI<barrier_op, 0b010, "tsb", []> {
1247 let CRm = 0b0010;
1248 let Inst{12} = 0;
1252 def DSBnXS : CRmSystemI<barrier_nxs_op, 0b001, "dsb"> {
1253 let CRm{1-0} = 0b11;
1254 let Inst{9-8} = 0b10;
1259 def WFET : RegInputSystemI<0b0000, 0b000, "wfet">;
1260 def WFIT : RegInputSystemI<0b0000, 0b001, "wfit">;
1263 // Branch Record Buffer two-word mnemonic instructions
1265 : SimpleSystemI<0, (ins), "brb", keyword>, Sched<[WriteSys]> {
1266 let Inst{31-8} = 0b110101010000100101110010;
1267 let Inst{7-5} = op2;
1270 def BRB_IALL: BRBEI<0b100, "\tiall">;
1271 def BRB_INJ: BRBEI<0b101, "\tinj">;
1280 // ARMv9.4-A Guarded Control Stack
1282 : SimpleSystemI<0, (ins), mnemonic, "">, Sched<[]> {
1283 let Inst{20-8} = 0b0100001110111;
1284 let Inst{7-5} = op2;
1287 def GCSPUSHX : GCSNoOp<0b100, "gcspushx">;
1288 def GCSPOPCX : GCSNoOp<0b101, "gcspopcx">;
1289 def GCSPOPX : GCSNoOp<0b110, "gcspopx">;
1293 : RtSystemI<0, (outs), (ins GPR64:$Rt), mnemonic, "\t$Rt", pattern> {
1294 let Inst{20-19} = 0b01;
1295 let Inst{18-16} = op1;
1296 let Inst{15-8} = 0b01110111;
1297 let Inst{7-5} = op2;
1303 def GCSSS1 : GCSRtIn<0b011, 0b010, "gcsss1">;
1305 def GCSPUSHM : GCSRtIn<0b011, 0b000, "gcspushm">;
1310 let Inst{20-19} = 0b01;
1311 let Inst{18-16} = op1;
1312 let Inst{15-8} = 0b01110111;
1313 let Inst{7-5} = op2;
1322 def GCSSS2 : GCSRtOut<0b011, 0b011, "gcsss2">;
1325 def GCSPOPM : GCSRtOut<0b011, 0b001, "gcspopm",
1329 def GCSB_DSYNC_disable : InstAlias<"gcsb\tdsync", (HINT 19), 0>;
1334 let Uses = [X16], Defs = [X16], CRm = 0b0101 in {
1335 def CHKFEAT : SystemNoOperands<0b000, "hint\t#40",
1338 def : InstAlias<"chkfeat\tx16", (CHKFEAT), 0>;
1345 let Inst{31-15} = 0b11011001000111110;
1346 let Inst{14-12} = op;
1347 let Inst{11-10} = 0b11;
1348 let Inst{9-5} = Rn;
1349 let Inst{4-0} = Rt;
1352 def GCSSTR : GCSSt<"gcsstr", 0b000>;
1353 def GCSSTTR : GCSSt<"gcssttr", 0b001>;
1355 // ARMv8.2-A Dot Product
1357 defm SDOT : SIMDThreeSameVectorDot<0, 0, "sdot", AArch64sdot>;
1358 defm UDOT : SIMDThreeSameVectorDot<1, 0, "udot", AArch64udot>;
1359 defm SDOTlane : SIMDThreeSameVectorDotIndex<0, 0, 0b10, "sdot", AArch64sdot>;
1360 defm UDOTlane : SIMDThreeSameVectorDotIndex<1, 0, 0b10, "udot", AArch64udot>;
1363 // ARMv8.6-A BFloat
1366 defm BF16DOTlane : SIMDThreeSameVectorBF16DotI<0, "bfdot">;
1368 def BFMLALB : SIMDBF16MLAL<0, "bfmlalb", int_aarch64_neon_bfmlalb>;
1370 def BFMLALBIdx : SIMDBF16MLALIndex<0, "bfmlalb", int_aarch64_neon_bfmlalb>;
1378 // Vector-scalar BFDOT:
1379 // The second source operand of the 64-bit variant of BF16DOTlane is a 128-bit
1380 // register (the instruction uses a single 32-bit lane from it), so the pattern
1389 (i64 0))))),
1392 (SUBREG_TO_REG (i32 0), V64:$Rm, dsub),
1404 def SMMLA : SIMDThreeSameVectorMatMul<0, 0, "smmla", int_aarch64_neon_smmla>;
1405 def UMMLA : SIMDThreeSameVectorMatMul<0, 1, "ummla", int_aarch64_neon_ummla>;
1406 def USMMLA : SIMDThreeSameVectorMatMul<1, 0, "usmmla", int_aarch64_neon_usmmla>;
1407 defm USDOT : SIMDThreeSameVectorDot<0, 1, "usdot", int_aarch64_neon_usdot>;
1408 defm USDOTlane : SIMDThreeSameVectorDotIndex<0, 1, 0b10, "usdot", int_aarch64_neon_usdot>;
1416 : BaseSIMDThreeSameVectorIndexS<Q, 0, 0b00, 0b1111, "sudot", dst_kind,
1428 def v8i8 : BaseSIMDSUDOTIndex<0, ".2s", ".8b", ".4b", V64, v2i32, v8i8>;
1436 // ARMv8.2-A FP16 Fused Multiply-Add Long
1438 defm FMLAL : SIMDThreeSameVectorFML<0, 1, 0b001, "fmlal", int_aarch64_neon_fmlal>;
1439 defm FMLSL : SIMDThreeSameVectorFML<0, 1, 0b101, "fmlsl", int_aarch64_neon_fmlsl>;
1440 defm FMLAL2 : SIMDThreeSameVectorFML<1, 0, 0b001, "fmlal2", int_aarch64_neon_fmlal2>;
1441 defm FMLSL2 : SIMDThreeSameVectorFML<1, 0, 0b101, "fmlsl2", int_aarch64_neon_fmlsl2>;
1442 defm FMLALlane : SIMDThreeSameVectorFMLIndex<0, 0b0000, "fmlal", int_aarch64_neon_fmlal>;
1443 defm FMLSLlane : SIMDThreeSameVectorFMLIndex<0, 0b0100, "fmlsl", int_aarch64_neon_fmlsl>;
1444 defm FMLAL2lane : SIMDThreeSameVectorFMLIndex<1, 0b1000, "fmlal2", int_aarch64_neon_fmlal2>;
1445 defm FMLSL2lane : SIMDThreeSameVectorFMLIndex<1, 0b1100, "fmlsl2", int_aarch64_neon_fmlsl2>;
1448 // Armv8.2-A Crypto extensions
1450 def SHA512H : CryptoRRRTied<0b0, 0b00, "sha512h">;
1451 def SHA512H2 : CryptoRRRTied<0b0, 0b01, "sha512h2">;
1452 def SHA512SU0 : CryptoRRTied_2D<0b0, 0b00, "sha512su0">;
1453 def SHA512SU1 : CryptoRRRTied_2D<0b0, 0b10, "sha512su1">;
1454 def RAX1 : CryptoRRR_2D<0b0,0b11, "rax1">;
1455 def EOR3 : CryptoRRRR_16B<0b00, "eor3">;
1456 def BCAX : CryptoRRRR_16B<0b01, "bcax">;
1520 def SM3TT1A : CryptoRRRi2Tied<0b0, 0b00, "sm3tt1a">;
1521 def SM3TT1B : CryptoRRRi2Tied<0b0, 0b01, "sm3tt1b">;
1522 def SM3TT2A : CryptoRRRi2Tied<0b0, 0b10, "sm3tt2a">;
1523 def SM3TT2B : CryptoRRRi2Tied<0b0, 0b11, "sm3tt2b">;
1524 def SM3SS1 : CryptoRRRR_4S<0b10, "sm3ss1">;
1525 def SM3PARTW1 : CryptoRRRTied_4S<0b1, 0b00, "sm3partw1">;
1526 def SM3PARTW2 : CryptoRRRTied_4S<0b1, 0b01, "sm3partw2">;
1527 def SM4ENCKEY : CryptoRRR_4S<0b1, 0b10, "sm4ekey">;
1528 def SM4E : CryptoRRTied_4S<0b0, 0b01, "sm4e">;
1559 def LDAPRB : RCPCLoad<0b00, "ldaprb", GPR32>;
1560 def LDAPRH : RCPCLoad<0b01, "ldaprh", GPR32>;
1561 def LDAPRW : RCPCLoad<0b10, "ldapr", GPR32>;
1562 def LDAPRX : RCPCLoad<0b11, "ldapr", GPR64>;
1565 // v8.3a complex add and multiply-accumulate. No predicate here, that is done
1567 defm FCMLA : SIMDThreeSameVectorTiedComplexHSD<1, 0b110, complexrotateop,
1569 defm FCADD : SIMDThreeSameVectorComplexHSD<1, 0b111, complexrotateopodd,
1571 defm FCMLA : SIMDIndexedTiedComplexHSD<0, 1, complexrotateop, "fcmla">;
1575 (FCADDv4f16 (v4f16 V64:$Rn), (v4f16 V64:$Rm), (i32 0))>;
1579 (FCADDv8f16 (v8f16 V128:$Rn), (v8f16 V128:$Rm), (i32 0))>;
1586 (FCADDv2f32 (v2f32 V64:$Rn), (v2f32 V64:$Rm), (i32 0))>;
1591 (!cast<Instruction>("FCADD"#Ty) (Ty V128:$Rn), (Ty V128:$Rm), (i32 0))>;
1599 (!cast<Instruction>("FCMLA" # ty) $Rd, $Rn, $Rm, 0)>;
1610 (!cast<Instruction>("FCMLA" # ty # "_indexed") $Rd, $Rn, $Rm, VectorIndexS:$idx, 0)>;
1644 def PACIAZ : SystemNoOperands<0b000, "hint\t#24">;
1645 def PACIBZ : SystemNoOperands<0b010, "hint\t#26">;
1647 def AUTIAZ : SystemNoOperands<0b100, "hint\t#28">;
1648 def AUTIBZ : SystemNoOperands<0b110, "hint\t#30">;
1652 def PACIASP : SystemNoOperands<0b001, "hint\t#25">;
1653 def PACIBSP : SystemNoOperands<0b011, "hint\t#27">;
1655 def AUTIASP : SystemNoOperands<0b101, "hint\t#29">;
1656 def AUTIBSP : SystemNoOperands<0b111, "hint\t#31">;
1659 let Uses = [X16, X17], Defs = [X17], CRm = 0b0001 in {
1660 def PACIA1716 : SystemNoOperands<0b000, "hint\t#8">;
1661 def PACIB1716 : SystemNoOperands<0b010, "hint\t#10">;
1663 def AUTIA1716 : SystemNoOperands<0b100, "hint\t#12">;
1664 def AUTIB1716 : SystemNoOperands<0b110, "hint\t#14">;
1668 let Uses = [LR], Defs = [LR], CRm = 0b0000 in {
1669 def XPACLRI : SystemNoOperands<0b111, "hint\t#7">;
1676 def : InstAlias<"paciaz", (PACIAZ), 0>;
1677 def : InstAlias<"pacibz", (PACIBZ), 0>;
1678 def : InstAlias<"autiaz", (AUTIAZ), 0>;
1679 def : InstAlias<"autibz", (AUTIBZ), 0>;
1680 def : InstAlias<"paciasp", (PACIASP), 0>;
1681 def : InstAlias<"pacibsp", (PACIBSP), 0>;
1682 def : InstAlias<"autiasp", (AUTIASP), 0>;
1683 def : InstAlias<"autibsp", (AUTIBSP), 0>;
1684 def : InstAlias<"pacia1716", (PACIA1716), 0>;
1685 def : InstAlias<"pacib1716", (PACIB1716), 0>;
1686 def : InstAlias<"autia1716", (AUTIA1716), 0>;
1687 def : InstAlias<"autib1716", (AUTIB1716), 0>;
1688 def : InstAlias<"xpaclri", (XPACLRI), 0>;
1724 def IA : SignAuthOneData<prefix, 0b00, !strconcat(asm, "ia"), op>;
1725 def IB : SignAuthOneData<prefix, 0b01, !strconcat(asm, "ib"), op>;
1726 def DA : SignAuthOneData<prefix, 0b10, !strconcat(asm, "da"), op>;
1727 def DB : SignAuthOneData<prefix, 0b11, !strconcat(asm, "db"), op>;
1728 def IZA : SignAuthZero<prefix_z, 0b00, !strconcat(asm, "iza"), op>;
1729 def DZA : SignAuthZero<prefix_z, 0b10, !strconcat(asm, "dza"), op>;
1730 def IZB : SignAuthZero<prefix_z, 0b01, !strconcat(asm, "izb"), op>;
1731 def DZB : SignAuthZero<prefix_z, 0b11, !strconcat(asm, "dzb"), op>;
1734 defm PAC : SignAuth<0b000, 0b010, "pac", int_ptrauth_sign>;
1735 defm AUT : SignAuth<0b001, 0b011, "aut", null_frag>;
1737 def XPACI : ClearAuth<0, "xpaci">;
1738 def : Pat<(int_ptrauth_strip GPR64:$Rd, 0), (XPACI GPR64:$Rd)>;
1745 def PACGA : SignAuthTwoOperand<0b1100, "pacga", int_ptrauth_sign_generic>;
1749 def BRAA : AuthBranchTwoOperands<0, 0, "braa">;
1750 def BRAB : AuthBranchTwoOperands<0, 1, "brab">;
1753 def BLRAA : AuthBranchTwoOperands<1, 0, "blraa">;
1758 def BRAAZ : AuthOneOperand<0b000, 0, "braaz">;
1759 def BRABZ : AuthOneOperand<0b000, 1, "brabz">;
1762 def BLRAAZ : AuthOneOperand<0b001, 0, "blraaz">;
1763 def BLRABZ : AuthOneOperand<0b001, 1, "blrabz">;
1778 let mayStore = 0;
1779 let mayLoad = 0;
1806 let mayStore = 0;
1807 let mayLoad = 0;
1817 def RETAA : AuthReturn<0b010, 0, "retaa">;
1818 def RETAB : AuthReturn<0b010, 1, "retab">;
1819 def ERETAA : AuthReturn<0b100, 0, "eretaa">;
1820 def ERETAB : AuthReturn<0b100, 1, "eretab">;
1823 defm LDRAA : AuthLoad<0, "ldraa", simm10Scaled>;
1833 let mayStore = 0;
1834 let mayLoad = 0;
1840 // AUT and re-PAC a value, using different keys/data.
1850 let mayStore = 0;
1851 let mayLoad = 0;
1916 // v9.5-A pointer authentication extensions
1919 // disassembling if we don't have the pauth-lr feature.
1920 let CRm = 0b0100 in {
1921 def PACM : SystemNoOperands<0b111, "hint\t#39">;
1923 def : InstAlias<"pacm", (PACM), 0>;
1928 def PACIASPPC : SignAuthFixedRegs<0b00001, 0b101000, "paciasppc">;
1929 def PACIBSPPC : SignAuthFixedRegs<0b00001, 0b101001, "pacibsppc">;
1930 def PACNBIASPPC : SignAuthFixedRegs<0b00001, 0b100000, "pacnbiasppc">;
1931 def PACNBIBSPPC : SignAuthFixedRegs<0b00001, 0b100001, "pacnbibsppc">;
1933 def AUTIASPPCi : SignAuthPCRel<0b00, "autiasppc">;
1934 def AUTIBSPPCi : SignAuthPCRel<0b01, "autibsppc">;
1936 def AUTIASPPCr : SignAuthOneReg<0b00001, 0b100100, "autiasppcr">;
1937 def AUTIBSPPCr : SignAuthOneReg<0b00001, 0b100101, "autibsppcr">;
1939 def PACIA171615 : SignAuthFixedRegs<0b00001, 0b100010, "pacia171615">;
1940 def PACIB171615 : SignAuthFixedRegs<0b00001, 0b100011, "pacib171615">;
1941 def AUTIA171615 : SignAuthFixedRegs<0b00001, 0b101110, "autia171615">;
1942 def AUTIB171615 : SignAuthFixedRegs<0b00001, 0b101111, "autib171615">;
1947 def RETAASPPCi : SignAuthReturnPCRel<0b000, 0b11111, "retaasppc">;
1948 def RETABSPPCi : SignAuthReturnPCRel<0b001, 0b11111, "retabsppc">;
1950 def RETAASPPCr : SignAuthReturnReg<0b000010, "retaasppcr">;
1951 def RETABSPPCr : SignAuthReturnReg<0b000011, "retabsppcr">;
1959 def FJCVTZS : BaseFPToIntegerUnscaled<0b01, 0b11, 0b110, FPR64, GPR32,
1963 let Inst{31} = 0;
1968 def CFINV : SimpleSystemI<0, (ins), "cfinv", "">, Sched<[WriteSys]> {
1969 let Inst{20-5} = 0b0000001000000000;
1971 def SETF8 : BaseFlagManipulation<0, 0, (ins GPR32:$Rn), "setf8", "{\t$Rn}">;
1972 def SETF16 : BaseFlagManipulation<0, 1, (ins GPR32:$Rn), "setf16", "{\t$Rn}">;
1981 let Inst{18-16} = 0b000;
1982 let Inst{11-8} = 0b0000;
1983 let Unpredictable{11-8} = 0b1111;
1984 let Inst{7-5} = 0b001;
1988 let Inst{18-16} = 0b000;
1989 let Inst{11-8} = 0b0000;
1990 let Unpredictable{11-8} = 0b1111;
1991 let Inst{7-5} = 0b010;
1996 // Armv8.5-A speculation barrier
1997 def SB : SimpleSystemI<0, (ins), "sb", "">, Sched<[]> {
1998 let Inst{20-5} = 0b0001100110000111;
1999 let Unpredictable{11-8} = 0b1111;
2004 def : InstAlias<"clrex", (CLREX 0xf)>;
2005 def : InstAlias<"isb", (ISB 0xf)>;
2006 def : InstAlias<"ssbb", (DSB 0)>;
2008 def : InstAlias<"dfb", (DSB 0b1100)>, Requires<[HasV8_0r]>;
2023 // This gets lowered into a 24-byte instruction sequence
2058 def : Pat<(readcyclecounter), (MRS 0xdf02)>;
2064 PseudoInstExpansion<(MRS GPR64:$dst, 0xda20)>,
2069 PseudoInstExpansion<(MSR 0xda20, GPR64:$val)>,
2075 PseudoInstExpansion<(MRS GPR64:$dst, 0xda21)>,
2080 PseudoInstExpansion<(MSR 0xda21, GPR64:$val)>,
2084 def SYSxt : SystemXtI<0, "sys">;
2094 def TSTART : TMSystemI<0b0000, "tstart",
2097 def TCOMMIT : TMSystemINoOperand<0b0000, "tcommit", [(int_aarch64_tcommit)]>;
2099 def TCANCEL : TMSystemException<0b011, "tcancel",
2102 def TTEST : TMSystemI<0b0001, "ttest", [(set GPR64:$Rt, (int_aarch64_ttest))]> {
2103 let mayLoad = 0;
2104 let mayStore = 0;
2108 //===----------------------------------------------------------------------===//
2110 //===----------------------------------------------------------------------===//
2112 defm MOVK : InsertImmediate<0b11, "movk">;
2113 defm MOVN : MoveImmediate<0b00, "movn">;
2116 defm MOVZ : MoveImmediate<0b10, "movz">;
2118 // First group of aliases covers an implicit "lsl #0".
2119 def : InstAlias<"movk $dst, $imm", (MOVKWi GPR32:$dst, timm32_0_65535:$imm, 0), 0>;
2120 def : InstAlias<"movk $dst, $imm", (MOVKXi GPR64:$dst, timm32_0_65535:$imm, 0), 0>;
2121 def : InstAlias<"movn $dst, $imm", (MOVNWi GPR32:$dst, timm32_0_65535:$imm, 0)>;
2122 def : InstAlias<"movn $dst, $imm", (MOVNXi GPR64:$dst, timm32_0_65535:$imm, 0)>;
2123 def : InstAlias<"movz $dst, $imm", (MOVZWi GPR32:$dst, timm32_0_65535:$imm, 0)>;
2124 def : InstAlias<"movz $dst, $imm", (MOVZXi GPR64:$dst, timm32_0_65535:$imm, 0)>;
2130 def : InstAlias<"movz $Rd, $sym", (MOVZXi GPR64:$Rd, movw_symbol_g0:$sym, 0)>;
2135 def : InstAlias<"movn $Rd, $sym", (MOVNXi GPR64:$Rd, movw_symbol_g0:$sym, 0)>;
2137 def : InstAlias<"movk $Rd, $sym", (MOVKXi GPR64:$Rd, movw_symbol_g3:$sym, 48), 0>;
2138 def : InstAlias<"movk $Rd, $sym", (MOVKXi GPR64:$Rd, movw_symbol_g2:$sym, 32), 0>;
2139 def : InstAlias<"movk $Rd, $sym", (MOVKXi GPR64:$Rd, movw_symbol_g1:$sym, 16), 0>;
2140 def : InstAlias<"movk $Rd, $sym", (MOVKXi GPR64:$Rd, movw_symbol_g0:$sym, 0), 0>;
2143 def : InstAlias<"movz $Rd, $sym", (MOVZWi GPR32:$Rd, movw_symbol_g0:$sym, 0)>;
2146 def : InstAlias<"movn $Rd, $sym", (MOVNWi GPR32:$Rd, movw_symbol_g0:$sym, 0)>;
2148 def : InstAlias<"movk $Rd, $sym", (MOVKWi GPR32:$Rd, movw_symbol_g1:$sym, 16), 0>;
2149 def : InstAlias<"movk $Rd, $sym", (MOVKWi GPR32:$Rd, movw_symbol_g0:$sym, 0), 0>;
2169 defm : movw_mov_alias<"MOVZ", MOVZWi, GPR32, 32, 0>;
2172 defm : movw_mov_alias<"MOVZ", MOVZXi, GPR64, 64, 0>;
2177 defm : movw_mov_alias<"MOVN", MOVNWi, GPR32, 32, 0>;
2180 defm : movw_mov_alias<"MOVN", MOVNXi, GPR64, 64, 0>;
2201 // If possible, we want to use MOVi32imm even for 64-bit moves. This gives the
2205 return (Imm & 0xffffffffULL) == static_cast<uint64_t>(Imm);
2215 return CurDAG->getTargetConstant(N->getZExtValue(), SDLoc(N), MVT::i32);
2222 // The SUBREG_TO_REG isn't eliminated at -O0, which can result in pointless
2225 (SUBREG_TO_REG (i64 0), (MOVi32imm (trunc_imm imm:$src)), sub_32)>;
2230 return CurDAG->getTargetConstant(
2231 N->getValueAPF().bitcastToAPInt().getZExtValue(), SDLoc(N), MVT::i32);
2235 return CurDAG->getTargetConstant(
2236 N->getValueAPF().bitcastToAPInt().getZExtValue(), SDLoc(N), MVT::i64);
2250 (MOVKXi (MOVKXi (MOVKXi (MOVZXi tglobaladdr:$g0, 0),
2257 (MOVKXi (MOVKXi (MOVKXi (MOVZXi tblockaddress:$g0, 0),
2264 (MOVKXi (MOVKXi (MOVKXi (MOVZXi tconstpool:$g0, 0),
2271 (MOVKXi (MOVKXi (MOVKXi (MOVZXi tjumptable:$g0, 0),
2277 //===----------------------------------------------------------------------===//
2279 //===----------------------------------------------------------------------===//
2282 defm ADC : AddSubCarry<0, "adc", "adcs", AArch64adc, AArch64adc_flag>;
2291 defm ADD : AddSub<0, "add", "sub", add>;
2295 (ADDWri GPR32sponly:$dst, GPR32sp:$src, 0, 0)>;
2297 (ADDWri GPR32sp:$dst, GPR32sponly:$src, 0, 0)>;
2299 (ADDXri GPR64sponly:$dst, GPR64sp:$src, 0, 0)>;
2301 (ADDXri GPR64sp:$dst, GPR64sponly:$src, 0, 0)>;
2303 defm ADDS : AddSubS<0, "adds", AArch64add_flag, "cmn", "subs", "cmp">;
2307 return N->getOpcode() == ISD::CopyFromReg &&
2308 cast<RegisterSDNode>(N->getOperand(1))->getReg() == AArch64::SP;
2333 // Because of the immediate format for add/sub-imm instructions, the
2334 // expression (add x, -1) must be transformed to (SUB{W,X}ri x, 1).
2347 // Because of the immediate format for add/sub-imm instructions, the
2348 // expression (add x, -1) must be transformed to (SUB{W,X}ri x, 1).
2361 def : InstAlias<"neg $dst, $src", (SUBWrs GPR32:$dst, WZR, GPR32:$src, 0), 3>;
2362 def : InstAlias<"neg $dst, $src", (SUBXrs GPR64:$dst, XZR, GPR64:$src, 0), 3>;
2368 def : InstAlias<"negs $dst, $src", (SUBSWrs GPR32:$dst, WZR, GPR32:$src, 0), 3>;
2369 def : InstAlias<"negs $dst, $src", (SUBSXrs GPR64:$dst, XZR, GPR64:$src, 0), 3>;
2377 defm UDIV : Div<0, "udiv", udiv>;
2386 defm ASRV : Shift<0b10, "asr", sra>;
2387 defm LSLV : Shift<0b00, "lsl", shl>;
2388 defm LSRV : Shift<0b01, "lsr", srl>;
2389 defm RORV : Shift<0b11, "ror", rotr>;
2400 // Multiply-add
2402 defm MADD : MulAccum<0, "madd">;
2421 def SMADDLrrr : WideMulAccum<0, 0b001, "smaddl", add, sext>;
2422 def SMSUBLrrr : WideMulAccum<1, 0b001, "smsubl", sub, sext>;
2423 def UMADDLrrr : WideMulAccum<0, 0b101, "umaddl", add, zext>;
2424 def UMSUBLrrr : WideMulAccum<1, 0b101, "umsubl", sub, zext>;
2432 def : Pat<(i64 (mul (and GPR64:$Rn, 0xFFFFFFFF), (and GPR64:$Rm, 0xFFFFFFFF))),
2434 def : Pat<(i64 (mul (and GPR64:$Rn, 0xFFFFFFFF), (zext GPR32:$Rm))),
2528 // Multiply-high
2529 def SMULHrr : MulHi<0b010, "smulh", mulhs>;
2530 def UMULHrr : MulHi<0b110, "umulh", mulhu>;
2533 def CRC32Brr : BaseCRC32<0, 0b00, 0, GPR32, int_aarch64_crc32b, "crc32b">;
2534 def CRC32Hrr : BaseCRC32<0, 0b01, 0, GPR32, int_aarch64_crc32h, "crc32h">;
2535 def CRC32Wrr : BaseCRC32<0, 0b10, 0, GPR32, int_aarch64_crc32w, "crc32w">;
2536 def CRC32Xrr : BaseCRC32<1, 0b11, 0, GPR64, int_aarch64_crc32x, "crc32x">;
2538 def CRC32CBrr : BaseCRC32<0, 0b00, 1, GPR32, int_aarch64_crc32cb, "crc32cb">;
2539 def CRC32CHrr : BaseCRC32<0, 0b01, 1, GPR32, int_aarch64_crc32ch, "crc32ch">;
2540 def CRC32CWrr : BaseCRC32<0, 0b10, 1, GPR32, int_aarch64_crc32cw, "crc32cw">;
2541 def CRC32CXrr : BaseCRC32<1, 0b11, 1, GPR64, int_aarch64_crc32cx, "crc32cx">;
2544 defm CAS : CompareAndSwap<0, 0, "">;
2545 defm CASA : CompareAndSwap<1, 0, "a">;
2546 defm CASL : CompareAndSwap<0, 1, "l">;
2550 defm CASP : CompareAndSwapPair<0, 0, "">;
2551 defm CASPA : CompareAndSwapPair<1, 0, "a">;
2552 defm CASPL : CompareAndSwapPair<0, 1, "l">;
2556 defm SWP : Swap<0, 0, "">;
2557 defm SWPA : Swap<1, 0, "a">;
2558 defm SWPL : Swap<0, 1, "l">;
2562 defm LDADD : LDOPregister<0b000, "add", 0, 0, "">;
2563 defm LDADDA : LDOPregister<0b000, "add", 1, 0, "a">;
2564 defm LDADDL : LDOPregister<0b000, "add", 0, 1, "l">;
2565 defm LDADDAL : LDOPregister<0b000, "add", 1, 1, "al">;
2567 defm LDCLR : LDOPregister<0b001, "clr", 0, 0, "">;
2568 defm LDCLRA : LDOPregister<0b001, "clr", 1, 0, "a">;
2569 defm LDCLRL : LDOPregister<0b001, "clr", 0, 1, "l">;
2570 defm LDCLRAL : LDOPregister<0b001, "clr", 1, 1, "al">;
2572 defm LDEOR : LDOPregister<0b010, "eor", 0, 0, "">;
2573 defm LDEORA : LDOPregister<0b010, "eor", 1, 0, "a">;
2574 defm LDEORL : LDOPregister<0b010, "eor", 0, 1, "l">;
2575 defm LDEORAL : LDOPregister<0b010, "eor", 1, 1, "al">;
2577 defm LDSET : LDOPregister<0b011, "set", 0, 0, "">;
2578 defm LDSETA : LDOPregister<0b011, "set", 1, 0, "a">;
2579 defm LDSETL : LDOPregister<0b011, "set", 0, 1, "l">;
2580 defm LDSETAL : LDOPregister<0b011, "set", 1, 1, "al">;
2582 defm LDSMAX : LDOPregister<0b100, "smax", 0, 0, "">;
2583 defm LDSMAXA : LDOPregister<0b100, "smax", 1, 0, "a">;
2584 defm LDSMAXL : LDOPregister<0b100, "smax", 0, 1, "l">;
2585 defm LDSMAXAL : LDOPregister<0b100, "smax", 1, 1, "al">;
2587 defm LDSMIN : LDOPregister<0b101, "smin", 0, 0, "">;
2588 defm LDSMINA : LDOPregister<0b101, "smin", 1, 0, "a">;
2589 defm LDSMINL : LDOPregister<0b101, "smin", 0, 1, "l">;
2590 defm LDSMINAL : LDOPregister<0b101, "smin", 1, 1, "al">;
2592 defm LDUMAX : LDOPregister<0b110, "umax", 0, 0, "">;
2593 defm LDUMAXA : LDOPregister<0b110, "umax", 1, 0, "a">;
2594 defm LDUMAXL : LDOPregister<0b110, "umax", 0, 1, "l">;
2595 defm LDUMAXAL : LDOPregister<0b110, "umax", 1, 1, "al">;
2597 defm LDUMIN : LDOPregister<0b111, "umin", 0, 0, "">;
2598 defm LDUMINA : LDOPregister<0b111, "umin", 1, 0, "a">;
2599 defm LDUMINL : LDOPregister<0b111, "umin", 0, 1, "l">;
2600 defm LDUMINAL : LDOPregister<0b111, "umin", 1, 1, "al">;
2615 def IRG : BaseTwoOperandRegReg<0b1, 0b0, 0b000100, GPR64sp, "irg",
2618 def GMI : BaseTwoOperandRegReg<0b1, 0b0, 0b000101, GPR64, "gmi",
2622 def ADDG : AddSubG<0, "addg", null_frag>;
2627 def SUBP : SUBP<0, "subp", int_aarch64_subp>, Sched<[]>;
2632 def : InstAlias<"cmpp $lhs, $rhs", (SUBPS XZR, GPR64sp:$lhs, GPR64sp:$rhs), 0>;
2641 def : InstAlias<"ldg $Rt, [$Rn]", (LDG GPR64:$Rt, GPR64sp:$Rn, 0), 1>;
2645 def STGM : MemTagVector<0, "stgm", "\t$Rt, [$Rn]",
2647 def STZGM : MemTagVector<0, "stzgm", "\t$Rt, [$Rn]",
2649 let Inst{23} = 0;
2652 defm STG : MemTagStore<0b00, "stg">;
2653 defm STZG : MemTagStore<0b01, "stzg">;
2654 defm ST2G : MemTagStore<0b10, "st2g">;
2655 defm STZ2G : MemTagStore<0b11, "stz2g">;
2666 defm STGP : StorePairOffset <0b01, 0, GPR64z, simm7s16, "stgp">;
2667 def STGPpre : StorePairPreIdx <0b01, 0, GPR64z, simm7s16, "stgp">;
2668 def STGPpost : StorePairPostIdx<0b01, 0, GPR64z, simm7s16, "stgp">;
2717 //===----------------------------------------------------------------------===//
2719 //===----------------------------------------------------------------------===//
2722 defm ANDS : LogicalImmS<0b11, "ands", AArch64and_flag, "bics">;
2723 defm AND : LogicalImm<0b00, "and", and, "bic">;
2724 defm EOR : LogicalImm<0b10, "eor", xor, "eon">;
2725 defm ORR : LogicalImm<0b01, "orr", or, "orn">;
2731 logical_imm32:$imm), 0>;
2733 logical_imm64:$imm), 0>;
2737 defm ANDS : LogicalRegS<0b11, 0, "ands", AArch64and_flag>;
2738 defm BICS : LogicalRegS<0b11, 1, "bics",
2740 defm AND : LogicalReg<0b00, 0, "and", and>;
2741 defm BIC : LogicalReg<0b00, 1, "bic",
2743 defm EON : LogicalReg<0b10, 1, "eon",
2745 defm EOR : LogicalReg<0b10, 0, "eor", xor>;
2746 defm ORN : LogicalReg<0b01, 1, "orn",
2748 defm ORR : LogicalReg<0b01, 0, "orr", or>;
2750 def : InstAlias<"mov $dst, $src", (ORRWrs GPR32:$dst, WZR, GPR32:$src, 0), 2>;
2751 def : InstAlias<"mov $dst, $src", (ORRXrs GPR64:$dst, XZR, GPR64:$src, 0), 2>;
2753 def : InstAlias<"mvn $Wd, $Wm", (ORNWrs GPR32:$Wd, WZR, GPR32:$Wm, 0), 3>;
2754 def : InstAlias<"mvn $Xd, $Xm", (ORNXrs GPR64:$Xd, XZR, GPR64:$Xm, 0), 3>;
2767 (ANDSWrs WZR, GPR32:$src1, GPR32:$src2, 0), 3>;
2769 (ANDSXrs XZR, GPR64:$src1, GPR64:$src2, 0), 3>;
2780 // Emit (and 0xFFFFFFFF) as a ORRWrr move which may be eliminated.
2782 def : Pat<(i64 (and GPR64:$Rn, 0xffffffff)),
2783 (SUBREG_TO_REG (i64 0), (ORRWrr WZR, (EXTRACT_SUBREG GPR64:$Rn, sub_32)), sub_32)>;
2786 //===----------------------------------------------------------------------===//
2788 //===----------------------------------------------------------------------===//
2790 defm CLS : OneOperandData<0b000101, "cls">;
2791 defm CLZ : OneOperandData<0b000100, "clz", ctlz>;
2792 defm RBIT : OneOperandData<0b000000, "rbit", bitreverse>;
2794 def REV16Wr : OneWRegData<0b000001, "rev16",
2796 def REV16Xr : OneXRegData<0b000001, "rev16", null_frag>;
2814 def REVWr : OneWRegData<0b000010, "rev", bswap>;
2815 def REVXr : OneXRegData<0b000011, "rev", bswap>;
2816 def REV32Xr : OneXRegData<0b000010, "rev32",
2819 def : InstAlias<"rev64 $Rd, $Rn", (REVXr GPR64:$Rd, GPR64:$Rn), 0>;
2826 // Match (srl (bswap x), C) -> revC if the upper bswap bits are known zero.
2830 def : Pat<(or (and (srl GPR64:$Rn, (i64 8)), (i64 0x00ff00ff00ff00ff)),
2831 (and (shl GPR64:$Rn, (i64 8)), (i64 0xff00ff00ff00ff00))),
2834 //===----------------------------------------------------------------------===//
2836 //===----------------------------------------------------------------------===//
2837 let hasSideEffects = 0 in
2849 //===----------------------------------------------------------------------===//
2851 //===----------------------------------------------------------------------===//
2852 let hasSideEffects = 0 in {
2853 defm BFM : BitfieldImmWith2RegArgs<0b01, "bfm">;
2854 defm SBFM : BitfieldImm<0b00, "sbfm">;
2855 defm UBFM : BitfieldImm<0b10, "ubfm">;
2859 uint64_t enc = (32 - N->getZExtValue()) & 0x1f;
2860 return CurDAG->getTargetConstant(enc, SDLoc(N), MVT::i64);
2864 uint64_t enc = 31 - N->getZExtValue();
2865 return CurDAG->getTargetConstant(enc, SDLoc(N), MVT::i64);
2868 // min(7, 31 - shift_amt)
2870 uint64_t enc = 31 - N->getZExtValue();
2872 return CurDAG->getTargetConstant(enc, SDLoc(N), MVT::i64);
2875 // min(15, 31 - shift_amt)
2877 uint64_t enc = 31 - N->getZExtValue();
2879 return CurDAG->getTargetConstant(enc, SDLoc(N), MVT::i64);
2883 uint64_t enc = (64 - N->getZExtValue()) & 0x3f;
2884 return CurDAG->getTargetConstant(enc, SDLoc(N), MVT::i64);
2888 uint64_t enc = 63 - N->getZExtValue();
2889 return CurDAG->getTargetConstant(enc, SDLoc(N), MVT::i64);
2892 // min(7, 63 - shift_amt)
2894 uint64_t enc = 63 - N->getZExtValue();
2896 return CurDAG->getTargetConstant(enc, SDLoc(N), MVT::i64);
2899 // min(15, 63 - shift_amt)
2901 uint64_t enc = 63 - N->getZExtValue();
2903 return CurDAG->getTargetConstant(enc, SDLoc(N), MVT::i64);
2906 // min(31, 63 - shift_amt)
2908 uint64_t enc = 63 - N->getZExtValue();
2910 return CurDAG->getTargetConstant(enc, SDLoc(N), MVT::i64);
2931 def : InstAlias<"sxtb $dst, $src", (SBFMWri GPR32:$dst, GPR32:$src, 0, 7)>;
2932 def : InstAlias<"sxtb $dst, $src", (SBFMXri GPR64:$dst, GPR64:$src, 0, 7)>;
2933 def : InstAlias<"sxth $dst, $src", (SBFMWri GPR32:$dst, GPR32:$src, 0, 15)>;
2934 def : InstAlias<"sxth $dst, $src", (SBFMXri GPR64:$dst, GPR64:$src, 0, 15)>;
2935 def : InstAlias<"sxtw $dst, $src", (SBFMXri GPR64:$dst, GPR64:$src, 0, 31)>;
2946 def : InstAlias<"uxtb $dst, $src", (UBFMWri GPR32:$dst, GPR32:$src, 0, 7)>;
2947 def : InstAlias<"uxtb $dst, $src", (UBFMXri GPR64:$dst, GPR64:$src, 0, 7)>;
2948 def : InstAlias<"uxth $dst, $src", (UBFMWri GPR32:$dst, GPR32:$src, 0, 15)>;
2949 def : InstAlias<"uxth $dst, $src", (UBFMXri GPR64:$dst, GPR64:$src, 0, 15)>;
2950 def : InstAlias<"uxtw $dst, $src", (UBFMXri GPR64:$dst, GPR64:$src, 0, 31)>;
2952 //===----------------------------------------------------------------------===//
2954 //===----------------------------------------------------------------------===//
2955 defm CCMN : CondComparison<0, "ccmn", AArch64ccmn>;
2958 //===----------------------------------------------------------------------===//
2960 //===----------------------------------------------------------------------===//
2961 defm CSEL : CondSelect<0, 0b00, "csel">;
2964 defm CSINC : CondSelectOp<0, 0b01, "csinc", inc>;
2965 defm CSINV : CondSelectOp<1, 0b00, "csinv", not>;
2966 defm CSNEG : CondSelectOp<1, 0b01, "csneg", ineg>;
2981 def : Pat<(AArch64csel (i32 0), (i32 1), (i32 imm:$cc), NZCV),
2983 def : Pat<(AArch64csel (i64 0), (i64 1), (i32 imm:$cc), NZCV),
2993 def : Pat<(AArch64csel (i32 0), (i32 -1), (i32 imm:$cc), NZCV),
2995 def : Pat<(AArch64csel (i64 0), (i64 -1), (i32 imm:$cc), NZCV),
2997 def : Pat<(AArch64csel GPR32:$tval, (i32 -1), (i32 imm:$cc), NZCV),
2999 def : Pat<(AArch64csel GPR64:$tval, (i64 -1), (i32 imm:$cc), NZCV),
3001 def : Pat<(AArch64csel (i32 -1), GPR32:$fval, (i32 imm:$cc), NZCV),
3003 def : Pat<(AArch64csel (i64 -1), GPR64:$fval, (i32 imm:$cc), NZCV),
3006 def : Pat<(add_and_or_is_add GPR32:$val, (AArch64csel (i32 0), (i32 1), (i32 imm:$cc), NZCV)),
3008 def : Pat<(add_and_or_is_add GPR64:$val, (zext (AArch64csel (i32 0), (i32 1), (i32 imm:$cc), NZCV))…
3011 def : Pat<(or (topbitsallzero32:$val), (AArch64csel (i32 0), (i32 1), (i32 imm:$cc), NZCV)),
3013 def : Pat<(or (topbitsallzero64:$val), (AArch64csel (i64 0), (i64 1), (i32 imm:$cc), NZCV)),
3015 def : Pat<(or (topbitsallzero64:$val), (zext (AArch64csel (i32 0), (i32 1), (i32 imm:$cc), NZCV))),
3018 def : Pat<(and (topbitsallzero32:$val), (AArch64csel (i32 0), (i32 1), (i32 imm:$cc), NZCV)),
3020 def : Pat<(and (topbitsallzero64:$val), (AArch64csel (i64 0), (i64 1), (i32 imm:$cc), NZCV)),
3022 def : Pat<(and (topbitsallzero64:$val), (zext (AArch64csel (i32 0), (i32 1), (i32 imm:$cc), NZCV))),
3053 //===----------------------------------------------------------------------===//
3054 // PC-relative instructions.
3055 //===----------------------------------------------------------------------===//
3057 let hasSideEffects = 0, mayStore = 0, mayLoad = 0 in {
3058 def ADR : ADRI<0, "adr", adrlabel,
3060 } // hasSideEffects = 0
3075 //===----------------------------------------------------------------------===//
3077 //===----------------------------------------------------------------------===//
3080 def RET : BranchReg<0b0010, "ret", []>;
3081 def DRPS : SpecialReturn<0b0101, "drps">;
3082 def ERET : SpecialReturn<0b0100, "eret">;
3089 def BLR : BranchReg<0b0001, "blr", []>;
3122 def BR : BranchReg<0b0000, "br", [(brind GPR64:$Rn)]>;
3125 // Create a separate pseudo-instruction for codegen to use so that we don't
3135 // This is a directive-like pseudo-instruction. The purpose is to insert an
3163 //===----------------------------------------------------------------------===//
3165 //===----------------------------------------------------------------------===//
3166 def Bcc : BranchCond<0, "b">;
3168 // Armv8.8-A variant form which hints to the branch predictor that
3173 //===----------------------------------------------------------------------===//
3174 // Compare-and-branch instructions.
3175 //===----------------------------------------------------------------------===//
3176 defm CBZ : CmpBranch<0, "cbz", AArch64cbz>;
3179 //===----------------------------------------------------------------------===//
3180 // Test-bit-and-branch instructions.
3181 //===----------------------------------------------------------------------===//
3182 defm TBZ : TestBranch<0, "tbz", AArch64tbz>;
3185 //===----------------------------------------------------------------------===//
3187 //===----------------------------------------------------------------------===//
3189 def B : BranchImm<0, "b", [(br bb:$addr)]>;
3197 //===----------------------------------------------------------------------===//
3199 //===----------------------------------------------------------------------===//
3201 def BRK : ExceptionGeneration<0b001, 0b00, "brk",
3204 def DCPS1 : ExceptionGeneration<0b101, 0b01, "dcps1">;
3205 def DCPS2 : ExceptionGeneration<0b101, 0b10, "dcps2">;
3206 def DCPS3 : ExceptionGeneration<0b101, 0b11, "dcps3">, Requires<[HasEL3]>;
3207 def HLT : ExceptionGeneration<0b010, 0b00, "hlt",
3209 def HVC : ExceptionGeneration<0b000, 0b10, "hvc">;
3210 def SMC : ExceptionGeneration<0b000, 0b11, "smc">, Requires<[HasEL3]>;
3211 def SVC : ExceptionGeneration<0b000, 0b01, "svc">;
3214 def : InstAlias<"dcps1", (DCPS1 0)>;
3215 def : InstAlias<"dcps2", (DCPS2 0)>;
3216 def : InstAlias<"dcps3", (DCPS3 0)>, Requires<[HasEL3]>;
3218 def UDF : UDFType<0, "udf">;
3220 //===----------------------------------------------------------------------===//
3222 //===----------------------------------------------------------------------===//
3225 defm LDPW : LoadPairOffset<0b00, 0, GPR32z, simm7s4, "ldp">;
3226 defm LDPX : LoadPairOffset<0b10, 0, GPR64z, simm7s8, "ldp">;
3228 defm LDPS : LoadPairOffset<0b00, 1, FPR32Op, simm7s4, "ldp">;
3229 defm LDPD : LoadPairOffset<0b01, 1, FPR64Op, simm7s8, "ldp">;
3230 defm LDPQ : LoadPairOffset<0b10, 1, FPR128Op, simm7s16, "ldp">;
3233 defm LDPSW : LoadPairOffset<0b01, 0, GPR64z, simm7s4, "ldpsw">;
3235 // Pair (pre-indexed)
3236 def LDPWpre : LoadPairPreIdx<0b00, 0, GPR32z, simm7s4, "ldp">;
3237 def LDPXpre : LoadPairPreIdx<0b10, 0, GPR64z, simm7s8, "ldp">;
3239 def LDPSpre : LoadPairPreIdx<0b00, 1, FPR32Op, simm7s4, "ldp">;
3240 def LDPDpre : LoadPairPreIdx<0b01, 1, FPR64Op, simm7s8, "ldp">;
3241 def LDPQpre : LoadPairPreIdx<0b10, 1, FPR128Op, simm7s16, "ldp">;
3244 def LDPSWpre : LoadPairPreIdx<0b01, 0, GPR64z, simm7s4, "ldpsw">;
3246 // Pair (post-indexed)
3247 def LDPWpost : LoadPairPostIdx<0b00, 0, GPR32z, simm7s4, "ldp">;
3248 def LDPXpost : LoadPairPostIdx<0b10, 0, GPR64z, simm7s8, "ldp">;
3250 def LDPSpost : LoadPairPostIdx<0b00, 1, FPR32Op, simm7s4, "ldp">;
3251 def LDPDpost : LoadPairPostIdx<0b01, 1, FPR64Op, simm7s8, "ldp">;
3252 def LDPQpost : LoadPairPostIdx<0b10, 1, FPR128Op, simm7s16, "ldp">;
3255 def LDPSWpost : LoadPairPostIdx<0b01, 0, GPR64z, simm7s4, "ldpsw">;
3259 defm LDNPW : LoadPairNoAlloc<0b00, 0, GPR32z, simm7s4, "ldnp">;
3260 defm LDNPX : LoadPairNoAlloc<0b10, 0, GPR64z, simm7s8, "ldnp">;
3262 defm LDNPS : LoadPairNoAlloc<0b00, 1, FPR32Op, simm7s4, "ldnp">;
3263 defm LDNPD : LoadPairNoAlloc<0b01, 1, FPR64Op, simm7s8, "ldnp">;
3264 defm LDNPQ : LoadPairNoAlloc<0b10, 1, FPR128Op, simm7s16, "ldnp">;
3272 //---
3274 //---
3277 defm LDRBB : Load8RO<0b00, 0, 0b01, GPR32, "ldrb", i32, zextloadi8>;
3278 defm LDRHH : Load16RO<0b01, 0, 0b01, GPR32, "ldrh", i32, zextloadi16>;
3279 defm LDRW : Load32RO<0b10, 0, 0b01, GPR32, "ldr", i32, load>;
3280 defm LDRX : Load64RO<0b11, 0, 0b01, GPR64, "ldr", i64, load>;
3282 // Floating-point
3284 defm LDRB : Load8RO<0b00, 1, 0b01, FPR8Op, "ldr", i8, load>;
3285 defm LDRH : Load16RO<0b01, 1, 0b01, FPR16Op, "ldr", f16, load>;
3286 defm LDRS : Load32RO<0b10, 1, 0b01, FPR32Op, "ldr", f32, load>;
3287 defm LDRD : Load64RO<0b11, 1, 0b01, FPR64Op, "ldr", f64, load>;
3288 defm LDRQ : Load128RO<0b00, 1, 0b11, FPR128Op, "ldr", f128, load>;
3291 // Load sign-extended half-word
3292 defm LDRSHW : Load16RO<0b01, 0, 0b11, GPR32, "ldrsh", i32, sextloadi16>;
3293 defm LDRSHX : Load16RO<0b01, 0, 0b10, GPR64, "ldrsh", i64, sextloadi16>;
3295 // Load sign-extended byte
3296 defm LDRSBW : Load8RO<0b00, 0, 0b11, GPR32, "ldrsb", i32, sextloadi8>;
3297 defm LDRSBX : Load8RO<0b00, 0, 0b10, GPR64, "ldrsb", i64, sextloadi8>;
3299 // Load sign-extended word
3300 defm LDRSW : Load32RO<0b10, 0, 0b10, GPR64, "ldrsw", i64, sextloadi32>;
3302 // Pre-fetch.
3303 defm PRFM : PrefetchRO<0b11, 0, 0b10, "prfm">;
3371 // We must do vector loads with LD1 in big-endian.
3385 // We must do vector loads with LD1 in big-endian.
3397 // zextload -> i64
3401 (SUBREG_TO_REG (i64 0),
3406 (SUBREG_TO_REG (i64 0),
3416 // zextloadi1 -> zextloadi8
3419 // extload -> zextload
3424 // extloadi1 -> zextloadi8
3429 // zextload -> i64
3441 // extload -> zextload
3446 // zextloadi1 -> zextloadi8
3450 //---
3452 //---
3453 defm LDRX : LoadUI<0b11, 0, 0b01, GPR64z, uimm12s8, "ldr",
3456 defm LDRW : LoadUI<0b10, 0, 0b01, GPR32z, uimm12s4, "ldr",
3460 defm LDRB : LoadUI<0b00, 1, 0b01, FPR8Op, uimm12s1, "ldr",
3463 defm LDRH : LoadUI<0b01, 1, 0b01, FPR16Op, uimm12s2, "ldr",
3466 defm LDRS : LoadUI<0b10, 1, 0b01, FPR32Op, uimm12s4, "ldr",
3469 defm LDRD : LoadUI<0b11, 1, 0b01, FPR64Op, uimm12s8, "ldr",
3472 defm LDRQ : LoadUI<0b00, 1, 0b11, FPR128Op, uimm12s16, "ldr",
3519 // We must use LD1 to perform vector loads in big-endian.
3540 // We must use LD1 to perform vector loads in big-endian.
3561 defm LDRHH : LoadUI<0b01, 0, 0b01, GPR32, uimm12s2, "ldrh",
3565 defm LDRBB : LoadUI<0b00, 0, 0b01, GPR32, uimm12s1, "ldrb",
3569 // zextload -> i64
3571 (SUBREG_TO_REG (i64 0), (LDRBBui GPR64sp:$Rn, uimm12s1:$offset), sub_32)>;
3573 (SUBREG_TO_REG (i64 0), (LDRHHui GPR64sp:$Rn, uimm12s2:$offset), sub_32)>;
3575 // zextloadi1 -> zextloadi8
3579 (SUBREG_TO_REG (i64 0), (LDRBBui GPR64sp:$Rn, uimm12s1:$offset), sub_32)>;
3581 // extload -> zextload
3589 (SUBREG_TO_REG (i64 0), (LDRWui GPR64sp:$Rn, uimm12s4:$offset), sub_32)>;
3591 (SUBREG_TO_REG (i64 0), (LDRHHui GPR64sp:$Rn, uimm12s2:$offset), sub_32)>;
3593 (SUBREG_TO_REG (i64 0), (LDRBBui GPR64sp:$Rn, uimm12s1:$offset), sub_32)>;
3595 (SUBREG_TO_REG (i64 0), (LDRBBui GPR64sp:$Rn, uimm12s1:$offset), sub_32)>;
3597 // load sign-extended half-word
3598 defm LDRSHW : LoadUI<0b01, 0, 0b11, GPR32, uimm12s2, "ldrsh",
3602 defm LDRSHX : LoadUI<0b01, 0, 0b10, GPR64, uimm12s2, "ldrsh",
3607 // load sign-extended byte
3608 defm LDRSBW : LoadUI<0b00, 0, 0b11, GPR32, uimm12s1, "ldrsb",
3612 defm LDRSBX : LoadUI<0b00, 0, 0b10, GPR64, uimm12s1, "ldrsb",
3617 // load sign-extended word
3618 defm LDRSW : LoadUI<0b10, 0, 0b10, GPR64, uimm12s4, "ldrsw",
3623 // load zero-extended word
3625 (SUBREG_TO_REG (i64 0), (LDRWui GPR64sp:$Rn, uimm12s4:$offset), sub_32)>;
3627 // Pre-fetch.
3628 def PRFMui : PrefetchUI<0b11, 0, 0b10, "prfm",
3633 def : InstAlias<"prfm $Rt, [$Rn]", (PRFMui prfop:$Rt, GPR64sp:$Rn, 0)>;
3635 //---
3640 const DataLayout &DL = MF->getDataLayout();
3641 Align Align = G->getGlobal()->getPointerAlignment(DL);
3642 return Align >= 4 && G->getOffset() % 4 == 0;
3645 return C->getAlign() >= 4 && C->getOffset() % 4 == 0;
3649 def LDRWl : LoadLiteral<0b00, 0, GPR32z, "ldr",
3651 def LDRXl : LoadLiteral<0b01, 0, GPR64z, "ldr",
3654 def LDRSl : LoadLiteral<0b00, 1, FPR32Op, "ldr",
3656 def LDRDl : LoadLiteral<0b01, 1, FPR64Op, "ldr",
3658 def LDRQl : LoadLiteral<0b10, 1, FPR128Op, "ldr",
3662 // load sign-extended word
3663 def LDRSWl : LoadLiteral<0b10, 0, GPR64z, "ldrsw",
3668 (SUBREG_TO_REG (i64 0), (LDRWl $label), sub_32)>;
3672 def PRFMl : PrefetchLiteral<0b11, 0, "prfm", []>;
3675 //---
3677 defm LDURX : LoadUnscaled<0b11, 0, 0b01, GPR64z, "ldur",
3680 defm LDURW : LoadUnscaled<0b10, 0, 0b01, GPR32z, "ldur",
3684 defm LDURB : LoadUnscaled<0b00, 1, 0b01, FPR8Op, "ldur",
3687 defm LDURH : LoadUnscaled<0b01, 1, 0b01, FPR16Op, "ldur",
3690 defm LDURS : LoadUnscaled<0b10, 1, 0b01, FPR32Op, "ldur",
3693 defm LDURD : LoadUnscaled<0b11, 1, 0b01, FPR64Op, "ldur",
3696 defm LDURQ : LoadUnscaled<0b00, 1, 0b11, FPR128Op, "ldur",
3702 : LoadUnscaled<0b01, 0, 0b01, GPR32, "ldurh",
3706 : LoadUnscaled<0b00, 0, 0b01, GPR32, "ldurb",
3750 // anyext -> zext
3758 (SUBREG_TO_REG (i64 0), (LDURWi GPR64sp:$Rn, simm9:$offset), sub_32)>;
3760 (SUBREG_TO_REG (i64 0), (LDURHHi GPR64sp:$Rn, simm9:$offset), sub_32)>;
3762 (SUBREG_TO_REG (i64 0), (LDURBBi GPR64sp:$Rn, simm9:$offset), sub_32)>;
3764 (SUBREG_TO_REG (i64 0), (LDURBBi GPR64sp:$Rn, simm9:$offset), sub_32)>;
3773 (SUBREG_TO_REG (i64 0), (LDURWi GPR64sp:$Rn, simm9:$offset), sub_32)>;
3775 (SUBREG_TO_REG (i64 0), (LDURHHi GPR64sp:$Rn, simm9:$offset), sub_32)>;
3777 (SUBREG_TO_REG (i64 0), (LDURBBi GPR64sp:$Rn, simm9:$offset), sub_32)>;
3779 (SUBREG_TO_REG (i64 0), (LDURBBi GPR64sp:$Rn, simm9:$offset), sub_32)>;
3782 //---
3818 (LDURXi GPR64:$Rt, GPR64sp:$Rn, simm9_offset_fb64:$offset), 0>;
3820 (LDURWi GPR32:$Rt, GPR64sp:$Rn, simm9_offset_fb32:$offset), 0>;
3823 (LDURBi FPR8Op:$Rt, GPR64sp:$Rn, simm9_offset_fb8:$offset), 0>;
3825 (LDURHi FPR16Op:$Rt, GPR64sp:$Rn, simm9_offset_fb16:$offset), 0>;
3827 (LDURSi FPR32Op:$Rt, GPR64sp:$Rn, simm9_offset_fb32:$offset), 0>;
3829 (LDURDi FPR64Op:$Rt, GPR64sp:$Rn, simm9_offset_fb64:$offset), 0>;
3831 (LDURQi FPR128Op:$Rt, GPR64sp:$Rn, simm9_offset_fb128:$offset), 0>;
3834 // zextload -> i64
3836 (SUBREG_TO_REG (i64 0), (LDURBBi GPR64sp:$Rn, simm9:$offset), sub_32)>;
3838 (SUBREG_TO_REG (i64 0), (LDURHHi GPR64sp:$Rn, simm9:$offset), sub_32)>;
3840 // load sign-extended half-word
3842 : LoadUnscaled<0b01, 0, 0b11, GPR32, "ldursh",
3846 : LoadUnscaled<0b01, 0, 0b10, GPR64, "ldursh",
3850 // load sign-extended byte
3852 : LoadUnscaled<0b00, 0, 0b11, GPR32, "ldursb",
3856 : LoadUnscaled<0b00, 0, 0b10, GPR64, "ldursb",
3860 // load sign-extended word
3862 : LoadUnscaled<0b10, 0, 0b10, GPR64, "ldursw",
3868 (LDURBBi GPR32:$Rt, GPR64sp:$Rn, simm9_offset_fb8:$offset), 0>;
3870 (LDURHHi GPR32:$Rt, GPR64sp:$Rn, simm9_offset_fb16:$offset), 0>;
3872 (LDURSBWi GPR32:$Rt, GPR64sp:$Rn, simm9_offset_fb8:$offset), 0>;
3874 (LDURSBXi GPR64:$Rt, GPR64sp:$Rn, simm9_offset_fb8:$offset), 0>;
3876 (LDURSHWi GPR32:$Rt, GPR64sp:$Rn, simm9_offset_fb16:$offset), 0>;
3878 (LDURSHXi GPR64:$Rt, GPR64sp:$Rn, simm9_offset_fb16:$offset), 0>;
3880 (LDURSWi GPR64:$Rt, GPR64sp:$Rn, simm9_offset_fb32:$offset), 0>;
3883 // load, 0) can use a single load.
3890 (ScalarVT (LoadOp (Addr GPR64sp:$Rn, AddrImm:$offset))), (i64 0)),
3891 (SUBREG_TO_REG (i64 0), (LoadInst GPR64sp:$Rn, AddrImm:$offset), SubReg)>;
3894 (ScalarVT (LoadOp (UnscaledAddr GPR64sp:$Rn, simm9:$offset))), (i64 0)),
3895 (SUBREG_TO_REG (i64 0), (UnscaledLoadInst GPR64sp:$Rn, simm9:$offset), SubReg)>;
3897 // Half-vector patterns
3899 (ScalarVT (LoadOp (Addr GPR64sp:$Rn, AddrImm:$offset))), (i64 0)),
3900 (SUBREG_TO_REG (i64 0), (LoadInst GPR64sp:$Rn, AddrImm:$offset), SubReg)>;
3903 (ScalarVT (LoadOp (UnscaledAddr GPR64sp:$Rn, simm9:$offset))), (i64 0)),
3904 (SUBREG_TO_REG (i64 0), (UnscaledLoadInst GPR64sp:$Rn, simm9:$offset), SubReg)>;
3908 (ScalarVT (LoadOp (Addr GPR64sp:$Rn, AddrImm:$offset))), (i64 0)),
3909 (SUBREG_TO_REG (i64 0), (LoadInst GPR64sp:$Rn, AddrImm:$offset), SubReg)>;
3912 (ScalarVT (LoadOp (UnscaledAddr GPR64sp:$Rn, simm9:$offset))), (i64 0)),
3913 (SUBREG_TO_REG (i64 0), (UnscaledLoadInst GPR64sp:$Rn, simm9:$offset), SubReg)>;
3933 // Pre-fetch.
3934 defm PRFUM : PrefetchUnscaled<0b11, 0, 0b10, "prfum",
3938 //---
3940 defm LDTRX : LoadUnprivileged<0b11, 0, 0b01, GPR64, "ldtr">;
3941 defm LDTRW : LoadUnprivileged<0b10, 0, 0b01, GPR32, "ldtr">;
3943 defm LDTRH : LoadUnprivileged<0b01, 0, 0b01, GPR32, "ldtrh">;
3944 defm LDTRB : LoadUnprivileged<0b00, 0, 0b01, GPR32, "ldtrb">;
3946 // load sign-extended half-word
3947 defm LDTRSHW : LoadUnprivileged<0b01, 0, 0b11, GPR32, "ldtrsh">;
3948 defm LDTRSHX : LoadUnprivileged<0b01, 0, 0b10, GPR64, "ldtrsh">;
3950 // load sign-extended byte
3951 defm LDTRSBW : LoadUnprivileged<0b00, 0, 0b11, GPR32, "ldtrsb">;
3952 defm LDTRSBX : LoadUnprivileged<0b00, 0, 0b10, GPR64, "ldtrsb">;
3954 // load sign-extended word
3955 defm LDTRSW : LoadUnprivileged<0b10, 0, 0b10, GPR64, "ldtrsw">;
3957 //---
3958 // (immediate pre-indexed)
3959 def LDRWpre : LoadPreIdx<0b10, 0, 0b01, GPR32z, "ldr">;
3960 def LDRXpre : LoadPreIdx<0b11, 0, 0b01, GPR64z, "ldr">;
3962 def LDRBpre : LoadPreIdx<0b00, 1, 0b01, FPR8Op, "ldr">;
3963 def LDRHpre : LoadPreIdx<0b01, 1, 0b01, FPR16Op, "ldr">;
3964 def LDRSpre : LoadPreIdx<0b10, 1, 0b01, FPR32Op, "ldr">;
3965 def LDRDpre : LoadPreIdx<0b11, 1, 0b01, FPR64Op, "ldr">;
3966 def LDRQpre : LoadPreIdx<0b00, 1, 0b11, FPR128Op, "ldr">;
3969 // load sign-extended half-word
3970 def LDRSHWpre : LoadPreIdx<0b01, 0, 0b11, GPR32z, "ldrsh">;
3971 def LDRSHXpre : LoadPreIdx<0b01, 0, 0b10, GPR64z, "ldrsh">;
3973 // load sign-extended byte
3974 def LDRSBWpre : LoadPreIdx<0b00, 0, 0b11, GPR32z, "ldrsb">;
3975 def LDRSBXpre : LoadPreIdx<0b00, 0, 0b10, GPR64z, "ldrsb">;
3977 // load zero-extended byte
3978 def LDRBBpre : LoadPreIdx<0b00, 0, 0b01, GPR32z, "ldrb">;
3979 def LDRHHpre : LoadPreIdx<0b01, 0, 0b01, GPR32z, "ldrh">;
3981 // load sign-extended word
3982 def LDRSWpre : LoadPreIdx<0b10, 0, 0b10, GPR64z, "ldrsw">;
3984 //---
3985 // (immediate post-indexed)
3986 def LDRWpost : LoadPostIdx<0b10, 0, 0b01, GPR32z, "ldr">;
3987 def LDRXpost : LoadPostIdx<0b11, 0, 0b01, GPR64z, "ldr">;
3989 def LDRBpost : LoadPostIdx<0b00, 1, 0b01, FPR8Op, "ldr">;
3990 def LDRHpost : LoadPostIdx<0b01, 1, 0b01, FPR16Op, "ldr">;
3991 def LDRSpost : LoadPostIdx<0b10, 1, 0b01, FPR32Op, "ldr">;
3992 def LDRDpost : LoadPostIdx<0b11, 1, 0b01, FPR64Op, "ldr">;
3993 def LDRQpost : LoadPostIdx<0b00, 1, 0b11, FPR128Op, "ldr">;
3996 // load sign-extended half-word
3997 def LDRSHWpost : LoadPostIdx<0b01, 0, 0b11, GPR32z, "ldrsh">;
3998 def LDRSHXpost : LoadPostIdx<0b01, 0, 0b10, GPR64z, "ldrsh">;
4000 // load sign-extended byte
4001 def LDRSBWpost : LoadPostIdx<0b00, 0, 0b11, GPR32z, "ldrsb">;
4002 def LDRSBXpost : LoadPostIdx<0b00, 0, 0b10, GPR64z, "ldrsb">;
4004 // load zero-extended byte
4005 def LDRBBpost : LoadPostIdx<0b00, 0, 0b01, GPR32z, "ldrb">;
4006 def LDRHHpost : LoadPostIdx<0b01, 0, 0b01, GPR32z, "ldrh">;
4008 // load sign-extended word
4009 def LDRSWpost : LoadPostIdx<0b10, 0, 0b10, GPR64z, "ldrsw">;
4011 //===----------------------------------------------------------------------===//
4013 //===----------------------------------------------------------------------===//
4016 // FIXME: Use dedicated range-checked addressing mode operand here.
4017 defm STPW : StorePairOffset<0b00, 0, GPR32z, simm7s4, "stp">;
4018 defm STPX : StorePairOffset<0b10, 0, GPR64z, simm7s8, "stp">;
4020 defm STPS : StorePairOffset<0b00, 1, FPR32Op, simm7s4, "stp">;
4021 defm STPD : StorePairOffset<0b01, 1, FPR64Op, simm7s8, "stp">;
4022 defm STPQ : StorePairOffset<0b10, 1, FPR128Op, simm7s16, "stp">;
4025 // Pair (pre-indexed)
4026 def STPWpre : StorePairPreIdx<0b00, 0, GPR32z, simm7s4, "stp">;
4027 def STPXpre : StorePairPreIdx<0b10, 0, GPR64z, simm7s8, "stp">;
4029 def STPSpre : StorePairPreIdx<0b00, 1, FPR32Op, simm7s4, "stp">;
4030 def STPDpre : StorePairPreIdx<0b01, 1, FPR64Op, simm7s8, "stp">;
4031 def STPQpre : StorePairPreIdx<0b10, 1, FPR128Op, simm7s16, "stp">;
4034 // Pair (post-indexed)
4035 def STPWpost : StorePairPostIdx<0b00, 0, GPR32z, simm7s4, "stp">;
4036 def STPXpost : StorePairPostIdx<0b10, 0, GPR64z, simm7s8, "stp">;
4038 def STPSpost : StorePairPostIdx<0b00, 1, FPR32Op, simm7s4, "stp">;
4039 def STPDpost : StorePairPostIdx<0b01, 1, FPR64Op, simm7s8, "stp">;
4040 def STPQpost : StorePairPostIdx<0b10, 1, FPR128Op, simm7s16, "stp">;
4044 defm STNPW : StorePairNoAlloc<0b00, 0, GPR32z, simm7s4, "stnp">;
4045 defm STNPX : StorePairNoAlloc<0b10, 0, GPR64z, simm7s8, "stnp">;
4047 defm STNPS : StorePairNoAlloc<0b00, 1, FPR32Op, simm7s4, "stnp">;
4048 defm STNPD : StorePairNoAlloc<0b01, 1, FPR64Op, simm7s8, "stnp">;
4049 defm STNPQ : StorePairNoAlloc<0b10, 1, FPR128Op, simm7s16, "stnp">;
4059 //---
4063 defm STRBB : Store8RO< 0b00, 0, 0b00, GPR32, "strb", i32, truncstorei8>;
4064 defm STRHH : Store16RO<0b01, 0, 0b00, GPR32, "strh", i32, truncstorei16>;
4065 defm STRW : Store32RO<0b10, 0, 0b00, GPR32, "str", i32, store>;
4066 defm STRX : Store64RO<0b11, 0, 0b00, GPR64, "str", i64, store>;
4069 // Floating-point
4071 defm STRB : Store8RO< 0b00, 1, 0b00, FPR8Op, "str", i8, store>;
4072 defm STRH : Store16RO<0b01, 1, 0b00, FPR16Op, "str", f16, store>;
4073 defm STRS : Store32RO<0b10, 1, 0b00, FPR32Op, "str", f32, store>;
4074 defm STRD : Store64RO<0b11, 1, 0b00, FPR64Op, "str", f64, store>;
4075 defm STRQ : Store128RO<0b00, 1, 0b10, FPR128Op, "str">;
4124 // We must use ST1 to store vectors in big-endian.
4138 // We must use ST1 to store vectors in big-endian.
4150 // Match stores from lane 0 to the appropriate subreg's store.
4157 def : Pat<(storeop (STy (vector_extract (VecTy VecListOne128:$Vt), (i64 0))),
4162 def : Pat<(storeop (STy (vector_extract (VecTy VecListOne128:$Vt), (i64 0))),
4177 //---
4179 defm STRX : StoreUIz<0b11, 0, 0b00, GPR64z, uimm12s8, "str",
4182 defm STRW : StoreUIz<0b10, 0, 0b00, GPR32z, uimm12s4, "str",
4186 defm STRB : StoreUI<0b00, 1, 0b00, FPR8Op, uimm12s1, "str",
4189 defm STRH : StoreUI<0b01, 1, 0b00, FPR16Op, uimm12s2, "str",
4192 defm STRS : StoreUI<0b10, 1, 0b00, FPR32Op, uimm12s4, "str",
4195 defm STRD : StoreUI<0b11, 1, 0b00, FPR64Op, uimm12s8, "str",
4198 defm STRQ : StoreUI<0b00, 1, 0b10, FPR128Op, uimm12s16, "str", []>;
4201 defm STRHH : StoreUIz<0b01, 0, 0b00, GPR32z, uimm12s2, "strh",
4205 defm STRBB : StoreUIz<0b00, 0, 0b00, GPR32z, uimm12s1, "strb",
4226 // We must use ST1 to store vectors in big-endian.
4253 // We must use ST1 to store vectors in big-endian.
4292 // Match stores from lane 0 to the appropriate subreg's store.
4298 def : Pat<(storeop (STy (vector_extract (VTy VecListOne128:$Vt), (i64 0))),
4313 //---
4315 defm STURX : StoreUnscaled<0b11, 0, 0b00, GPR64z, "stur",
4318 defm STURW : StoreUnscaled<0b10, 0, 0b00, GPR32z, "stur",
4322 defm STURB : StoreUnscaled<0b00, 1, 0b00, FPR8Op, "stur",
4325 defm STURH : StoreUnscaled<0b01, 1, 0b00, FPR16Op, "stur",
4328 defm STURS : StoreUnscaled<0b10, 1, 0b00, FPR32Op, "stur",
4331 defm STURD : StoreUnscaled<0b11, 1, 0b00, FPR64Op, "stur",
4334 defm STURQ : StoreUnscaled<0b00, 1, 0b10, FPR128Op, "stur",
4338 defm STURHH : StoreUnscaled<0b01, 0, 0b00, GPR32z, "sturh",
4341 defm STURBB : StoreUnscaled<0b00, 0, 0b00, GPR32z, "sturb",
4353 defm STLURB : BaseStoreUnscaleV84<"stlurb", 0b00, 0b00, GPR32>;
4354 defm STLURH : BaseStoreUnscaleV84<"stlurh", 0b01, 0b00, GPR32>;
4355 defm STLURW : BaseStoreUnscaleV84<"stlur", 0b10, 0b00, GPR32>;
4356 defm STLURX : BaseStoreUnscaleV84<"stlur", 0b11, 0b00, GPR64>;
4357 defm LDAPURB : BaseLoadUnscaleV84<"ldapurb", 0b00, 0b01, GPR32>;
4358 defm LDAPURSBW : BaseLoadUnscaleV84<"ldapursb", 0b00, 0b11, GPR32>;
4359 defm LDAPURSBX : BaseLoadUnscaleV84<"ldapursb", 0b00, 0b10, GPR64>;
4360 defm LDAPURH : BaseLoadUnscaleV84<"ldapurh", 0b01, 0b01, GPR32>;
4361 defm LDAPURSHW : BaseLoadUnscaleV84<"ldapursh", 0b01, 0b11, GPR32>;
4362 defm LDAPURSHX : BaseLoadUnscaleV84<"ldapursh", 0b01, 0b10, GPR64>;
4363 defm LDAPUR : BaseLoadUnscaleV84<"ldapur", 0b10, 0b01, GPR32>;
4364 defm LDAPURSW : BaseLoadUnscaleV84<"ldapursw", 0b10, 0b10, GPR64>;
4365 defm LDAPURX : BaseLoadUnscaleV84<"ldapur", 0b11, 0b01, GPR64>;
4377 // We must use ST1 to store vectors in big-endian.
4403 // We must use ST1 to store vectors in big-endian.
4443 // Match stores from lane 0 to the appropriate subreg's store.
4460 //---
4463 (STURXi GPR64:$Rt, GPR64sp:$Rn, simm9_offset_fb64:$offset), 0>;
4465 (STURWi GPR32:$Rt, GPR64sp:$Rn, simm9_offset_fb32:$offset), 0>;
4468 (STURBi FPR8Op:$Rt, GPR64sp:$Rn, simm9_offset_fb8:$offset), 0>;
4470 (STURHi FPR16Op:$Rt, GPR64sp:$Rn, simm9_offset_fb16:$offset), 0>;
4472 (STURSi FPR32Op:$Rt, GPR64sp:$Rn, simm9_offset_fb32:$offset), 0>;
4474 (STURDi FPR64Op:$Rt, GPR64sp:$Rn, simm9_offset_fb64:$offset), 0>;
4476 (STURQi FPR128Op:$Rt, GPR64sp:$Rn, simm9_offset_fb128:$offset), 0>;
4480 (STURBBi GPR32:$Rt, GPR64sp:$Rn, simm9_offset_fb8:$offset), 0>;
4482 (STURHHi GPR32:$Rt, GPR64sp:$Rn, simm9_offset_fb16:$offset), 0>;
4484 //---
4486 defm STTRW : StoreUnprivileged<0b10, 0, 0b00, GPR32, "sttr">;
4487 defm STTRX : StoreUnprivileged<0b11, 0, 0b00, GPR64, "sttr">;
4489 defm STTRH : StoreUnprivileged<0b01, 0, 0b00, GPR32, "sttrh">;
4490 defm STTRB : StoreUnprivileged<0b00, 0, 0b00, GPR32, "sttrb">;
4492 //---
4493 // (immediate pre-indexed)
4494 def STRWpre : StorePreIdx<0b10, 0, 0b00, GPR32z, "str", pre_store, i32>;
4495 def STRXpre : StorePreIdx<0b11, 0, 0b00, GPR64z, "str", pre_store, i64>;
4497 def STRBpre : StorePreIdx<0b00, 1, 0b00, FPR8Op, "str", pre_store, i8>;
4498 def STRHpre : StorePreIdx<0b01, 1, 0b00, FPR16Op, "str", pre_store, f16>;
4499 def STRSpre : StorePreIdx<0b10, 1, 0b00, FPR32Op, "str", pre_store, f32>;
4500 def STRDpre : StorePreIdx<0b11, 1, 0b00, FPR64Op, "str", pre_store, f64>;
4501 def STRQpre : StorePreIdx<0b00, 1, 0b10, FPR128Op, "str", pre_store, f128>;
4504 def STRBBpre : StorePreIdx<0b00, 0, 0b00, GPR32z, "strb", pre_truncsti8, i32>;
4505 def STRHHpre : StorePreIdx<0b01, 0, 0b00, GPR32z, "strh", pre_truncsti16, i32>;
4548 //---
4549 // (immediate post-indexed)
4550 def STRWpost : StorePostIdx<0b10, 0, 0b00, GPR32z, "str", post_store, i32>;
4551 def STRXpost : StorePostIdx<0b11, 0, 0b00, GPR64z, "str", post_store, i64>;
4553 def STRBpost : StorePostIdx<0b00, 1, 0b00, FPR8Op, "str", post_store, i8>;
4554 def STRHpost : StorePostIdx<0b01, 1, 0b00, FPR16Op, "str", post_store, f16>;
4555 def STRSpost : StorePostIdx<0b10, 1, 0b00, FPR32Op, "str", post_store, f32>;
4556 def STRDpost : StorePostIdx<0b11, 1, 0b00, FPR64Op, "str", post_store, f64>;
4557 def STRQpost : StorePostIdx<0b00, 1, 0b10, FPR128Op, "str", post_store, f128>;
4560 def STRBBpost : StorePostIdx<0b00, 0, 0b00, GPR32z, "strb", post_truncsti8, i32>;
4561 def STRHHpost : StorePostIdx<0b01, 0, 0b00, GPR32z, "strh", post_truncsti16, i32>;
4611 //===----------------------------------------------------------------------===//
4613 //===----------------------------------------------------------------------===//
4615 def LDARW : LoadAcquire <0b10, 1, 1, 0, 1, GPR32, "ldar">;
4616 def LDARX : LoadAcquire <0b11, 1, 1, 0, 1, GPR64, "ldar">;
4617 def LDARB : LoadAcquire <0b00, 1, 1, 0, 1, GPR32, "ldarb">;
4618 def LDARH : LoadAcquire <0b01, 1, 1, 0, 1, GPR32, "ldarh">;
4620 def LDAXRW : LoadExclusive <0b10, 0, 1, 0, 1, GPR32, "ldaxr">;
4621 def LDAXRX : LoadExclusive <0b11, 0, 1, 0, 1, GPR64, "ldaxr">;
4622 def LDAXRB : LoadExclusive <0b00, 0, 1, 0, 1, GPR32, "ldaxrb">;
4623 def LDAXRH : LoadExclusive <0b01, 0, 1, 0, 1, GPR32, "ldaxrh">;
4625 def LDXRW : LoadExclusive <0b10, 0, 1, 0, 0, GPR32, "ldxr">;
4626 def LDXRX : LoadExclusive <0b11, 0, 1, 0, 0, GPR64, "ldxr">;
4627 def LDXRB : LoadExclusive <0b00, 0, 1, 0, 0, GPR32, "ldxrb">;
4628 def LDXRH : LoadExclusive <0b01, 0, 1, 0, 0, GPR32, "ldxrh">;
4630 def STLRW : StoreRelease <0b10, 1, 0, 0, 1, GPR32, "stlr">;
4631 def STLRX : StoreRelease <0b11, 1, 0, 0, 1, GPR64, "stlr">;
4632 def STLRB : StoreRelease <0b00, 1, 0, 0, 1, GPR32, "stlrb">;
4633 def STLRH : StoreRelease <0b01, 1, 0, 0, 1, GPR32, "stlrh">;
4636 Aliases for when offset=0. Note that in contrast to LoadAcquire which has a $Rn
4638 alias for the case of immediate #0. This is because new STLR versions (from
4639 LRCPC3 extension) do have a non-zero immediate value, so GPR64sp0 is not
4641 case for LoadAcquire because the new LRCPC3 LDAR instructions are post-indexed,
4645 def STLRW0 : InstAlias<"stlr\t$Rt, [$Rn, #0]" , (STLRW GPR32: $Rt, GPR64sp:$Rn)>;
4646 def STLRX0 : InstAlias<"stlr\t$Rt, [$Rn, #0]" , (STLRX GPR64: $Rt, GPR64sp:$Rn)>;
4647 def STLRB0 : InstAlias<"stlrb\t$Rt, [$Rn, #0]", (STLRB GPR32: $Rt, GPR64sp:$Rn)>;
4648 def STLRH0 : InstAlias<"stlrh\t$Rt, [$Rn, #0]", (STLRH GPR32: $Rt, GPR64sp:$Rn)>;
4650 def STLXRW : StoreExclusive<0b10, 0, 0, 0, 1, GPR32, "stlxr">;
4651 def STLXRX : StoreExclusive<0b11, 0, 0, 0, 1, GPR64, "stlxr">;
4652 def STLXRB : StoreExclusive<0b00, 0, 0, 0, 1, GPR32, "stlxrb">;
4653 def STLXRH : StoreExclusive<0b01, 0, 0, 0, 1, GPR32, "stlxrh">;
4655 def STXRW : StoreExclusive<0b10, 0, 0, 0, 0, GPR32, "stxr">;
4656 def STXRX : StoreExclusive<0b11, 0, 0, 0, 0, GPR64, "stxr">;
4657 def STXRB : StoreExclusive<0b00, 0, 0, 0, 0, GPR32, "stxrb">;
4658 def STXRH : StoreExclusive<0b01, 0, 0, 0, 0, GPR32, "stxrh">;
4660 def LDAXPW : LoadExclusivePair<0b10, 0, 1, 1, 1, GPR32, "ldaxp">;
4661 def LDAXPX : LoadExclusivePair<0b11, 0, 1, 1, 1, GPR64, "ldaxp">;
4663 def LDXPW : LoadExclusivePair<0b10, 0, 1, 1, 0, GPR32, "ldxp">;
4664 def LDXPX : LoadExclusivePair<0b11, 0, 1, 1, 0, GPR64, "ldxp">;
4666 def STLXPW : StoreExclusivePair<0b10, 0, 0, 1, 1, GPR32, "stlxp">;
4667 def STLXPX : StoreExclusivePair<0b11, 0, 0, 1, 1, GPR64, "stlxp">;
4669 def STXPW : StoreExclusivePair<0b10, 0, 0, 1, 0, GPR32, "stxp">;
4670 def STXPX : StoreExclusivePair<0b11, 0, 0, 1, 0, GPR64, "stxp">;
4673 // v8.1a "Limited Order Region" extension load-acquire instructions
4674 def LDLARW : LoadAcquire <0b10, 1, 1, 0, 0, GPR32, "ldlar">;
4675 def LDLARX : LoadAcquire <0b11, 1, 1, 0, 0, GPR64, "ldlar">;
4676 def LDLARB : LoadAcquire <0b00, 1, 1, 0, 0, GPR32, "ldlarb">;
4677 def LDLARH : LoadAcquire <0b01, 1, 1, 0, 0, GPR32, "ldlarh">;
4679 // v8.1a "Limited Order Region" extension store-release instructions
4680 def STLLRW : StoreRelease <0b10, 1, 0, 0, 0, GPR32, "stllr">;
4681 def STLLRX : StoreRelease <0b11, 1, 0, 0, 0, GPR64, "stllr">;
4682 def STLLRB : StoreRelease <0b00, 1, 0, 0, 0, GPR32, "stllrb">;
4683 def STLLRH : StoreRelease <0b01, 1, 0, 0, 0, GPR32, "stllrh">;
4685 // Aliases for when offset=0
4686 def STLLRW0 : InstAlias<"stllr\t$Rt, [$Rn, #0]", (STLLRW GPR32: $Rt, GPR64sp:$Rn)>;
4687 def STLLRX0 : InstAlias<"stllr\t$Rt, [$Rn, #0]", (STLLRX GPR64: $Rt, GPR64sp:$Rn)>;
4688 def STLLRB0 : InstAlias<"stllrb\t$Rt, [$Rn, #0]", (STLLRB GPR32: $Rt, GPR64sp:$Rn)>;
4689 def STLLRH0 : InstAlias<"stllrh\t$Rt, [$Rn, #0]", (STLLRH GPR32: $Rt, GPR64sp:$Rn)>;
4692 //===----------------------------------------------------------------------===//
4694 //===----------------------------------------------------------------------===//
4696 defm FCVTAS : FPToIntegerUnscaled<0b00, 0b100, "fcvtas", int_aarch64_neon_fcvtas>;
4697 defm FCVTAU : FPToIntegerUnscaled<0b00, 0b101, "fcvtau", int_aarch64_neon_fcvtau>;
4698 defm FCVTMS : FPToIntegerUnscaled<0b10, 0b000, "fcvtms", int_aarch64_neon_fcvtms>;
4699 defm FCVTMU : FPToIntegerUnscaled<0b10, 0b001, "fcvtmu", int_aarch64_neon_fcvtmu>;
4700 defm FCVTNS : FPToIntegerUnscaled<0b00, 0b000, "fcvtns", int_aarch64_neon_fcvtns>;
4701 defm FCVTNU : FPToIntegerUnscaled<0b00, 0b001, "fcvtnu", int_aarch64_neon_fcvtnu>;
4702 defm FCVTPS : FPToIntegerUnscaled<0b01, 0b000, "fcvtps", int_aarch64_neon_fcvtps>;
4703 defm FCVTPU : FPToIntegerUnscaled<0b01, 0b001, "fcvtpu", int_aarch64_neon_fcvtpu>;
4704 defm FCVTZS : FPToIntegerUnscaled<0b11, 0b000, "fcvtzs", any_fp_to_sint>;
4705 defm FCVTZU : FPToIntegerUnscaled<0b11, 0b001, "fcvtzu", any_fp_to_uint>;
4706 defm FCVTZS : FPToIntegerScaled<0b11, 0b000, "fcvtzs", any_fp_to_sint>;
4707 defm FCVTZU : FPToIntegerScaled<0b11, 0b001, "fcvtzu", any_fp_to_uint>;
4833 //===----------------------------------------------------------------------===//
4835 //===----------------------------------------------------------------------===//
4837 defm SCVTF : IntegerToFP<0, "scvtf", any_sint_to_fp>;
4868 //===----------------------------------------------------------------------===//
4870 //===----------------------------------------------------------------------===//
4874 // Add pseudo ops for FMOV 0 so we can mark them as isReMaterializable
4886 def : InstAlias<"fmov $Rd, #0.0", (FMOVWHr FPR16:$Rd, WZR), 0>,
4889 def : InstAlias<"fmov $Rd, #0.0", (FMOVWSr FPR32:$Rd, WZR), 0>;
4890 def : InstAlias<"fmov $Rd, #0.0", (FMOVXDr FPR64:$Rd, XZR), 0>;
4905 //===----------------------------------------------------------------------===//
4907 //===----------------------------------------------------------------------===//
4920 // Pattern for bf16 -> fp32.
4923 // Pattern for bf16 -> fp64.
4927 //===----------------------------------------------------------------------===//
4929 //===----------------------------------------------------------------------===//
4931 defm FABS : SingleOperandFPDataNoException<0b0001, "fabs", fabs>;
4932 defm FMOV : SingleOperandFPDataNoException<0b0000, "fmov">;
4933 defm FNEG : SingleOperandFPDataNoException<0b0010, "fneg", fneg>;
4934 defm FRINTA : SingleOperandFPData<0b1100, "frinta", any_fround>;
4935 defm FRINTI : SingleOperandFPData<0b1111, "frinti", any_fnearbyint>;
4936 defm FRINTM : SingleOperandFPData<0b1010, "frintm", any_ffloor>;
4937 defm FRINTN : SingleOperandFPData<0b1000, "frintn", any_froundeven>;
4938 defm FRINTP : SingleOperandFPData<0b1001, "frintp", any_fceil>;
4940 defm FRINTX : SingleOperandFPData<0b1110, "frintx", any_frint>;
4941 defm FRINTZ : SingleOperandFPData<0b1011, "frintz", any_ftrunc>;
4944 defm FSQRT : SingleOperandFPData<0b0011, "fsqrt", any_fsqrt>;
4948 defm FRINT32Z : FRIntNNT<0b00, "frint32z", int_aarch64_frint32z>;
4949 defm FRINT64Z : FRIntNNT<0b10, "frint64z", int_aarch64_frint64z>;
4950 defm FRINT32X : FRIntNNT<0b01, "frint32x", int_aarch64_frint32x>;
4951 defm FRINT64X : FRIntNNT<0b11, "frint64x", int_aarch64_frint64x>;
4989 //===----------------------------------------------------------------------===//
4991 //===----------------------------------------------------------------------===//
4993 defm FADD : TwoOperandFPData<0b0010, "fadd", any_fadd>;
4995 defm FDIV : TwoOperandFPData<0b0001, "fdiv", any_fdiv>;
4997 defm FMAXNM : TwoOperandFPData<0b0110, "fmaxnm", any_fmaxnum>;
4998 defm FMAX : TwoOperandFPData<0b0100, "fmax", any_fmaximum>;
4999 defm FMINNM : TwoOperandFPData<0b0111, "fminnm", any_fminnum>;
5000 defm FMIN : TwoOperandFPData<0b0101, "fmin", any_fminimum>;
5002 defm FMUL : TwoOperandFPData<0b0000, "fmul", any_fmul>;
5003 defm FNMUL : TwoOperandFPDataNeg<0b1000, "fnmul", any_fmul>;
5005 defm FSUB : TwoOperandFPData<0b0011, "fsub", any_fsub>;
5015 (f16 (vector_extract (v8f16 V128:$Rm), (i64 0))))),
5021 (f32 (vector_extract (v4f32 V128:$Rm), (i64 0))))),
5025 (f64 (vector_extract (v2f64 V128:$Rm), (i64 0))))),
5052 //===----------------------------------------------------------------------===//
5054 //===----------------------------------------------------------------------===//
5056 defm FMADD : ThreeOperandFPData<0, 0, "fmadd", any_fma>;
5057 defm FMSUB : ThreeOperandFPData<0, 1, "fmsub",
5059 defm FNMADD : ThreeOperandFPData<1, 0, "fnmadd",
5070 // Here we handle first -(a + b*c) for FNMADD:
5082 // Now it's time for "(-a) + (-b)*c"
5094 //===----------------------------------------------------------------------===//
5096 //===----------------------------------------------------------------------===//
5099 defm FCMP : FPComparison<0, "fcmp", AArch64any_fcmp>;
5101 //===----------------------------------------------------------------------===//
5103 //===----------------------------------------------------------------------===//
5106 defm FCCMP : FPCondComparison<0, "fccmp", AArch64fccmp>;
5108 //===----------------------------------------------------------------------===//
5110 //===----------------------------------------------------------------------===//
5119 // pseudo-instruction since the eventual code will need to introduce basic
5132 //===----------------------------------------------------------------------===//
5134 //===----------------------------------------------------------------------===//
5159 //===----------------------------------------------------------------------===//
5176 //===----------------------------------------------------------------------===//
5178 //===----------------------------------------------------------------------===//
5189 //===----------------------------------------------------------------------===//
5191 //===----------------------------------------------------------------------===//
5193 defm UABDL : SIMDLongThreeVectorBHSabdl<1, 0b0111, "uabdl",
5195 // Match UABDL in log2-shuffle patterns.
5215 defm ABS : SIMDTwoVectorBHSD<0, 0b01011, "abs", abs>;
5216 defm CLS : SIMDTwoVectorBHS<0, 0b00100, "cls", int_aarch64_neon_cls>;
5217 defm CLZ : SIMDTwoVectorBHS<1, 0b00100, "clz", ctlz>;
5218 defm CMEQ : SIMDCmpTwoVector<0, 0b01001, "cmeq", AArch64cmeqz>;
5219 defm CMGE : SIMDCmpTwoVector<1, 0b01000, "cmge", AArch64cmgez>;
5220 defm CMGT : SIMDCmpTwoVector<0, 0b01000, "cmgt", AArch64cmgtz>;
5221 defm CMLE : SIMDCmpTwoVector<1, 0b01001, "cmle", AArch64cmlez>;
5222 defm CMLT : SIMDCmpTwoVector<0, 0b01010, "cmlt", AArch64cmltz>;
5223 defm CNT : SIMDTwoVectorB<0, 0b00, 0b00101, "cnt", ctpop>;
5224 defm FABS : SIMDTwoVectorFPNoException<0, 1, 0b01111, "fabs", fabs>;
5241 defm FCMEQ : SIMDFPCmpTwoVector<0, 1, 0b01101, "fcmeq", AArch64fcmeqz>;
5242 defm FCMGE : SIMDFPCmpTwoVector<1, 1, 0b01100, "fcmge", AArch64fcmgez>;
5243 defm FCMGT : SIMDFPCmpTwoVector<0, 1, 0b01100, "fcmgt", AArch64fcmgtz>;
5244 defm FCMLE : SIMDFPCmpTwoVector<1, 1, 0b01101, "fcmle", AArch64fcmlez>;
5245 defm FCMLT : SIMDFPCmpTwoVector<0, 1, 0b01110, "fcmlt", AArch64fcmltz>;
5246 defm FCVTAS : SIMDTwoVectorFPToInt<0,0,0b11100, "fcvtas",int_aarch64_neon_fcvtas>;
5247 defm FCVTAU : SIMDTwoVectorFPToInt<1,0,0b11100, "fcvtau",int_aarch64_neon_fcvtau>;
5248 defm FCVTL : SIMDFPWidenTwoVector<0, 0, 0b10111, "fcvtl">;
5263 defm FCVTMS : SIMDTwoVectorFPToInt<0,0,0b11011, "fcvtms",int_aarch64_neon_fcvtms>;
5264 defm FCVTMU : SIMDTwoVectorFPToInt<1,0,0b11011, "fcvtmu",int_aarch64_neon_fcvtmu>;
5265 defm FCVTNS : SIMDTwoVectorFPToInt<0,0,0b11010, "fcvtns",int_aarch64_neon_fcvtns>;
5266 defm FCVTNU : SIMDTwoVectorFPToInt<1,0,0b11010, "fcvtnu",int_aarch64_neon_fcvtnu>;
5267 defm FCVTN : SIMDFPNarrowTwoVector<0, 0, 0b10110, "fcvtn">;
5281 defm FCVTPS : SIMDTwoVectorFPToInt<0,1,0b11010, "fcvtps",int_aarch64_neon_fcvtps>;
5282 defm FCVTPU : SIMDTwoVectorFPToInt<1,1,0b11010, "fcvtpu",int_aarch64_neon_fcvtpu>;
5283 defm FCVTXN : SIMDFPInexactCvtTwoVector<1, 0, 0b10110, "fcvtxn",
5285 defm FCVTZS : SIMDTwoVectorFPToInt<0, 1, 0b11011, "fcvtzs", any_fp_to_sint>;
5286 defm FCVTZU : SIMDTwoVectorFPToInt<1, 1, 0b11011, "fcvtzu", any_fp_to_uint>;
5318 defm FNEG : SIMDTwoVectorFPNoException<1, 1, 0b01111, "fneg", fneg>;
5319 defm FRECPE : SIMDTwoVectorFP<0, 1, 0b11101, "frecpe", int_aarch64_neon_frecpe>;
5320 defm FRINTA : SIMDTwoVectorFP<1, 0, 0b11000, "frinta", any_fround>;
5321 defm FRINTI : SIMDTwoVectorFP<1, 1, 0b11001, "frinti", any_fnearbyint>;
5322 defm FRINTM : SIMDTwoVectorFP<0, 0, 0b11001, "frintm", any_ffloor>;
5323 defm FRINTN : SIMDTwoVectorFP<0, 0, 0b11000, "frintn", any_froundeven>;
5324 defm FRINTP : SIMDTwoVectorFP<0, 1, 0b11000, "frintp", any_fceil>;
5325 defm FRINTX : SIMDTwoVectorFP<1, 0, 0b11001, "frintx", any_frint>;
5326 defm FRINTZ : SIMDTwoVectorFP<0, 1, 0b11001, "frintz", any_ftrunc>;
5329 defm FRINT32Z : FRIntNNTVector<0, 0, "frint32z", int_aarch64_neon_frint32z>;
5330 defm FRINT64Z : FRIntNNTVector<0, 1, "frint64z", int_aarch64_neon_frint64z>;
5331 defm FRINT32X : FRIntNNTVector<1, 0, "frint32x", int_aarch64_neon_frint32x>;
5335 defm FRSQRTE: SIMDTwoVectorFP<1, 1, 0b11101, "frsqrte", int_aarch64_neon_frsqrte>;
5336 defm FSQRT : SIMDTwoVectorFP<1, 1, 0b11111, "fsqrt", any_fsqrt>;
5337 defm NEG : SIMDTwoVectorBHSD<1, 0b01011, "neg",
5339 defm NOT : SIMDTwoVectorB<1, 0b00, 0b00101, "not", vnot>;
5340 // Aliases for MVN -> NOT.
5355 defm RBIT : SIMDTwoVectorB<1, 0b01, 0b00101, "rbit", bitreverse>;
5356 defm REV16 : SIMDTwoVectorB<0, 0b00, 0b00001, "rev16", AArch64rev16>;
5357 defm REV32 : SIMDTwoVectorBH<1, 0b00000, "rev32", AArch64rev32>;
5358 defm REV64 : SIMDTwoVectorBHS<0, 0b00000, "rev64", AArch64rev64>;
5359 defm SADALP : SIMDLongTwoVectorTied<0, 0b00110, "sadalp",
5361 defm SADDLP : SIMDLongTwoVector<0, 0b00010, "saddlp", AArch64saddlp>;
5362 defm SCVTF : SIMDTwoVectorIntToFP<0, 0, 0b11101, "scvtf", any_sint_to_fp>;
5364 defm SQABS : SIMDTwoVectorBHSD<0, 0b00111, "sqabs", int_aarch64_neon_sqabs>;
5365 defm SQNEG : SIMDTwoVectorBHSD<1, 0b00111, "sqneg", int_aarch64_neon_sqneg>;
5366 defm SQXTN : SIMDMixedTwoVector<0, 0b10100, "sqxtn", int_aarch64_neon_sqxtn>;
5367 defm SQXTUN : SIMDMixedTwoVector<1, 0b10010, "sqxtun", int_aarch64_neon_sqxtun>;
5368 defm SUQADD : SIMDTwoVectorBHSDTied<0, 0b00011, "suqadd",int_aarch64_neon_suqadd>;
5369 defm UADALP : SIMDLongTwoVectorTied<1, 0b00110, "uadalp",
5371 defm UADDLP : SIMDLongTwoVector<1, 0b00010, "uaddlp", AArch64uaddlp>;
5372 defm UCVTF : SIMDTwoVectorIntToFP<1, 0, 0b11101, "ucvtf", any_uint_to_fp>;
5373 defm UQXTN : SIMDMixedTwoVector<1, 0b10100, "uqxtn", int_aarch64_neon_uqxtn>;
5374 defm URECPE : SIMDTwoVectorS<0, 1, 0b11100, "urecpe", int_aarch64_neon_urecpe>;
5375 defm URSQRTE: SIMDTwoVectorS<1, 1, 0b11100, "ursqrte", int_aarch64_neon_ursqrte>;
5376 defm USQADD : SIMDTwoVectorBHSDTied<1, 0b00011, "usqadd",int_aarch64_neon_usqadd>;
5377 defm XTN : SIMDMixedTwoVector<0, 0b10010, "xtn", trunc>;
5415 def VImm7F: PatLeaf<(AArch64movi_shift (i32 127), (i32 0))>;
5416 def VImm80: PatLeaf<(AArch64mvni_shift (i32 127), (i32 0))>;
5420 // trunc(umin(X, 255)) -> UQXTRN v8i8
5423 // trunc(umin(X, 65535)) -> UQXTRN v4i16
5426 // trunc(smin(smax(X, -128), 128)) -> SQXTRN
5434 // trunc(smin(smax(X, -32768), 32767)) -> SQXTRN
5443 // concat_vectors(Vd, trunc(umin(X, 255))) -> UQXTRN(Vd, Vn)
5448 // concat_vectors(Vd, trunc(umin(X, 65535))) -> UQXTRN(Vd, Vn)
5454 // concat_vectors(Vd, trunc(smin(smax Vm, -128), 127) ~> SQXTN2(Vd, Vn)
5467 // concat_vectors(Vd, trunc(smin(smax Vm, -32768), 32767) ~> SQXTN2(Vd, Vn)
5492 //===----------------------------------------------------------------------===//
5494 //===----------------------------------------------------------------------===//
5496 defm ADD : SIMDThreeSameVector<0, 0b10000, "add", add>;
5497 defm ADDP : SIMDThreeSameVector<0, 0b10111, "addp", AArch64addp>;
5498 defm CMEQ : SIMDThreeSameVector<1, 0b10001, "cmeq", AArch64cmeq>;
5499 defm CMGE : SIMDThreeSameVector<0, 0b00111, "cmge", AArch64cmge>;
5500 defm CMGT : SIMDThreeSameVector<0, 0b00110, "cmgt", AArch64cmgt>;
5501 defm CMHI : SIMDThreeSameVector<1, 0b00110, "cmhi", AArch64cmhi>;
5502 defm CMHS : SIMDThreeSameVector<1, 0b00111, "cmhs", AArch64cmhs>;
5503 defm CMTST : SIMDThreeSameVector<0, 0b10001, "cmtst", AArch64cmtst>;
5507 defm FABD : SIMDThreeSameVectorFP<1,1,0b010,"fabd", int_aarch64_neon_fabd>;
5516 defm FACGE : SIMDThreeSameVectorFPCmp<1,0,0b101,"facge",AArch64facge>;
5517 defm FACGT : SIMDThreeSameVectorFPCmp<1,1,0b101,"facgt",AArch64facgt>;
5518 defm FADDP : SIMDThreeSameVectorFP<1,0,0b010,"faddp", AArch64faddp>;
5519 defm FADD : SIMDThreeSameVectorFP<0,0,0b010,"fadd", any_fadd>;
5520 defm FCMEQ : SIMDThreeSameVectorFPCmp<0, 0, 0b100, "fcmeq", AArch64fcmeq>;
5521 defm FCMGE : SIMDThreeSameVectorFPCmp<1, 0, 0b100, "fcmge", AArch64fcmge>;
5522 defm FCMGT : SIMDThreeSameVectorFPCmp<1, 1, 0b100, "fcmgt", AArch64fcmgt>;
5523 defm FDIV : SIMDThreeSameVectorFP<1,0,0b111,"fdiv", any_fdiv>;
5524 defm FMAXNMP : SIMDThreeSameVectorFP<1,0,0b000,"fmaxnmp", int_aarch64_neon_fmaxnmp>;
5525 defm FMAXNM : SIMDThreeSameVectorFP<0,0,0b000,"fmaxnm", any_fmaxnum>;
5526 defm FMAXP : SIMDThreeSameVectorFP<1,0,0b110,"fmaxp", int_aarch64_neon_fmaxp>;
5527 defm FMAX : SIMDThreeSameVectorFP<0,0,0b110,"fmax", any_fmaximum>;
5528 defm FMINNMP : SIMDThreeSameVectorFP<1,1,0b000,"fminnmp", int_aarch64_neon_fminnmp>;
5529 defm FMINNM : SIMDThreeSameVectorFP<0,1,0b000,"fminnm", any_fminnum>;
5530 defm FMINP : SIMDThreeSameVectorFP<1,1,0b110,"fminp", int_aarch64_neon_fminp>;
5531 defm FMIN : SIMDThreeSameVectorFP<0,1,0b110,"fmin", any_fminimum>;
5535 defm FMLA : SIMDThreeSameVectorFPTied<0, 0, 0b001, "fmla",
5537 defm FMLS : SIMDThreeSameVectorFPTied<0, 1, 0b001, "fmls",
5540 defm FMULX : SIMDThreeSameVectorFP<0,0,0b011,"fmulx", int_aarch64_neon_fmulx>;
5541 defm FMUL : SIMDThreeSameVectorFP<1,0,0b011,"fmul", any_fmul>;
5542 defm FRECPS : SIMDThreeSameVectorFP<0,0,0b111,"frecps", int_aarch64_neon_frecps>;
5543 defm FRSQRTS : SIMDThreeSameVectorFP<0,1,0b111,"frsqrts", int_aarch64_neon_frsqrts>;
5544 defm FSUB : SIMDThreeSameVectorFP<0,1,0b010,"fsub", any_fsub>;
5547 defm MLA : SIMDThreeSameVectorBHSTied<0, 0b10010, "mla", null_frag>;
5548 defm MLS : SIMDThreeSameVectorBHSTied<1, 0b10010, "mls", null_frag>;
5550 defm MUL : SIMDThreeSameVectorBHS<0, 0b10011, "mul", mul>;
5551 defm PMUL : SIMDThreeSameVectorB<1, 0b10011, "pmul", int_aarch64_neon_pmul>;
5552 defm SABA : SIMDThreeSameVectorBHSTied<0, 0b01111, "saba",
5554 defm SABD : SIMDThreeSameVectorBHS<0,0b01110,"sabd", AArch64sabd>;
5555 defm SHADD : SIMDThreeSameVectorBHS<0,0b00000,"shadd", avgfloors>;
5556 defm SHSUB : SIMDThreeSameVectorBHS<0,0b00100,"shsub", int_aarch64_neon_shsub>;
5557 defm SMAXP : SIMDThreeSameVectorBHS<0,0b10100,"smaxp", int_aarch64_neon_smaxp>;
5558 defm SMAX : SIMDThreeSameVectorBHS<0,0b01100,"smax", smax>;
5559 defm SMINP : SIMDThreeSameVectorBHS<0,0b10101,"sminp", int_aarch64_neon_sminp>;
5560 defm SMIN : SIMDThreeSameVectorBHS<0,0b01101,"smin", smin>;
5561 defm SQADD : SIMDThreeSameVector<0,0b00001,"sqadd", int_aarch64_neon_sqadd>;
5562 defm SQDMULH : SIMDThreeSameVectorHS<0,0b10110,"sqdmulh",int_aarch64_neon_sqdmulh>;
5563 defm SQRDMULH : SIMDThreeSameVectorHS<1,0b10110,"sqrdmulh",int_aarch64_neon_sqrdmulh>;
5564 defm SQRSHL : SIMDThreeSameVector<0,0b01011,"sqrshl", int_aarch64_neon_sqrshl>;
5565 defm SQSHL : SIMDThreeSameVector<0,0b01001,"sqshl", int_aarch64_neon_sqshl>;
5566 defm SQSUB : SIMDThreeSameVector<0,0b00101,"sqsub", int_aarch64_neon_sqsub>;
5567 defm SRHADD : SIMDThreeSameVectorBHS<0,0b00010,"srhadd", avgceils>;
5568 defm SRSHL : SIMDThreeSameVector<0,0b01010,"srshl", int_aarch64_neon_srshl>;
5569 defm SSHL : SIMDThreeSameVector<0,0b01000,"sshl", int_aarch64_neon_sshl>;
5570 defm SUB : SIMDThreeSameVector<1,0b10000,"sub", sub>;
5571 defm UABA : SIMDThreeSameVectorBHSTied<1, 0b01111, "uaba",
5573 defm UABD : SIMDThreeSameVectorBHS<1,0b01110,"uabd", AArch64uabd>;
5574 defm UHADD : SIMDThreeSameVectorBHS<1,0b00000,"uhadd", avgflooru>;
5575 defm UHSUB : SIMDThreeSameVectorBHS<1,0b00100,"uhsub", int_aarch64_neon_uhsub>;
5576 defm UMAXP : SIMDThreeSameVectorBHS<1,0b10100,"umaxp", int_aarch64_neon_umaxp>;
5577 defm UMAX : SIMDThreeSameVectorBHS<1,0b01100,"umax", umax>;
5578 defm UMINP : SIMDThreeSameVectorBHS<1,0b10101,"uminp", int_aarch64_neon_uminp>;
5579 defm UMIN : SIMDThreeSameVectorBHS<1,0b01101,"umin", umin>;
5580 defm UQADD : SIMDThreeSameVector<1,0b00001,"uqadd", int_aarch64_neon_uqadd>;
5581 defm UQRSHL : SIMDThreeSameVector<1,0b01011,"uqrshl", int_aarch64_neon_uqrshl>;
5582 defm UQSHL : SIMDThreeSameVector<1,0b01001,"uqshl", int_aarch64_neon_uqshl>;
5583 defm UQSUB : SIMDThreeSameVector<1,0b00101,"uqsub", int_aarch64_neon_uqsub>;
5584 defm URHADD : SIMDThreeSameVectorBHS<1,0b00010,"urhadd", avgceilu>;
5585 defm URSHL : SIMDThreeSameVector<1,0b01010,"urshl", int_aarch64_neon_urshl>;
5586 defm USHL : SIMDThreeSameVector<1,0b01000,"ushl", int_aarch64_neon_ushl>;
5587 defm SQRDMLAH : SIMDThreeSameVectorSQRDMLxHTiedHS<1,0b10000,"sqrdmlah",
5589 defm SQRDMLSH : SIMDThreeSameVectorSQRDMLxHTiedHS<1,0b10001,"sqrdmlsh",
5598 defm AND : SIMDLogicalThreeVector<0, 0b00, "and", and>;
5599 defm BIC : SIMDLogicalThreeVector<0, 0b01, "bic",
5601 defm EOR : SIMDLogicalThreeVector<1, 0b00, "eor", xor>;
5602 defm ORN : SIMDLogicalThreeVector<0, 0b11, "orn",
5604 defm ORR : SIMDLogicalThreeVector<0, 0b10, "orr", or>;
5610 defm BSL : SIMDLogicalThreeVectorTied<1, 0b01, "bsl">;
5611 defm BIT : SIMDLogicalThreeVectorTied<1, 0b10, "bit">;
5612 defm BIF : SIMDLogicalThreeVectorTied<1, 0b11, "bif">;
5728 (ORRv16i8 V128:$dst, V128:$src, V128:$src), 0>;
5730 (ORRv16i8 V128:$dst, V128:$src, V128:$src), 0>;
5732 (ORRv16i8 V128:$dst, V128:$src, V128:$src), 0>;
5737 (ORRv8i8 V64:$dst, V64:$src, V64:$src), 0>;
5739 (ORRv8i8 V64:$dst, V64:$src, V64:$src), 0>;
5741 (ORRv8i8 V64:$dst, V64:$src, V64:$src), 0>;
5745 (CMHSv8i8 V64:$dst, V64:$src2, V64:$src1), 0>;
5748 (CMHSv16i8 V128:$dst, V128:$src2, V128:$src1), 0>;
5751 (CMHSv4i16 V64:$dst, V64:$src2, V64:$src1), 0>;
5754 (CMHSv8i16 V128:$dst, V128:$src2, V128:$src1), 0>;
5757 (CMHSv2i32 V64:$dst, V64:$src2, V64:$src1), 0>;
5760 (CMHSv4i32 V128:$dst, V128:$src2, V128:$src1), 0>;
5763 (CMHSv2i64 V128:$dst, V128:$src2, V128:$src1), 0>;
5767 (CMHIv8i8 V64:$dst, V64:$src2, V64:$src1), 0>;
5770 (CMHIv16i8 V128:$dst, V128:$src2, V128:$src1), 0>;
5773 (CMHIv4i16 V64:$dst, V64:$src2, V64:$src1), 0>;
5776 (CMHIv8i16 V128:$dst, V128:$src2, V128:$src1), 0>;
5779 (CMHIv2i32 V64:$dst, V64:$src2, V64:$src1), 0>;
5782 (CMHIv4i32 V128:$dst, V128:$src2, V128:$src1), 0>;
5785 (CMHIv2i64 V128:$dst, V128:$src2, V128:$src1), 0>;
5789 (CMGEv8i8 V64:$dst, V64:$src2, V64:$src1), 0>;
5792 (CMGEv16i8 V128:$dst, V128:$src2, V128:$src1), 0>;
5795 (CMGEv4i16 V64:$dst, V64:$src2, V64:$src1), 0>;
5798 (CMGEv8i16 V128:$dst, V128:$src2, V128:$src1), 0>;
5801 (CMGEv2i32 V64:$dst, V64:$src2, V64:$src1), 0>;
5804 (CMGEv4i32 V128:$dst, V128:$src2, V128:$src1), 0>;
5807 (CMGEv2i64 V128:$dst, V128:$src2, V128:$src1), 0>;
5811 (CMGTv8i8 V64:$dst, V64:$src2, V64:$src1), 0>;
5814 (CMGTv16i8 V128:$dst, V128:$src2, V128:$src1), 0>;
5817 (CMGTv4i16 V64:$dst, V64:$src2, V64:$src1), 0>;
5820 (CMGTv8i16 V128:$dst, V128:$src2, V128:$src1), 0>;
5823 (CMGTv2i32 V64:$dst, V64:$src2, V64:$src1), 0>;
5826 (CMGTv4i32 V128:$dst, V128:$src2, V128:$src1), 0>;
5829 (CMGTv2i64 V128:$dst, V128:$src2, V128:$src1), 0>;
5834 (FCMGEv4f16 V64:$dst, V64:$src2, V64:$src1), 0>;
5837 (FCMGEv8f16 V128:$dst, V128:$src2, V128:$src1), 0>;
5841 (FCMGEv2f32 V64:$dst, V64:$src2, V64:$src1), 0>;
5844 (FCMGEv4f32 V128:$dst, V128:$src2, V128:$src1), 0>;
5847 (FCMGEv2f64 V128:$dst, V128:$src2, V128:$src1), 0>;
5852 (FCMGTv4f16 V64:$dst, V64:$src2, V64:$src1), 0>;
5855 (FCMGTv8f16 V128:$dst, V128:$src2, V128:$src1), 0>;
5859 (FCMGTv2f32 V64:$dst, V64:$src2, V64:$src1), 0>;
5862 (FCMGTv4f32 V128:$dst, V128:$src2, V128:$src1), 0>;
5865 (FCMGTv2f64 V128:$dst, V128:$src2, V128:$src1), 0>;
5870 (FACGEv4f16 V64:$dst, V64:$src2, V64:$src1), 0>;
5873 (FACGEv8f16 V128:$dst, V128:$src2, V128:$src1), 0>;
5877 (FACGEv2f32 V64:$dst, V64:$src2, V64:$src1), 0>;
5880 (FACGEv4f32 V128:$dst, V128:$src2, V128:$src1), 0>;
5883 (FACGEv2f64 V128:$dst, V128:$src2, V128:$src1), 0>;
5888 (FACGTv4f16 V64:$dst, V64:$src2, V64:$src1), 0>;
5891 (FACGTv8f16 V128:$dst, V128:$src2, V128:$src1), 0>;
5895 (FACGTv2f32 V64:$dst, V64:$src2, V64:$src1), 0>;
5898 (FACGTv4f32 V128:$dst, V128:$src2, V128:$src1), 0>;
5901 (FACGTv2f64 V128:$dst, V128:$src2, V128:$src1), 0>;
5904 //===----------------------------------------------------------------------===//
5906 //===----------------------------------------------------------------------===//
5908 defm ADD : SIMDThreeScalarD<0, 0b10000, "add", add>;
5909 defm CMEQ : SIMDThreeScalarD<1, 0b10001, "cmeq", AArch64cmeq>;
5910 defm CMGE : SIMDThreeScalarD<0, 0b00111, "cmge", AArch64cmge>;
5911 defm CMGT : SIMDThreeScalarD<0, 0b00110, "cmgt", AArch64cmgt>;
5912 defm CMHI : SIMDThreeScalarD<1, 0b00110, "cmhi", AArch64cmhi>;
5913 defm CMHS : SIMDThreeScalarD<1, 0b00111, "cmhs", AArch64cmhs>;
5914 defm CMTST : SIMDThreeScalarD<0, 0b10001, "cmtst", AArch64cmtst>;
5915 defm FABD : SIMDFPThreeScalar<1, 1, 0b010, "fabd", int_aarch64_sisd_fabd>;
5925 defm FACGE : SIMDThreeScalarFPCmp<1, 0, 0b101, "facge",
5927 defm FACGT : SIMDThreeScalarFPCmp<1, 1, 0b101, "facgt",
5929 defm FCMEQ : SIMDThreeScalarFPCmp<0, 0, 0b100, "fcmeq", AArch64fcmeq>;
5930 defm FCMGE : SIMDThreeScalarFPCmp<1, 0, 0b100, "fcmge", AArch64fcmge>;
5931 defm FCMGT : SIMDThreeScalarFPCmp<1, 1, 0b100, "fcmgt", AArch64fcmgt>;
5932 defm FMULX : SIMDFPThreeScalar<0, 0, 0b011, "fmulx", int_aarch64_neon_fmulx, HasNEONandIsStreami…
5933 defm FRECPS : SIMDFPThreeScalar<0, 0, 0b111, "frecps", int_aarch64_neon_frecps, HasNEONandIsStrea…
5934 defm FRSQRTS : SIMDFPThreeScalar<0, 1, 0b111, "frsqrts", int_aarch64_neon_frsqrts, HasNEONandIsStr…
5935 defm SQADD : SIMDThreeScalarBHSD<0, 0b00001, "sqadd", int_aarch64_neon_sqadd>;
5936 defm SQDMULH : SIMDThreeScalarHS< 0, 0b10110, "sqdmulh", int_aarch64_neon_sqdmulh>;
5937 defm SQRDMULH : SIMDThreeScalarHS< 1, 0b10110, "sqrdmulh", int_aarch64_neon_sqrdmulh>;
5938 defm SQRSHL : SIMDThreeScalarBHSD<0, 0b01011, "sqrshl",int_aarch64_neon_sqrshl>;
5939 defm SQSHL : SIMDThreeScalarBHSD<0, 0b01001, "sqshl", int_aarch64_neon_sqshl>;
5940 defm SQSUB : SIMDThreeScalarBHSD<0, 0b00101, "sqsub", int_aarch64_neon_sqsub>;
5941 defm SRSHL : SIMDThreeScalarD< 0, 0b01010, "srshl", int_aarch64_neon_srshl>;
5942 defm SSHL : SIMDThreeScalarD< 0, 0b01000, "sshl", int_aarch64_neon_sshl>;
5943 defm SUB : SIMDThreeScalarD< 1, 0b10000, "sub", sub>;
5944 defm UQADD : SIMDThreeScalarBHSD<1, 0b00001, "uqadd", int_aarch64_neon_uqadd>;
5945 defm UQRSHL : SIMDThreeScalarBHSD<1, 0b01011, "uqrshl",int_aarch64_neon_uqrshl>;
5946 defm UQSHL : SIMDThreeScalarBHSD<1, 0b01001, "uqshl", int_aarch64_neon_uqshl>;
5947 defm UQSUB : SIMDThreeScalarBHSD<1, 0b00101, "uqsub", int_aarch64_neon_uqsub>;
5948 defm URSHL : SIMDThreeScalarD< 1, 0b01010, "urshl", int_aarch64_neon_urshl>;
5949 defm USHL : SIMDThreeScalarD< 1, 0b01000, "ushl", int_aarch64_neon_ushl>;
5951 defm SQRDMLAH : SIMDThreeScalarHSTied<1, 0, 0b10000, "sqrdmlah">;
5952 defm SQRDMLSH : SIMDThreeScalarHSTied<1, 0, 0b10001, "sqrdmlsh">;
5967 (CMHSv1i64 FPR64:$dst, FPR64:$src2, FPR64:$src1), 0>;
5969 (CMGEv1i64 FPR64:$dst, FPR64:$src2, FPR64:$src1), 0>;
5971 (CMHIv1i64 FPR64:$dst, FPR64:$src2, FPR64:$src1), 0>;
5973 (CMGTv1i64 FPR64:$dst, FPR64:$src2, FPR64:$src1), 0>;
5977 (FCMGE32 FPR32:$dst, FPR32:$src2, FPR32:$src1), 0>;
5979 (FCMGE64 FPR64:$dst, FPR64:$src2, FPR64:$src1), 0>;
5981 (FCMGT32 FPR32:$dst, FPR32:$src2, FPR32:$src1), 0>;
5983 (FCMGT64 FPR64:$dst, FPR64:$src2, FPR64:$src1), 0>;
5985 (FACGE32 FPR32:$dst, FPR32:$src2, FPR32:$src1), 0>;
5987 (FACGE64 FPR64:$dst, FPR64:$src2, FPR64:$src1), 0>;
5989 (FACGT32 FPR32:$dst, FPR32:$src2, FPR32:$src1), 0>;
5991 (FACGT64 FPR64:$dst, FPR64:$src2, FPR64:$src1), 0>;
5994 //===----------------------------------------------------------------------===//
5996 //===----------------------------------------------------------------------===//
5997 defm SQDMULL : SIMDThreeScalarMixedHS<0, 0b11010, "sqdmull",
5999 defm SQDMLAL : SIMDThreeScalarMixedTiedHS<0, 0b10010, "sqdmlal">;
6000 defm SQDMLSL : SIMDThreeScalarMixedTiedHS<0, 0b10110, "sqdmlsl">;
6011 //===----------------------------------------------------------------------===//
6013 //===----------------------------------------------------------------------===//
6015 defm ABS : SIMDTwoScalarD< 0, 0b01011, "abs", abs, [HasNoCSSC]>;
6016 defm CMEQ : SIMDCmpTwoScalarD< 0, 0b01001, "cmeq", AArch64cmeqz>;
6017 defm CMGE : SIMDCmpTwoScalarD< 1, 0b01000, "cmge", AArch64cmgez>;
6018 defm CMGT : SIMDCmpTwoScalarD< 0, 0b01000, "cmgt", AArch64cmgtz>;
6019 defm CMLE : SIMDCmpTwoScalarD< 1, 0b01001, "cmle", AArch64cmlez>;
6020 defm CMLT : SIMDCmpTwoScalarD< 0, 0b01010, "cmlt", AArch64cmltz>;
6021 defm FCMEQ : SIMDFPCmpTwoScalar<0, 1, 0b01101, "fcmeq", AArch64fcmeqz>;
6022 defm FCMGE : SIMDFPCmpTwoScalar<1, 1, 0b01100, "fcmge", AArch64fcmgez>;
6023 defm FCMGT : SIMDFPCmpTwoScalar<0, 1, 0b01100, "fcmgt", AArch64fcmgtz>;
6024 defm FCMLE : SIMDFPCmpTwoScalar<1, 1, 0b01101, "fcmle", AArch64fcmlez>;
6025 defm FCMLT : SIMDFPCmpTwoScalar<0, 1, 0b01110, "fcmlt", AArch64fcmltz>;
6026 defm FCVTAS : SIMDFPTwoScalar< 0, 0, 0b11100, "fcvtas">;
6027 defm FCVTAU : SIMDFPTwoScalar< 1, 0, 0b11100, "fcvtau">;
6028 defm FCVTMS : SIMDFPTwoScalar< 0, 0, 0b11011, "fcvtms">;
6029 defm FCVTMU : SIMDFPTwoScalar< 1, 0, 0b11011, "fcvtmu">;
6030 defm FCVTNS : SIMDFPTwoScalar< 0, 0, 0b11010, "fcvtns">;
6031 defm FCVTNU : SIMDFPTwoScalar< 1, 0, 0b11010, "fcvtnu">;
6032 defm FCVTPS : SIMDFPTwoScalar< 0, 1, 0b11010, "fcvtps">;
6033 defm FCVTPU : SIMDFPTwoScalar< 1, 1, 0b11010, "fcvtpu">;
6034 def FCVTXNv1i64 : SIMDInexactCvtTwoScalar<0b10110, "fcvtxn">;
6035 defm FCVTZS : SIMDFPTwoScalar< 0, 1, 0b11011, "fcvtzs">;
6036 defm FCVTZU : SIMDFPTwoScalar< 1, 1, 0b11011, "fcvtzu">;
6037 defm FRECPE : SIMDFPTwoScalar< 0, 1, 0b11101, "frecpe">;
6038 defm FRECPX : SIMDFPTwoScalar< 0, 1, 0b11111, "frecpx">;
6039 defm FRSQRTE : SIMDFPTwoScalar< 1, 1, 0b11101, "frsqrte">;
6040 defm NEG : SIMDTwoScalarD< 1, 0b01011, "neg",
6042 defm SCVTF : SIMDFPTwoScalarCVT< 0, 0, 0b11101, "scvtf", AArch64sitof>;
6043 defm SQABS : SIMDTwoScalarBHSD< 0, 0b00111, "sqabs", int_aarch64_neon_sqabs>;
6044 defm SQNEG : SIMDTwoScalarBHSD< 1, 0b00111, "sqneg", int_aarch64_neon_sqneg>;
6045 defm SQXTN : SIMDTwoScalarMixedBHS< 0, 0b10100, "sqxtn", int_aarch64_neon_scalar_sqxtn>;
6046 defm SQXTUN : SIMDTwoScalarMixedBHS< 1, 0b10010, "sqxtun", int_aarch64_neon_scalar_sqxtun>;
6047 defm SUQADD : SIMDTwoScalarBHSDTied< 0, 0b00011, "suqadd",
6049 defm UCVTF : SIMDFPTwoScalarCVT< 1, 0, 0b11101, "ucvtf", AArch64uitof>;
6050 defm UQXTN : SIMDTwoScalarMixedBHS<1, 0b10100, "uqxtn", int_aarch64_neon_scalar_uqxtn>;
6051 defm USQADD : SIMDTwoScalarBHSDTied< 1, 0b00011, "usqadd",
6156 // Some float -> int -> float conversion patterns for which we want to keep the
6158 // avoid more costly int <-> fp register transfers.
6176 // int -> float conversion of value in lane 0 of simd vector should use
6177 // correct cvtf variant to avoid costly fpr <-> gpr register transfers.
6178 def : Pat<(f32 (sint_to_fp (i32 (vector_extract (v4i32 FPR128:$Rn), (i64 0))))),
6181 def : Pat<(f32 (uint_to_fp (i32 (vector_extract (v4i32 FPR128:$Rn), (i64 0))))),
6184 def : Pat<(f64 (sint_to_fp (i64 (vector_extract (v2i64 FPR128:$Rn), (i64 0))))),
6187 def : Pat<(f64 (uint_to_fp (i64 (vector_extract (v2i64 FPR128:$Rn), (i64 0))))),
6190 // fp16: integer extraction from vector must be at least 32-bits to be legal.
6191 // Actual extraction result is then an in-reg sign-extension of lower 16-bits.
6194 (v8i16 FPR128:$Rn), (i64 0))), i16)))),
6197 // unsigned 32-bit extracted element is truncated to 16-bits using AND
6199 (v8i16 FPR128:$Rn), (i64 0))), (i32 65535))))),
6205 // Here are the patterns for 8 and 16-bits to float.
6206 // 8-bits -> float.
6236 // 16-bits -> float.
6247 // 32-bits are handled in target specific dag combine:
6249 // 64-bits integer to 32-bits floating point, not possible with
6253 // Here are the patterns for 8, 16, 32, and 64-bits to double.
6254 // 8-bits -> double.
6265 // 16-bits -> double.
6276 // 32-bits -> double.
6287 // 64-bits -> double are handled in target specific dag combine:
6291 //===----------------------------------------------------------------------===//
6292 // Advanced SIMD three different-sized vector instructions.
6293 //===----------------------------------------------------------------------===//
6295 defm ADDHN : SIMDNarrowThreeVectorBHS<0,0b0100,"addhn", int_aarch64_neon_addhn>;
6296 defm SUBHN : SIMDNarrowThreeVectorBHS<0,0b0110,"subhn", int_aarch64_neon_subhn>;
6297 defm RADDHN : SIMDNarrowThreeVectorBHS<1,0b0100,"raddhn",int_aarch64_neon_raddhn>;
6298 defm RSUBHN : SIMDNarrowThreeVectorBHS<1,0b0110,"rsubhn",int_aarch64_neon_rsubhn>;
6299 defm PMULL : SIMDDifferentThreeVectorBD<0,0b1110,"pmull", AArch64pmull>;
6300 defm SABAL : SIMDLongThreeVectorTiedBHSabal<0,0b0101,"sabal",
6302 defm SABDL : SIMDLongThreeVectorBHSabdl<0, 0b0111, "sabdl",
6304 defm SADDL : SIMDLongThreeVectorBHS< 0, 0b0000, "saddl",
6306 defm SADDW : SIMDWideThreeVectorBHS< 0, 0b0001, "saddw",
6308 defm SMLAL : SIMDLongThreeVectorTiedBHS<0, 0b1000, "smlal",
6310 defm SMLSL : SIMDLongThreeVectorTiedBHS<0, 0b1010, "smlsl",
6312 defm SMULL : SIMDLongThreeVectorBHS<0, 0b1100, "smull", AArch64smull>;
6313 defm SQDMLAL : SIMDLongThreeVectorSQDMLXTiedHS<0, 0b1001, "sqdmlal",
6315 defm SQDMLSL : SIMDLongThreeVectorSQDMLXTiedHS<0, 0b1011, "sqdmlsl",
6317 defm SQDMULL : SIMDLongThreeVectorHS<0, 0b1101, "sqdmull",
6319 defm SSUBL : SIMDLongThreeVectorBHS<0, 0b0010, "ssubl",
6321 defm SSUBW : SIMDWideThreeVectorBHS<0, 0b0011, "ssubw",
6323 defm UABAL : SIMDLongThreeVectorTiedBHSabal<1, 0b0101, "uabal",
6325 defm UADDL : SIMDLongThreeVectorBHS<1, 0b0000, "uaddl",
6327 defm UADDW : SIMDWideThreeVectorBHS<1, 0b0001, "uaddw",
6329 defm UMLAL : SIMDLongThreeVectorTiedBHS<1, 0b1000, "umlal",
6331 defm UMLSL : SIMDLongThreeVectorTiedBHS<1, 0b1010, "umlsl",
6333 defm UMULL : SIMDLongThreeVectorBHS<1, 0b1100, "umull", AArch64umull>;
6334 defm USUBL : SIMDLongThreeVectorBHS<1, 0b0010, "usubl",
6336 defm USUBW : SIMDWideThreeVectorBHS< 1, 0b0011, "usubw",
6346 (i64 0))))),
6354 (i64 0))))),
6362 (i64 0))))),
6379 def : Pat<(v4i16 (opnode (extract_subvector (ext (v8i8 V64:$Rn)), (i64 0)),
6380 (extract_subvector (ext (v8i8 V64:$Rm)), (i64 0)))),
6382 def : Pat<(v2i32 (opnode (extract_subvector (ext (v4i16 V64:$Rn)), (i64 0)),
6383 (extract_subvector (ext (v4i16 V64:$Rm)), (i64 0)))),
6385 def : Pat<(v1i64 (opnode (extract_subvector (ext (v2i32 V64:$Rn)), (i64 0)),
6386 (extract_subvector (ext (v2i32 V64:$Rm)), (i64 0)))),
6390 (extract_subvector (ext (v8i8 V64:$Rm)), (i64 0)))),
6393 (extract_subvector (ext (v4i16 V64:$Rm)), (i64 0)))),
6396 (extract_subvector (ext (v2i32 V64:$Rm)), (i64 0)))),
6423 (ADDHNv8i16_v16i8 (SUBREG_TO_REG (i32 0), V64:$Rd, dsub),
6428 (ADDHNv4i32_v8i16 (SUBREG_TO_REG (i32 0), V64:$Rd, dsub),
6433 (ADDHNv2i64_v4i32 (SUBREG_TO_REG (i32 0), V64:$Rd, dsub),
6448 (SUBHNv8i16_v16i8 (SUBREG_TO_REG (i32 0), V64:$Rd, dsub),
6453 (SUBHNv4i32_v8i16 (SUBREG_TO_REG (i32 0), V64:$Rd, dsub),
6458 (SUBHNv2i64_v4i32 (SUBREG_TO_REG (i32 0), V64:$Rd, dsub),
6463 //----------------------------------------------------------------------------
6465 //----------------------------------------------------------------------------
6470 return CurDAG->getTargetConstant(8 + N->getZExtValue(), SDLoc(N), MVT::i32);
6477 // We use EXT to handle extract_subvector to copy the upper 64-bits of a
6478 // 128-bit vector.
6481 // A 64-bit EXT of two halves of the same 128-bit register can be done as a
6482 // single 128-bit EXT.
6483 def : Pat<(VT64 (AArch64ext (extract_subvector V128:$Rn, (i64 0)),
6487 // A 64-bit EXT of the high half of a 128-bit register can be done using a
6488 // 128-bit EXT of the whole register with an adjustment to the immediate. The
6495 (SUBREG_TO_REG (i32 0), V64:$Rm, dsub),
6508 //----------------------------------------------------------------------------
6510 //----------------------------------------------------------------------------
6512 defm TRN1 : SIMDZipVector<0b010, "trn1", AArch64trn1>;
6513 defm TRN2 : SIMDZipVector<0b110, "trn2", AArch64trn2>;
6514 defm UZP1 : SIMDZipVector<0b001, "uzp1", AArch64uzp1>;
6515 defm UZP2 : SIMDZipVector<0b101, "uzp2", AArch64uzp2>;
6516 defm ZIP1 : SIMDZipVector<0b011, "zip1", AArch64zip1>;
6517 defm ZIP2 : SIMDZipVector<0b111, "zip2", AArch64zip2>;
6524 // concat_vectors(trunc(x), trunc(y)) -> uzp1(x, y)
6525 // concat_vectors(assertzext(trunc(x)), assertzext(trunc(y))) -> uzp1(x, y)
6526 // concat_vectors(assertsext(trunc(x)), assertsext(trunc(y))) -> uzp1(x, y)
6535 // trunc(concat_vectors(trunc(x), trunc(y))) -> xtn(uzp1(x, y))
6536 // trunc(concat_vectors(assertzext(trunc(x)), assertzext(trunc(y)))) -> xtn(uzp1(x, y))
6537 // trunc(concat_vectors(assertsext(trunc(x)), assertsext(trunc(y)))) -> xtn(uzp1(x, y))
6566 //----------------------------------------------------------------------------
6568 //----------------------------------------------------------------------------
6570 defm TBL : SIMDTableLookup< 0, "tbl">;
6585 //----------------------------------------------------------------------------
6587 //----------------------------------------------------------------------------
6593 //----------------------------------------------------------------------------
6595 //----------------------------------------------------------------------------
6599 //----------------------------------------------------------------------------
6601 //----------------------------------------------------------------------------
6603 defm ADDP : SIMDPairwiseScalarD<0, 0b11011, "addp">;
6604 defm FADDP : SIMDFPPairwiseScalar<0, 0b01101, "faddp">;
6605 defm FMAXNMP : SIMDFPPairwiseScalar<0, 0b01100, "fmaxnmp">;
6606 defm FMAXP : SIMDFPPairwiseScalar<0, 0b01111, "fmaxp">;
6607 defm FMINNMP : SIMDFPPairwiseScalar<1, 0b01100, "fminnmp">;
6608 defm FMINP : SIMDFPPairwiseScalar<1, 0b01111, "fminp">;
6611 // below, so the second operand does not matter. Re-use the first input
6659 //----------------------------------------------------------------------------
6661 //----------------------------------------------------------------------------
6663 def DUPv8i8gpr : SIMDDupFromMain<0, {?,?,?,?,1}, ".8b", v8i8, V64, GPR32>;
6665 def DUPv4i16gpr : SIMDDupFromMain<0, {?,?,?,1,0}, ".4h", v4i16, V64, GPR32>;
6666 def DUPv8i16gpr : SIMDDupFromMain<1, {?,?,?,1,0}, ".8h", v8i16, V128, GPR32>;
6667 def DUPv2i32gpr : SIMDDupFromMain<0, {?,?,1,0,0}, ".2s", v2i32, V64, GPR32>;
6668 def DUPv4i32gpr : SIMDDupFromMain<1, {?,?,1,0,0}, ".4s", v4i32, V128, GPR32>;
6669 def DUPv2i64gpr : SIMDDupFromMain<1, {?,1,0,0,0}, ".2d", v2i64, V128, GPR64>;
6672 def DUPv2i32lane : SIMDDup32FromElement<0, ".2s", v2i32, V64>;
6674 def DUPv4i16lane : SIMDDup16FromElement<0, ".4h", v4i16, V64>;
6676 def DUPv8i8lane : SIMDDup8FromElement <0, ".8b", v8i8, V64>;
6679 // DUP from a 64-bit register to a 64-bit register is just a copy
6688 (i64 0)))>;
6692 (i64 0)))>;
6696 (i64 0)))>;
6700 (i64 0)))>;
6704 (i64 0)))>;
6708 (i64 0)))>;
6712 (i64 0)))>;
6735 return CurDAG->getTargetConstant(2 * N->getZExtValue(), SDLoc(N), MVT::i64);
6738 return CurDAG->getTargetConstant(4 * N->getZExtValue(), SDLoc(N), MVT::i64);
6741 return CurDAG->getTargetConstant(8 * N->getZExtValue(), SDLoc(N), MVT::i64);
6753 (DUP (SUBREG_TO_REG (i64 0), V64:$Rn, dsub), (IdxXFORM imm:$idx))>;
6772 (DUP (SUBREG_TO_REG (i64 0), V64:$Rn, dsub), (IdxXFORM imm:$idx))>;
6807 // Extracting i8 or i16 elements will have the zero-extend transformed to
6812 (i32 0xff)),
6815 (i32 0xffff)),
6819 VectorIndexB:$idx)))), (i64 0xff))),
6820 (SUBREG_TO_REG (i64 0), (i32 (UMOVvi8 V128:$Rn, VectorIndexB:$idx)), sub_32)>;
6822 VectorIndexH:$idx)))), (i64 0xffff))),
6823 (SUBREG_TO_REG (i64 0), (i32 (UMOVvi16 V128:$Rn, VectorIndexH:$idx)), sub_32)>;
6828 (SUBREG_TO_REG (i32 0),
6831 (SUBREG_TO_REG (i32 0),
6836 (SUBREG_TO_REG (i32 0), (f32 (FMOVWSr GPR32:$Rn)), ssub)>;
6839 (SUBREG_TO_REG (i32 0),
6842 (SUBREG_TO_REG (i32 0),
6891 (i64 0)),
6910 (i64 0))>;
6919 (i64 0)),
6927 (i64 0))>;
6936 (i64 0)),
6943 (i64 0))>;
6949 (i64 0))>;
6970 … VectorIndexB:$imm, (v16i8 (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)), FPR8:$Rm, bsub)), (i64 0)),
6974 (v16i8 (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)), FPR8:$Rm, bsub)), (i64 0))>;
7016 (SUBREG_TO_REG (i64 0), V64:$Rn, dsub), imm:$Immn)>;
7021 (EXTRACT_SUBREG (INS (SUBREG_TO_REG (i64 0), V64:$src, dsub),
7029 (INS (SUBREG_TO_REG (i64 0), V64:$src, dsub), imm:$Immd,
7030 (SUBREG_TO_REG (i64 0), V64:$Rn, dsub), imm:$Immn),
7045 // vector_insert(bitcast(f32 src), n, lane) -> INSvi32lane(src, lane, INSERT_SUBREG(-, n), 0)
7047 (INSvi32lane V128:$src, imm:$Immd, (INSERT_SUBREG (IMPLICIT_DEF), FPR32:$Sn, ssub), 0)>;
7051 imm:$Immd, (INSERT_SUBREG (IMPLICIT_DEF), FPR32:$Sn, ssub), 0),
7054 (INSvi64lane V128:$src, imm:$Immd, (INSERT_SUBREG (IMPLICIT_DEF), FPR64:$Sn, dsub), 0)>;
7057 // f32 bitcast(vector_extract(v4i32 src, lane)) -> EXTRACT_SUBREG(INSvi32lane(-, 0, src, lane))
7059 (EXTRACT_SUBREG (INSvi32lane (IMPLICIT_DEF), 0, V128:$src, imm:$Immd), ssub)>;
7060 def : Pat<(f32 (bitconvert (i32 (vector_extract v4i32:$src, (i64 0))))),
7063 (EXTRACT_SUBREG (INSvi64lane (IMPLICIT_DEF), 0, V128:$src, imm:$Immd), dsub)>;
7064 def : Pat<(f64 (bitconvert (i64 (vector_extract v2i64:$src, (i64 0))))),
7070 def : Pat<(f64 (vector_extract (v2f64 V128:$Rn), (i64 0))),
7072 def : Pat<(f32 (vector_extract (v4f32 V128:$Rn), (i64 0))),
7074 def : Pat<(f16 (vector_extract (v8f16 V128:$Rn), (i64 0))),
7076 def : Pat<(bf16 (vector_extract (v8bf16 V128:$Rn), (i64 0))),
7095 (INSERT_SUBREG (IMPLICIT_DEF), V64:$Rn, dsub), 0)>;
7097 // If the high lanes are zero we can instead emit a d->d register mov, which
7100 (SUBREG_TO_REG (i64 0), (FMOVDr V64:$Rn), dsub)>;
7116 //----------------------------------------------------------------------------
7118 //----------------------------------------------------------------------------
7120 defm ADDV : SIMDAcrossLanesBHS<0, 0b11011, "addv">;
7121 defm SMAXV : SIMDAcrossLanesBHS<0, 0b01010, "smaxv">;
7122 defm SMINV : SIMDAcrossLanesBHS<0, 0b11010, "sminv">;
7123 defm UMAXV : SIMDAcrossLanesBHS<1, 0b01010, "umaxv">;
7124 defm UMINV : SIMDAcrossLanesBHS<1, 0b11010, "uminv">;
7125 defm SADDLV : SIMDAcrossLanesHSD<0, 0b00011, "saddlv">;
7126 defm UADDLV : SIMDAcrossLanesHSD<1, 0b00011, "uaddlv">;
7127 defm FMAXNMV : SIMDFPAcrossLanes<0b01100, 0, "fmaxnmv", AArch64fmaxnmv>;
7128 defm FMAXV : SIMDFPAcrossLanes<0b01111, 0, "fmaxv", AArch64fmaxv>;
7129 defm FMINNMV : SIMDFPAcrossLanes<0b01100, 1, "fminnmv", AArch64fminnmv>;
7130 defm FMINV : SIMDFPAcrossLanes<0b01111, 1, "fminv", AArch64fminv>;
7136 (i64 0))), (i64 0))),
7139 def : Pat<(i32 (vector_extract (v8i16 (AArch64uaddv (v8i16 (addlp (v16i8 V128:$op))))), (i64 0))),
7178 (v4i32 (SUBREG_TO_REG (i64 0), (UADDLVv8i16v V128:$op), ssub)),
7183 (v8i16 (SUBREG_TO_REG (i64 0), (UADDLVv16i8v V128:$op), hsub)),
7187 (v2i64 (SUBREG_TO_REG (i64 0), (UADDLVv8i16v V128:$op), ssub))>;
7190 (v4i32 (SUBREG_TO_REG (i64 0), (UADDLVv16i8v V128:$op), hsub))>;
7193 (v4i32 (SUBREG_TO_REG (i64 0), (UADDLVv8i8v V64:$op), hsub))>;
7197 (v4i32 (SUBREG_TO_REG (i64 0), (!cast<Instruction>(Opc#"v8i8v") V64:$Rn), hsub))>;
7200 (v4i32 (SUBREG_TO_REG (i64 0), (!cast<Instruction>(Opc#"v4i16v") V64:$Rn), ssub))>;
7203 (v4i32 (SUBREG_TO_REG (i64 0), (!cast<Instruction>(Opc#"v16i8v") V128:$Rn), hsub))>;
7206 (v4i32 (SUBREG_TO_REG (i64 0), (!cast<Instruction>(Opc#"v8i16v") V128:$Rn), ssub))>;
7209 (v2i64 (SUBREG_TO_REG (i64 0), (!cast<Instruction>(Opc#"v4i32v") V128:$Rn), dsub))>;
7215 // Patterns for across-vector intrinsics, that have a node equivalent, that
7241 (i64 0)), (i64 0))),
7245 def : Pat<(i32 (vector_extract (v16i8 (opNode V128:$Rn)), (i64 0))),
7250 (v4i16 (opNode V64:$Rn)), (i64 0)), (i64 0))),
7254 def : Pat<(i32 (vector_extract (v8i16 (opNode V128:$Rn)), (i64 0))),
7258 def : Pat<(i32 (vector_extract (v4i32 (opNode V128:$Rn)), (i64 0))),
7271 (opNode (v8i8 V64:$Rn)), (i64 0)), (i64 0))), i8)),
7275 (i64 0)))>;
7277 (opNode (v16i8 V128:$Rn)), (i64 0))), i8)),
7281 (i64 0)))>;
7283 (opNode (v4i16 V64:$Rn)), (i64 0)), (i64 0))), i16)),
7287 (i64 0)))>;
7289 (opNode (v8i16 V128:$Rn)), (i64 0))), i16)),
7293 (i64 0)))>;
7302 (opNode (v8i8 V64:$Rn)), (i64 0)), (i64 0))), maski8_or_more)),
7307 def : Pat<(i32 (and (i32 (vector_extract (opNode (v16i8 V128:$Rn)), (i64 0))),
7314 (opNode (v4i16 V64:$Rn)), (i64 0)), (i64 0))), maski16_or_more)),
7319 def : Pat<(i32 (and (i32 (vector_extract (opNode (v8i16 V128:$Rn)), (i64 0))),
7344 // vaddv_[su]32 is special; -> ADDP Vd.2S,Vn.2S,Vm.2S; return Vd.s[0];Vn==Vm
7349 // vaddv_[su]32 is special; -> ADDP Vd.2S,Vn.2S,Vm.2S; return Vd.s[0];Vn==Vm
7411 (i64 0)))>;
7416 (i64 0)))>;
7483 //------------------------------------------------------------------------------
7485 //------------------------------------------------------------------------------
7488 defm BIC : SIMDModifiedImmVectorShiftTied<1, 0b11, 0b01, "bic", AArch64bici>;
7490 defm ORR : SIMDModifiedImmVectorShiftTied<0, 0b11, 0b01, "orr", AArch64orri>;
7493 def : InstAlias<"bic $Vd.4h, $imm", (BICv4i16 V64:$Vd, imm0_255:$imm, 0)>;
7494 def : InstAlias<"bic $Vd.8h, $imm", (BICv8i16 V128:$Vd, imm0_255:$imm, 0)>;
7495 def : InstAlias<"bic $Vd.2s, $imm", (BICv2i32 V64:$Vd, imm0_255:$imm, 0)>;
7496 def : InstAlias<"bic $Vd.4s, $imm", (BICv4i32 V128:$Vd, imm0_255:$imm, 0)>;
7498 def : InstAlias<"bic.4h $Vd, $imm", (BICv4i16 V64:$Vd, imm0_255:$imm, 0)>;
7499 def : InstAlias<"bic.8h $Vd, $imm", (BICv8i16 V128:$Vd, imm0_255:$imm, 0)>;
7500 def : InstAlias<"bic.2s $Vd, $imm", (BICv2i32 V64:$Vd, imm0_255:$imm, 0)>;
7501 def : InstAlias<"bic.4s $Vd, $imm", (BICv4i32 V128:$Vd, imm0_255:$imm, 0)>;
7503 def : InstAlias<"orr $Vd.4h, $imm", (ORRv4i16 V64:$Vd, imm0_255:$imm, 0)>;
7504 def : InstAlias<"orr $Vd.8h, $imm", (ORRv8i16 V128:$Vd, imm0_255:$imm, 0)>;
7505 def : InstAlias<"orr $Vd.2s, $imm", (ORRv2i32 V64:$Vd, imm0_255:$imm, 0)>;
7506 def : InstAlias<"orr $Vd.4s, $imm", (ORRv4i32 V128:$Vd, imm0_255:$imm, 0)>;
7508 def : InstAlias<"orr.4h $Vd, $imm", (ORRv4i16 V64:$Vd, imm0_255:$imm, 0)>;
7509 def : InstAlias<"orr.8h $Vd, $imm", (ORRv8i16 V128:$Vd, imm0_255:$imm, 0)>;
7510 def : InstAlias<"orr.2s $Vd, $imm", (ORRv2i32 V64:$Vd, imm0_255:$imm, 0)>;
7511 def : InstAlias<"orr.4s $Vd, $imm", (ORRv4i32 V128:$Vd, imm0_255:$imm, 0)>;
7515 def FMOVv2f64_ns : SIMDModifiedImmVectorNoShift<1, 1, 0, 0b1111, V128, fpimm8,
7518 def FMOVv2f32_ns : SIMDModifiedImmVectorNoShift<0, 0, 0, 0b1111, V64, fpimm8,
7521 def FMOVv4f32_ns : SIMDModifiedImmVectorNoShift<1, 0, 0, 0b1111, V128, fpimm8,
7525 def FMOVv4f16_ns : SIMDModifiedImmVectorNoShift<0, 0, 1, 0b1111, V64, fpimm8,
7528 def FMOVv8f16_ns : SIMDModifiedImmVectorNoShift<1, 0, 1, 0b1111, V128, fpimm8,
7537 def MOVID : SIMDModifiedImmScalarNoShift<0, 1, 0b1110, "movi",
7549 def MOVIv2d_ns : SIMDModifiedImmVectorNoShift<1, 1, 0, 0b1110, V128,
7554 def : Pat<(v2i64 immAllZerosV), (MOVIv2d_ns (i32 0))>;
7555 def : Pat<(v4i32 immAllZerosV), (MOVIv2d_ns (i32 0))>;
7556 def : Pat<(v8i16 immAllZerosV), (MOVIv2d_ns (i32 0))>;
7557 def : Pat<(v16i8 immAllZerosV), (MOVIv2d_ns (i32 0))>;
7558 def : Pat<(v2f64 immAllZerosV), (MOVIv2d_ns (i32 0))>;
7559 def : Pat<(v4f32 immAllZerosV), (MOVIv2d_ns (i32 0))>;
7560 def : Pat<(v8f16 immAllZerosV), (MOVIv2d_ns (i32 0))>;
7561 def : Pat<(v8bf16 immAllZerosV), (MOVIv2d_ns (i32 0))>;
7568 // Set 64-bit vectors to all 0/1 by extracting from a 128-bit register as the
7570 def : Pat<(v1i64 immAllZerosV), (EXTRACT_SUBREG (MOVIv2d_ns (i32 0)), dsub)>;
7571 def : Pat<(v2i32 immAllZerosV), (EXTRACT_SUBREG (MOVIv2d_ns (i32 0)), dsub)>;
7572 def : Pat<(v4i16 immAllZerosV), (EXTRACT_SUBREG (MOVIv2d_ns (i32 0)), dsub)>;
7573 def : Pat<(v8i8 immAllZerosV), (EXTRACT_SUBREG (MOVIv2d_ns (i32 0)), dsub)>;
7574 def : Pat<(v1f64 immAllZerosV), (MOVID (i32 0))>;
7575 def : Pat<(v2f32 immAllZerosV), (MOVID (i32 0))>;
7576 def : Pat<(v4f16 immAllZerosV), (MOVID (i32 0))>;
7577 def : Pat<(v4bf16 immAllZerosV), (MOVID (i32 0))>;
7586 defm MOVI : SIMDModifiedImmVectorShift<0, 0b10, 0b00, "movi">;
7597 def : InstAlias<"movi $Vd.4h, $imm", (MOVIv4i16 V64:$Vd, imm0_255:$imm, 0), 0>;
7598 def : InstAlias<"movi $Vd.8h, $imm", (MOVIv8i16 V128:$Vd, imm0_255:$imm, 0), 0>;
7599 def : InstAlias<"movi $Vd.2s, $imm", (MOVIv2i32 V64:$Vd, imm0_255:$imm, 0), 0>;
7600 def : InstAlias<"movi $Vd.4s, $imm", (MOVIv4i32 V128:$Vd, imm0_255:$imm, 0), 0>;
7602 def : InstAlias<"movi.4h $Vd, $imm", (MOVIv4i16 V64:$Vd, imm0_255:$imm, 0), 0>;
7603 def : InstAlias<"movi.8h $Vd, $imm", (MOVIv8i16 V128:$Vd, imm0_255:$imm, 0), 0>;
7604 def : InstAlias<"movi.2s $Vd, $imm", (MOVIv2i32 V64:$Vd, imm0_255:$imm, 0), 0>;
7605 def : InstAlias<"movi.4s $Vd, $imm", (MOVIv4i32 V128:$Vd, imm0_255:$imm, 0), 0>;
7619 def MOVIv2s_msl : SIMDModifiedImmMoveMSL<0, 0, {1,1,0,?}, V64, "movi", ".2s",
7622 def MOVIv4s_msl : SIMDModifiedImmMoveMSL<1, 0, {1,1,0,?}, V128, "movi", ".4s",
7627 def MOVIv8b_ns : SIMDModifiedImmVectorNoShift<0, 0, 0, 0b1110, V64, imm0_255,
7631 def MOVIv16b_ns : SIMDModifiedImmVectorNoShift<1, 0, 0, 0b1110, V128, imm0_255,
7640 defm MVNI : SIMDModifiedImmVectorShift<1, 0b10, 0b00, "mvni">;
7643 def : InstAlias<"mvni $Vd.4h, $imm", (MVNIv4i16 V64:$Vd, imm0_255:$imm, 0), 0>;
7644 def : InstAlias<"mvni $Vd.8h, $imm", (MVNIv8i16 V128:$Vd, imm0_255:$imm, 0), 0>;
7645 def : InstAlias<"mvni $Vd.2s, $imm", (MVNIv2i32 V64:$Vd, imm0_255:$imm, 0), 0>;
7646 def : InstAlias<"mvni $Vd.4s, $imm", (MVNIv4i32 V128:$Vd, imm0_255:$imm, 0), 0>;
7648 def : InstAlias<"mvni.4h $Vd, $imm", (MVNIv4i16 V64:$Vd, imm0_255:$imm, 0), 0>;
7649 def : InstAlias<"mvni.8h $Vd, $imm", (MVNIv8i16 V128:$Vd, imm0_255:$imm, 0), 0>;
7650 def : InstAlias<"mvni.2s $Vd, $imm", (MVNIv2i32 V64:$Vd, imm0_255:$imm, 0), 0>;
7651 def : InstAlias<"mvni.4s $Vd, $imm", (MVNIv4i32 V128:$Vd, imm0_255:$imm, 0), 0>;
7665 def MVNIv2s_msl : SIMDModifiedImmMoveMSL<0, 1, {1,1,0,?}, V64, "mvni", ".2s",
7668 def MVNIv4s_msl : SIMDModifiedImmMoveMSL<1, 1, {1,1,0,?}, V128, "mvni", ".4s",
7673 //----------------------------------------------------------------------------
7675 //----------------------------------------------------------------------------
7677 let hasSideEffects = 0 in {
7678 defm FMLA : SIMDFPIndexedTied<0, 0b0001, "fmla">;
7679 defm FMLS : SIMDFPIndexedTied<0, 0b0101, "fmls">;
7686 // the commutativity of multiplication and the fact that (-x) * y = x * (-y).
7702 // 3 variants for the .2s version: DUPLANE from 128-bit, DUPLANE from 64-bit
7712 (i64 0))),
7715 (SUBREG_TO_REG (i32 0), V64:$Rm, dsub),
7720 (SUBREG_TO_REG (i32 0), FPR32Op:$Rm, ssub), (i64 0))>;
7722 // 3 variants for the .4s version: DUPLANE from 128-bit, DUPLANE from 64-bit
7733 (i64 0))),
7736 (SUBREG_TO_REG (i32 0), V64:$Rm, dsub),
7741 (SUBREG_TO_REG (i32 0), FPR32Op:$Rm, ssub), (i64 0))>;
7743 // 2 variants for the .2d version: DUPLANE from 128-bit, and DUP scalar
7744 // (DUPLANE from 64-bit would be trivial).
7753 (SUBREG_TO_REG (i32 0), FPR64Op:$Rm, dsub), (i64 0))>;
7755 // 2 variants for 32-bit scalar version: extract from .2s or from .4s
7764 (i64 0))),
7767 (SUBREG_TO_REG (i32 0), V64:$Rm, dsub), VectorIndexS:$idx)>;
7769 // 1 variant for 64-bit scalar version: extract from .1d or from .2d
7782 defm FMULX : SIMDFPIndexed<1, 0b1001, "fmulx", int_aarch64_neon_fmulx>;
7783 defm FMUL : SIMDFPIndexed<0, 0b1001, "fmul", any_fmul>;
7788 (i64 0))>;
7792 (i64 0))>;
7796 (i64 0))>;
7798 defm SQDMULH : SIMDIndexedHS<0, 0b1100, "sqdmulh", int_aarch64_neon_sqdmulh>;
7799 defm SQRDMULH : SIMDIndexedHS<0, 0b1101, "sqrdmulh", int_aarch64_neon_sqrdmulh>;
7807 defm MLA : SIMDVectorIndexedHSTied<1, 0b0000, "mla", null_frag>;
7808 defm MLS : SIMDVectorIndexedHSTied<1, 0b0100, "mls", null_frag>;
7810 defm MUL : SIMDVectorIndexedHS<0, 0b1000, "mul", mul>;
7811 defm SMLAL : SIMDVectorIndexedLongSDTied<0, 0b0010, "smlal",
7813 defm SMLSL : SIMDVectorIndexedLongSDTied<0, 0b0110, "smlsl",
7815 defm SMULL : SIMDVectorIndexedLongSD<0, 0b1010, "smull", AArch64smull>;
7816 defm SQDMLAL : SIMDIndexedLongSQDMLXSDTied<0, 0b0011, "sqdmlal",
7818 defm SQDMLSL : SIMDIndexedLongSQDMLXSDTied<0, 0b0111, "sqdmlsl",
7820 defm SQRDMLAH : SIMDIndexedSQRDMLxHSDTied<1, 0b1101, "sqrdmlah",
7822 defm SQRDMLSH : SIMDIndexedSQRDMLxHSDTied<1, 0b1111, "sqrdmlsh",
7824 defm SQDMULL : SIMDIndexedLongSD<0, 0b1011, "sqdmull", int_aarch64_neon_sqdmull>;
7825 defm UMLAL : SIMDVectorIndexedLongSDTied<1, 0b0010, "umlal",
7827 defm UMLSL : SIMDVectorIndexedLongSDTied<1, 0b0110, "umlsl",
7829 defm UMULL : SIMDVectorIndexedLongSD<1, 0b1010, "umull", AArch64umull>;
7838 //----------------------------------------------------------------------------
7840 //----------------------------------------------------------------------------
7841 defm FCVTZS : SIMDFPScalarRShift<0, 0b11111, "fcvtzs">;
7842 defm FCVTZU : SIMDFPScalarRShift<1, 0b11111, "fcvtzu">;
7843 defm SCVTF : SIMDFPScalarRShift<0, 0b11100, "scvtf">;
7844 defm UCVTF : SIMDFPScalarRShift<1, 0b11100, "ucvtf">;
7847 // Having the same base pattern for fp <--> int totally freaks it out.
7877 // Patterns for FP16 Intrinsics - requires reg copy to/from as i16s not supported.
7924 defm SHL : SIMDScalarLShiftD< 0, 0b01010, "shl", AArch64vshl>;
7925 defm SLI : SIMDScalarLShiftDTied<1, 0b01010, "sli">;
7926 defm SQRSHRN : SIMDScalarRShiftBHS< 0, 0b10011, "sqrshrn",
7928 defm SQRSHRUN : SIMDScalarRShiftBHS< 1, 0b10001, "sqrshrun",
7930 defm SQSHLU : SIMDScalarLShiftBHSD<1, 0b01100, "sqshlu", AArch64sqshlui>;
7931 defm SQSHL : SIMDScalarLShiftBHSD<0, 0b01110, "sqshl", AArch64sqshli>;
7932 defm SQSHRN : SIMDScalarRShiftBHS< 0, 0b10010, "sqshrn",
7934 defm SQSHRUN : SIMDScalarRShiftBHS< 1, 0b10000, "sqshrun",
7936 defm SRI : SIMDScalarRShiftDTied< 1, 0b01000, "sri">;
7937 defm SRSHR : SIMDScalarRShiftD< 0, 0b00100, "srshr", AArch64srshri>;
7938 defm SRSRA : SIMDScalarRShiftDTied< 0, 0b00110, "srsra",
7941 defm SSHR : SIMDScalarRShiftD< 0, 0b00000, "sshr", AArch64vashr>;
7942 defm SSRA : SIMDScalarRShiftDTied< 0, 0b00010, "ssra",
7945 defm UQRSHRN : SIMDScalarRShiftBHS< 1, 0b10011, "uqrshrn",
7947 defm UQSHL : SIMDScalarLShiftBHSD<1, 0b01110, "uqshl", AArch64uqshli>;
7948 defm UQSHRN : SIMDScalarRShiftBHS< 1, 0b10010, "uqshrn",
7950 defm URSHR : SIMDScalarRShiftD< 1, 0b00100, "urshr", AArch64urshri>;
7951 defm URSRA : SIMDScalarRShiftDTied< 1, 0b00110, "ursra",
7954 defm USHR : SIMDScalarRShiftD< 1, 0b00000, "ushr", AArch64vlshr>;
7955 defm USRA : SIMDScalarRShiftDTied< 1, 0b00010, "usra",
7959 //----------------------------------------------------------------------------
7961 //----------------------------------------------------------------------------
7962 defm FCVTZS:SIMDVectorRShiftSD<0, 0b11111, "fcvtzs", int_aarch64_neon_vcvtfp2fxs>;
7963 defm FCVTZU:SIMDVectorRShiftSD<1, 0b11111, "fcvtzu", int_aarch64_neon_vcvtfp2fxu>;
7964 defm SCVTF: SIMDVectorRShiftToFP<0, 0b11100, "scvtf",
7966 defm RSHRN : SIMDVectorRShiftNarrowBHS<0, 0b10001, "rshrn", AArch64rshrn>;
7967 defm SHL : SIMDVectorLShiftBHSD<0, 0b01010, "shl", AArch64vshl>;
8001 defm SHRN : SIMDVectorRShiftNarrowBHS<0, 0b10000, "shrn",
8003 defm SLI : SIMDVectorLShiftBHSDTied<1, 0b01010, "sli", AArch64vsli>;
8007 defm SQRSHRN : SIMDVectorRShiftNarrowBHS<0, 0b10011, "sqrshrn",
8009 defm SQRSHRUN: SIMDVectorRShiftNarrowBHS<1, 0b10001, "sqrshrun",
8011 defm SQSHLU : SIMDVectorLShiftBHSD<1, 0b01100, "sqshlu", AArch64sqshlui>;
8012 defm SQSHL : SIMDVectorLShiftBHSD<0, 0b01110, "sqshl", AArch64sqshli>;
8013 defm SQSHRN : SIMDVectorRShiftNarrowBHS<0, 0b10010, "sqshrn",
8015 defm SQSHRUN : SIMDVectorRShiftNarrowBHS<1, 0b10000, "sqshrun",
8017 defm SRI : SIMDVectorRShiftBHSDTied<1, 0b01000, "sri", AArch64vsri>;
8021 defm SRSHR : SIMDVectorRShiftBHSD<0, 0b00100, "srshr", AArch64srshri>;
8022 defm SRSRA : SIMDVectorRShiftBHSDTied<0, 0b00110, "srsra",
8025 defm SSHLL : SIMDVectorLShiftLongBHSD<0, 0b10100, "sshll",
8028 defm SSHR : SIMDVectorRShiftBHSD<0, 0b00000, "sshr", AArch64vashr>;
8029 defm SSRA : SIMDVectorRShiftBHSDTied<0, 0b00010, "ssra",
8031 defm UCVTF : SIMDVectorRShiftToFP<1, 0b11100, "ucvtf",
8033 defm UQRSHRN : SIMDVectorRShiftNarrowBHS<1, 0b10011, "uqrshrn",
8035 defm UQSHL : SIMDVectorLShiftBHSD<1, 0b01110, "uqshl", AArch64uqshli>;
8036 defm UQSHRN : SIMDVectorRShiftNarrowBHS<1, 0b10010, "uqshrn",
8038 defm URSHR : SIMDVectorRShiftBHSD<1, 0b00100, "urshr", AArch64urshri>;
8039 defm URSRA : SIMDVectorRShiftBHSDTied<1, 0b00110, "ursra",
8042 defm USHLL : SIMDVectorLShiftLongBHSD<1, 0b10100, "ushll",
8044 defm USHR : SIMDVectorRShiftBHSD<1, 0b00000, "ushr", AArch64vlshr>;
8045 defm USRA : SIMDVectorRShiftBHSDTied<1, 0b00010, "usra",
8048 def VImm0080: PatLeaf<(AArch64movi_shift (i32 128), (i32 0))>;
8054 (RADDHNv8i16_v8i8 V128:$Vn, (v8i16 (MOVIv2d_ns (i32 0))))>;
8056 (RADDHNv4i32_v4i16 V128:$Vn, (v4i32 (MOVIv2d_ns (i32 0))))>;
8059 (RADDHNv2i64_v2i32 V128:$Vn, (v2i64 (MOVIv2d_ns (i32 0))))>;
8061 (RADDHNv8i16_v8i8 V128:$Vn, (v8i16 (MOVIv2d_ns (i32 0))))>;
8063 (RADDHNv4i32_v4i16 V128:$Vn, (v4i32 (MOVIv2d_ns (i32 0))))>;
8065 (RADDHNv2i64_v2i32 V128:$Vn, (v2i64 (MOVIv2d_ns (i32 0))))>;
8073 (v8i16 (MOVIv2d_ns (i32 0))))>;
8079 (v4i32 (MOVIv2d_ns (i32 0))))>;
8086 (v2i64 (MOVIv2d_ns (i32 0))))>;
8092 (v8i16 (MOVIv2d_ns (i32 0))))>;
8098 (v4i32 (MOVIv2d_ns (i32 0))))>;
8104 (v2i64 (MOVIv2d_ns (i32 0))))>;
8134 def : Pat<(v8i16 (sext (v8i8 V64:$Rn))), (SSHLLv8i8_shift V64:$Rn, (i32 0))>;
8135 def : Pat<(v8i16 (zext (v8i8 V64:$Rn))), (USHLLv8i8_shift V64:$Rn, (i32 0))>;
8136 def : Pat<(v8i16 (anyext (v8i8 V64:$Rn))), (USHLLv8i8_shift V64:$Rn, (i32 0))>;
8137 def : Pat<(v4i32 (sext (v4i16 V64:$Rn))), (SSHLLv4i16_shift V64:$Rn, (i32 0))>;
8138 def : Pat<(v4i32 (zext (v4i16 V64:$Rn))), (USHLLv4i16_shift V64:$Rn, (i32 0))>;
8139 def : Pat<(v4i32 (anyext (v4i16 V64:$Rn))), (USHLLv4i16_shift V64:$Rn, (i32 0))>;
8140 def : Pat<(v2i64 (sext (v2i32 V64:$Rn))), (SSHLLv2i32_shift V64:$Rn, (i32 0))>;
8141 def : Pat<(v2i64 (zext (v2i32 V64:$Rn))), (USHLLv2i32_shift V64:$Rn, (i32 0))>;
8142 def : Pat<(v2i64 (anyext (v2i32 V64:$Rn))), (USHLLv2i32_shift V64:$Rn, (i32 0))>;
8143 // Vector bf16 -> fp32 is implemented morally as a zext + shift.
8147 (USHLLv16i8_shift V128:$Rn, (i32 0))>;
8149 (USHLLv16i8_shift V128:$Rn, (i32 0))>;
8151 (SSHLLv16i8_shift V128:$Rn, (i32 0))>;
8153 (USHLLv8i16_shift V128:$Rn, (i32 0))>;
8155 (USHLLv8i16_shift V128:$Rn, (i32 0))>;
8157 (SSHLLv8i16_shift V128:$Rn, (i32 0))>;
8159 (USHLLv4i32_shift V128:$Rn, (i32 0))>;
8161 (USHLLv4i32_shift V128:$Rn, (i32 0))>;
8163 (SSHLLv4i32_shift V128:$Rn, (i32 0))>;
8168 (SSHLLv8i8_shift V128:$dst, V64:$src1, 0)>;
8170 (SSHLLv8i8_shift V128:$dst, V64:$src1, 0)>;
8172 (SSHLLv4i16_shift V128:$dst, V64:$src1, 0)>;
8174 (SSHLLv4i16_shift V128:$dst, V64:$src1, 0)>;
8176 (SSHLLv2i32_shift V128:$dst, V64:$src1, 0)>;
8178 (SSHLLv2i32_shift V128:$dst, V64:$src1, 0)>;
8182 (SSHLLv16i8_shift V128:$dst, V128:$src1, 0)>;
8184 (SSHLLv16i8_shift V128:$dst, V128:$src1, 0)>;
8186 (SSHLLv8i16_shift V128:$dst, V128:$src1, 0)>;
8188 (SSHLLv8i16_shift V128:$dst, V128:$src1, 0)>;
8190 (SSHLLv4i32_shift V128:$dst, V128:$src1, 0)>;
8192 (SSHLLv4i32_shift V128:$dst, V128:$src1, 0)>;
8196 (USHLLv8i8_shift V128:$dst, V64:$src1, 0)>;
8198 (USHLLv8i8_shift V128:$dst, V64:$src1, 0)>;
8200 (USHLLv4i16_shift V128:$dst, V64:$src1, 0)>;
8202 (USHLLv4i16_shift V128:$dst, V64:$src1, 0)>;
8204 (USHLLv2i32_shift V128:$dst, V64:$src1, 0)>;
8206 (USHLLv2i32_shift V128:$dst, V64:$src1, 0)>;
8210 (USHLLv16i8_shift V128:$dst, V128:$src1, 0)>;
8212 (USHLLv16i8_shift V128:$dst, V128:$src1, 0)>;
8214 (USHLLv8i16_shift V128:$dst, V128:$src1, 0)>;
8216 (USHLLv8i16_shift V128:$dst, V128:$src1, 0)>;
8218 (USHLLv4i32_shift V128:$dst, V128:$src1, 0)>;
8220 (USHLLv4i32_shift V128:$dst, V128:$src1, 0)>;
8229 (i32 (logical_imm32_XFORM(i32 0x7fff))))),
8241 (i32 (logical_imm32_XFORM(i32 0x8000))))),
8254 … (v4bf16 V64:$Rn))), (v4bf16 (EORv8i8 (v4bf16 V64:$Rn), (v4i16 (MOVIv4i16 (i32 0x80), (i32 8)))))>;
8256 …8bf16 V128:$Rn))), (v8bf16 (EORv16i8 (v8bf16 V128:$Rn), (v8i16 (MOVIv8i16 (i32 0x80), (i32 8)))))>;
8264 // one step: byte-to-half, half-to-word, word-to-doubleword.
8265 // SCVTF GPR -> FPR is 9 cycles.
8266 // SCVTF FPR -> FPR is 4 cyclces.
8267 // (sign extension with lengthen) SXTL FPR -> FPR is 2 cycles.
8268 // Therefore, we can do 2 sign extensions and one SCVTF FPR -> FPR
8271 // 8-bits -> float. 2 sizes step-up.
8282 0),
8284 0),
8297 // 16-bits -> float. 1 size step-up.
8305 0),
8318 // 32-bits to 32-bits are handled in target specific dag combine:
8320 // 64-bits integer to 32-bits floating point, not possible with
8324 // Here are the patterns for 8, 16, 32, and 64-bits to double.
8325 // 8-bits -> double. 3 size step-up: give up.
8326 // 16-bits -> double. 2 size step.
8337 0),
8339 0),
8351 // 32-bits -> double. 1 size step-up.
8359 0),
8372 // 64-bits -> double are handled in target specific dag combine:
8376 //----------------------------------------------------------------------------
8377 // AdvSIMD Load-Store Structure
8378 //----------------------------------------------------------------------------
8414 //---
8415 // Single-element
8416 //---
8418 defm LD1R : SIMDLdR<0, 0b110, 0, "ld1r", "One", 1, 2, 4, 8>;
8419 defm LD2R : SIMDLdR<1, 0b110, 0, "ld2r", "Two", 2, 4, 8, 16>;
8420 defm LD3R : SIMDLdR<0, 0b111, 0, "ld3r", "Three", 3, 6, 12, 24>;
8421 defm LD4R : SIMDLdR<1, 0b111, 0, "ld4r", "Four", 4, 8, 16, 32>;
8422 let mayLoad = 1, hasSideEffects = 0 in {
8423 defm LD1 : SIMDLdSingleBTied<0, 0b000, "ld1", VecListOneb, GPR64pi1>;
8424 defm LD1 : SIMDLdSingleHTied<0, 0b010, 0, "ld1", VecListOneh, GPR64pi2>;
8425 defm LD1 : SIMDLdSingleSTied<0, 0b100, 0b00, "ld1", VecListOnes, GPR64pi4>;
8426 defm LD1 : SIMDLdSingleDTied<0, 0b100, 0b01, "ld1", VecListOned, GPR64pi8>;
8427 defm LD2 : SIMDLdSingleBTied<1, 0b000, "ld2", VecListTwob, GPR64pi2>;
8428 defm LD2 : SIMDLdSingleHTied<1, 0b010, 0, "ld2", VecListTwoh, GPR64pi4>;
8429 defm LD2 : SIMDLdSingleSTied<1, 0b100, 0b00, "ld2", VecListTwos, GPR64pi8>;
8430 defm LD2 : SIMDLdSingleDTied<1, 0b100, 0b01, "ld2", VecListTwod, GPR64pi16>;
8431 defm LD3 : SIMDLdSingleBTied<0, 0b001, "ld3", VecListThreeb, GPR64pi3>;
8432 defm LD3 : SIMDLdSingleHTied<0, 0b011, 0, "ld3", VecListThreeh, GPR64pi6>;
8433 defm LD3 : SIMDLdSingleSTied<0, 0b101, 0b00, "ld3", VecListThrees, GPR64pi12>;
8434 defm LD3 : SIMDLdSingleDTied<0, 0b101, 0b01, "ld3", VecListThreed, GPR64pi24>;
8435 defm LD4 : SIMDLdSingleBTied<1, 0b001, "ld4", VecListFourb, GPR64pi4>;
8436 defm LD4 : SIMDLdSingleHTied<1, 0b011, 0, "ld4", VecListFourh, GPR64pi8>;
8437 defm LD4 : SIMDLdSingleSTied<1, 0b101, 0b00, "ld4", VecListFours, GPR64pi16>;
8438 defm LD4 : SIMDLdSingleDTied<1, 0b101, 0b01, "ld4", VecListFourd, GPR64pi32>;
8458 …8 (AArch64duplane8 (v16i8 (insert_subvector undef, (v8i8 (load GPR64sp:$Rn)), (i64 0))), (i64 0))),
8460 def : Pat<(v16i8 (AArch64duplane8 (v16i8 (load GPR64sp:$Rn)), (i64 0))),
8462 …(AArch64duplane16 (v8i16 (insert_subvector undef, (v4i16 (load GPR64sp:$Rn)), (i64 0))), (i64 0))),
8464 def : Pat<(v8i16 (AArch64duplane16 (v8i16 (load GPR64sp:$Rn)), (i64 0))),
8466 …(AArch64duplane32 (v4i32 (insert_subvector undef, (v2i32 (load GPR64sp:$Rn)), (i64 0))), (i64 0))),
8468 def : Pat<(v4i32 (AArch64duplane32 (v4i32 (load GPR64sp:$Rn)), (i64 0))),
8470 def : Pat<(v2i64 (AArch64duplane64 (v2i64 (load GPR64sp:$Rn)), (i64 0))),
8526 (LD1 (SUBREG_TO_REG (i32 0), VecListOne64:$Rd, dsub),
8531 return CurDAG->getTargetConstant(N->getZExtValue() * 2, SDLoc(N), MVT::i64);
8534 return CurDAG->getTargetConstant(N->getZExtValue() * 4, SDLoc(N), MVT::i64);
8537 return CurDAG->getTargetConstant(N->getZExtValue() * 2, SDLoc(N), MVT::i64);
8555 (LD1 (VecTy (IMPLICIT_DEF)), 0, GPR64sp:$Rn), dsub))>;
8566 (LD1 (SUBREG_TO_REG (i32 0), VecListOne64:$Rd, dsub),
8584 defm ST1 : SIMDStSingleB<0, 0b000, "st1", VecListOneb, GPR64pi1>;
8585 defm ST1 : SIMDStSingleH<0, 0b010, 0, "st1", VecListOneh, GPR64pi2>;
8586 defm ST1 : SIMDStSingleS<0, 0b100, 0b00, "st1", VecListOnes, GPR64pi4>;
8587 defm ST1 : SIMDStSingleD<0, 0b100, 0b01, "st1", VecListOned, GPR64pi8>;
8612 (ST1 (SUBREG_TO_REG (i32 0), VecListOne64:$Vt, dsub),
8628 (ST1 (SUBREG_TO_REG (i32 0), VecListOne64:$Vt, dsub),
8634 (ST1 (SUBREG_TO_REG (i32 0), VecListOne64:$Vt, dsub),
8673 let mayStore = 1, hasSideEffects = 0 in {
8674 defm ST2 : SIMDStSingleB<1, 0b000, "st2", VecListTwob, GPR64pi2>;
8675 defm ST2 : SIMDStSingleH<1, 0b010, 0, "st2", VecListTwoh, GPR64pi4>;
8676 defm ST2 : SIMDStSingleS<1, 0b100, 0b00, "st2", VecListTwos, GPR64pi8>;
8677 defm ST2 : SIMDStSingleD<1, 0b100, 0b01, "st2", VecListTwod, GPR64pi16>;
8678 defm ST3 : SIMDStSingleB<0, 0b001, "st3", VecListThreeb, GPR64pi3>;
8679 defm ST3 : SIMDStSingleH<0, 0b011, 0, "st3", VecListThreeh, GPR64pi6>;
8680 defm ST3 : SIMDStSingleS<0, 0b101, 0b00, "st3", VecListThrees, GPR64pi12>;
8681 defm ST3 : SIMDStSingleD<0, 0b101, 0b01, "st3", VecListThreed, GPR64pi24>;
8682 defm ST4 : SIMDStSingleB<1, 0b001, "st4", VecListFourb, GPR64pi4>;
8683 defm ST4 : SIMDStSingleH<1, 0b011, 0, "st4", VecListFourh, GPR64pi8>;
8684 defm ST4 : SIMDStSingleS<1, 0b101, 0b00, "st4", VecListFours, GPR64pi16>;
8685 defm ST4 : SIMDStSingleD<1, 0b101, 0b01, "st4", VecListFourd, GPR64pi32>;
8693 //----------------------------------------------------------------------------
8695 //----------------------------------------------------------------------------
8699 def AESErr : AESTiedInst<0b0100, "aese", int_aarch64_crypto_aese>;
8700 def AESDrr : AESTiedInst<0b0101, "aesd", int_aarch64_crypto_aesd>;
8702 def AESMCrr : AESInst< 0b0110, "aesmc", int_aarch64_crypto_aesmc>;
8703 def AESIMCrr : AESInst< 0b0111, "aesimc", int_aarch64_crypto_aesimc>;
8708 let hasSideEffects = 0, mayStore = 0, mayLoad = 0, Predicates = [HasAES] in {
8732 def SHA1Crrr : SHATiedInstQSV<0b000, "sha1c", int_aarch64_crypto_sha1c>;
8733 def SHA1Prrr : SHATiedInstQSV<0b001, "sha1p", int_aarch64_crypto_sha1p>;
8734 def SHA1Mrrr : SHATiedInstQSV<0b010, "sha1m", int_aarch64_crypto_sha1m>;
8735 def SHA1SU0rrr : SHATiedInstVVV<0b011, "sha1su0", int_aarch64_crypto_sha1su0>;
8736 def SHA256Hrrr : SHATiedInstQQV<0b100, "sha256h", int_aarch64_crypto_sha256h>;
8737 def SHA256H2rrr : SHATiedInstQQV<0b101, "sha256h2",int_aarch64_crypto_sha256h2>;
8738 def SHA256SU1rrr :SHATiedInstVVV<0b110, "sha256su1",int_aarch64_crypto_sha256su1>;
8740 def SHA1Hrr : SHAInstSS< 0b0000, "sha1h", int_aarch64_crypto_sha1h>;
8741 def SHA1SU1rr : SHATiedInstVV<0b0001, "sha1su1", int_aarch64_crypto_sha1su1>;
8742 def SHA256SU0rr : SHATiedInstVV<0b0010, "sha256su0",int_aarch64_crypto_sha256su0>;
8745 //----------------------------------------------------------------------------
8746 // Compiler-pseudos
8747 //----------------------------------------------------------------------------
8755 // When we need to explicitly zero-extend, we use a 32-bit MOV instruction and
8758 (SUBREG_TO_REG (i32 0), (ORRWrs WZR, GPR32:$src, 0), sub_32)>;
8761 // containing super-reg.
8763 (SBFMXri (INSERT_SUBREG (i64 (IMPLICIT_DEF)), GPR32:$src, sub_32), 0, 31)>;
8764 def : Pat<(i64 (sext_inreg GPR64:$src, i32)), (SBFMXri GPR64:$src, 0, 31)>;
8765 def : Pat<(i64 (sext_inreg GPR64:$src, i16)), (SBFMXri GPR64:$src, 0, 15)>;
8766 def : Pat<(i64 (sext_inreg GPR64:$src, i8)), (SBFMXri GPR64:$src, 0, 7)>;
8767 def : Pat<(i64 (sext_inreg GPR64:$src, i1)), (SBFMXri GPR64:$src, 0, 0)>;
8768 def : Pat<(i32 (sext_inreg GPR32:$src, i16)), (SBFMWri GPR32:$src, 0, 15)>;
8769 def : Pat<(i32 (sext_inreg GPR32:$src, i8)), (SBFMWri GPR32:$src, 0, 7)>;
8770 def : Pat<(i32 (sext_inreg GPR32:$src, i1)), (SBFMWri GPR32:$src, 0, 0)>;
8802 // bitwidth-1 bits.
8824 def : Pat<(debugtrap), (BRK 0xF000)>;
8827 return CurDAG->getTargetConstant(N->getZExtValue() | ('U' << 8), SDLoc(N), MVT::i32);
8884 // STR do a 64-bit byte swap, whereas LD1/ST1 do a byte swap per lane - that
8885 // is, they treat the vector as a sequence of elements to be byte-swapped.
8897 // But this is now broken - the value stored is different to the value loaded
8912 // There is also no 128-bit REV instruction. This must be synthesized with an
8916 // a) Identity conversions - vNfX <-> vNiX
8917 // b) Single-lane-to-scalar - v1fX <-> fX or v1iX <-> iX
9489 def : Pat<(v4i16 (extract_subvector V128:$Rn, (i64 0))),
9491 def : Pat<(v8i8 (extract_subvector V128:$Rn, (i64 0))),
9493 def : Pat<(v2f32 (extract_subvector V128:$Rn, (i64 0))),
9495 def : Pat<(v4f16 (extract_subvector V128:$Rn, (i64 0))),
9497 def : Pat<(v4bf16 (extract_subvector V128:$Rn, (i64 0))),
9499 def : Pat<(v2i32 (extract_subvector V128:$Rn, (i64 0))),
9501 def : Pat<(v1i64 (extract_subvector V128:$Rn, (i64 0))),
9503 def : Pat<(v1f64 (extract_subvector V128:$Rn, (i64 0))),
9515 // A 64-bit subvector insert to the first 128-bit vector position
9518 def : Pat<(insert_subvector undef, (v1i64 FPR64:$src), (Ty 0)),
9520 def : Pat<(insert_subvector undef, (v1f64 FPR64:$src), (Ty 0)),
9522 def : Pat<(insert_subvector undef, (v2i32 FPR64:$src), (Ty 0)),
9524 def : Pat<(insert_subvector undef, (v2f32 FPR64:$src), (Ty 0)),
9526 def : Pat<(insert_subvector undef, (v4i16 FPR64:$src), (Ty 0)),
9528 def : Pat<(insert_subvector undef, (v4f16 FPR64:$src), (Ty 0)),
9530 def : Pat<(insert_subvector undef, (v4bf16 FPR64:$src), (Ty 0)),
9532 def : Pat<(insert_subvector undef, (v8i8 FPR64:$src), (Ty 0)),
9539 // Use pair-wise add instructions when summing up the lanes for v2f64, v2i64
9541 def : Pat<(i64 (add (vector_extract (v2i64 FPR128:$Rn), (i64 0)),
9544 def : Pat<(f64 (any_fadd (vector_extract (v2f64 FPR128:$Rn), (i64 0)),
9547 // vector_extract on 64-bit vectors gets promoted to a 128 bit vector,
9550 def : Pat<(any_fadd (vector_extract (v4f32 FPR128:$Rn), (i64 0)),
9553 def : Pat<(any_fadd (vector_extract (v8f16 FPR128:$Rn), (i64 0)),
9559 def : Pat<(AArch64addp (v2i32 (extract_subvector (v4i32 FPR128:$Rn), (i64 0))),
9562 def : Pat<(AArch64addp (v4i16 (extract_subvector (v8i16 FPR128:$Rn), (i64 0))),
9565 def : Pat<(AArch64addp (v8i8 (extract_subvector (v16i8 FPR128:$Rn), (i64 0))),
9569 def : Pat<(AArch64faddp (v2f32 (extract_subvector (v4f32 FPR128:$Rn), (i64 0))),
9572 def : Pat<(AArch64faddp (v4f16 (extract_subvector (v8f16 FPR128:$Rn), (i64 0))),
9576 // add(uzp1(X, Y), uzp2(X, Y)) -> addp(X, Y)
9601 // Scalar 64-bit shifts in FPR64 registers.
9611 // Patterns for nontemporal/no-allocate stores.
9612 // We have to resort to tricks to turn a single-input store into a store pair,
9613 // because there is no single-input nontemporal store, only STNP.
9632 (DUPi32 (SUBREG_TO_REG (i64 0), FPR64:$Rt, dsub), (i64 1)),
9650 // Tail call return handling. These are all compiler pseudo-instructions,
9657 // Indirect tail-call with any register allowed, used by MachineOutliner when
9664 // Indirect tail-calls with reduced register classes, needed for BTI and
9700 def : Pat<(i64 (extractelt (v2i64 V128:$V), (i64 0))), (EXTRACT_SUBREG V128:$V, dsub)>;
9701 def : Pat<(i32 (extractelt (v4i32 V128:$V), (i64 0))), (EXTRACT_SUBREG V128:$V, ssub)>;
9702 def : Pat<(i32 (extractelt (v2i32 V64:$V), (i64 0))), (EXTRACT_SUBREG V64:$V, ssub)>;
9718 (i32 (COPY_TO_REGCLASS (LDRWui node:$R, (i64 0)), FPR32)),
9745 (add (mul_v8i8<mulop, extend> node:$M, node:$N, (i64 0)),
9747 (i64 0)))>;
9749 // vaddv_[su]32 is special; -> ADDP Vd.2S,Vn.2S,Vm.2S; return Vd.s[0];Vn==Vm
9785 (add (mul_v16i8<m, x> node:$M, node:$N, (i64 0), (i64 0)),
9786 (mul_v16i8<m, x> node:$M, node:$N, (i64 8), (i64 0))),
9787 (add (mul_v16i8<m, x> node:$M, node:$N, (i64 0), (i64 4)),
9789 (i64 0)))>;
9811 // FIXME: add SVE dot-product patterns.
9814 // Custom DAG nodes and isel rules to make a 64-byte block out of eight GPRs,
9823 foreach i = 0-7 in {
9829 def LD64B: LoadStore64B<0b101, "ld64b", (ins GPR64sp:$Rn),
9831 def ST64B: LoadStore64B<0b001, "st64b", (ins GPR64x8:$Rt, GPR64sp:$Rn),
9833 def ST64BV: Store64BV<0b011, "st64bv">;
9834 def ST64BV0: Store64BV<0b010, "st64bv0">;
9847 defm CPYFP : MOPSMemoryCopyInsns<0b00, "cpyfp">;
9849 defm CPYP : MOPSMemoryMoveInsns<0b00, "cpyp">;
9851 defm SETP : MOPSMemorySetInsns<0b00, "setp">;
9854 defm CPYFM : MOPSMemoryCopyInsns<0b01, "cpyfm">;
9855 defm CPYFE : MOPSMemoryCopyInsns<0b10, "cpyfe">;
9857 defm CPYM : MOPSMemoryMoveInsns<0b01, "cpym">;
9858 defm CPYE : MOPSMemoryMoveInsns<0b10, "cpye">;
9860 defm SETM : MOPSMemorySetInsns<0b01, "setm">;
9861 defm SETE : MOPSMemorySetInsns<0b10, "sete">;
9866 defm SETGP : MOPSMemorySetTaggingInsns<0b00, "setgp">;
9869 defm SETGM : MOPSMemorySetTaggingInsns<0b01, "setgm">;
9871 defm MOPSSETGE : MOPSMemorySetTaggingInsns<0b10, "setge">;
9875 // MOPS Node operands: 0: Dst, 1: Src or Value, 2: Size, 3: Chain
9876 // MOPS Node results: 0: Dst writeback, 1: Size writeback, 2: Chain
9877 def SDT_AArch64mops : SDTypeProfile<2, 3, [ SDTCisInt<0>, SDTCisInt<1>, SDTCisInt<2> ]>;
9883 // MOPS operations always contain three 4-byte instructions
9893 let mayLoad = 0 in {
9899 let Predicates = [HasMOPS, HasMTE], Defs = [NZCV], Size = 12, mayLoad = 0, mayStore = 1 in {
9905 //-----------------------------------------------------------------------------
9913 //-----------------------------------------------------------------------------
9925 //===----------------------------===//
9927 //===----------------------------===//
9929 def : InstAlias<"clrbhb", (HINT 22), 0>;
9934 //===----------------------------------------------------------------------===//
9936 //===----------------------------------------------------------------------===//
9939 defm RCWCLR : ReadCheckWriteOperation<0b001, "clr">;
9940 defm RCWSET : ReadCheckWriteOperation<0b011, "set">;
9941 defm RCWSWP : ReadCheckWriteOperation<0b010, "swp">;
9943 //===----------------------------------------------------------------------===//
9944 // General Data-Processing Instructions (FEAT_V94_DP)
9945 //===----------------------------------------------------------------------===//
9946 defm ABS : OneOperandData<0b001000, "abs", abs>, Requires<[HasCSSC]>;
9947 defm CNT : OneOperandData<0b000111, "cnt", ctpop>, Requires<[HasCSSC]>;
9948 defm CTZ : OneOperandData<0b000110, "ctz", cttz>, Requires<[HasCSSC]>;
9950 defm SMAX : ComparisonOp<0, 0, "smax", smax>, Requires<[HasCSSC]>;
9951 defm SMIN : ComparisonOp<0, 1, "smin", smin>, Requires<[HasCSSC]>;
9952 defm UMAX : ComparisonOp<1, 0, "umax", umax>, Requires<[HasCSSC]>;
9962 let Inst{2-0} = Rt{2-0};
9963 let Inst{4-3} = 0b11;
9964 let Inst{9-5} = Rn;
9965 let Inst{11-10} = 0b10;
9966 let Inst{13-12} = Rt{4-3};
9967 let Inst{14} = 0b1;
9969 let Inst{20-16} = Rm;
9970 let Inst{31-21} = 0b11111000101;
9971 let mayLoad = 0;
9972 let mayStore = 0;
9980 //===----------------------------------------------------------------------===//
9981 // 128-bit Atomics (FEAT_LSE128)
9982 //===----------------------------------------------------------------------===//
9984 def SWPP : LSE128Base<0b000, 0b00, 0b1, "swpp">;
9985 def SWPPA : LSE128Base<0b000, 0b10, 0b1, "swppa">;
9986 def SWPPAL : LSE128Base<0b000, 0b11, 0b1, "swppal">;
9987 def SWPPL : LSE128Base<0b000, 0b01, 0b1, "swppl">;
9988 def LDCLRP : LSE128Base<0b001, 0b00, 0b0, "ldclrp">;
9989 def LDCLRPA : LSE128Base<0b001, 0b10, 0b0, "ldclrpa">;
9990 def LDCLRPAL : LSE128Base<0b001, 0b11, 0b0, "ldclrpal">;
9991 def LDCLRPL : LSE128Base<0b001, 0b01, 0b0, "ldclrpl">;
9992 def LDSETP : LSE128Base<0b011, 0b00, 0b0, "ldsetp">;
9993 def LDSETPA : LSE128Base<0b011, 0b10, 0b0, "ldsetpa">;
9994 def LDSETPAL : LSE128Base<0b011, 0b11, 0b0, "ldsetpal">;
9995 def LDSETPL : LSE128Base<0b011, 0b01, 0b0, "ldsetpl">;
9998 //===----------------------------------------------------------------------===//
10000 //===----------------------------------------------------------------------===//
10004 …egerLoadStorePair<0b10, 0b00, 0b0000, (outs GPR64sp:$wback), (ins GPR32:$Rt, GPR32:$Rt2, GPR64sp:$…
10005 …egerLoadStorePair<0b11, 0b00, 0b0000, (outs GPR64sp:$wback), (ins GPR64:$Rt, GPR64:$Rt2, GPR64sp:$…
10006 …def STILPW: BaseLRCPC3IntegerLoadStorePair<0b10, 0b00, 0b0001, (outs), (ins GPR32:$Rt, GPR32:…
10007 …def STILPX: BaseLRCPC3IntegerLoadStorePair<0b11, 0b00, 0b0001, (outs), (ins GPR64:$Rt, GPR64:…
10008 …def LDIAPPWpost: BaseLRCPC3IntegerLoadStorePair<0b10, 0b01, 0b0000, (outs GPR64sp:$wback, GPR32:$R…
10009 …def LDIAPPXpost: BaseLRCPC3IntegerLoadStorePair<0b11, 0b01, 0b0000, (outs GPR64sp:$wback, GPR64:$R…
10010 …def LDIAPPW: BaseLRCPC3IntegerLoadStorePair<0b10, 0b01, 0b0001, (outs GPR32:$Rt, GPR32:$Rt2), …
10011 …def LDIAPPX: BaseLRCPC3IntegerLoadStorePair<0b11, 0b01, 0b0001, (outs GPR64:$Rt, GPR64:$Rt2), …
10016 // Aliases for when offset=0
10017 def : InstAlias<"stilp\t$Rt, $Rt2, [$Rn, #0]", (STILPW GPR32: $Rt, GPR32: $Rt2, GPR64sp:$Rn)>;
10018 def : InstAlias<"stilp\t$Rt, $Rt2, [$Rn, #0]", (STILPX GPR64: $Rt, GPR64: $Rt2, GPR64sp:$Rn)>;
10021 …aseLRCPC3IntegerLoadStore<0b10, 0b10, (outs GPR64sp:$wback), (ins GPR32:$Rt, GPR64sp:$R…
10022 …aseLRCPC3IntegerLoadStore<0b11, 0b10, (outs GPR64sp:$wback), (ins GPR64:$Rt, GPR64sp:$R…
10023 …def LDAPRWpost: BaseLRCPC3IntegerLoadStore<0b10, 0b11, (outs GPR64sp:$wback, GPR32:$Rt), (ins GPR6…
10024 …def LDAPRXpost: BaseLRCPC3IntegerLoadStore<0b11, 0b11, (outs GPR64sp:$wback, GPR64:$Rt), (ins GPR6…
10029 …defm STLURb: LRCPC3NEONLoadStoreUnscaledOffset<0b00, 0b00, FPR8 , (outs), (ins FPR8 :$Rt, GPR64…
10030 …defm STLURh: LRCPC3NEONLoadStoreUnscaledOffset<0b01, 0b00, FPR16 , (outs), (ins FPR16 :$Rt, GPR64…
10031 …defm STLURs: LRCPC3NEONLoadStoreUnscaledOffset<0b10, 0b00, FPR32 , (outs), (ins FPR32 :$Rt, GPR64…
10032 …defm STLURd: LRCPC3NEONLoadStoreUnscaledOffset<0b11, 0b00, FPR64 , (outs), (ins FPR64 :$Rt, GPR64…
10033 …defm STLURq: LRCPC3NEONLoadStoreUnscaledOffset<0b00, 0b10, FPR128, (outs), (ins FPR128:$Rt, GPR64…
10034 …defm LDAPURb: LRCPC3NEONLoadStoreUnscaledOffset<0b00, 0b01, FPR8 , (outs FPR8 :$Rt), (ins GPR64s…
10035 …defm LDAPURh: LRCPC3NEONLoadStoreUnscaledOffset<0b01, 0b01, FPR16 , (outs FPR16 :$Rt), (ins GPR64s…
10036 …defm LDAPURs: LRCPC3NEONLoadStoreUnscaledOffset<0b10, 0b01, FPR32 , (outs FPR32 :$Rt), (ins GPR64s…
10037 …defm LDAPURd: LRCPC3NEONLoadStoreUnscaledOffset<0b11, 0b01, FPR64 , (outs FPR64 :$Rt), (ins GPR64s…
10038 …defm LDAPURq: LRCPC3NEONLoadStoreUnscaledOffset<0b00, 0b11, FPR128, (outs FPR128:$Rt), (ins GPR64s…
10041 …def STL1: LRCPC3NEONLdStSingle<0b0, (outs), (ins VecListOned:$Vt, VectorIndexD:$Q, GPR64sp:$Rn) ,…
10042 …def LDAP1: LRCPC3NEONLdStSingle<0b1, (outs VecListOned:$dst), (ins VecListOned:$Vt, VectorIndexD:$…
10044 // Aliases for when offset=0
10045 def : InstAlias<"stl1\t$Vt$Q, [$Rn, #0]", (STL1 VecListOned:$Vt, VectorIndexD:$Q, GPR64sp:$Rn)>;
10048 //===----------------------------------------------------------------------===//
10049 // 128-bit System Instructions (FEAT_SYSINSTR128)
10050 //===----------------------------------------------------------------------===//
10052 def SYSPxt : SystemPXtI<0, "sysp">;
10055 : BaseSystemI<0, (outs),
10065 // I could add a bits<5> xzr_pair. But without a way to constrain it to 0b11111 here it would
10072 let Inst{22} = 0b1; // override BaseSystemI
10073 let Inst{20-19} = 0b01;
10074 let Inst{18-16} = op1;
10075 let Inst{15-12} = Cn;
10076 let Inst{11-8} = Cm;
10077 let Inst{7-5} = op2;
10078 let Inst{4-0} = 0b11111;
10085 //---
10086 // 128-bit System Registers (FEAT_SYSREG128)
10087 //---
10091 // 31 22|21|20|19|18 16|15 12|11 8|7 5|4 0
10093 // MSRR 1101010101| 0| 1|o0| op1| Cn| Cm|op2| Rt
10108 let Inst{20-5} = systemreg;
10111 def MSRR : RtSystemI128<0,
10116 let Inst{20-5} = systemreg;
10120 //===----------------------------===//
10122 //===----------------------------===//
10125 defm F1CVTL : SIMDMixedTwoVectorFP8<0b00, "f1cvtl">;
10126 defm F2CVTL : SIMDMixedTwoVectorFP8<0b01, "f2cvtl">;
10127 defm BF1CVTL : SIMDMixedTwoVectorFP8<0b10, "bf1cvtl">;
10128 defm BF2CVTL : SIMDMixedTwoVectorFP8<0b11, "bf2cvtl">;
10131 defm FSCALE : SIMDThreeSameVectorFP<0b1, 0b1, 0b111, "fscale", null_frag>;
10135 defm FAMAX : SIMDThreeSameVectorFP<0b0, 0b1, 0b011, "famax", null_frag>;
10136 defm FAMIN : SIMDThreeSameVectorFP<0b1, 0b1, 0b011, "famin", null_frag>;
10140 defm FMLALBlane : SIMDThreeSameVectorMLAIndex<0b0, "fmlalb">;
10141 defm FMLALTlane : SIMDThreeSameVectorMLAIndex<0b1, "fmlalt">;
10142 defm FMLALLBBlane : SIMDThreeSameVectorMLALIndex<0b0, 0b00, "fmlallbb">;
10143 defm FMLALLBTlane : SIMDThreeSameVectorMLALIndex<0b0, 0b01, "fmlallbt">;
10144 defm FMLALLTBlane : SIMDThreeSameVectorMLALIndex<0b1, 0b00, "fmlalltb">;
10145 defm FMLALLTTlane : SIMDThreeSameVectorMLALIndex<0b1, 0b01, "fmlalltt">;
10147 defm FMLALB : SIMDThreeSameVectorMLA<0b0, "fmlalb">;
10148 defm FMLALT : SIMDThreeSameVectorMLA<0b1, "fmlalt">;
10149 defm FMLALLBB : SIMDThreeSameVectorMLAL<0b0, 0b00, "fmlallbb">;
10150 defm FMLALLBT : SIMDThreeSameVectorMLAL<0b0, 0b01, "fmlallbt">;
10151 defm FMLALLTB : SIMDThreeSameVectorMLAL<0b1, 0b00, "fmlalltb">;
10152 defm FMLALLTT : SIMDThreeSameVectorMLAL<0b1, 0b01, "fmlalltt">;
10165 //===----------------------------------------------------------------------===//
10167 //===----------------------------------------------------------------------===//
10170 defm ADDPT : AddSubCPA<0, "addpt">;
10173 // Scalar multiply-add/subtract
10174 def MADDPT : MulAccumCPA<0, "maddpt">;
10186 (MOVIv4i32 (i32 1), (i32 0)))),