1 //===- AArch64LegalizerInfo.cpp ----------------------------------*- C++ -*-==// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 /// \file 9 /// This file implements the targeting of the Machinelegalizer class for 10 /// AArch64. 11 /// \todo This should be generated by TableGen. 12 //===----------------------------------------------------------------------===// 13 14 #include "AArch64LegalizerInfo.h" 15 #include "AArch64Subtarget.h" 16 #include "llvm/CodeGen/GlobalISel/LegalizerHelper.h" 17 #include "llvm/CodeGen/GlobalISel/LegalizerInfo.h" 18 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" 19 #include "llvm/CodeGen/GlobalISel/Utils.h" 20 #include "llvm/CodeGen/MachineInstr.h" 21 #include "llvm/CodeGen/MachineRegisterInfo.h" 22 #include "llvm/CodeGen/TargetOpcodes.h" 23 #include "llvm/CodeGen/ValueTypes.h" 24 #include "llvm/IR/DerivedTypes.h" 25 #include "llvm/IR/Type.h" 26 #include <initializer_list> 27 #include "llvm/Support/MathExtras.h" 28 29 #define DEBUG_TYPE "aarch64-legalinfo" 30 31 using namespace llvm; 32 using namespace LegalizeActions; 33 using namespace LegalizeMutations; 34 using namespace LegalityPredicates; 35 36 AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) 37 : ST(&ST) { 38 using namespace TargetOpcode; 39 const LLT p0 = LLT::pointer(0, 64); 40 const LLT s1 = LLT::scalar(1); 41 const LLT s8 = LLT::scalar(8); 42 const LLT s16 = LLT::scalar(16); 43 const LLT s32 = LLT::scalar(32); 44 const LLT s64 = LLT::scalar(64); 45 const LLT s128 = LLT::scalar(128); 46 const LLT s256 = LLT::scalar(256); 47 const LLT s512 = LLT::scalar(512); 48 const LLT v16s8 = LLT::vector(16, 8); 49 const LLT v8s8 = LLT::vector(8, 8); 50 const LLT v4s8 = LLT::vector(4, 8); 51 const LLT v8s16 = LLT::vector(8, 16); 52 const LLT v4s16 = LLT::vector(4, 16); 53 const LLT v2s16 = LLT::vector(2, 16); 54 const LLT v2s32 = LLT::vector(2, 32); 55 const LLT v4s32 = LLT::vector(4, 32); 56 const LLT v2s64 = LLT::vector(2, 64); 57 const LLT v2p0 = LLT::vector(2, p0); 58 59 std::initializer_list<LLT> PackedVectorAllTypeList = {/* Begin 128bit types */ 60 v16s8, v8s16, v4s32, 61 v2s64, v2p0, 62 /* End 128bit types */ 63 /* Begin 64bit types */ 64 v8s8, v4s16, v2s32}; 65 66 const TargetMachine &TM = ST.getTargetLowering()->getTargetMachine(); 67 68 // FIXME: support subtargets which have neon/fp-armv8 disabled. 69 if (!ST.hasNEON() || !ST.hasFPARMv8()) { 70 computeTables(); 71 return; 72 } 73 74 // Some instructions only support s16 if the subtarget has full 16-bit FP 75 // support. 76 const bool HasFP16 = ST.hasFullFP16(); 77 const LLT &MinFPScalar = HasFP16 ? s16 : s32; 78 79 getActionDefinitionsBuilder({G_IMPLICIT_DEF, G_FREEZE}) 80 .legalFor({p0, s1, s8, s16, s32, s64}) 81 .legalFor(PackedVectorAllTypeList) 82 .clampScalar(0, s1, s64) 83 .widenScalarToNextPow2(0, 8) 84 .fewerElementsIf( 85 [=](const LegalityQuery &Query) { 86 return Query.Types[0].isVector() && 87 (Query.Types[0].getElementType() != s64 || 88 Query.Types[0].getNumElements() != 2); 89 }, 90 [=](const LegalityQuery &Query) { 91 LLT EltTy = Query.Types[0].getElementType(); 92 if (EltTy == s64) 93 return std::make_pair(0, LLT::vector(2, 64)); 94 return std::make_pair(0, EltTy); 95 }); 96 97 getActionDefinitionsBuilder(G_PHI).legalFor({p0, s16, s32, s64}) 98 .legalFor(PackedVectorAllTypeList) 99 .clampScalar(0, s16, s64) 100 .widenScalarToNextPow2(0); 101 102 getActionDefinitionsBuilder(G_BSWAP) 103 .legalFor({s32, s64, v4s32, v2s32, v2s64}) 104 .clampScalar(0, s32, s64) 105 .widenScalarToNextPow2(0); 106 107 getActionDefinitionsBuilder({G_ADD, G_SUB, G_MUL, G_AND, G_OR, G_XOR}) 108 .legalFor({s32, s64, v2s32, v4s32, v4s16, v8s16, v16s8, v8s8}) 109 .scalarizeIf( 110 [=](const LegalityQuery &Query) { 111 return Query.Opcode == G_MUL && Query.Types[0] == v2s64; 112 }, 113 0) 114 .legalFor({v2s64}) 115 .clampScalar(0, s32, s64) 116 .widenScalarToNextPow2(0) 117 .clampNumElements(0, v2s32, v4s32) 118 .clampNumElements(0, v2s64, v2s64) 119 .moreElementsToNextPow2(0); 120 121 getActionDefinitionsBuilder({G_SHL, G_ASHR, G_LSHR}) 122 .customIf([=](const LegalityQuery &Query) { 123 const auto &SrcTy = Query.Types[0]; 124 const auto &AmtTy = Query.Types[1]; 125 return !SrcTy.isVector() && SrcTy.getSizeInBits() == 32 && 126 AmtTy.getSizeInBits() == 32; 127 }) 128 .legalFor({ 129 {s32, s32}, 130 {s32, s64}, 131 {s64, s64}, 132 {v8s8, v8s8}, 133 {v16s8, v16s8}, 134 {v4s16, v4s16}, 135 {v8s16, v8s16}, 136 {v2s32, v2s32}, 137 {v4s32, v4s32}, 138 {v2s64, v2s64}, 139 }) 140 .clampScalar(1, s32, s64) 141 .clampScalar(0, s32, s64) 142 .widenScalarToNextPow2(0) 143 .clampNumElements(0, v2s32, v4s32) 144 .clampNumElements(0, v2s64, v2s64) 145 .moreElementsToNextPow2(0) 146 .minScalarSameAs(1, 0); 147 148 getActionDefinitionsBuilder(G_PTR_ADD) 149 .legalFor({{p0, s64}, {v2p0, v2s64}}) 150 .clampScalar(1, s64, s64); 151 152 getActionDefinitionsBuilder(G_PTRMASK).legalFor({{p0, s64}}); 153 154 getActionDefinitionsBuilder({G_SDIV, G_UDIV}) 155 .legalFor({s32, s64}) 156 .libcallFor({s128}) 157 .clampScalar(0, s32, s64) 158 .widenScalarToNextPow2(0) 159 .scalarize(0); 160 161 getActionDefinitionsBuilder({G_SREM, G_UREM}) 162 .lowerFor({s1, s8, s16, s32, s64}); 163 164 getActionDefinitionsBuilder({G_SMULO, G_UMULO}).lowerFor({{s64, s1}}); 165 166 getActionDefinitionsBuilder({G_SMULH, G_UMULH}).legalFor({s32, s64}); 167 168 getActionDefinitionsBuilder( 169 {G_UADDE, G_USUBE, G_SADDO, G_SSUBO, G_UADDO, G_USUBO}) 170 .legalFor({{s32, s1}, {s64, s1}}) 171 .minScalar(0, s32); 172 173 getActionDefinitionsBuilder({G_FADD, G_FSUB, G_FMUL, G_FDIV, G_FNEG}) 174 .legalFor({s32, s64, v2s64, v4s32, v2s32}) 175 .clampNumElements(0, v2s32, v4s32) 176 .clampNumElements(0, v2s64, v2s64); 177 178 getActionDefinitionsBuilder(G_FREM).libcallFor({s32, s64}); 179 180 getActionDefinitionsBuilder({G_FCEIL, G_FABS, G_FSQRT, G_FFLOOR, G_FRINT, 181 G_FMA, G_INTRINSIC_TRUNC, G_INTRINSIC_ROUND, 182 G_FNEARBYINT, G_INTRINSIC_LRINT}) 183 // If we don't have full FP16 support, then scalarize the elements of 184 // vectors containing fp16 types. 185 .fewerElementsIf( 186 [=, &ST](const LegalityQuery &Query) { 187 const auto &Ty = Query.Types[0]; 188 return Ty.isVector() && Ty.getElementType() == s16 && 189 !ST.hasFullFP16(); 190 }, 191 [=](const LegalityQuery &Query) { return std::make_pair(0, s16); }) 192 // If we don't have full FP16 support, then widen s16 to s32 if we 193 // encounter it. 194 .widenScalarIf( 195 [=, &ST](const LegalityQuery &Query) { 196 return Query.Types[0] == s16 && !ST.hasFullFP16(); 197 }, 198 [=](const LegalityQuery &Query) { return std::make_pair(0, s32); }) 199 .legalFor({s16, s32, s64, v2s32, v4s32, v2s64, v2s16, v4s16, v8s16}); 200 201 getActionDefinitionsBuilder( 202 {G_FCOS, G_FSIN, G_FLOG10, G_FLOG, G_FLOG2, G_FEXP, G_FEXP2, G_FPOW}) 203 // We need a call for these, so we always need to scalarize. 204 .scalarize(0) 205 // Regardless of FP16 support, widen 16-bit elements to 32-bits. 206 .minScalar(0, s32) 207 .libcallFor({s32, s64, v2s32, v4s32, v2s64}); 208 209 getActionDefinitionsBuilder(G_INSERT) 210 .unsupportedIf([=](const LegalityQuery &Query) { 211 return Query.Types[0].getSizeInBits() <= Query.Types[1].getSizeInBits(); 212 }) 213 .legalIf([=](const LegalityQuery &Query) { 214 const LLT &Ty0 = Query.Types[0]; 215 const LLT &Ty1 = Query.Types[1]; 216 if (Ty0 != s32 && Ty0 != s64 && Ty0 != p0) 217 return false; 218 return isPowerOf2_32(Ty1.getSizeInBits()) && 219 (Ty1.getSizeInBits() == 1 || Ty1.getSizeInBits() >= 8); 220 }) 221 .clampScalar(0, s32, s64) 222 .widenScalarToNextPow2(0) 223 .maxScalarIf(typeInSet(0, {s32}), 1, s16) 224 .maxScalarIf(typeInSet(0, {s64}), 1, s32) 225 .widenScalarToNextPow2(1); 226 227 getActionDefinitionsBuilder(G_EXTRACT) 228 .unsupportedIf([=](const LegalityQuery &Query) { 229 return Query.Types[0].getSizeInBits() >= Query.Types[1].getSizeInBits(); 230 }) 231 .legalIf([=](const LegalityQuery &Query) { 232 const LLT &Ty0 = Query.Types[0]; 233 const LLT &Ty1 = Query.Types[1]; 234 if (Ty1 != s32 && Ty1 != s64 && Ty1 != s128) 235 return false; 236 if (Ty1 == p0) 237 return true; 238 return isPowerOf2_32(Ty0.getSizeInBits()) && 239 (Ty0.getSizeInBits() == 1 || Ty0.getSizeInBits() >= 8); 240 }) 241 .clampScalar(1, s32, s128) 242 .widenScalarToNextPow2(1) 243 .maxScalarIf(typeInSet(1, {s32}), 0, s16) 244 .maxScalarIf(typeInSet(1, {s64}), 0, s32) 245 .widenScalarToNextPow2(0); 246 247 getActionDefinitionsBuilder({G_SEXTLOAD, G_ZEXTLOAD}) 248 .legalForTypesWithMemDesc({{s32, p0, 8, 8}, 249 {s32, p0, 16, 8}, 250 {s32, p0, 32, 8}, 251 {s64, p0, 8, 2}, 252 {s64, p0, 16, 2}, 253 {s64, p0, 32, 4}, 254 {s64, p0, 64, 8}, 255 {p0, p0, 64, 8}, 256 {v2s32, p0, 64, 8}}) 257 .clampScalar(0, s32, s64) 258 .widenScalarToNextPow2(0) 259 // TODO: We could support sum-of-pow2's but the lowering code doesn't know 260 // how to do that yet. 261 .unsupportedIfMemSizeNotPow2() 262 // Lower anything left over into G_*EXT and G_LOAD 263 .lower(); 264 265 auto IsPtrVecPred = [=](const LegalityQuery &Query) { 266 const LLT &ValTy = Query.Types[0]; 267 if (!ValTy.isVector()) 268 return false; 269 const LLT EltTy = ValTy.getElementType(); 270 return EltTy.isPointer() && EltTy.getAddressSpace() == 0; 271 }; 272 273 getActionDefinitionsBuilder(G_LOAD) 274 .legalForTypesWithMemDesc({{s8, p0, 8, 8}, 275 {s16, p0, 16, 8}, 276 {s32, p0, 32, 8}, 277 {s64, p0, 64, 8}, 278 {p0, p0, 64, 8}, 279 {s128, p0, 128, 8}, 280 {v8s8, p0, 64, 8}, 281 {v16s8, p0, 128, 8}, 282 {v4s16, p0, 64, 8}, 283 {v8s16, p0, 128, 8}, 284 {v2s32, p0, 64, 8}, 285 {v4s32, p0, 128, 8}, 286 {v2s64, p0, 128, 8}}) 287 // These extends are also legal 288 .legalForTypesWithMemDesc({{s32, p0, 8, 8}, {s32, p0, 16, 8}}) 289 .clampScalar(0, s8, s64) 290 .lowerIfMemSizeNotPow2() 291 // Lower any any-extending loads left into G_ANYEXT and G_LOAD 292 .lowerIf([=](const LegalityQuery &Query) { 293 return Query.Types[0].getSizeInBits() != Query.MMODescrs[0].SizeInBits; 294 }) 295 .widenScalarToNextPow2(0) 296 .clampMaxNumElements(0, s32, 2) 297 .clampMaxNumElements(0, s64, 1) 298 .customIf(IsPtrVecPred); 299 300 getActionDefinitionsBuilder(G_STORE) 301 .legalForTypesWithMemDesc({{s8, p0, 8, 8}, 302 {s16, p0, 16, 8}, 303 {s32, p0, 8, 8}, 304 {s32, p0, 16, 8}, 305 {s32, p0, 32, 8}, 306 {s64, p0, 64, 8}, 307 {p0, p0, 64, 8}, 308 {s128, p0, 128, 8}, 309 {v16s8, p0, 128, 8}, 310 {v8s8, p0, 64, 8}, 311 {v4s16, p0, 64, 8}, 312 {v8s16, p0, 128, 8}, 313 {v2s32, p0, 64, 8}, 314 {v4s32, p0, 128, 8}, 315 {v2s64, p0, 128, 8}}) 316 .clampScalar(0, s8, s64) 317 .lowerIfMemSizeNotPow2() 318 .lowerIf([=](const LegalityQuery &Query) { 319 return Query.Types[0].isScalar() && 320 Query.Types[0].getSizeInBits() != Query.MMODescrs[0].SizeInBits; 321 }) 322 .clampMaxNumElements(0, s32, 2) 323 .clampMaxNumElements(0, s64, 1) 324 .customIf(IsPtrVecPred); 325 326 // Constants 327 getActionDefinitionsBuilder(G_CONSTANT) 328 .legalFor({p0, s8, s16, s32, s64}) 329 .clampScalar(0, s8, s64) 330 .widenScalarToNextPow2(0); 331 getActionDefinitionsBuilder(G_FCONSTANT) 332 .legalIf([=](const LegalityQuery &Query) { 333 const auto &Ty = Query.Types[0]; 334 if (HasFP16 && Ty == s16) 335 return true; 336 return Ty == s32 || Ty == s64 || Ty == s128; 337 }) 338 .clampScalar(0, MinFPScalar, s128); 339 340 getActionDefinitionsBuilder({G_ICMP, G_FCMP}) 341 .legalFor({{s32, s32}, 342 {s32, s64}, 343 {s32, p0}, 344 {v4s32, v4s32}, 345 {v2s32, v2s32}, 346 {v2s64, v2s64}, 347 {v2s64, v2p0}, 348 {v4s16, v4s16}, 349 {v8s16, v8s16}, 350 {v8s8, v8s8}, 351 {v16s8, v16s8}}) 352 .clampScalar(1, s32, s64) 353 .clampScalar(0, s32, s32) 354 .minScalarEltSameAsIf( 355 [=](const LegalityQuery &Query) { 356 const LLT &Ty = Query.Types[0]; 357 const LLT &SrcTy = Query.Types[1]; 358 return Ty.isVector() && !SrcTy.getElementType().isPointer() && 359 Ty.getElementType() != SrcTy.getElementType(); 360 }, 361 0, 1) 362 .minScalarOrEltIf( 363 [=](const LegalityQuery &Query) { return Query.Types[1] == v2s16; }, 364 1, s32) 365 .minScalarOrEltIf( 366 [=](const LegalityQuery &Query) { return Query.Types[1] == v2p0; }, 0, 367 s64) 368 .widenScalarOrEltToNextPow2(1) 369 .clampNumElements(0, v2s32, v4s32); 370 371 // Extensions 372 auto ExtLegalFunc = [=](const LegalityQuery &Query) { 373 unsigned DstSize = Query.Types[0].getSizeInBits(); 374 375 if (DstSize == 128 && !Query.Types[0].isVector()) 376 return false; // Extending to a scalar s128 needs narrowing. 377 378 // Make sure that we have something that will fit in a register, and 379 // make sure it's a power of 2. 380 if (DstSize < 8 || DstSize > 128 || !isPowerOf2_32(DstSize)) 381 return false; 382 383 const LLT &SrcTy = Query.Types[1]; 384 385 // Special case for s1. 386 if (SrcTy == s1) 387 return true; 388 389 // Make sure we fit in a register otherwise. Don't bother checking that 390 // the source type is below 128 bits. We shouldn't be allowing anything 391 // through which is wider than the destination in the first place. 392 unsigned SrcSize = SrcTy.getSizeInBits(); 393 if (SrcSize < 8 || !isPowerOf2_32(SrcSize)) 394 return false; 395 396 return true; 397 }; 398 getActionDefinitionsBuilder({G_ZEXT, G_SEXT, G_ANYEXT}) 399 .legalIf(ExtLegalFunc) 400 .clampScalar(0, s64, s64); // Just for s128, others are handled above. 401 402 getActionDefinitionsBuilder(G_TRUNC) 403 .minScalarOrEltIf( 404 [=](const LegalityQuery &Query) { return Query.Types[0].isVector(); }, 405 0, s8) 406 .customIf([=](const LegalityQuery &Query) { 407 LLT DstTy = Query.Types[0]; 408 LLT SrcTy = Query.Types[1]; 409 return DstTy == v8s8 && SrcTy.getSizeInBits() > 128; 410 }) 411 .alwaysLegal(); 412 413 getActionDefinitionsBuilder(G_SEXT_INREG).legalFor({s32, s64}).lower(); 414 415 // FP conversions 416 getActionDefinitionsBuilder(G_FPTRUNC) 417 .legalFor( 418 {{s16, s32}, {s16, s64}, {s32, s64}, {v4s16, v4s32}, {v2s32, v2s64}}) 419 .clampMaxNumElements(0, s32, 2); 420 getActionDefinitionsBuilder(G_FPEXT) 421 .legalFor( 422 {{s32, s16}, {s64, s16}, {s64, s32}, {v4s32, v4s16}, {v2s64, v2s32}}) 423 .clampMaxNumElements(0, s64, 2); 424 425 // Conversions 426 getActionDefinitionsBuilder({G_FPTOSI, G_FPTOUI}) 427 .legalForCartesianProduct({s32, s64, v2s64, v4s32, v2s32}) 428 .clampScalar(0, s32, s64) 429 .widenScalarToNextPow2(0) 430 .clampScalar(1, s32, s64) 431 .widenScalarToNextPow2(1); 432 433 getActionDefinitionsBuilder({G_SITOFP, G_UITOFP}) 434 .legalForCartesianProduct({s32, s64, v2s64, v4s32, v2s32}) 435 .clampScalar(1, s32, s64) 436 .minScalarSameAs(1, 0) 437 .clampScalar(0, s32, s64) 438 .widenScalarToNextPow2(0); 439 440 // Control-flow 441 getActionDefinitionsBuilder(G_BRCOND).legalFor({s1, s8, s16, s32}); 442 getActionDefinitionsBuilder(G_BRINDIRECT).legalFor({p0}); 443 444 getActionDefinitionsBuilder(G_SELECT) 445 .legalFor({{s32, s1}, {s64, s1}, {p0, s1}}) 446 .clampScalar(0, s32, s64) 447 .widenScalarToNextPow2(0) 448 .minScalarEltSameAsIf(all(isVector(0), isVector(1)), 1, 0) 449 .lowerIf(isVector(0)); 450 451 // Pointer-handling 452 getActionDefinitionsBuilder(G_FRAME_INDEX).legalFor({p0}); 453 454 if (TM.getCodeModel() == CodeModel::Small) 455 getActionDefinitionsBuilder(G_GLOBAL_VALUE).custom(); 456 else 457 getActionDefinitionsBuilder(G_GLOBAL_VALUE).legalFor({p0}); 458 459 getActionDefinitionsBuilder(G_PTRTOINT) 460 .legalForCartesianProduct({s1, s8, s16, s32, s64}, {p0}) 461 .maxScalar(0, s64) 462 .widenScalarToNextPow2(0, /*Min*/ 8); 463 464 getActionDefinitionsBuilder(G_INTTOPTR) 465 .unsupportedIf([&](const LegalityQuery &Query) { 466 return Query.Types[0].getSizeInBits() != Query.Types[1].getSizeInBits(); 467 }) 468 .legalFor({{p0, s64}}); 469 470 // Casts for 32 and 64-bit width type are just copies. 471 // Same for 128-bit width type, except they are on the FPR bank. 472 getActionDefinitionsBuilder(G_BITCAST) 473 // FIXME: This is wrong since G_BITCAST is not allowed to change the 474 // number of bits but it's what the previous code described and fixing 475 // it breaks tests. 476 .legalForCartesianProduct({s1, s8, s16, s32, s64, s128, v16s8, v8s8, v4s8, 477 v8s16, v4s16, v2s16, v4s32, v2s32, v2s64, 478 v2p0}); 479 480 getActionDefinitionsBuilder(G_VASTART).legalFor({p0}); 481 482 // va_list must be a pointer, but most sized types are pretty easy to handle 483 // as the destination. 484 getActionDefinitionsBuilder(G_VAARG) 485 .customForCartesianProduct({s8, s16, s32, s64, p0}, {p0}) 486 .clampScalar(0, s8, s64) 487 .widenScalarToNextPow2(0, /*Min*/ 8); 488 489 if (ST.hasLSE()) { 490 getActionDefinitionsBuilder(G_ATOMIC_CMPXCHG_WITH_SUCCESS) 491 .lowerIf(all( 492 typeInSet(0, {s8, s16, s32, s64}), typeIs(1, s1), typeIs(2, p0), 493 atomicOrderingAtLeastOrStrongerThan(0, AtomicOrdering::Monotonic))); 494 495 getActionDefinitionsBuilder( 496 {G_ATOMICRMW_XCHG, G_ATOMICRMW_ADD, G_ATOMICRMW_SUB, G_ATOMICRMW_AND, 497 G_ATOMICRMW_OR, G_ATOMICRMW_XOR, G_ATOMICRMW_MIN, G_ATOMICRMW_MAX, 498 G_ATOMICRMW_UMIN, G_ATOMICRMW_UMAX, G_ATOMIC_CMPXCHG}) 499 .legalIf(all( 500 typeInSet(0, {s8, s16, s32, s64}), typeIs(1, p0), 501 atomicOrderingAtLeastOrStrongerThan(0, AtomicOrdering::Monotonic))); 502 } 503 504 getActionDefinitionsBuilder(G_BLOCK_ADDR).legalFor({p0}); 505 506 // Merge/Unmerge 507 for (unsigned Op : {G_MERGE_VALUES, G_UNMERGE_VALUES}) { 508 unsigned BigTyIdx = Op == G_MERGE_VALUES ? 0 : 1; 509 unsigned LitTyIdx = Op == G_MERGE_VALUES ? 1 : 0; 510 511 auto notValidElt = [](const LegalityQuery &Query, unsigned TypeIdx) { 512 const LLT &Ty = Query.Types[TypeIdx]; 513 if (Ty.isVector()) { 514 const LLT &EltTy = Ty.getElementType(); 515 if (EltTy.getSizeInBits() < 8 || EltTy.getSizeInBits() > 64) 516 return true; 517 if (!isPowerOf2_32(EltTy.getSizeInBits())) 518 return true; 519 } 520 return false; 521 }; 522 523 // FIXME: This rule is horrible, but specifies the same as what we had 524 // before with the particularly strange definitions removed (e.g. 525 // s8 = G_MERGE_VALUES s32, s32). 526 // Part of the complexity comes from these ops being extremely flexible. For 527 // example, you can build/decompose vectors with it, concatenate vectors, 528 // etc. and in addition to this you can also bitcast with it at the same 529 // time. We've been considering breaking it up into multiple ops to make it 530 // more manageable throughout the backend. 531 getActionDefinitionsBuilder(Op) 532 // Break up vectors with weird elements into scalars 533 .fewerElementsIf( 534 [=](const LegalityQuery &Query) { return notValidElt(Query, 0); }, 535 scalarize(0)) 536 .fewerElementsIf( 537 [=](const LegalityQuery &Query) { return notValidElt(Query, 1); }, 538 scalarize(1)) 539 // Clamp the big scalar to s8-s512 and make it either a power of 2, 192, 540 // or 384. 541 .clampScalar(BigTyIdx, s8, s512) 542 .widenScalarIf( 543 [=](const LegalityQuery &Query) { 544 const LLT &Ty = Query.Types[BigTyIdx]; 545 return !isPowerOf2_32(Ty.getSizeInBits()) && 546 Ty.getSizeInBits() % 64 != 0; 547 }, 548 [=](const LegalityQuery &Query) { 549 // Pick the next power of 2, or a multiple of 64 over 128. 550 // Whichever is smaller. 551 const LLT &Ty = Query.Types[BigTyIdx]; 552 unsigned NewSizeInBits = 1 553 << Log2_32_Ceil(Ty.getSizeInBits() + 1); 554 if (NewSizeInBits >= 256) { 555 unsigned RoundedTo = alignTo<64>(Ty.getSizeInBits() + 1); 556 if (RoundedTo < NewSizeInBits) 557 NewSizeInBits = RoundedTo; 558 } 559 return std::make_pair(BigTyIdx, LLT::scalar(NewSizeInBits)); 560 }) 561 // Clamp the little scalar to s8-s256 and make it a power of 2. It's not 562 // worth considering the multiples of 64 since 2*192 and 2*384 are not 563 // valid. 564 .clampScalar(LitTyIdx, s8, s256) 565 .widenScalarToNextPow2(LitTyIdx, /*Min*/ 8) 566 // So at this point, we have s8, s16, s32, s64, s128, s192, s256, s384, 567 // s512, <X x s8>, <X x s16>, <X x s32>, or <X x s64>. 568 // At this point it's simple enough to accept the legal types. 569 .legalIf([=](const LegalityQuery &Query) { 570 const LLT &BigTy = Query.Types[BigTyIdx]; 571 const LLT &LitTy = Query.Types[LitTyIdx]; 572 if (BigTy.isVector() && BigTy.getSizeInBits() < 32) 573 return false; 574 if (LitTy.isVector() && LitTy.getSizeInBits() < 32) 575 return false; 576 return BigTy.getSizeInBits() % LitTy.getSizeInBits() == 0; 577 }) 578 // Any vectors left are the wrong size. Scalarize them. 579 .scalarize(0) 580 .scalarize(1); 581 } 582 583 getActionDefinitionsBuilder(G_EXTRACT_VECTOR_ELT) 584 .unsupportedIf([=](const LegalityQuery &Query) { 585 const LLT &EltTy = Query.Types[1].getElementType(); 586 return Query.Types[0] != EltTy; 587 }) 588 .minScalar(2, s64) 589 .legalIf([=](const LegalityQuery &Query) { 590 const LLT &VecTy = Query.Types[1]; 591 return VecTy == v2s16 || VecTy == v4s16 || VecTy == v8s16 || 592 VecTy == v4s32 || VecTy == v2s64 || VecTy == v2s32 || 593 VecTy == v16s8 || VecTy == v2s32 || VecTy == v2p0; 594 }) 595 .minScalarOrEltIf( 596 [=](const LegalityQuery &Query) { 597 // We want to promote to <M x s1> to <M x s64> if that wouldn't 598 // cause the total vec size to be > 128b. 599 return Query.Types[1].getNumElements() <= 2; 600 }, 601 0, s64) 602 .minScalarOrEltIf( 603 [=](const LegalityQuery &Query) { 604 return Query.Types[1].getNumElements() <= 4; 605 }, 606 0, s32) 607 .minScalarOrEltIf( 608 [=](const LegalityQuery &Query) { 609 return Query.Types[1].getNumElements() <= 8; 610 }, 611 0, s16) 612 .minScalarOrEltIf( 613 [=](const LegalityQuery &Query) { 614 return Query.Types[1].getNumElements() <= 16; 615 }, 616 0, s8) 617 .minScalarOrElt(0, s8); // Worst case, we need at least s8. 618 619 getActionDefinitionsBuilder(G_INSERT_VECTOR_ELT) 620 .legalIf(typeInSet(0, {v8s16, v2s32, v4s32, v2s64})); 621 622 getActionDefinitionsBuilder(G_BUILD_VECTOR) 623 .legalFor({{v8s8, s8}, 624 {v16s8, s8}, 625 {v4s16, s16}, 626 {v8s16, s16}, 627 {v2s32, s32}, 628 {v4s32, s32}, 629 {v2p0, p0}, 630 {v2s64, s64}}) 631 .clampNumElements(0, v4s32, v4s32) 632 .clampNumElements(0, v2s64, v2s64) 633 634 // Deal with larger scalar types, which will be implicitly truncated. 635 .legalIf([=](const LegalityQuery &Query) { 636 return Query.Types[0].getScalarSizeInBits() < 637 Query.Types[1].getSizeInBits(); 638 }) 639 .minScalarSameAs(1, 0); 640 641 getActionDefinitionsBuilder(G_CTLZ) 642 .legalForCartesianProduct( 643 {s32, s64, v8s8, v16s8, v4s16, v8s16, v2s32, v4s32}) 644 .scalarize(1); 645 646 getActionDefinitionsBuilder(G_SHUFFLE_VECTOR) 647 .legalIf([=](const LegalityQuery &Query) { 648 const LLT &DstTy = Query.Types[0]; 649 const LLT &SrcTy = Query.Types[1]; 650 // For now just support the TBL2 variant which needs the source vectors 651 // to be the same size as the dest. 652 if (DstTy != SrcTy) 653 return false; 654 for (auto &Ty : {v2s32, v4s32, v2s64, v2p0, v16s8, v8s16}) { 655 if (DstTy == Ty) 656 return true; 657 } 658 return false; 659 }) 660 // G_SHUFFLE_VECTOR can have scalar sources (from 1 x s vectors), we 661 // just want those lowered into G_BUILD_VECTOR 662 .lowerIf([=](const LegalityQuery &Query) { 663 return !Query.Types[1].isVector(); 664 }) 665 .clampNumElements(0, v4s32, v4s32) 666 .clampNumElements(0, v2s64, v2s64); 667 668 getActionDefinitionsBuilder(G_CONCAT_VECTORS) 669 .legalFor({{v4s32, v2s32}, {v8s16, v4s16}}); 670 671 getActionDefinitionsBuilder(G_JUMP_TABLE).legalFor({{p0}, {s64}}); 672 673 getActionDefinitionsBuilder(G_BRJT).legalIf([=](const LegalityQuery &Query) { 674 return Query.Types[0] == p0 && Query.Types[1] == s64; 675 }); 676 677 getActionDefinitionsBuilder(G_DYN_STACKALLOC).lower(); 678 679 getActionDefinitionsBuilder({G_MEMCPY, G_MEMMOVE, G_MEMSET}).libcall(); 680 681 getActionDefinitionsBuilder(G_ABS).lowerIf( 682 [=](const LegalityQuery &Query) { return Query.Types[0].isScalar(); }); 683 684 getActionDefinitionsBuilder(G_VECREDUCE_FADD) 685 // We only have FADDP to do reduction-like operations. Lower the rest. 686 .legalFor({{s32, v2s32}, {s64, v2s64}}) 687 .lower(); 688 689 getActionDefinitionsBuilder(G_VECREDUCE_ADD) 690 .legalFor({{s8, v16s8}, {s16, v8s16}, {s32, v4s32}, {s64, v2s64}}) 691 .lower(); 692 693 computeTables(); 694 verify(*ST.getInstrInfo()); 695 } 696 697 bool AArch64LegalizerInfo::legalizeCustom(LegalizerHelper &Helper, 698 MachineInstr &MI) const { 699 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder; 700 MachineRegisterInfo &MRI = *MIRBuilder.getMRI(); 701 GISelChangeObserver &Observer = Helper.Observer; 702 switch (MI.getOpcode()) { 703 default: 704 // No idea what to do. 705 return false; 706 case TargetOpcode::G_VAARG: 707 return legalizeVaArg(MI, MRI, MIRBuilder); 708 case TargetOpcode::G_LOAD: 709 case TargetOpcode::G_STORE: 710 return legalizeLoadStore(MI, MRI, MIRBuilder, Observer); 711 case TargetOpcode::G_SHL: 712 case TargetOpcode::G_ASHR: 713 case TargetOpcode::G_LSHR: 714 return legalizeShlAshrLshr(MI, MRI, MIRBuilder, Observer); 715 case TargetOpcode::G_GLOBAL_VALUE: 716 return legalizeSmallCMGlobalValue(MI, MRI, MIRBuilder, Observer); 717 case TargetOpcode::G_TRUNC: 718 return legalizeVectorTrunc(MI, Helper); 719 } 720 721 llvm_unreachable("expected switch to return"); 722 } 723 724 static void extractParts(Register Reg, MachineRegisterInfo &MRI, 725 MachineIRBuilder &MIRBuilder, LLT Ty, int NumParts, 726 SmallVectorImpl<Register> &VRegs) { 727 for (int I = 0; I < NumParts; ++I) 728 VRegs.push_back(MRI.createGenericVirtualRegister(Ty)); 729 MIRBuilder.buildUnmerge(VRegs, Reg); 730 } 731 732 bool AArch64LegalizerInfo::legalizeVectorTrunc( 733 MachineInstr &MI, LegalizerHelper &Helper) const { 734 MachineIRBuilder &MIRBuilder = Helper.MIRBuilder; 735 MachineRegisterInfo &MRI = *MIRBuilder.getMRI(); 736 // Similar to how operand splitting is done in SelectiondDAG, we can handle 737 // %res(v8s8) = G_TRUNC %in(v8s32) by generating: 738 // %inlo(<4x s32>), %inhi(<4 x s32>) = G_UNMERGE %in(<8 x s32>) 739 // %lo16(<4 x s16>) = G_TRUNC %inlo 740 // %hi16(<4 x s16>) = G_TRUNC %inhi 741 // %in16(<8 x s16>) = G_CONCAT_VECTORS %lo16, %hi16 742 // %res(<8 x s8>) = G_TRUNC %in16 743 744 Register DstReg = MI.getOperand(0).getReg(); 745 Register SrcReg = MI.getOperand(1).getReg(); 746 LLT DstTy = MRI.getType(DstReg); 747 LLT SrcTy = MRI.getType(SrcReg); 748 assert(isPowerOf2_32(DstTy.getSizeInBits()) && 749 isPowerOf2_32(SrcTy.getSizeInBits())); 750 751 // Split input type. 752 LLT SplitSrcTy = SrcTy.changeNumElements(SrcTy.getNumElements() / 2); 753 // First, split the source into two smaller vectors. 754 SmallVector<Register, 2> SplitSrcs; 755 extractParts(SrcReg, MRI, MIRBuilder, SplitSrcTy, 2, SplitSrcs); 756 757 // Truncate the splits into intermediate narrower elements. 758 LLT InterTy = SplitSrcTy.changeElementSize(DstTy.getScalarSizeInBits() * 2); 759 for (unsigned I = 0; I < SplitSrcs.size(); ++I) 760 SplitSrcs[I] = MIRBuilder.buildTrunc(InterTy, SplitSrcs[I]).getReg(0); 761 762 auto Concat = MIRBuilder.buildConcatVectors( 763 DstTy.changeElementSize(DstTy.getScalarSizeInBits() * 2), SplitSrcs); 764 765 Helper.Observer.changingInstr(MI); 766 MI.getOperand(1).setReg(Concat.getReg(0)); 767 Helper.Observer.changedInstr(MI); 768 return true; 769 } 770 771 bool AArch64LegalizerInfo::legalizeSmallCMGlobalValue( 772 MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &MIRBuilder, 773 GISelChangeObserver &Observer) const { 774 assert(MI.getOpcode() == TargetOpcode::G_GLOBAL_VALUE); 775 // We do this custom legalization to convert G_GLOBAL_VALUE into target ADRP + 776 // G_ADD_LOW instructions. 777 // By splitting this here, we can optimize accesses in the small code model by 778 // folding in the G_ADD_LOW into the load/store offset. 779 auto GV = MI.getOperand(1).getGlobal(); 780 if (GV->isThreadLocal()) 781 return true; // Don't want to modify TLS vars. 782 783 auto &TM = ST->getTargetLowering()->getTargetMachine(); 784 unsigned OpFlags = ST->ClassifyGlobalReference(GV, TM); 785 786 if (OpFlags & AArch64II::MO_GOT) 787 return true; 788 789 Register DstReg = MI.getOperand(0).getReg(); 790 auto ADRP = MIRBuilder.buildInstr(AArch64::ADRP, {LLT::pointer(0, 64)}, {}) 791 .addGlobalAddress(GV, 0, OpFlags | AArch64II::MO_PAGE); 792 // Set the regclass on the dest reg too. 793 MRI.setRegClass(ADRP.getReg(0), &AArch64::GPR64RegClass); 794 795 // MO_TAGGED on the page indicates a tagged address. Set the tag now. We do so 796 // by creating a MOVK that sets bits 48-63 of the register to (global address 797 // + 0x100000000 - PC) >> 48. The additional 0x100000000 offset here is to 798 // prevent an incorrect tag being generated during relocation when the the 799 // global appears before the code section. Without the offset, a global at 800 // `0x0f00'0000'0000'1000` (i.e. at `0x1000` with tag `0xf`) that's referenced 801 // by code at `0x2000` would result in `0x0f00'0000'0000'1000 - 0x2000 = 802 // 0x0eff'ffff'ffff'f000`, meaning the tag would be incorrectly set to `0xe` 803 // instead of `0xf`. 804 // This assumes that we're in the small code model so we can assume a binary 805 // size of <= 4GB, which makes the untagged PC relative offset positive. The 806 // binary must also be loaded into address range [0, 2^48). Both of these 807 // properties need to be ensured at runtime when using tagged addresses. 808 if (OpFlags & AArch64II::MO_TAGGED) { 809 ADRP = MIRBuilder.buildInstr(AArch64::MOVKXi, {LLT::pointer(0, 64)}, {ADRP}) 810 .addGlobalAddress(GV, 0x100000000, 811 AArch64II::MO_PREL | AArch64II::MO_G3) 812 .addImm(48); 813 MRI.setRegClass(ADRP.getReg(0), &AArch64::GPR64RegClass); 814 } 815 816 MIRBuilder.buildInstr(AArch64::G_ADD_LOW, {DstReg}, {ADRP}) 817 .addGlobalAddress(GV, 0, 818 OpFlags | AArch64II::MO_PAGEOFF | AArch64II::MO_NC); 819 MI.eraseFromParent(); 820 return true; 821 } 822 823 bool AArch64LegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper, 824 MachineInstr &MI) const { 825 return true; 826 } 827 828 bool AArch64LegalizerInfo::legalizeShlAshrLshr( 829 MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &MIRBuilder, 830 GISelChangeObserver &Observer) const { 831 assert(MI.getOpcode() == TargetOpcode::G_ASHR || 832 MI.getOpcode() == TargetOpcode::G_LSHR || 833 MI.getOpcode() == TargetOpcode::G_SHL); 834 // If the shift amount is a G_CONSTANT, promote it to a 64 bit type so the 835 // imported patterns can select it later. Either way, it will be legal. 836 Register AmtReg = MI.getOperand(2).getReg(); 837 auto VRegAndVal = getConstantVRegValWithLookThrough(AmtReg, MRI); 838 if (!VRegAndVal) 839 return true; 840 // Check the shift amount is in range for an immediate form. 841 int64_t Amount = VRegAndVal->Value.getSExtValue(); 842 if (Amount > 31) 843 return true; // This will have to remain a register variant. 844 auto ExtCst = MIRBuilder.buildConstant(LLT::scalar(64), Amount); 845 Observer.changingInstr(MI); 846 MI.getOperand(2).setReg(ExtCst.getReg(0)); 847 Observer.changedInstr(MI); 848 return true; 849 } 850 851 bool AArch64LegalizerInfo::legalizeLoadStore( 852 MachineInstr &MI, MachineRegisterInfo &MRI, MachineIRBuilder &MIRBuilder, 853 GISelChangeObserver &Observer) const { 854 assert(MI.getOpcode() == TargetOpcode::G_STORE || 855 MI.getOpcode() == TargetOpcode::G_LOAD); 856 // Here we just try to handle vector loads/stores where our value type might 857 // have pointer elements, which the SelectionDAG importer can't handle. To 858 // allow the existing patterns for s64 to fire for p0, we just try to bitcast 859 // the value to use s64 types. 860 861 // Custom legalization requires the instruction, if not deleted, must be fully 862 // legalized. In order to allow further legalization of the inst, we create 863 // a new instruction and erase the existing one. 864 865 Register ValReg = MI.getOperand(0).getReg(); 866 const LLT ValTy = MRI.getType(ValReg); 867 868 if (!ValTy.isVector() || !ValTy.getElementType().isPointer() || 869 ValTy.getElementType().getAddressSpace() != 0) { 870 LLVM_DEBUG(dbgs() << "Tried to do custom legalization on wrong load/store"); 871 return false; 872 } 873 874 unsigned PtrSize = ValTy.getElementType().getSizeInBits(); 875 const LLT NewTy = LLT::vector(ValTy.getNumElements(), PtrSize); 876 auto &MMO = **MI.memoperands_begin(); 877 if (MI.getOpcode() == TargetOpcode::G_STORE) { 878 auto Bitcast = MIRBuilder.buildBitcast(NewTy, ValReg); 879 MIRBuilder.buildStore(Bitcast.getReg(0), MI.getOperand(1), MMO); 880 } else { 881 auto NewLoad = MIRBuilder.buildLoad(NewTy, MI.getOperand(1), MMO); 882 MIRBuilder.buildBitcast(ValReg, NewLoad); 883 } 884 MI.eraseFromParent(); 885 return true; 886 } 887 888 bool AArch64LegalizerInfo::legalizeVaArg(MachineInstr &MI, 889 MachineRegisterInfo &MRI, 890 MachineIRBuilder &MIRBuilder) const { 891 MachineFunction &MF = MIRBuilder.getMF(); 892 Align Alignment(MI.getOperand(2).getImm()); 893 Register Dst = MI.getOperand(0).getReg(); 894 Register ListPtr = MI.getOperand(1).getReg(); 895 896 LLT PtrTy = MRI.getType(ListPtr); 897 LLT IntPtrTy = LLT::scalar(PtrTy.getSizeInBits()); 898 899 const unsigned PtrSize = PtrTy.getSizeInBits() / 8; 900 const Align PtrAlign = Align(PtrSize); 901 auto List = MIRBuilder.buildLoad( 902 PtrTy, ListPtr, 903 *MF.getMachineMemOperand(MachinePointerInfo(), MachineMemOperand::MOLoad, 904 PtrSize, PtrAlign)); 905 906 MachineInstrBuilder DstPtr; 907 if (Alignment > PtrAlign) { 908 // Realign the list to the actual required alignment. 909 auto AlignMinus1 = 910 MIRBuilder.buildConstant(IntPtrTy, Alignment.value() - 1); 911 auto ListTmp = MIRBuilder.buildPtrAdd(PtrTy, List, AlignMinus1.getReg(0)); 912 DstPtr = MIRBuilder.buildMaskLowPtrBits(PtrTy, ListTmp, Log2(Alignment)); 913 } else 914 DstPtr = List; 915 916 uint64_t ValSize = MRI.getType(Dst).getSizeInBits() / 8; 917 MIRBuilder.buildLoad( 918 Dst, DstPtr, 919 *MF.getMachineMemOperand(MachinePointerInfo(), MachineMemOperand::MOLoad, 920 ValSize, std::max(Alignment, PtrAlign))); 921 922 auto Size = MIRBuilder.buildConstant(IntPtrTy, alignTo(ValSize, PtrAlign)); 923 924 auto NewList = MIRBuilder.buildPtrAdd(PtrTy, DstPtr, Size.getReg(0)); 925 926 MIRBuilder.buildStore(NewList, ListPtr, 927 *MF.getMachineMemOperand(MachinePointerInfo(), 928 MachineMemOperand::MOStore, 929 PtrSize, PtrAlign)); 930 931 MI.eraseFromParent(); 932 return true; 933 } 934