1 //===-- SystemZISelDAGToDAG.cpp - A dag to dag inst selector for SystemZ --===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file defines an instruction selector for the SystemZ target. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "SystemZTargetMachine.h" 14 #include "SystemZISelLowering.h" 15 #include "llvm/Analysis/AliasAnalysis.h" 16 #include "llvm/CodeGen/SelectionDAGISel.h" 17 #include "llvm/Support/Debug.h" 18 #include "llvm/Support/KnownBits.h" 19 #include "llvm/Support/raw_ostream.h" 20 21 using namespace llvm; 22 23 #define DEBUG_TYPE "systemz-isel" 24 #define PASS_NAME "SystemZ DAG->DAG Pattern Instruction Selection" 25 26 namespace { 27 // Used to build addressing modes. 28 struct SystemZAddressingMode { 29 // The shape of the address. 30 enum AddrForm { 31 // base+displacement 32 FormBD, 33 34 // base+displacement+index for load and store operands 35 FormBDXNormal, 36 37 // base+displacement+index for load address operands 38 FormBDXLA, 39 40 // base+displacement+index+ADJDYNALLOC 41 FormBDXDynAlloc 42 }; 43 AddrForm Form; 44 45 // The type of displacement. The enum names here correspond directly 46 // to the definitions in SystemZOperand.td. We could split them into 47 // flags -- single/pair, 128-bit, etc. -- but it hardly seems worth it. 48 enum DispRange { 49 Disp12Only, 50 Disp12Pair, 51 Disp20Only, 52 Disp20Only128, 53 Disp20Pair 54 }; 55 DispRange DR; 56 57 // The parts of the address. The address is equivalent to: 58 // 59 // Base + Disp + Index + (IncludesDynAlloc ? ADJDYNALLOC : 0) 60 SDValue Base; 61 int64_t Disp; 62 SDValue Index; 63 bool IncludesDynAlloc; 64 65 SystemZAddressingMode(AddrForm form, DispRange dr) 66 : Form(form), DR(dr), Disp(0), IncludesDynAlloc(false) {} 67 68 // True if the address can have an index register. 69 bool hasIndexField() { return Form != FormBD; } 70 71 // True if the address can (and must) include ADJDYNALLOC. 72 bool isDynAlloc() { return Form == FormBDXDynAlloc; } 73 74 void dump(const llvm::SelectionDAG *DAG) { 75 errs() << "SystemZAddressingMode " << this << '\n'; 76 77 errs() << " Base "; 78 if (Base.getNode()) 79 Base.getNode()->dump(DAG); 80 else 81 errs() << "null\n"; 82 83 if (hasIndexField()) { 84 errs() << " Index "; 85 if (Index.getNode()) 86 Index.getNode()->dump(DAG); 87 else 88 errs() << "null\n"; 89 } 90 91 errs() << " Disp " << Disp; 92 if (IncludesDynAlloc) 93 errs() << " + ADJDYNALLOC"; 94 errs() << '\n'; 95 } 96 }; 97 98 // Return a mask with Count low bits set. 99 static uint64_t allOnes(unsigned int Count) { 100 assert(Count <= 64); 101 if (Count > 63) 102 return UINT64_MAX; 103 return (uint64_t(1) << Count) - 1; 104 } 105 106 // Represents operands 2 to 5 of the ROTATE AND ... SELECTED BITS operation 107 // given by Opcode. The operands are: Input (R2), Start (I3), End (I4) and 108 // Rotate (I5). The combined operand value is effectively: 109 // 110 // (or (rotl Input, Rotate), ~Mask) 111 // 112 // for RNSBG and: 113 // 114 // (and (rotl Input, Rotate), Mask) 115 // 116 // otherwise. The output value has BitSize bits, although Input may be 117 // narrower (in which case the upper bits are don't care), or wider (in which 118 // case the result will be truncated as part of the operation). 119 struct RxSBGOperands { 120 RxSBGOperands(unsigned Op, SDValue N) 121 : Opcode(Op), BitSize(N.getValueSizeInBits()), 122 Mask(allOnes(BitSize)), Input(N), Start(64 - BitSize), End(63), 123 Rotate(0) {} 124 125 unsigned Opcode; 126 unsigned BitSize; 127 uint64_t Mask; 128 SDValue Input; 129 unsigned Start; 130 unsigned End; 131 unsigned Rotate; 132 }; 133 134 class SystemZDAGToDAGISel : public SelectionDAGISel { 135 const SystemZSubtarget *Subtarget; 136 137 // Used by SystemZOperands.td to create integer constants. 138 inline SDValue getImm(const SDNode *Node, uint64_t Imm) const { 139 return CurDAG->getTargetConstant(Imm, SDLoc(Node), Node->getValueType(0)); 140 } 141 142 const SystemZTargetMachine &getTargetMachine() const { 143 return static_cast<const SystemZTargetMachine &>(TM); 144 } 145 146 const SystemZInstrInfo *getInstrInfo() const { 147 return Subtarget->getInstrInfo(); 148 } 149 150 // Try to fold more of the base or index of AM into AM, where IsBase 151 // selects between the base and index. 152 bool expandAddress(SystemZAddressingMode &AM, bool IsBase) const; 153 154 // Try to describe N in AM, returning true on success. 155 bool selectAddress(SDValue N, SystemZAddressingMode &AM) const; 156 157 // Extract individual target operands from matched address AM. 158 void getAddressOperands(const SystemZAddressingMode &AM, EVT VT, 159 SDValue &Base, SDValue &Disp) const; 160 void getAddressOperands(const SystemZAddressingMode &AM, EVT VT, 161 SDValue &Base, SDValue &Disp, SDValue &Index) const; 162 163 // Try to match Addr as a FormBD address with displacement type DR. 164 // Return true on success, storing the base and displacement in 165 // Base and Disp respectively. 166 bool selectBDAddr(SystemZAddressingMode::DispRange DR, SDValue Addr, 167 SDValue &Base, SDValue &Disp) const; 168 169 // Try to match Addr as a FormBDX address with displacement type DR. 170 // Return true on success and if the result had no index. Store the 171 // base and displacement in Base and Disp respectively. 172 bool selectMVIAddr(SystemZAddressingMode::DispRange DR, SDValue Addr, 173 SDValue &Base, SDValue &Disp) const; 174 175 // Try to match Addr as a FormBDX* address of form Form with 176 // displacement type DR. Return true on success, storing the base, 177 // displacement and index in Base, Disp and Index respectively. 178 bool selectBDXAddr(SystemZAddressingMode::AddrForm Form, 179 SystemZAddressingMode::DispRange DR, SDValue Addr, 180 SDValue &Base, SDValue &Disp, SDValue &Index) const; 181 182 // PC-relative address matching routines used by SystemZOperands.td. 183 bool selectPCRelAddress(SDValue Addr, SDValue &Target) const { 184 if (SystemZISD::isPCREL(Addr.getOpcode())) { 185 Target = Addr.getOperand(0); 186 return true; 187 } 188 return false; 189 } 190 191 // BD matching routines used by SystemZOperands.td. 192 bool selectBDAddr12Only(SDValue Addr, SDValue &Base, SDValue &Disp) const { 193 return selectBDAddr(SystemZAddressingMode::Disp12Only, Addr, Base, Disp); 194 } 195 bool selectBDAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const { 196 return selectBDAddr(SystemZAddressingMode::Disp12Pair, Addr, Base, Disp); 197 } 198 bool selectBDAddr20Only(SDValue Addr, SDValue &Base, SDValue &Disp) const { 199 return selectBDAddr(SystemZAddressingMode::Disp20Only, Addr, Base, Disp); 200 } 201 bool selectBDAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const { 202 return selectBDAddr(SystemZAddressingMode::Disp20Pair, Addr, Base, Disp); 203 } 204 205 // MVI matching routines used by SystemZOperands.td. 206 bool selectMVIAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const { 207 return selectMVIAddr(SystemZAddressingMode::Disp12Pair, Addr, Base, Disp); 208 } 209 bool selectMVIAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const { 210 return selectMVIAddr(SystemZAddressingMode::Disp20Pair, Addr, Base, Disp); 211 } 212 213 // BDX matching routines used by SystemZOperands.td. 214 bool selectBDXAddr12Only(SDValue Addr, SDValue &Base, SDValue &Disp, 215 SDValue &Index) const { 216 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal, 217 SystemZAddressingMode::Disp12Only, 218 Addr, Base, Disp, Index); 219 } 220 bool selectBDXAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp, 221 SDValue &Index) const { 222 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal, 223 SystemZAddressingMode::Disp12Pair, 224 Addr, Base, Disp, Index); 225 } 226 bool selectDynAlloc12Only(SDValue Addr, SDValue &Base, SDValue &Disp, 227 SDValue &Index) const { 228 return selectBDXAddr(SystemZAddressingMode::FormBDXDynAlloc, 229 SystemZAddressingMode::Disp12Only, 230 Addr, Base, Disp, Index); 231 } 232 bool selectBDXAddr20Only(SDValue Addr, SDValue &Base, SDValue &Disp, 233 SDValue &Index) const { 234 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal, 235 SystemZAddressingMode::Disp20Only, 236 Addr, Base, Disp, Index); 237 } 238 bool selectBDXAddr20Only128(SDValue Addr, SDValue &Base, SDValue &Disp, 239 SDValue &Index) const { 240 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal, 241 SystemZAddressingMode::Disp20Only128, 242 Addr, Base, Disp, Index); 243 } 244 bool selectBDXAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp, 245 SDValue &Index) const { 246 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal, 247 SystemZAddressingMode::Disp20Pair, 248 Addr, Base, Disp, Index); 249 } 250 bool selectLAAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp, 251 SDValue &Index) const { 252 return selectBDXAddr(SystemZAddressingMode::FormBDXLA, 253 SystemZAddressingMode::Disp12Pair, 254 Addr, Base, Disp, Index); 255 } 256 bool selectLAAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp, 257 SDValue &Index) const { 258 return selectBDXAddr(SystemZAddressingMode::FormBDXLA, 259 SystemZAddressingMode::Disp20Pair, 260 Addr, Base, Disp, Index); 261 } 262 263 // Try to match Addr as an address with a base, 12-bit displacement 264 // and index, where the index is element Elem of a vector. 265 // Return true on success, storing the base, displacement and vector 266 // in Base, Disp and Index respectively. 267 bool selectBDVAddr12Only(SDValue Addr, SDValue Elem, SDValue &Base, 268 SDValue &Disp, SDValue &Index) const; 269 270 // Check whether (or Op (and X InsertMask)) is effectively an insertion 271 // of X into bits InsertMask of some Y != Op. Return true if so and 272 // set Op to that Y. 273 bool detectOrAndInsertion(SDValue &Op, uint64_t InsertMask) const; 274 275 // Try to update RxSBG so that only the bits of RxSBG.Input in Mask are used. 276 // Return true on success. 277 bool refineRxSBGMask(RxSBGOperands &RxSBG, uint64_t Mask) const; 278 279 // Try to fold some of RxSBG.Input into other fields of RxSBG. 280 // Return true on success. 281 bool expandRxSBG(RxSBGOperands &RxSBG) const; 282 283 // Return an undefined value of type VT. 284 SDValue getUNDEF(const SDLoc &DL, EVT VT) const; 285 286 // Convert N to VT, if it isn't already. 287 SDValue convertTo(const SDLoc &DL, EVT VT, SDValue N) const; 288 289 // Try to implement AND or shift node N using RISBG with the zero flag set. 290 // Return the selected node on success, otherwise return null. 291 bool tryRISBGZero(SDNode *N); 292 293 // Try to use RISBG or Opcode to implement OR or XOR node N. 294 // Return the selected node on success, otherwise return null. 295 bool tryRxSBG(SDNode *N, unsigned Opcode); 296 297 // If Op0 is null, then Node is a constant that can be loaded using: 298 // 299 // (Opcode UpperVal LowerVal) 300 // 301 // If Op0 is nonnull, then Node can be implemented using: 302 // 303 // (Opcode (Opcode Op0 UpperVal) LowerVal) 304 void splitLargeImmediate(unsigned Opcode, SDNode *Node, SDValue Op0, 305 uint64_t UpperVal, uint64_t LowerVal); 306 307 void loadVectorConstant(const SystemZVectorConstantInfo &VCI, 308 SDNode *Node); 309 310 SDNode *loadPoolVectorConstant(APInt Val, EVT VT, SDLoc DL); 311 312 // Try to use gather instruction Opcode to implement vector insertion N. 313 bool tryGather(SDNode *N, unsigned Opcode); 314 315 // Try to use scatter instruction Opcode to implement store Store. 316 bool tryScatter(StoreSDNode *Store, unsigned Opcode); 317 318 // Change a chain of {load; op; store} of the same value into a simple op 319 // through memory of that value, if the uses of the modified value and its 320 // address are suitable. 321 bool tryFoldLoadStoreIntoMemOperand(SDNode *Node); 322 323 // Return true if Load and Store are loads and stores of the same size 324 // and are guaranteed not to overlap. Such operations can be implemented 325 // using block (SS-format) instructions. 326 // 327 // Partial overlap would lead to incorrect code, since the block operations 328 // are logically bytewise, even though they have a fast path for the 329 // non-overlapping case. We also need to avoid full overlap (i.e. two 330 // addresses that might be equal at run time) because although that case 331 // would be handled correctly, it might be implemented by millicode. 332 bool canUseBlockOperation(StoreSDNode *Store, LoadSDNode *Load) const; 333 334 // N is a (store (load Y), X) pattern. Return true if it can use an MVC 335 // from Y to X. 336 bool storeLoadCanUseMVC(SDNode *N) const; 337 338 // N is a (store (op (load A[0]), (load A[1])), X) pattern. Return true 339 // if A[1 - I] == X and if N can use a block operation like NC from A[I] 340 // to X. 341 bool storeLoadCanUseBlockBinary(SDNode *N, unsigned I) const; 342 343 // Return true if N (a load or a store) fullfills the alignment 344 // requirements for a PC-relative access. 345 bool storeLoadIsAligned(SDNode *N) const; 346 347 // Try to expand a boolean SELECT_CCMASK using an IPM sequence. 348 SDValue expandSelectBoolean(SDNode *Node); 349 350 public: 351 static char ID; 352 353 SystemZDAGToDAGISel() = delete; 354 355 SystemZDAGToDAGISel(SystemZTargetMachine &TM, CodeGenOptLevel OptLevel) 356 : SelectionDAGISel(ID, TM, OptLevel) {} 357 358 bool runOnMachineFunction(MachineFunction &MF) override { 359 const Function &F = MF.getFunction(); 360 if (F.getFnAttribute("fentry-call").getValueAsString() != "true") { 361 if (F.hasFnAttribute("mnop-mcount")) 362 report_fatal_error("mnop-mcount only supported with fentry-call"); 363 if (F.hasFnAttribute("mrecord-mcount")) 364 report_fatal_error("mrecord-mcount only supported with fentry-call"); 365 } 366 367 Subtarget = &MF.getSubtarget<SystemZSubtarget>(); 368 return SelectionDAGISel::runOnMachineFunction(MF); 369 } 370 371 // Override SelectionDAGISel. 372 void Select(SDNode *Node) override; 373 bool SelectInlineAsmMemoryOperand(const SDValue &Op, 374 InlineAsm::ConstraintCode ConstraintID, 375 std::vector<SDValue> &OutOps) override; 376 bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const override; 377 void PreprocessISelDAG() override; 378 379 // Include the pieces autogenerated from the target description. 380 #include "SystemZGenDAGISel.inc" 381 }; 382 } // end anonymous namespace 383 384 char SystemZDAGToDAGISel::ID = 0; 385 386 INITIALIZE_PASS(SystemZDAGToDAGISel, DEBUG_TYPE, PASS_NAME, false, false) 387 388 FunctionPass *llvm::createSystemZISelDag(SystemZTargetMachine &TM, 389 CodeGenOptLevel OptLevel) { 390 return new SystemZDAGToDAGISel(TM, OptLevel); 391 } 392 393 // Return true if Val should be selected as a displacement for an address 394 // with range DR. Here we're interested in the range of both the instruction 395 // described by DR and of any pairing instruction. 396 static bool selectDisp(SystemZAddressingMode::DispRange DR, int64_t Val) { 397 switch (DR) { 398 case SystemZAddressingMode::Disp12Only: 399 return isUInt<12>(Val); 400 401 case SystemZAddressingMode::Disp12Pair: 402 case SystemZAddressingMode::Disp20Only: 403 case SystemZAddressingMode::Disp20Pair: 404 return isInt<20>(Val); 405 406 case SystemZAddressingMode::Disp20Only128: 407 return isInt<20>(Val) && isInt<20>(Val + 8); 408 } 409 llvm_unreachable("Unhandled displacement range"); 410 } 411 412 // Change the base or index in AM to Value, where IsBase selects 413 // between the base and index. 414 static void changeComponent(SystemZAddressingMode &AM, bool IsBase, 415 SDValue Value) { 416 if (IsBase) 417 AM.Base = Value; 418 else 419 AM.Index = Value; 420 } 421 422 // The base or index of AM is equivalent to Value + ADJDYNALLOC, 423 // where IsBase selects between the base and index. Try to fold the 424 // ADJDYNALLOC into AM. 425 static bool expandAdjDynAlloc(SystemZAddressingMode &AM, bool IsBase, 426 SDValue Value) { 427 if (AM.isDynAlloc() && !AM.IncludesDynAlloc) { 428 changeComponent(AM, IsBase, Value); 429 AM.IncludesDynAlloc = true; 430 return true; 431 } 432 return false; 433 } 434 435 // The base of AM is equivalent to Base + Index. Try to use Index as 436 // the index register. 437 static bool expandIndex(SystemZAddressingMode &AM, SDValue Base, 438 SDValue Index) { 439 if (AM.hasIndexField() && !AM.Index.getNode()) { 440 AM.Base = Base; 441 AM.Index = Index; 442 return true; 443 } 444 return false; 445 } 446 447 // The base or index of AM is equivalent to Op0 + Op1, where IsBase selects 448 // between the base and index. Try to fold Op1 into AM's displacement. 449 static bool expandDisp(SystemZAddressingMode &AM, bool IsBase, 450 SDValue Op0, uint64_t Op1) { 451 // First try adjusting the displacement. 452 int64_t TestDisp = AM.Disp + Op1; 453 if (selectDisp(AM.DR, TestDisp)) { 454 changeComponent(AM, IsBase, Op0); 455 AM.Disp = TestDisp; 456 return true; 457 } 458 459 // We could consider forcing the displacement into a register and 460 // using it as an index, but it would need to be carefully tuned. 461 return false; 462 } 463 464 bool SystemZDAGToDAGISel::expandAddress(SystemZAddressingMode &AM, 465 bool IsBase) const { 466 SDValue N = IsBase ? AM.Base : AM.Index; 467 unsigned Opcode = N.getOpcode(); 468 // Look through no-op truncations. 469 if (Opcode == ISD::TRUNCATE && N.getOperand(0).getValueSizeInBits() <= 64) { 470 N = N.getOperand(0); 471 Opcode = N.getOpcode(); 472 } 473 if (Opcode == ISD::ADD || CurDAG->isBaseWithConstantOffset(N)) { 474 SDValue Op0 = N.getOperand(0); 475 SDValue Op1 = N.getOperand(1); 476 477 unsigned Op0Code = Op0->getOpcode(); 478 unsigned Op1Code = Op1->getOpcode(); 479 480 if (Op0Code == SystemZISD::ADJDYNALLOC) 481 return expandAdjDynAlloc(AM, IsBase, Op1); 482 if (Op1Code == SystemZISD::ADJDYNALLOC) 483 return expandAdjDynAlloc(AM, IsBase, Op0); 484 485 if (Op0Code == ISD::Constant) 486 return expandDisp(AM, IsBase, Op1, 487 cast<ConstantSDNode>(Op0)->getSExtValue()); 488 if (Op1Code == ISD::Constant) 489 return expandDisp(AM, IsBase, Op0, 490 cast<ConstantSDNode>(Op1)->getSExtValue()); 491 492 if (IsBase && expandIndex(AM, Op0, Op1)) 493 return true; 494 } 495 if (Opcode == SystemZISD::PCREL_OFFSET) { 496 SDValue Full = N.getOperand(0); 497 SDValue Base = N.getOperand(1); 498 SDValue Anchor = Base.getOperand(0); 499 uint64_t Offset = (cast<GlobalAddressSDNode>(Full)->getOffset() - 500 cast<GlobalAddressSDNode>(Anchor)->getOffset()); 501 return expandDisp(AM, IsBase, Base, Offset); 502 } 503 return false; 504 } 505 506 // Return true if an instruction with displacement range DR should be 507 // used for displacement value Val. selectDisp(DR, Val) must already hold. 508 static bool isValidDisp(SystemZAddressingMode::DispRange DR, int64_t Val) { 509 assert(selectDisp(DR, Val) && "Invalid displacement"); 510 switch (DR) { 511 case SystemZAddressingMode::Disp12Only: 512 case SystemZAddressingMode::Disp20Only: 513 case SystemZAddressingMode::Disp20Only128: 514 return true; 515 516 case SystemZAddressingMode::Disp12Pair: 517 // Use the other instruction if the displacement is too large. 518 return isUInt<12>(Val); 519 520 case SystemZAddressingMode::Disp20Pair: 521 // Use the other instruction if the displacement is small enough. 522 return !isUInt<12>(Val); 523 } 524 llvm_unreachable("Unhandled displacement range"); 525 } 526 527 // Return true if Base + Disp + Index should be performed by LA(Y). 528 static bool shouldUseLA(SDNode *Base, int64_t Disp, SDNode *Index) { 529 // Don't use LA(Y) for constants. 530 if (!Base) 531 return false; 532 533 // Always use LA(Y) for frame addresses, since we know that the destination 534 // register is almost always (perhaps always) going to be different from 535 // the frame register. 536 if (Base->getOpcode() == ISD::FrameIndex) 537 return true; 538 539 if (Disp) { 540 // Always use LA(Y) if there is a base, displacement and index. 541 if (Index) 542 return true; 543 544 // Always use LA if the displacement is small enough. It should always 545 // be no worse than AGHI (and better if it avoids a move). 546 if (isUInt<12>(Disp)) 547 return true; 548 549 // For similar reasons, always use LAY if the constant is too big for AGHI. 550 // LAY should be no worse than AGFI. 551 if (!isInt<16>(Disp)) 552 return true; 553 } else { 554 // Don't use LA for plain registers. 555 if (!Index) 556 return false; 557 558 // Don't use LA for plain addition if the index operand is only used 559 // once. It should be a natural two-operand addition in that case. 560 if (Index->hasOneUse()) 561 return false; 562 563 // Prefer addition if the second operation is sign-extended, in the 564 // hope of using AGF. 565 unsigned IndexOpcode = Index->getOpcode(); 566 if (IndexOpcode == ISD::SIGN_EXTEND || 567 IndexOpcode == ISD::SIGN_EXTEND_INREG) 568 return false; 569 } 570 571 // Don't use LA for two-operand addition if either operand is only 572 // used once. The addition instructions are better in that case. 573 if (Base->hasOneUse()) 574 return false; 575 576 return true; 577 } 578 579 // Return true if Addr is suitable for AM, updating AM if so. 580 bool SystemZDAGToDAGISel::selectAddress(SDValue Addr, 581 SystemZAddressingMode &AM) const { 582 // Start out assuming that the address will need to be loaded separately, 583 // then try to extend it as much as we can. 584 AM.Base = Addr; 585 586 // First try treating the address as a constant. 587 if (Addr.getOpcode() == ISD::Constant && 588 expandDisp(AM, true, SDValue(), 589 cast<ConstantSDNode>(Addr)->getSExtValue())) 590 ; 591 // Also see if it's a bare ADJDYNALLOC. 592 else if (Addr.getOpcode() == SystemZISD::ADJDYNALLOC && 593 expandAdjDynAlloc(AM, true, SDValue())) 594 ; 595 else 596 // Otherwise try expanding each component. 597 while (expandAddress(AM, true) || 598 (AM.Index.getNode() && expandAddress(AM, false))) 599 continue; 600 601 // Reject cases where it isn't profitable to use LA(Y). 602 if (AM.Form == SystemZAddressingMode::FormBDXLA && 603 !shouldUseLA(AM.Base.getNode(), AM.Disp, AM.Index.getNode())) 604 return false; 605 606 // Reject cases where the other instruction in a pair should be used. 607 if (!isValidDisp(AM.DR, AM.Disp)) 608 return false; 609 610 // Make sure that ADJDYNALLOC is included where necessary. 611 if (AM.isDynAlloc() && !AM.IncludesDynAlloc) 612 return false; 613 614 LLVM_DEBUG(AM.dump(CurDAG)); 615 return true; 616 } 617 618 // Insert a node into the DAG at least before Pos. This will reposition 619 // the node as needed, and will assign it a node ID that is <= Pos's ID. 620 // Note that this does *not* preserve the uniqueness of node IDs! 621 // The selection DAG must no longer depend on their uniqueness when this 622 // function is used. 623 static void insertDAGNode(SelectionDAG *DAG, SDNode *Pos, SDValue N) { 624 if (N->getNodeId() == -1 || 625 (SelectionDAGISel::getUninvalidatedNodeId(N.getNode()) > 626 SelectionDAGISel::getUninvalidatedNodeId(Pos))) { 627 DAG->RepositionNode(Pos->getIterator(), N.getNode()); 628 // Mark Node as invalid for pruning as after this it may be a successor to a 629 // selected node but otherwise be in the same position of Pos. 630 // Conservatively mark it with the same -abs(Id) to assure node id 631 // invariant is preserved. 632 N->setNodeId(Pos->getNodeId()); 633 SelectionDAGISel::InvalidateNodeId(N.getNode()); 634 } 635 } 636 637 void SystemZDAGToDAGISel::getAddressOperands(const SystemZAddressingMode &AM, 638 EVT VT, SDValue &Base, 639 SDValue &Disp) const { 640 Base = AM.Base; 641 if (!Base.getNode()) 642 // Register 0 means "no base". This is mostly useful for shifts. 643 Base = CurDAG->getRegister(0, VT); 644 else if (Base.getOpcode() == ISD::FrameIndex) { 645 // Lower a FrameIndex to a TargetFrameIndex. 646 int64_t FrameIndex = cast<FrameIndexSDNode>(Base)->getIndex(); 647 Base = CurDAG->getTargetFrameIndex(FrameIndex, VT); 648 } else if (Base.getValueType() != VT) { 649 // Truncate values from i64 to i32, for shifts. 650 assert(VT == MVT::i32 && Base.getValueType() == MVT::i64 && 651 "Unexpected truncation"); 652 SDLoc DL(Base); 653 SDValue Trunc = CurDAG->getNode(ISD::TRUNCATE, DL, VT, Base); 654 insertDAGNode(CurDAG, Base.getNode(), Trunc); 655 Base = Trunc; 656 } 657 658 // Lower the displacement to a TargetConstant. 659 Disp = CurDAG->getTargetConstant(AM.Disp, SDLoc(Base), VT); 660 } 661 662 void SystemZDAGToDAGISel::getAddressOperands(const SystemZAddressingMode &AM, 663 EVT VT, SDValue &Base, 664 SDValue &Disp, 665 SDValue &Index) const { 666 getAddressOperands(AM, VT, Base, Disp); 667 668 Index = AM.Index; 669 if (!Index.getNode()) 670 // Register 0 means "no index". 671 Index = CurDAG->getRegister(0, VT); 672 } 673 674 bool SystemZDAGToDAGISel::selectBDAddr(SystemZAddressingMode::DispRange DR, 675 SDValue Addr, SDValue &Base, 676 SDValue &Disp) const { 677 SystemZAddressingMode AM(SystemZAddressingMode::FormBD, DR); 678 if (!selectAddress(Addr, AM)) 679 return false; 680 681 getAddressOperands(AM, Addr.getValueType(), Base, Disp); 682 return true; 683 } 684 685 bool SystemZDAGToDAGISel::selectMVIAddr(SystemZAddressingMode::DispRange DR, 686 SDValue Addr, SDValue &Base, 687 SDValue &Disp) const { 688 SystemZAddressingMode AM(SystemZAddressingMode::FormBDXNormal, DR); 689 if (!selectAddress(Addr, AM) || AM.Index.getNode()) 690 return false; 691 692 getAddressOperands(AM, Addr.getValueType(), Base, Disp); 693 return true; 694 } 695 696 bool SystemZDAGToDAGISel::selectBDXAddr(SystemZAddressingMode::AddrForm Form, 697 SystemZAddressingMode::DispRange DR, 698 SDValue Addr, SDValue &Base, 699 SDValue &Disp, SDValue &Index) const { 700 SystemZAddressingMode AM(Form, DR); 701 if (!selectAddress(Addr, AM)) 702 return false; 703 704 getAddressOperands(AM, Addr.getValueType(), Base, Disp, Index); 705 return true; 706 } 707 708 bool SystemZDAGToDAGISel::selectBDVAddr12Only(SDValue Addr, SDValue Elem, 709 SDValue &Base, 710 SDValue &Disp, 711 SDValue &Index) const { 712 SDValue Regs[2]; 713 if (selectBDXAddr12Only(Addr, Regs[0], Disp, Regs[1]) && 714 Regs[0].getNode() && Regs[1].getNode()) { 715 for (unsigned int I = 0; I < 2; ++I) { 716 Base = Regs[I]; 717 Index = Regs[1 - I]; 718 // We can't tell here whether the index vector has the right type 719 // for the access; the caller needs to do that instead. 720 if (Index.getOpcode() == ISD::ZERO_EXTEND) 721 Index = Index.getOperand(0); 722 if (Index.getOpcode() == ISD::EXTRACT_VECTOR_ELT && 723 Index.getOperand(1) == Elem) { 724 Index = Index.getOperand(0); 725 return true; 726 } 727 } 728 } 729 return false; 730 } 731 732 bool SystemZDAGToDAGISel::detectOrAndInsertion(SDValue &Op, 733 uint64_t InsertMask) const { 734 // We're only interested in cases where the insertion is into some operand 735 // of Op, rather than into Op itself. The only useful case is an AND. 736 if (Op.getOpcode() != ISD::AND) 737 return false; 738 739 // We need a constant mask. 740 auto *MaskNode = dyn_cast<ConstantSDNode>(Op.getOperand(1).getNode()); 741 if (!MaskNode) 742 return false; 743 744 // It's not an insertion of Op.getOperand(0) if the two masks overlap. 745 uint64_t AndMask = MaskNode->getZExtValue(); 746 if (InsertMask & AndMask) 747 return false; 748 749 // It's only an insertion if all bits are covered or are known to be zero. 750 // The inner check covers all cases but is more expensive. 751 uint64_t Used = allOnes(Op.getValueSizeInBits()); 752 if (Used != (AndMask | InsertMask)) { 753 KnownBits Known = CurDAG->computeKnownBits(Op.getOperand(0)); 754 if (Used != (AndMask | InsertMask | Known.Zero.getZExtValue())) 755 return false; 756 } 757 758 Op = Op.getOperand(0); 759 return true; 760 } 761 762 bool SystemZDAGToDAGISel::refineRxSBGMask(RxSBGOperands &RxSBG, 763 uint64_t Mask) const { 764 const SystemZInstrInfo *TII = getInstrInfo(); 765 if (RxSBG.Rotate != 0) 766 Mask = (Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate)); 767 Mask &= RxSBG.Mask; 768 if (TII->isRxSBGMask(Mask, RxSBG.BitSize, RxSBG.Start, RxSBG.End)) { 769 RxSBG.Mask = Mask; 770 return true; 771 } 772 return false; 773 } 774 775 // Return true if any bits of (RxSBG.Input & Mask) are significant. 776 static bool maskMatters(RxSBGOperands &RxSBG, uint64_t Mask) { 777 // Rotate the mask in the same way as RxSBG.Input is rotated. 778 if (RxSBG.Rotate != 0) 779 Mask = ((Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate))); 780 return (Mask & RxSBG.Mask) != 0; 781 } 782 783 bool SystemZDAGToDAGISel::expandRxSBG(RxSBGOperands &RxSBG) const { 784 SDValue N = RxSBG.Input; 785 unsigned Opcode = N.getOpcode(); 786 switch (Opcode) { 787 case ISD::TRUNCATE: { 788 if (RxSBG.Opcode == SystemZ::RNSBG) 789 return false; 790 if (N.getOperand(0).getValueSizeInBits() > 64) 791 return false; 792 uint64_t BitSize = N.getValueSizeInBits(); 793 uint64_t Mask = allOnes(BitSize); 794 if (!refineRxSBGMask(RxSBG, Mask)) 795 return false; 796 RxSBG.Input = N.getOperand(0); 797 return true; 798 } 799 case ISD::AND: { 800 if (RxSBG.Opcode == SystemZ::RNSBG) 801 return false; 802 803 auto *MaskNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode()); 804 if (!MaskNode) 805 return false; 806 807 SDValue Input = N.getOperand(0); 808 uint64_t Mask = MaskNode->getZExtValue(); 809 if (!refineRxSBGMask(RxSBG, Mask)) { 810 // If some bits of Input are already known zeros, those bits will have 811 // been removed from the mask. See if adding them back in makes the 812 // mask suitable. 813 KnownBits Known = CurDAG->computeKnownBits(Input); 814 Mask |= Known.Zero.getZExtValue(); 815 if (!refineRxSBGMask(RxSBG, Mask)) 816 return false; 817 } 818 RxSBG.Input = Input; 819 return true; 820 } 821 822 case ISD::OR: { 823 if (RxSBG.Opcode != SystemZ::RNSBG) 824 return false; 825 826 auto *MaskNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode()); 827 if (!MaskNode) 828 return false; 829 830 SDValue Input = N.getOperand(0); 831 uint64_t Mask = ~MaskNode->getZExtValue(); 832 if (!refineRxSBGMask(RxSBG, Mask)) { 833 // If some bits of Input are already known ones, those bits will have 834 // been removed from the mask. See if adding them back in makes the 835 // mask suitable. 836 KnownBits Known = CurDAG->computeKnownBits(Input); 837 Mask &= ~Known.One.getZExtValue(); 838 if (!refineRxSBGMask(RxSBG, Mask)) 839 return false; 840 } 841 RxSBG.Input = Input; 842 return true; 843 } 844 845 case ISD::ROTL: { 846 // Any 64-bit rotate left can be merged into the RxSBG. 847 if (RxSBG.BitSize != 64 || N.getValueType() != MVT::i64) 848 return false; 849 auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode()); 850 if (!CountNode) 851 return false; 852 853 RxSBG.Rotate = (RxSBG.Rotate + CountNode->getZExtValue()) & 63; 854 RxSBG.Input = N.getOperand(0); 855 return true; 856 } 857 858 case ISD::ANY_EXTEND: 859 // Bits above the extended operand are don't-care. 860 RxSBG.Input = N.getOperand(0); 861 return true; 862 863 case ISD::ZERO_EXTEND: 864 if (RxSBG.Opcode != SystemZ::RNSBG) { 865 // Restrict the mask to the extended operand. 866 unsigned InnerBitSize = N.getOperand(0).getValueSizeInBits(); 867 if (!refineRxSBGMask(RxSBG, allOnes(InnerBitSize))) 868 return false; 869 870 RxSBG.Input = N.getOperand(0); 871 return true; 872 } 873 [[fallthrough]]; 874 875 case ISD::SIGN_EXTEND: { 876 // Check that the extension bits are don't-care (i.e. are masked out 877 // by the final mask). 878 unsigned BitSize = N.getValueSizeInBits(); 879 unsigned InnerBitSize = N.getOperand(0).getValueSizeInBits(); 880 if (maskMatters(RxSBG, allOnes(BitSize) - allOnes(InnerBitSize))) { 881 // In the case where only the sign bit is active, increase Rotate with 882 // the extension width. 883 if (RxSBG.Mask == 1 && RxSBG.Rotate == 1) 884 RxSBG.Rotate += (BitSize - InnerBitSize); 885 else 886 return false; 887 } 888 889 RxSBG.Input = N.getOperand(0); 890 return true; 891 } 892 893 case ISD::SHL: { 894 auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode()); 895 if (!CountNode) 896 return false; 897 898 uint64_t Count = CountNode->getZExtValue(); 899 unsigned BitSize = N.getValueSizeInBits(); 900 if (Count < 1 || Count >= BitSize) 901 return false; 902 903 if (RxSBG.Opcode == SystemZ::RNSBG) { 904 // Treat (shl X, count) as (rotl X, size-count) as long as the bottom 905 // count bits from RxSBG.Input are ignored. 906 if (maskMatters(RxSBG, allOnes(Count))) 907 return false; 908 } else { 909 // Treat (shl X, count) as (and (rotl X, count), ~0<<count). 910 if (!refineRxSBGMask(RxSBG, allOnes(BitSize - Count) << Count)) 911 return false; 912 } 913 914 RxSBG.Rotate = (RxSBG.Rotate + Count) & 63; 915 RxSBG.Input = N.getOperand(0); 916 return true; 917 } 918 919 case ISD::SRL: 920 case ISD::SRA: { 921 auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode()); 922 if (!CountNode) 923 return false; 924 925 uint64_t Count = CountNode->getZExtValue(); 926 unsigned BitSize = N.getValueSizeInBits(); 927 if (Count < 1 || Count >= BitSize) 928 return false; 929 930 if (RxSBG.Opcode == SystemZ::RNSBG || Opcode == ISD::SRA) { 931 // Treat (srl|sra X, count) as (rotl X, size-count) as long as the top 932 // count bits from RxSBG.Input are ignored. 933 if (maskMatters(RxSBG, allOnes(Count) << (BitSize - Count))) 934 return false; 935 } else { 936 // Treat (srl X, count), mask) as (and (rotl X, size-count), ~0>>count), 937 // which is similar to SLL above. 938 if (!refineRxSBGMask(RxSBG, allOnes(BitSize - Count))) 939 return false; 940 } 941 942 RxSBG.Rotate = (RxSBG.Rotate - Count) & 63; 943 RxSBG.Input = N.getOperand(0); 944 return true; 945 } 946 default: 947 return false; 948 } 949 } 950 951 SDValue SystemZDAGToDAGISel::getUNDEF(const SDLoc &DL, EVT VT) const { 952 SDNode *N = CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, VT); 953 return SDValue(N, 0); 954 } 955 956 SDValue SystemZDAGToDAGISel::convertTo(const SDLoc &DL, EVT VT, 957 SDValue N) const { 958 if (N.getValueType() == MVT::i32 && VT == MVT::i64) 959 return CurDAG->getTargetInsertSubreg(SystemZ::subreg_l32, 960 DL, VT, getUNDEF(DL, MVT::i64), N); 961 if (N.getValueType() == MVT::i64 && VT == MVT::i32) 962 return CurDAG->getTargetExtractSubreg(SystemZ::subreg_l32, DL, VT, N); 963 assert(N.getValueType() == VT && "Unexpected value types"); 964 return N; 965 } 966 967 bool SystemZDAGToDAGISel::tryRISBGZero(SDNode *N) { 968 SDLoc DL(N); 969 EVT VT = N->getValueType(0); 970 if (!VT.isInteger() || VT.getSizeInBits() > 64) 971 return false; 972 RxSBGOperands RISBG(SystemZ::RISBG, SDValue(N, 0)); 973 unsigned Count = 0; 974 while (expandRxSBG(RISBG)) 975 // The widening or narrowing is expected to be free. 976 // Counting widening or narrowing as a saved operation will result in 977 // preferring an R*SBG over a simple shift/logical instruction. 978 if (RISBG.Input.getOpcode() != ISD::ANY_EXTEND && 979 RISBG.Input.getOpcode() != ISD::TRUNCATE) 980 Count += 1; 981 if (Count == 0 || isa<ConstantSDNode>(RISBG.Input)) 982 return false; 983 984 // Prefer to use normal shift instructions over RISBG, since they can handle 985 // all cases and are sometimes shorter. 986 if (Count == 1 && N->getOpcode() != ISD::AND) 987 return false; 988 989 // Prefer register extensions like LLC over RISBG. Also prefer to start 990 // out with normal ANDs if one instruction would be enough. We can convert 991 // these ANDs into an RISBG later if a three-address instruction is useful. 992 if (RISBG.Rotate == 0) { 993 bool PreferAnd = false; 994 // Prefer AND for any 32-bit and-immediate operation. 995 if (VT == MVT::i32) 996 PreferAnd = true; 997 // As well as for any 64-bit operation that can be implemented via LLC(R), 998 // LLH(R), LLGT(R), or one of the and-immediate instructions. 999 else if (RISBG.Mask == 0xff || 1000 RISBG.Mask == 0xffff || 1001 RISBG.Mask == 0x7fffffff || 1002 SystemZ::isImmLF(~RISBG.Mask) || 1003 SystemZ::isImmHF(~RISBG.Mask)) 1004 PreferAnd = true; 1005 // And likewise for the LLZRGF instruction, which doesn't have a register 1006 // to register version. 1007 else if (auto *Load = dyn_cast<LoadSDNode>(RISBG.Input)) { 1008 if (Load->getMemoryVT() == MVT::i32 && 1009 (Load->getExtensionType() == ISD::EXTLOAD || 1010 Load->getExtensionType() == ISD::ZEXTLOAD) && 1011 RISBG.Mask == 0xffffff00 && 1012 Subtarget->hasLoadAndZeroRightmostByte()) 1013 PreferAnd = true; 1014 } 1015 if (PreferAnd) { 1016 // Replace the current node with an AND. Note that the current node 1017 // might already be that same AND, in which case it is already CSE'd 1018 // with it, and we must not call ReplaceNode. 1019 SDValue In = convertTo(DL, VT, RISBG.Input); 1020 SDValue Mask = CurDAG->getConstant(RISBG.Mask, DL, VT); 1021 SDValue New = CurDAG->getNode(ISD::AND, DL, VT, In, Mask); 1022 if (N != New.getNode()) { 1023 insertDAGNode(CurDAG, N, Mask); 1024 insertDAGNode(CurDAG, N, New); 1025 ReplaceNode(N, New.getNode()); 1026 N = New.getNode(); 1027 } 1028 // Now, select the machine opcode to implement this operation. 1029 if (!N->isMachineOpcode()) 1030 SelectCode(N); 1031 return true; 1032 } 1033 } 1034 1035 unsigned Opcode = SystemZ::RISBG; 1036 // Prefer RISBGN if available, since it does not clobber CC. 1037 if (Subtarget->hasMiscellaneousExtensions()) 1038 Opcode = SystemZ::RISBGN; 1039 EVT OpcodeVT = MVT::i64; 1040 if (VT == MVT::i32 && Subtarget->hasHighWord() && 1041 // We can only use the 32-bit instructions if all source bits are 1042 // in the low 32 bits without wrapping, both after rotation (because 1043 // of the smaller range for Start and End) and before rotation 1044 // (because the input value is truncated). 1045 RISBG.Start >= 32 && RISBG.End >= RISBG.Start && 1046 ((RISBG.Start + RISBG.Rotate) & 63) >= 32 && 1047 ((RISBG.End + RISBG.Rotate) & 63) >= 1048 ((RISBG.Start + RISBG.Rotate) & 63)) { 1049 Opcode = SystemZ::RISBMux; 1050 OpcodeVT = MVT::i32; 1051 RISBG.Start &= 31; 1052 RISBG.End &= 31; 1053 } 1054 SDValue Ops[5] = { 1055 getUNDEF(DL, OpcodeVT), 1056 convertTo(DL, OpcodeVT, RISBG.Input), 1057 CurDAG->getTargetConstant(RISBG.Start, DL, MVT::i32), 1058 CurDAG->getTargetConstant(RISBG.End | 128, DL, MVT::i32), 1059 CurDAG->getTargetConstant(RISBG.Rotate, DL, MVT::i32) 1060 }; 1061 SDValue New = convertTo( 1062 DL, VT, SDValue(CurDAG->getMachineNode(Opcode, DL, OpcodeVT, Ops), 0)); 1063 ReplaceNode(N, New.getNode()); 1064 return true; 1065 } 1066 1067 bool SystemZDAGToDAGISel::tryRxSBG(SDNode *N, unsigned Opcode) { 1068 SDLoc DL(N); 1069 EVT VT = N->getValueType(0); 1070 if (!VT.isInteger() || VT.getSizeInBits() > 64) 1071 return false; 1072 // Try treating each operand of N as the second operand of the RxSBG 1073 // and see which goes deepest. 1074 RxSBGOperands RxSBG[] = { 1075 RxSBGOperands(Opcode, N->getOperand(0)), 1076 RxSBGOperands(Opcode, N->getOperand(1)) 1077 }; 1078 unsigned Count[] = { 0, 0 }; 1079 for (unsigned I = 0; I < 2; ++I) 1080 while (RxSBG[I].Input->hasOneUse() && expandRxSBG(RxSBG[I])) 1081 // In cases of multiple users it seems better to keep the simple 1082 // instruction as they are one cycle faster, and it also helps in cases 1083 // where both inputs share a common node. 1084 // The widening or narrowing is expected to be free. Counting widening 1085 // or narrowing as a saved operation will result in preferring an R*SBG 1086 // over a simple shift/logical instruction. 1087 if (RxSBG[I].Input.getOpcode() != ISD::ANY_EXTEND && 1088 RxSBG[I].Input.getOpcode() != ISD::TRUNCATE) 1089 Count[I] += 1; 1090 1091 // Do nothing if neither operand is suitable. 1092 if (Count[0] == 0 && Count[1] == 0) 1093 return false; 1094 1095 // Pick the deepest second operand. 1096 unsigned I = Count[0] > Count[1] ? 0 : 1; 1097 SDValue Op0 = N->getOperand(I ^ 1); 1098 1099 // Prefer IC for character insertions from memory. 1100 if (Opcode == SystemZ::ROSBG && (RxSBG[I].Mask & 0xff) == 0) 1101 if (auto *Load = dyn_cast<LoadSDNode>(Op0.getNode())) 1102 if (Load->getMemoryVT() == MVT::i8) 1103 return false; 1104 1105 // See whether we can avoid an AND in the first operand by converting 1106 // ROSBG to RISBG. 1107 if (Opcode == SystemZ::ROSBG && detectOrAndInsertion(Op0, RxSBG[I].Mask)) { 1108 Opcode = SystemZ::RISBG; 1109 // Prefer RISBGN if available, since it does not clobber CC. 1110 if (Subtarget->hasMiscellaneousExtensions()) 1111 Opcode = SystemZ::RISBGN; 1112 } 1113 1114 SDValue Ops[5] = { 1115 convertTo(DL, MVT::i64, Op0), 1116 convertTo(DL, MVT::i64, RxSBG[I].Input), 1117 CurDAG->getTargetConstant(RxSBG[I].Start, DL, MVT::i32), 1118 CurDAG->getTargetConstant(RxSBG[I].End, DL, MVT::i32), 1119 CurDAG->getTargetConstant(RxSBG[I].Rotate, DL, MVT::i32) 1120 }; 1121 SDValue New = convertTo( 1122 DL, VT, SDValue(CurDAG->getMachineNode(Opcode, DL, MVT::i64, Ops), 0)); 1123 ReplaceNode(N, New.getNode()); 1124 return true; 1125 } 1126 1127 void SystemZDAGToDAGISel::splitLargeImmediate(unsigned Opcode, SDNode *Node, 1128 SDValue Op0, uint64_t UpperVal, 1129 uint64_t LowerVal) { 1130 EVT VT = Node->getValueType(0); 1131 SDLoc DL(Node); 1132 SDValue Upper = CurDAG->getConstant(UpperVal, DL, VT); 1133 if (Op0.getNode()) 1134 Upper = CurDAG->getNode(Opcode, DL, VT, Op0, Upper); 1135 1136 { 1137 // When we haven't passed in Op0, Upper will be a constant. In order to 1138 // prevent folding back to the large immediate in `Or = getNode(...)` we run 1139 // SelectCode first and end up with an opaque machine node. This means that 1140 // we need to use a handle to keep track of Upper in case it gets CSE'd by 1141 // SelectCode. 1142 // 1143 // Note that in the case where Op0 is passed in we could just call 1144 // SelectCode(Upper) later, along with the SelectCode(Or), and avoid needing 1145 // the handle at all, but it's fine to do it here. 1146 // 1147 // TODO: This is a pretty hacky way to do this. Can we do something that 1148 // doesn't require a two paragraph explanation? 1149 HandleSDNode Handle(Upper); 1150 SelectCode(Upper.getNode()); 1151 Upper = Handle.getValue(); 1152 } 1153 1154 SDValue Lower = CurDAG->getConstant(LowerVal, DL, VT); 1155 SDValue Or = CurDAG->getNode(Opcode, DL, VT, Upper, Lower); 1156 1157 ReplaceNode(Node, Or.getNode()); 1158 1159 SelectCode(Or.getNode()); 1160 } 1161 1162 void SystemZDAGToDAGISel::loadVectorConstant( 1163 const SystemZVectorConstantInfo &VCI, SDNode *Node) { 1164 assert((VCI.Opcode == SystemZISD::BYTE_MASK || 1165 VCI.Opcode == SystemZISD::REPLICATE || 1166 VCI.Opcode == SystemZISD::ROTATE_MASK) && 1167 "Bad opcode!"); 1168 assert(VCI.VecVT.getSizeInBits() == 128 && "Expected a vector type"); 1169 EVT VT = Node->getValueType(0); 1170 SDLoc DL(Node); 1171 SmallVector<SDValue, 2> Ops; 1172 for (unsigned OpVal : VCI.OpVals) 1173 Ops.push_back(CurDAG->getTargetConstant(OpVal, DL, MVT::i32)); 1174 SDValue Op = CurDAG->getNode(VCI.Opcode, DL, VCI.VecVT, Ops); 1175 1176 if (VCI.VecVT == VT.getSimpleVT()) 1177 ReplaceNode(Node, Op.getNode()); 1178 else if (VT.getSizeInBits() == 128) { 1179 SDValue BitCast = CurDAG->getNode(ISD::BITCAST, DL, VT, Op); 1180 ReplaceNode(Node, BitCast.getNode()); 1181 SelectCode(BitCast.getNode()); 1182 } else { // float or double 1183 unsigned SubRegIdx = 1184 (VT.getSizeInBits() == 32 ? SystemZ::subreg_h32 : SystemZ::subreg_h64); 1185 ReplaceNode( 1186 Node, CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, Op).getNode()); 1187 } 1188 SelectCode(Op.getNode()); 1189 } 1190 1191 SDNode *SystemZDAGToDAGISel::loadPoolVectorConstant(APInt Val, EVT VT, SDLoc DL) { 1192 SDNode *ResNode; 1193 assert (VT.getSizeInBits() == 128); 1194 1195 SDValue CP = CurDAG->getTargetConstantPool( 1196 ConstantInt::get(Type::getInt128Ty(*CurDAG->getContext()), Val), 1197 TLI->getPointerTy(CurDAG->getDataLayout())); 1198 1199 EVT PtrVT = CP.getValueType(); 1200 SDValue Ops[] = { 1201 SDValue(CurDAG->getMachineNode(SystemZ::LARL, DL, PtrVT, CP), 0), 1202 CurDAG->getTargetConstant(0, DL, PtrVT), 1203 CurDAG->getRegister(0, PtrVT), 1204 CurDAG->getEntryNode() 1205 }; 1206 ResNode = CurDAG->getMachineNode(SystemZ::VL, DL, VT, MVT::Other, Ops); 1207 1208 // Annotate ResNode with memory operand information so that MachineInstr 1209 // queries work properly. This e.g. gives the register allocation the 1210 // required information for rematerialization. 1211 MachineFunction& MF = CurDAG->getMachineFunction(); 1212 MachineMemOperand *MemOp = 1213 MF.getMachineMemOperand(MachinePointerInfo::getConstantPool(MF), 1214 MachineMemOperand::MOLoad, 16, Align(8)); 1215 1216 CurDAG->setNodeMemRefs(cast<MachineSDNode>(ResNode), {MemOp}); 1217 return ResNode; 1218 } 1219 1220 bool SystemZDAGToDAGISel::tryGather(SDNode *N, unsigned Opcode) { 1221 SDValue ElemV = N->getOperand(2); 1222 auto *ElemN = dyn_cast<ConstantSDNode>(ElemV); 1223 if (!ElemN) 1224 return false; 1225 1226 unsigned Elem = ElemN->getZExtValue(); 1227 EVT VT = N->getValueType(0); 1228 if (Elem >= VT.getVectorNumElements()) 1229 return false; 1230 1231 auto *Load = dyn_cast<LoadSDNode>(N->getOperand(1)); 1232 if (!Load || !Load->hasNUsesOfValue(1, 0)) 1233 return false; 1234 if (Load->getMemoryVT().getSizeInBits() != 1235 Load->getValueType(0).getSizeInBits()) 1236 return false; 1237 1238 SDValue Base, Disp, Index; 1239 if (!selectBDVAddr12Only(Load->getBasePtr(), ElemV, Base, Disp, Index) || 1240 Index.getValueType() != VT.changeVectorElementTypeToInteger()) 1241 return false; 1242 1243 SDLoc DL(Load); 1244 SDValue Ops[] = { 1245 N->getOperand(0), Base, Disp, Index, 1246 CurDAG->getTargetConstant(Elem, DL, MVT::i32), Load->getChain() 1247 }; 1248 SDNode *Res = CurDAG->getMachineNode(Opcode, DL, VT, MVT::Other, Ops); 1249 ReplaceUses(SDValue(Load, 1), SDValue(Res, 1)); 1250 ReplaceNode(N, Res); 1251 return true; 1252 } 1253 1254 bool SystemZDAGToDAGISel::tryScatter(StoreSDNode *Store, unsigned Opcode) { 1255 SDValue Value = Store->getValue(); 1256 if (Value.getOpcode() != ISD::EXTRACT_VECTOR_ELT) 1257 return false; 1258 if (Store->getMemoryVT().getSizeInBits() != Value.getValueSizeInBits()) 1259 return false; 1260 1261 SDValue ElemV = Value.getOperand(1); 1262 auto *ElemN = dyn_cast<ConstantSDNode>(ElemV); 1263 if (!ElemN) 1264 return false; 1265 1266 SDValue Vec = Value.getOperand(0); 1267 EVT VT = Vec.getValueType(); 1268 unsigned Elem = ElemN->getZExtValue(); 1269 if (Elem >= VT.getVectorNumElements()) 1270 return false; 1271 1272 SDValue Base, Disp, Index; 1273 if (!selectBDVAddr12Only(Store->getBasePtr(), ElemV, Base, Disp, Index) || 1274 Index.getValueType() != VT.changeVectorElementTypeToInteger()) 1275 return false; 1276 1277 SDLoc DL(Store); 1278 SDValue Ops[] = { 1279 Vec, Base, Disp, Index, CurDAG->getTargetConstant(Elem, DL, MVT::i32), 1280 Store->getChain() 1281 }; 1282 ReplaceNode(Store, CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops)); 1283 return true; 1284 } 1285 1286 // Check whether or not the chain ending in StoreNode is suitable for doing 1287 // the {load; op; store} to modify transformation. 1288 static bool isFusableLoadOpStorePattern(StoreSDNode *StoreNode, 1289 SDValue StoredVal, SelectionDAG *CurDAG, 1290 LoadSDNode *&LoadNode, 1291 SDValue &InputChain) { 1292 // Is the stored value result 0 of the operation? 1293 if (StoredVal.getResNo() != 0) 1294 return false; 1295 1296 // Are there other uses of the loaded value than the operation? 1297 if (!StoredVal.getNode()->hasNUsesOfValue(1, 0)) 1298 return false; 1299 1300 // Is the store non-extending and non-indexed? 1301 if (!ISD::isNormalStore(StoreNode) || StoreNode->isNonTemporal()) 1302 return false; 1303 1304 SDValue Load = StoredVal->getOperand(0); 1305 // Is the stored value a non-extending and non-indexed load? 1306 if (!ISD::isNormalLoad(Load.getNode())) 1307 return false; 1308 1309 // Return LoadNode by reference. 1310 LoadNode = cast<LoadSDNode>(Load); 1311 1312 // Is store the only read of the loaded value? 1313 if (!Load.hasOneUse()) 1314 return false; 1315 1316 // Is the address of the store the same as the load? 1317 if (LoadNode->getBasePtr() != StoreNode->getBasePtr() || 1318 LoadNode->getOffset() != StoreNode->getOffset()) 1319 return false; 1320 1321 // Check if the chain is produced by the load or is a TokenFactor with 1322 // the load output chain as an operand. Return InputChain by reference. 1323 SDValue Chain = StoreNode->getChain(); 1324 1325 bool ChainCheck = false; 1326 if (Chain == Load.getValue(1)) { 1327 ChainCheck = true; 1328 InputChain = LoadNode->getChain(); 1329 } else if (Chain.getOpcode() == ISD::TokenFactor) { 1330 SmallVector<SDValue, 4> ChainOps; 1331 SmallVector<const SDNode *, 4> LoopWorklist; 1332 SmallPtrSet<const SDNode *, 16> Visited; 1333 const unsigned int Max = 1024; 1334 for (unsigned i = 0, e = Chain.getNumOperands(); i != e; ++i) { 1335 SDValue Op = Chain.getOperand(i); 1336 if (Op == Load.getValue(1)) { 1337 ChainCheck = true; 1338 // Drop Load, but keep its chain. No cycle check necessary. 1339 ChainOps.push_back(Load.getOperand(0)); 1340 continue; 1341 } 1342 LoopWorklist.push_back(Op.getNode()); 1343 ChainOps.push_back(Op); 1344 } 1345 1346 if (ChainCheck) { 1347 // Add the other operand of StoredVal to worklist. 1348 for (SDValue Op : StoredVal->ops()) 1349 if (Op.getNode() != LoadNode) 1350 LoopWorklist.push_back(Op.getNode()); 1351 1352 // Check if Load is reachable from any of the nodes in the worklist. 1353 if (SDNode::hasPredecessorHelper(Load.getNode(), Visited, LoopWorklist, Max, 1354 true)) 1355 return false; 1356 1357 // Make a new TokenFactor with all the other input chains except 1358 // for the load. 1359 InputChain = CurDAG->getNode(ISD::TokenFactor, SDLoc(Chain), 1360 MVT::Other, ChainOps); 1361 } 1362 } 1363 if (!ChainCheck) 1364 return false; 1365 1366 return true; 1367 } 1368 1369 // Change a chain of {load; op; store} of the same value into a simple op 1370 // through memory of that value, if the uses of the modified value and its 1371 // address are suitable. 1372 // 1373 // The tablegen pattern memory operand pattern is currently not able to match 1374 // the case where the CC on the original operation are used. 1375 // 1376 // See the equivalent routine in X86ISelDAGToDAG for further comments. 1377 bool SystemZDAGToDAGISel::tryFoldLoadStoreIntoMemOperand(SDNode *Node) { 1378 StoreSDNode *StoreNode = cast<StoreSDNode>(Node); 1379 SDValue StoredVal = StoreNode->getOperand(1); 1380 unsigned Opc = StoredVal->getOpcode(); 1381 SDLoc DL(StoreNode); 1382 1383 // Before we try to select anything, make sure this is memory operand size 1384 // and opcode we can handle. Note that this must match the code below that 1385 // actually lowers the opcodes. 1386 EVT MemVT = StoreNode->getMemoryVT(); 1387 unsigned NewOpc = 0; 1388 bool NegateOperand = false; 1389 switch (Opc) { 1390 default: 1391 return false; 1392 case SystemZISD::SSUBO: 1393 NegateOperand = true; 1394 [[fallthrough]]; 1395 case SystemZISD::SADDO: 1396 if (MemVT == MVT::i32) 1397 NewOpc = SystemZ::ASI; 1398 else if (MemVT == MVT::i64) 1399 NewOpc = SystemZ::AGSI; 1400 else 1401 return false; 1402 break; 1403 case SystemZISD::USUBO: 1404 NegateOperand = true; 1405 [[fallthrough]]; 1406 case SystemZISD::UADDO: 1407 if (MemVT == MVT::i32) 1408 NewOpc = SystemZ::ALSI; 1409 else if (MemVT == MVT::i64) 1410 NewOpc = SystemZ::ALGSI; 1411 else 1412 return false; 1413 break; 1414 } 1415 1416 LoadSDNode *LoadNode = nullptr; 1417 SDValue InputChain; 1418 if (!isFusableLoadOpStorePattern(StoreNode, StoredVal, CurDAG, LoadNode, 1419 InputChain)) 1420 return false; 1421 1422 SDValue Operand = StoredVal.getOperand(1); 1423 auto *OperandC = dyn_cast<ConstantSDNode>(Operand); 1424 if (!OperandC) 1425 return false; 1426 auto OperandV = OperandC->getAPIntValue(); 1427 if (NegateOperand) 1428 OperandV = -OperandV; 1429 if (OperandV.getSignificantBits() > 8) 1430 return false; 1431 Operand = CurDAG->getTargetConstant(OperandV, DL, MemVT); 1432 1433 SDValue Base, Disp; 1434 if (!selectBDAddr20Only(StoreNode->getBasePtr(), Base, Disp)) 1435 return false; 1436 1437 SDValue Ops[] = { Base, Disp, Operand, InputChain }; 1438 MachineSDNode *Result = 1439 CurDAG->getMachineNode(NewOpc, DL, MVT::i32, MVT::Other, Ops); 1440 CurDAG->setNodeMemRefs( 1441 Result, {StoreNode->getMemOperand(), LoadNode->getMemOperand()}); 1442 1443 ReplaceUses(SDValue(StoreNode, 0), SDValue(Result, 1)); 1444 ReplaceUses(SDValue(StoredVal.getNode(), 1), SDValue(Result, 0)); 1445 CurDAG->RemoveDeadNode(Node); 1446 return true; 1447 } 1448 1449 bool SystemZDAGToDAGISel::canUseBlockOperation(StoreSDNode *Store, 1450 LoadSDNode *Load) const { 1451 // Check that the two memory operands have the same size. 1452 if (Load->getMemoryVT() != Store->getMemoryVT()) 1453 return false; 1454 1455 // Volatility stops an access from being decomposed. 1456 if (Load->isVolatile() || Store->isVolatile()) 1457 return false; 1458 1459 // There's no chance of overlap if the load is invariant. 1460 if (Load->isInvariant() && Load->isDereferenceable()) 1461 return true; 1462 1463 // Otherwise we need to check whether there's an alias. 1464 const Value *V1 = Load->getMemOperand()->getValue(); 1465 const Value *V2 = Store->getMemOperand()->getValue(); 1466 if (!V1 || !V2) 1467 return false; 1468 1469 // Reject equality. 1470 uint64_t Size = Load->getMemoryVT().getStoreSize(); 1471 int64_t End1 = Load->getSrcValueOffset() + Size; 1472 int64_t End2 = Store->getSrcValueOffset() + Size; 1473 if (V1 == V2 && End1 == End2) 1474 return false; 1475 1476 return AA->isNoAlias(MemoryLocation(V1, End1, Load->getAAInfo()), 1477 MemoryLocation(V2, End2, Store->getAAInfo())); 1478 } 1479 1480 bool SystemZDAGToDAGISel::storeLoadCanUseMVC(SDNode *N) const { 1481 auto *Store = cast<StoreSDNode>(N); 1482 auto *Load = cast<LoadSDNode>(Store->getValue()); 1483 1484 // Prefer not to use MVC if either address can use ... RELATIVE LONG 1485 // instructions. 1486 uint64_t Size = Load->getMemoryVT().getStoreSize(); 1487 if (Size > 1 && Size <= 8) { 1488 // Prefer LHRL, LRL and LGRL. 1489 if (SystemZISD::isPCREL(Load->getBasePtr().getOpcode())) 1490 return false; 1491 // Prefer STHRL, STRL and STGRL. 1492 if (SystemZISD::isPCREL(Store->getBasePtr().getOpcode())) 1493 return false; 1494 } 1495 1496 return canUseBlockOperation(Store, Load); 1497 } 1498 1499 bool SystemZDAGToDAGISel::storeLoadCanUseBlockBinary(SDNode *N, 1500 unsigned I) const { 1501 auto *StoreA = cast<StoreSDNode>(N); 1502 auto *LoadA = cast<LoadSDNode>(StoreA->getValue().getOperand(1 - I)); 1503 auto *LoadB = cast<LoadSDNode>(StoreA->getValue().getOperand(I)); 1504 return !LoadA->isVolatile() && LoadA->getMemoryVT() == LoadB->getMemoryVT() && 1505 canUseBlockOperation(StoreA, LoadB); 1506 } 1507 1508 bool SystemZDAGToDAGISel::storeLoadIsAligned(SDNode *N) const { 1509 1510 auto *MemAccess = cast<LSBaseSDNode>(N); 1511 TypeSize StoreSize = MemAccess->getMemoryVT().getStoreSize(); 1512 SDValue BasePtr = MemAccess->getBasePtr(); 1513 MachineMemOperand *MMO = MemAccess->getMemOperand(); 1514 assert(MMO && "Expected a memory operand."); 1515 1516 // The memory access must have a proper alignment and no index register. 1517 if (MemAccess->getAlign().value() < StoreSize || 1518 !MemAccess->getOffset().isUndef()) 1519 return false; 1520 1521 // The MMO must not have an unaligned offset. 1522 if (MMO->getOffset() % StoreSize != 0) 1523 return false; 1524 1525 // An access to GOT or the Constant Pool is aligned. 1526 if (const PseudoSourceValue *PSV = MMO->getPseudoValue()) 1527 if ((PSV->isGOT() || PSV->isConstantPool())) 1528 return true; 1529 1530 // Check the alignment of a Global Address. 1531 if (BasePtr.getNumOperands()) 1532 if (GlobalAddressSDNode *GA = 1533 dyn_cast<GlobalAddressSDNode>(BasePtr.getOperand(0))) { 1534 // The immediate offset must be aligned. 1535 if (GA->getOffset() % StoreSize != 0) 1536 return false; 1537 1538 // The alignment of the symbol itself must be at least the store size. 1539 const GlobalValue *GV = GA->getGlobal(); 1540 const DataLayout &DL = GV->getParent()->getDataLayout(); 1541 if (GV->getPointerAlignment(DL).value() < StoreSize) 1542 return false; 1543 } 1544 1545 return true; 1546 } 1547 1548 void SystemZDAGToDAGISel::Select(SDNode *Node) { 1549 // If we have a custom node, we already have selected! 1550 if (Node->isMachineOpcode()) { 1551 LLVM_DEBUG(errs() << "== "; Node->dump(CurDAG); errs() << "\n"); 1552 Node->setNodeId(-1); 1553 return; 1554 } 1555 1556 unsigned Opcode = Node->getOpcode(); 1557 switch (Opcode) { 1558 case ISD::OR: 1559 if (Node->getOperand(1).getOpcode() != ISD::Constant) 1560 if (tryRxSBG(Node, SystemZ::ROSBG)) 1561 return; 1562 goto or_xor; 1563 1564 case ISD::XOR: 1565 if (Node->getOperand(1).getOpcode() != ISD::Constant) 1566 if (tryRxSBG(Node, SystemZ::RXSBG)) 1567 return; 1568 // Fall through. 1569 or_xor: 1570 // If this is a 64-bit operation in which both 32-bit halves are nonzero, 1571 // split the operation into two. If both operands here happen to be 1572 // constant, leave this to common code to optimize. 1573 if (Node->getValueType(0) == MVT::i64 && 1574 Node->getOperand(0).getOpcode() != ISD::Constant) 1575 if (auto *Op1 = dyn_cast<ConstantSDNode>(Node->getOperand(1))) { 1576 uint64_t Val = Op1->getZExtValue(); 1577 // Don't split the operation if we can match one of the combined 1578 // logical operations provided by miscellaneous-extensions-3. 1579 if (Subtarget->hasMiscellaneousExtensions3()) { 1580 unsigned ChildOpcode = Node->getOperand(0).getOpcode(); 1581 // Check whether this expression matches NAND/NOR/NXOR. 1582 if (Val == (uint64_t)-1 && Opcode == ISD::XOR) 1583 if (ChildOpcode == ISD::AND || ChildOpcode == ISD::OR || 1584 ChildOpcode == ISD::XOR) 1585 break; 1586 // Check whether this expression matches OR-with-complement 1587 // (or matches an alternate pattern for NXOR). 1588 if (ChildOpcode == ISD::XOR) { 1589 auto Op0 = Node->getOperand(0); 1590 if (auto *Op0Op1 = dyn_cast<ConstantSDNode>(Op0->getOperand(1))) 1591 if (Op0Op1->getZExtValue() == (uint64_t)-1) 1592 break; 1593 } 1594 } 1595 // Don't split an XOR with -1 as LCGR/AGHI is more compact. 1596 if (Opcode == ISD::XOR && Op1->isAllOnes()) 1597 break; 1598 if (!SystemZ::isImmLF(Val) && !SystemZ::isImmHF(Val)) { 1599 splitLargeImmediate(Opcode, Node, Node->getOperand(0), 1600 Val - uint32_t(Val), uint32_t(Val)); 1601 return; 1602 } 1603 } 1604 break; 1605 1606 case ISD::AND: 1607 if (Node->getOperand(1).getOpcode() != ISD::Constant) 1608 if (tryRxSBG(Node, SystemZ::RNSBG)) 1609 return; 1610 [[fallthrough]]; 1611 case ISD::ROTL: 1612 case ISD::SHL: 1613 case ISD::SRL: 1614 case ISD::ZERO_EXTEND: 1615 if (tryRISBGZero(Node)) 1616 return; 1617 break; 1618 1619 case ISD::BSWAP: 1620 if (Node->getValueType(0) == MVT::i128) { 1621 SDLoc DL(Node); 1622 SDValue Src = Node->getOperand(0); 1623 Src = CurDAG->getNode(ISD::BITCAST, DL, MVT::v16i8, Src); 1624 1625 uint64_t Bytes[2] = { 0x0706050403020100ULL, 0x0f0e0d0c0b0a0908ULL }; 1626 SDNode *Mask = loadPoolVectorConstant(APInt(128, Bytes), MVT::v16i8, DL); 1627 SDValue Ops[] = { Src, Src, SDValue(Mask, 0) }; 1628 SDValue Res = SDValue(CurDAG->getMachineNode(SystemZ::VPERM, DL, 1629 MVT::v16i8, Ops), 0); 1630 1631 Res = CurDAG->getNode(ISD::BITCAST, DL, MVT::i128, Res); 1632 SDNode *ResNode = Res.getNode(); 1633 ReplaceNode(Node, ResNode); 1634 SelectCode(Src.getNode()); 1635 SelectCode(ResNode); 1636 return; 1637 } 1638 break; 1639 1640 case ISD::Constant: 1641 // If this is a 64-bit constant that is out of the range of LLILF, 1642 // LLIHF and LGFI, split it into two 32-bit pieces. 1643 if (Node->getValueType(0) == MVT::i64) { 1644 uint64_t Val = Node->getAsZExtVal(); 1645 if (!SystemZ::isImmLF(Val) && !SystemZ::isImmHF(Val) && !isInt<32>(Val)) { 1646 splitLargeImmediate(ISD::OR, Node, SDValue(), Val - uint32_t(Val), 1647 uint32_t(Val)); 1648 return; 1649 } 1650 } 1651 if (Node->getValueType(0) == MVT::i128) { 1652 const APInt &Val = Node->getAsAPIntVal(); 1653 SystemZVectorConstantInfo VCI(Val); 1654 if (VCI.isVectorConstantLegal(*Subtarget)) { 1655 loadVectorConstant(VCI, Node); 1656 return; 1657 } 1658 // If we can't materialize the constant we need to use a literal pool. 1659 SDNode *ResNode = loadPoolVectorConstant(Val, MVT::i128, SDLoc(Node)); 1660 ReplaceNode(Node, ResNode); 1661 return; 1662 } 1663 break; 1664 1665 case SystemZISD::SELECT_CCMASK: { 1666 SDValue Op0 = Node->getOperand(0); 1667 SDValue Op1 = Node->getOperand(1); 1668 // Prefer to put any load first, so that it can be matched as a 1669 // conditional load. Likewise for constants in range for LOCHI. 1670 if ((Op1.getOpcode() == ISD::LOAD && Op0.getOpcode() != ISD::LOAD) || 1671 (Subtarget->hasLoadStoreOnCond2() && 1672 Node->getValueType(0).isInteger() && 1673 Node->getValueType(0).getSizeInBits() <= 64 && 1674 Op1.getOpcode() == ISD::Constant && 1675 isInt<16>(cast<ConstantSDNode>(Op1)->getSExtValue()) && 1676 !(Op0.getOpcode() == ISD::Constant && 1677 isInt<16>(cast<ConstantSDNode>(Op0)->getSExtValue())))) { 1678 SDValue CCValid = Node->getOperand(2); 1679 SDValue CCMask = Node->getOperand(3); 1680 uint64_t ConstCCValid = CCValid.getNode()->getAsZExtVal(); 1681 uint64_t ConstCCMask = CCMask.getNode()->getAsZExtVal(); 1682 // Invert the condition. 1683 CCMask = CurDAG->getTargetConstant(ConstCCValid ^ ConstCCMask, 1684 SDLoc(Node), CCMask.getValueType()); 1685 SDValue Op4 = Node->getOperand(4); 1686 SDNode *UpdatedNode = 1687 CurDAG->UpdateNodeOperands(Node, Op1, Op0, CCValid, CCMask, Op4); 1688 if (UpdatedNode != Node) { 1689 // In case this node already exists then replace Node with it. 1690 ReplaceNode(Node, UpdatedNode); 1691 Node = UpdatedNode; 1692 } 1693 } 1694 break; 1695 } 1696 1697 case ISD::INSERT_VECTOR_ELT: { 1698 EVT VT = Node->getValueType(0); 1699 unsigned ElemBitSize = VT.getScalarSizeInBits(); 1700 if (ElemBitSize == 32) { 1701 if (tryGather(Node, SystemZ::VGEF)) 1702 return; 1703 } else if (ElemBitSize == 64) { 1704 if (tryGather(Node, SystemZ::VGEG)) 1705 return; 1706 } 1707 break; 1708 } 1709 1710 case ISD::BUILD_VECTOR: { 1711 auto *BVN = cast<BuildVectorSDNode>(Node); 1712 SystemZVectorConstantInfo VCI(BVN); 1713 if (VCI.isVectorConstantLegal(*Subtarget)) { 1714 loadVectorConstant(VCI, Node); 1715 return; 1716 } 1717 break; 1718 } 1719 1720 case ISD::ConstantFP: { 1721 APFloat Imm = cast<ConstantFPSDNode>(Node)->getValueAPF(); 1722 if (Imm.isZero() || Imm.isNegZero()) 1723 break; 1724 SystemZVectorConstantInfo VCI(Imm); 1725 bool Success = VCI.isVectorConstantLegal(*Subtarget); (void)Success; 1726 assert(Success && "Expected legal FP immediate"); 1727 loadVectorConstant(VCI, Node); 1728 return; 1729 } 1730 1731 case ISD::STORE: { 1732 if (tryFoldLoadStoreIntoMemOperand(Node)) 1733 return; 1734 auto *Store = cast<StoreSDNode>(Node); 1735 unsigned ElemBitSize = Store->getValue().getValueSizeInBits(); 1736 if (ElemBitSize == 32) { 1737 if (tryScatter(Store, SystemZ::VSCEF)) 1738 return; 1739 } else if (ElemBitSize == 64) { 1740 if (tryScatter(Store, SystemZ::VSCEG)) 1741 return; 1742 } 1743 break; 1744 } 1745 } 1746 1747 SelectCode(Node); 1748 } 1749 1750 bool SystemZDAGToDAGISel::SelectInlineAsmMemoryOperand( 1751 const SDValue &Op, InlineAsm::ConstraintCode ConstraintID, 1752 std::vector<SDValue> &OutOps) { 1753 SystemZAddressingMode::AddrForm Form; 1754 SystemZAddressingMode::DispRange DispRange; 1755 SDValue Base, Disp, Index; 1756 1757 switch(ConstraintID) { 1758 default: 1759 llvm_unreachable("Unexpected asm memory constraint"); 1760 case InlineAsm::ConstraintCode::i: 1761 case InlineAsm::ConstraintCode::Q: 1762 case InlineAsm::ConstraintCode::ZQ: 1763 // Accept an address with a short displacement, but no index. 1764 Form = SystemZAddressingMode::FormBD; 1765 DispRange = SystemZAddressingMode::Disp12Only; 1766 break; 1767 case InlineAsm::ConstraintCode::R: 1768 case InlineAsm::ConstraintCode::ZR: 1769 // Accept an address with a short displacement and an index. 1770 Form = SystemZAddressingMode::FormBDXNormal; 1771 DispRange = SystemZAddressingMode::Disp12Only; 1772 break; 1773 case InlineAsm::ConstraintCode::S: 1774 case InlineAsm::ConstraintCode::ZS: 1775 // Accept an address with a long displacement, but no index. 1776 Form = SystemZAddressingMode::FormBD; 1777 DispRange = SystemZAddressingMode::Disp20Only; 1778 break; 1779 case InlineAsm::ConstraintCode::T: 1780 case InlineAsm::ConstraintCode::m: 1781 case InlineAsm::ConstraintCode::o: 1782 case InlineAsm::ConstraintCode::p: 1783 case InlineAsm::ConstraintCode::ZT: 1784 // Accept an address with a long displacement and an index. 1785 // m works the same as T, as this is the most general case. 1786 // We don't really have any special handling of "offsettable" 1787 // memory addresses, so just treat o the same as m. 1788 Form = SystemZAddressingMode::FormBDXNormal; 1789 DispRange = SystemZAddressingMode::Disp20Only; 1790 break; 1791 } 1792 1793 if (selectBDXAddr(Form, DispRange, Op, Base, Disp, Index)) { 1794 const TargetRegisterClass *TRC = 1795 Subtarget->getRegisterInfo()->getPointerRegClass(*MF); 1796 SDLoc DL(Base); 1797 SDValue RC = CurDAG->getTargetConstant(TRC->getID(), DL, MVT::i32); 1798 1799 // Make sure that the base address doesn't go into %r0. 1800 // If it's a TargetFrameIndex or a fixed register, we shouldn't do anything. 1801 if (Base.getOpcode() != ISD::TargetFrameIndex && 1802 Base.getOpcode() != ISD::Register) { 1803 Base = 1804 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, 1805 DL, Base.getValueType(), 1806 Base, RC), 0); 1807 } 1808 1809 // Make sure that the index register isn't assigned to %r0 either. 1810 if (Index.getOpcode() != ISD::Register) { 1811 Index = 1812 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, 1813 DL, Index.getValueType(), 1814 Index, RC), 0); 1815 } 1816 1817 OutOps.push_back(Base); 1818 OutOps.push_back(Disp); 1819 OutOps.push_back(Index); 1820 return false; 1821 } 1822 1823 return true; 1824 } 1825 1826 // IsProfitableToFold - Returns true if is profitable to fold the specific 1827 // operand node N of U during instruction selection that starts at Root. 1828 bool 1829 SystemZDAGToDAGISel::IsProfitableToFold(SDValue N, SDNode *U, 1830 SDNode *Root) const { 1831 // We want to avoid folding a LOAD into an ICMP node if as a result 1832 // we would be forced to spill the condition code into a GPR. 1833 if (N.getOpcode() == ISD::LOAD && U->getOpcode() == SystemZISD::ICMP) { 1834 if (!N.hasOneUse() || !U->hasOneUse()) 1835 return false; 1836 1837 // The user of the CC value will usually be a CopyToReg into the 1838 // physical CC register, which in turn is glued and chained to the 1839 // actual instruction that uses the CC value. Bail out if we have 1840 // anything else than that. 1841 SDNode *CCUser = *U->use_begin(); 1842 SDNode *CCRegUser = nullptr; 1843 if (CCUser->getOpcode() == ISD::CopyToReg || 1844 cast<RegisterSDNode>(CCUser->getOperand(1))->getReg() == SystemZ::CC) { 1845 for (auto *U : CCUser->uses()) { 1846 if (CCRegUser == nullptr) 1847 CCRegUser = U; 1848 else if (CCRegUser != U) 1849 return false; 1850 } 1851 } 1852 if (CCRegUser == nullptr) 1853 return false; 1854 1855 // If the actual instruction is a branch, the only thing that remains to be 1856 // checked is whether the CCUser chain is a predecessor of the load. 1857 if (CCRegUser->isMachineOpcode() && 1858 CCRegUser->getMachineOpcode() == SystemZ::BRC) 1859 return !N->isPredecessorOf(CCUser->getOperand(0).getNode()); 1860 1861 // Otherwise, the instruction may have multiple operands, and we need to 1862 // verify that none of them are a predecessor of the load. This is exactly 1863 // the same check that would be done by common code if the CC setter were 1864 // glued to the CC user, so simply invoke that check here. 1865 if (!IsLegalToFold(N, U, CCRegUser, OptLevel, false)) 1866 return false; 1867 } 1868 1869 return true; 1870 } 1871 1872 namespace { 1873 // Represents a sequence for extracting a 0/1 value from an IPM result: 1874 // (((X ^ XORValue) + AddValue) >> Bit) 1875 struct IPMConversion { 1876 IPMConversion(unsigned xorValue, int64_t addValue, unsigned bit) 1877 : XORValue(xorValue), AddValue(addValue), Bit(bit) {} 1878 1879 int64_t XORValue; 1880 int64_t AddValue; 1881 unsigned Bit; 1882 }; 1883 } // end anonymous namespace 1884 1885 // Return a sequence for getting a 1 from an IPM result when CC has a 1886 // value in CCMask and a 0 when CC has a value in CCValid & ~CCMask. 1887 // The handling of CC values outside CCValid doesn't matter. 1888 static IPMConversion getIPMConversion(unsigned CCValid, unsigned CCMask) { 1889 // Deal with cases where the result can be taken directly from a bit 1890 // of the IPM result. 1891 if (CCMask == (CCValid & (SystemZ::CCMASK_1 | SystemZ::CCMASK_3))) 1892 return IPMConversion(0, 0, SystemZ::IPM_CC); 1893 if (CCMask == (CCValid & (SystemZ::CCMASK_2 | SystemZ::CCMASK_3))) 1894 return IPMConversion(0, 0, SystemZ::IPM_CC + 1); 1895 1896 // Deal with cases where we can add a value to force the sign bit 1897 // to contain the right value. Putting the bit in 31 means we can 1898 // use SRL rather than RISBG(L), and also makes it easier to get a 1899 // 0/-1 value, so it has priority over the other tests below. 1900 // 1901 // These sequences rely on the fact that the upper two bits of the 1902 // IPM result are zero. 1903 uint64_t TopBit = uint64_t(1) << 31; 1904 if (CCMask == (CCValid & SystemZ::CCMASK_0)) 1905 return IPMConversion(0, -(1 << SystemZ::IPM_CC), 31); 1906 if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_1))) 1907 return IPMConversion(0, -(2 << SystemZ::IPM_CC), 31); 1908 if (CCMask == (CCValid & (SystemZ::CCMASK_0 1909 | SystemZ::CCMASK_1 1910 | SystemZ::CCMASK_2))) 1911 return IPMConversion(0, -(3 << SystemZ::IPM_CC), 31); 1912 if (CCMask == (CCValid & SystemZ::CCMASK_3)) 1913 return IPMConversion(0, TopBit - (3 << SystemZ::IPM_CC), 31); 1914 if (CCMask == (CCValid & (SystemZ::CCMASK_1 1915 | SystemZ::CCMASK_2 1916 | SystemZ::CCMASK_3))) 1917 return IPMConversion(0, TopBit - (1 << SystemZ::IPM_CC), 31); 1918 1919 // Next try inverting the value and testing a bit. 0/1 could be 1920 // handled this way too, but we dealt with that case above. 1921 if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_2))) 1922 return IPMConversion(-1, 0, SystemZ::IPM_CC); 1923 1924 // Handle cases where adding a value forces a non-sign bit to contain 1925 // the right value. 1926 if (CCMask == (CCValid & (SystemZ::CCMASK_1 | SystemZ::CCMASK_2))) 1927 return IPMConversion(0, 1 << SystemZ::IPM_CC, SystemZ::IPM_CC + 1); 1928 if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_3))) 1929 return IPMConversion(0, -(1 << SystemZ::IPM_CC), SystemZ::IPM_CC + 1); 1930 1931 // The remaining cases are 1, 2, 0/1/3 and 0/2/3. All these are 1932 // can be done by inverting the low CC bit and applying one of the 1933 // sign-based extractions above. 1934 if (CCMask == (CCValid & SystemZ::CCMASK_1)) 1935 return IPMConversion(1 << SystemZ::IPM_CC, -(1 << SystemZ::IPM_CC), 31); 1936 if (CCMask == (CCValid & SystemZ::CCMASK_2)) 1937 return IPMConversion(1 << SystemZ::IPM_CC, 1938 TopBit - (3 << SystemZ::IPM_CC), 31); 1939 if (CCMask == (CCValid & (SystemZ::CCMASK_0 1940 | SystemZ::CCMASK_1 1941 | SystemZ::CCMASK_3))) 1942 return IPMConversion(1 << SystemZ::IPM_CC, -(3 << SystemZ::IPM_CC), 31); 1943 if (CCMask == (CCValid & (SystemZ::CCMASK_0 1944 | SystemZ::CCMASK_2 1945 | SystemZ::CCMASK_3))) 1946 return IPMConversion(1 << SystemZ::IPM_CC, 1947 TopBit - (1 << SystemZ::IPM_CC), 31); 1948 1949 llvm_unreachable("Unexpected CC combination"); 1950 } 1951 1952 SDValue SystemZDAGToDAGISel::expandSelectBoolean(SDNode *Node) { 1953 auto *TrueOp = dyn_cast<ConstantSDNode>(Node->getOperand(0)); 1954 auto *FalseOp = dyn_cast<ConstantSDNode>(Node->getOperand(1)); 1955 if (!TrueOp || !FalseOp) 1956 return SDValue(); 1957 if (FalseOp->getZExtValue() != 0) 1958 return SDValue(); 1959 if (TrueOp->getSExtValue() != 1 && TrueOp->getSExtValue() != -1) 1960 return SDValue(); 1961 1962 auto *CCValidOp = dyn_cast<ConstantSDNode>(Node->getOperand(2)); 1963 auto *CCMaskOp = dyn_cast<ConstantSDNode>(Node->getOperand(3)); 1964 if (!CCValidOp || !CCMaskOp) 1965 return SDValue(); 1966 int CCValid = CCValidOp->getZExtValue(); 1967 int CCMask = CCMaskOp->getZExtValue(); 1968 1969 SDLoc DL(Node); 1970 SDValue CCReg = Node->getOperand(4); 1971 IPMConversion IPM = getIPMConversion(CCValid, CCMask); 1972 SDValue Result = CurDAG->getNode(SystemZISD::IPM, DL, MVT::i32, CCReg); 1973 1974 if (IPM.XORValue) 1975 Result = CurDAG->getNode(ISD::XOR, DL, MVT::i32, Result, 1976 CurDAG->getConstant(IPM.XORValue, DL, MVT::i32)); 1977 1978 if (IPM.AddValue) 1979 Result = CurDAG->getNode(ISD::ADD, DL, MVT::i32, Result, 1980 CurDAG->getConstant(IPM.AddValue, DL, MVT::i32)); 1981 1982 EVT VT = Node->getValueType(0); 1983 if (VT == MVT::i32 && IPM.Bit == 31) { 1984 unsigned ShiftOp = TrueOp->getSExtValue() == 1 ? ISD::SRL : ISD::SRA; 1985 Result = CurDAG->getNode(ShiftOp, DL, MVT::i32, Result, 1986 CurDAG->getConstant(IPM.Bit, DL, MVT::i32)); 1987 } else { 1988 if (VT != MVT::i32) 1989 Result = CurDAG->getNode(ISD::ANY_EXTEND, DL, VT, Result); 1990 1991 if (TrueOp->getSExtValue() == 1) { 1992 // The SHR/AND sequence should get optimized to an RISBG. 1993 Result = CurDAG->getNode(ISD::SRL, DL, VT, Result, 1994 CurDAG->getConstant(IPM.Bit, DL, MVT::i32)); 1995 Result = CurDAG->getNode(ISD::AND, DL, VT, Result, 1996 CurDAG->getConstant(1, DL, VT)); 1997 } else { 1998 // Sign-extend from IPM.Bit using a pair of shifts. 1999 int ShlAmt = VT.getSizeInBits() - 1 - IPM.Bit; 2000 int SraAmt = VT.getSizeInBits() - 1; 2001 Result = CurDAG->getNode(ISD::SHL, DL, VT, Result, 2002 CurDAG->getConstant(ShlAmt, DL, MVT::i32)); 2003 Result = CurDAG->getNode(ISD::SRA, DL, VT, Result, 2004 CurDAG->getConstant(SraAmt, DL, MVT::i32)); 2005 } 2006 } 2007 2008 return Result; 2009 } 2010 2011 void SystemZDAGToDAGISel::PreprocessISelDAG() { 2012 // If we have conditional immediate loads, we always prefer 2013 // using those over an IPM sequence. 2014 if (Subtarget->hasLoadStoreOnCond2()) 2015 return; 2016 2017 bool MadeChange = false; 2018 2019 for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(), 2020 E = CurDAG->allnodes_end(); 2021 I != E;) { 2022 SDNode *N = &*I++; 2023 if (N->use_empty()) 2024 continue; 2025 2026 SDValue Res; 2027 switch (N->getOpcode()) { 2028 default: break; 2029 case SystemZISD::SELECT_CCMASK: 2030 Res = expandSelectBoolean(N); 2031 break; 2032 } 2033 2034 if (Res) { 2035 LLVM_DEBUG(dbgs() << "SystemZ DAG preprocessing replacing:\nOld: "); 2036 LLVM_DEBUG(N->dump(CurDAG)); 2037 LLVM_DEBUG(dbgs() << "\nNew: "); 2038 LLVM_DEBUG(Res.getNode()->dump(CurDAG)); 2039 LLVM_DEBUG(dbgs() << "\n"); 2040 2041 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Res); 2042 MadeChange = true; 2043 } 2044 } 2045 2046 if (MadeChange) 2047 CurDAG->RemoveDeadNodes(); 2048 } 2049