1 //===-- SystemZISelDAGToDAG.cpp - A dag to dag inst selector for SystemZ --===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file defines an instruction selector for the SystemZ target. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "SystemZTargetMachine.h" 14 #include "SystemZISelLowering.h" 15 #include "llvm/Analysis/AliasAnalysis.h" 16 #include "llvm/CodeGen/SelectionDAGISel.h" 17 #include "llvm/Support/Debug.h" 18 #include "llvm/Support/KnownBits.h" 19 #include "llvm/Support/raw_ostream.h" 20 21 using namespace llvm; 22 23 #define DEBUG_TYPE "systemz-isel" 24 25 namespace { 26 // Used to build addressing modes. 27 struct SystemZAddressingMode { 28 // The shape of the address. 29 enum AddrForm { 30 // base+displacement 31 FormBD, 32 33 // base+displacement+index for load and store operands 34 FormBDXNormal, 35 36 // base+displacement+index for load address operands 37 FormBDXLA, 38 39 // base+displacement+index+ADJDYNALLOC 40 FormBDXDynAlloc 41 }; 42 AddrForm Form; 43 44 // The type of displacement. The enum names here correspond directly 45 // to the definitions in SystemZOperand.td. We could split them into 46 // flags -- single/pair, 128-bit, etc. -- but it hardly seems worth it. 47 enum DispRange { 48 Disp12Only, 49 Disp12Pair, 50 Disp20Only, 51 Disp20Only128, 52 Disp20Pair 53 }; 54 DispRange DR; 55 56 // The parts of the address. The address is equivalent to: 57 // 58 // Base + Disp + Index + (IncludesDynAlloc ? ADJDYNALLOC : 0) 59 SDValue Base; 60 int64_t Disp; 61 SDValue Index; 62 bool IncludesDynAlloc; 63 64 SystemZAddressingMode(AddrForm form, DispRange dr) 65 : Form(form), DR(dr), Base(), Disp(0), Index(), 66 IncludesDynAlloc(false) {} 67 68 // True if the address can have an index register. 69 bool hasIndexField() { return Form != FormBD; } 70 71 // True if the address can (and must) include ADJDYNALLOC. 72 bool isDynAlloc() { return Form == FormBDXDynAlloc; } 73 74 void dump(const llvm::SelectionDAG *DAG) { 75 errs() << "SystemZAddressingMode " << this << '\n'; 76 77 errs() << " Base "; 78 if (Base.getNode()) 79 Base.getNode()->dump(DAG); 80 else 81 errs() << "null\n"; 82 83 if (hasIndexField()) { 84 errs() << " Index "; 85 if (Index.getNode()) 86 Index.getNode()->dump(DAG); 87 else 88 errs() << "null\n"; 89 } 90 91 errs() << " Disp " << Disp; 92 if (IncludesDynAlloc) 93 errs() << " + ADJDYNALLOC"; 94 errs() << '\n'; 95 } 96 }; 97 98 // Return a mask with Count low bits set. 99 static uint64_t allOnes(unsigned int Count) { 100 assert(Count <= 64); 101 if (Count > 63) 102 return UINT64_MAX; 103 return (uint64_t(1) << Count) - 1; 104 } 105 106 // Represents operands 2 to 5 of the ROTATE AND ... SELECTED BITS operation 107 // given by Opcode. The operands are: Input (R2), Start (I3), End (I4) and 108 // Rotate (I5). The combined operand value is effectively: 109 // 110 // (or (rotl Input, Rotate), ~Mask) 111 // 112 // for RNSBG and: 113 // 114 // (and (rotl Input, Rotate), Mask) 115 // 116 // otherwise. The output value has BitSize bits, although Input may be 117 // narrower (in which case the upper bits are don't care), or wider (in which 118 // case the result will be truncated as part of the operation). 119 struct RxSBGOperands { 120 RxSBGOperands(unsigned Op, SDValue N) 121 : Opcode(Op), BitSize(N.getValueSizeInBits()), 122 Mask(allOnes(BitSize)), Input(N), Start(64 - BitSize), End(63), 123 Rotate(0) {} 124 125 unsigned Opcode; 126 unsigned BitSize; 127 uint64_t Mask; 128 SDValue Input; 129 unsigned Start; 130 unsigned End; 131 unsigned Rotate; 132 }; 133 134 class SystemZDAGToDAGISel : public SelectionDAGISel { 135 const SystemZSubtarget *Subtarget; 136 137 // Used by SystemZOperands.td to create integer constants. 138 inline SDValue getImm(const SDNode *Node, uint64_t Imm) const { 139 return CurDAG->getTargetConstant(Imm, SDLoc(Node), Node->getValueType(0)); 140 } 141 142 const SystemZTargetMachine &getTargetMachine() const { 143 return static_cast<const SystemZTargetMachine &>(TM); 144 } 145 146 const SystemZInstrInfo *getInstrInfo() const { 147 return Subtarget->getInstrInfo(); 148 } 149 150 // Try to fold more of the base or index of AM into AM, where IsBase 151 // selects between the base and index. 152 bool expandAddress(SystemZAddressingMode &AM, bool IsBase) const; 153 154 // Try to describe N in AM, returning true on success. 155 bool selectAddress(SDValue N, SystemZAddressingMode &AM) const; 156 157 // Extract individual target operands from matched address AM. 158 void getAddressOperands(const SystemZAddressingMode &AM, EVT VT, 159 SDValue &Base, SDValue &Disp) const; 160 void getAddressOperands(const SystemZAddressingMode &AM, EVT VT, 161 SDValue &Base, SDValue &Disp, SDValue &Index) const; 162 163 // Try to match Addr as a FormBD address with displacement type DR. 164 // Return true on success, storing the base and displacement in 165 // Base and Disp respectively. 166 bool selectBDAddr(SystemZAddressingMode::DispRange DR, SDValue Addr, 167 SDValue &Base, SDValue &Disp) const; 168 169 // Try to match Addr as a FormBDX address with displacement type DR. 170 // Return true on success and if the result had no index. Store the 171 // base and displacement in Base and Disp respectively. 172 bool selectMVIAddr(SystemZAddressingMode::DispRange DR, SDValue Addr, 173 SDValue &Base, SDValue &Disp) const; 174 175 // Try to match Addr as a FormBDX* address of form Form with 176 // displacement type DR. Return true on success, storing the base, 177 // displacement and index in Base, Disp and Index respectively. 178 bool selectBDXAddr(SystemZAddressingMode::AddrForm Form, 179 SystemZAddressingMode::DispRange DR, SDValue Addr, 180 SDValue &Base, SDValue &Disp, SDValue &Index) const; 181 182 // PC-relative address matching routines used by SystemZOperands.td. 183 bool selectPCRelAddress(SDValue Addr, SDValue &Target) const { 184 if (SystemZISD::isPCREL(Addr.getOpcode())) { 185 Target = Addr.getOperand(0); 186 return true; 187 } 188 return false; 189 } 190 191 // BD matching routines used by SystemZOperands.td. 192 bool selectBDAddr12Only(SDValue Addr, SDValue &Base, SDValue &Disp) const { 193 return selectBDAddr(SystemZAddressingMode::Disp12Only, Addr, Base, Disp); 194 } 195 bool selectBDAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const { 196 return selectBDAddr(SystemZAddressingMode::Disp12Pair, Addr, Base, Disp); 197 } 198 bool selectBDAddr20Only(SDValue Addr, SDValue &Base, SDValue &Disp) const { 199 return selectBDAddr(SystemZAddressingMode::Disp20Only, Addr, Base, Disp); 200 } 201 bool selectBDAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const { 202 return selectBDAddr(SystemZAddressingMode::Disp20Pair, Addr, Base, Disp); 203 } 204 205 // MVI matching routines used by SystemZOperands.td. 206 bool selectMVIAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const { 207 return selectMVIAddr(SystemZAddressingMode::Disp12Pair, Addr, Base, Disp); 208 } 209 bool selectMVIAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp) const { 210 return selectMVIAddr(SystemZAddressingMode::Disp20Pair, Addr, Base, Disp); 211 } 212 213 // BDX matching routines used by SystemZOperands.td. 214 bool selectBDXAddr12Only(SDValue Addr, SDValue &Base, SDValue &Disp, 215 SDValue &Index) const { 216 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal, 217 SystemZAddressingMode::Disp12Only, 218 Addr, Base, Disp, Index); 219 } 220 bool selectBDXAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp, 221 SDValue &Index) const { 222 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal, 223 SystemZAddressingMode::Disp12Pair, 224 Addr, Base, Disp, Index); 225 } 226 bool selectDynAlloc12Only(SDValue Addr, SDValue &Base, SDValue &Disp, 227 SDValue &Index) const { 228 return selectBDXAddr(SystemZAddressingMode::FormBDXDynAlloc, 229 SystemZAddressingMode::Disp12Only, 230 Addr, Base, Disp, Index); 231 } 232 bool selectBDXAddr20Only(SDValue Addr, SDValue &Base, SDValue &Disp, 233 SDValue &Index) const { 234 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal, 235 SystemZAddressingMode::Disp20Only, 236 Addr, Base, Disp, Index); 237 } 238 bool selectBDXAddr20Only128(SDValue Addr, SDValue &Base, SDValue &Disp, 239 SDValue &Index) const { 240 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal, 241 SystemZAddressingMode::Disp20Only128, 242 Addr, Base, Disp, Index); 243 } 244 bool selectBDXAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp, 245 SDValue &Index) const { 246 return selectBDXAddr(SystemZAddressingMode::FormBDXNormal, 247 SystemZAddressingMode::Disp20Pair, 248 Addr, Base, Disp, Index); 249 } 250 bool selectLAAddr12Pair(SDValue Addr, SDValue &Base, SDValue &Disp, 251 SDValue &Index) const { 252 return selectBDXAddr(SystemZAddressingMode::FormBDXLA, 253 SystemZAddressingMode::Disp12Pair, 254 Addr, Base, Disp, Index); 255 } 256 bool selectLAAddr20Pair(SDValue Addr, SDValue &Base, SDValue &Disp, 257 SDValue &Index) const { 258 return selectBDXAddr(SystemZAddressingMode::FormBDXLA, 259 SystemZAddressingMode::Disp20Pair, 260 Addr, Base, Disp, Index); 261 } 262 263 // Try to match Addr as an address with a base, 12-bit displacement 264 // and index, where the index is element Elem of a vector. 265 // Return true on success, storing the base, displacement and vector 266 // in Base, Disp and Index respectively. 267 bool selectBDVAddr12Only(SDValue Addr, SDValue Elem, SDValue &Base, 268 SDValue &Disp, SDValue &Index) const; 269 270 // Check whether (or Op (and X InsertMask)) is effectively an insertion 271 // of X into bits InsertMask of some Y != Op. Return true if so and 272 // set Op to that Y. 273 bool detectOrAndInsertion(SDValue &Op, uint64_t InsertMask) const; 274 275 // Try to update RxSBG so that only the bits of RxSBG.Input in Mask are used. 276 // Return true on success. 277 bool refineRxSBGMask(RxSBGOperands &RxSBG, uint64_t Mask) const; 278 279 // Try to fold some of RxSBG.Input into other fields of RxSBG. 280 // Return true on success. 281 bool expandRxSBG(RxSBGOperands &RxSBG) const; 282 283 // Return an undefined value of type VT. 284 SDValue getUNDEF(const SDLoc &DL, EVT VT) const; 285 286 // Convert N to VT, if it isn't already. 287 SDValue convertTo(const SDLoc &DL, EVT VT, SDValue N) const; 288 289 // Try to implement AND or shift node N using RISBG with the zero flag set. 290 // Return the selected node on success, otherwise return null. 291 bool tryRISBGZero(SDNode *N); 292 293 // Try to use RISBG or Opcode to implement OR or XOR node N. 294 // Return the selected node on success, otherwise return null. 295 bool tryRxSBG(SDNode *N, unsigned Opcode); 296 297 // If Op0 is null, then Node is a constant that can be loaded using: 298 // 299 // (Opcode UpperVal LowerVal) 300 // 301 // If Op0 is nonnull, then Node can be implemented using: 302 // 303 // (Opcode (Opcode Op0 UpperVal) LowerVal) 304 void splitLargeImmediate(unsigned Opcode, SDNode *Node, SDValue Op0, 305 uint64_t UpperVal, uint64_t LowerVal); 306 307 void loadVectorConstant(const SystemZVectorConstantInfo &VCI, 308 SDNode *Node); 309 310 // Try to use gather instruction Opcode to implement vector insertion N. 311 bool tryGather(SDNode *N, unsigned Opcode); 312 313 // Try to use scatter instruction Opcode to implement store Store. 314 bool tryScatter(StoreSDNode *Store, unsigned Opcode); 315 316 // Change a chain of {load; op; store} of the same value into a simple op 317 // through memory of that value, if the uses of the modified value and its 318 // address are suitable. 319 bool tryFoldLoadStoreIntoMemOperand(SDNode *Node); 320 321 // Return true if Load and Store are loads and stores of the same size 322 // and are guaranteed not to overlap. Such operations can be implemented 323 // using block (SS-format) instructions. 324 // 325 // Partial overlap would lead to incorrect code, since the block operations 326 // are logically bytewise, even though they have a fast path for the 327 // non-overlapping case. We also need to avoid full overlap (i.e. two 328 // addresses that might be equal at run time) because although that case 329 // would be handled correctly, it might be implemented by millicode. 330 bool canUseBlockOperation(StoreSDNode *Store, LoadSDNode *Load) const; 331 332 // N is a (store (load Y), X) pattern. Return true if it can use an MVC 333 // from Y to X. 334 bool storeLoadCanUseMVC(SDNode *N) const; 335 336 // N is a (store (op (load A[0]), (load A[1])), X) pattern. Return true 337 // if A[1 - I] == X and if N can use a block operation like NC from A[I] 338 // to X. 339 bool storeLoadCanUseBlockBinary(SDNode *N, unsigned I) const; 340 341 // Return true if N (a load or a store) fullfills the alignment 342 // requirements for a PC-relative access. 343 bool storeLoadIsAligned(SDNode *N) const; 344 345 // Try to expand a boolean SELECT_CCMASK using an IPM sequence. 346 SDValue expandSelectBoolean(SDNode *Node); 347 348 public: 349 SystemZDAGToDAGISel(SystemZTargetMachine &TM, CodeGenOpt::Level OptLevel) 350 : SelectionDAGISel(TM, OptLevel) {} 351 352 bool runOnMachineFunction(MachineFunction &MF) override { 353 const Function &F = MF.getFunction(); 354 if (F.getFnAttribute("fentry-call").getValueAsString() != "true") { 355 if (F.hasFnAttribute("mnop-mcount")) 356 report_fatal_error("mnop-mcount only supported with fentry-call"); 357 if (F.hasFnAttribute("mrecord-mcount")) 358 report_fatal_error("mrecord-mcount only supported with fentry-call"); 359 } 360 361 Subtarget = &MF.getSubtarget<SystemZSubtarget>(); 362 return SelectionDAGISel::runOnMachineFunction(MF); 363 } 364 365 // Override MachineFunctionPass. 366 StringRef getPassName() const override { 367 return "SystemZ DAG->DAG Pattern Instruction Selection"; 368 } 369 370 // Override SelectionDAGISel. 371 void Select(SDNode *Node) override; 372 bool SelectInlineAsmMemoryOperand(const SDValue &Op, unsigned ConstraintID, 373 std::vector<SDValue> &OutOps) override; 374 bool IsProfitableToFold(SDValue N, SDNode *U, SDNode *Root) const override; 375 void PreprocessISelDAG() override; 376 377 // Include the pieces autogenerated from the target description. 378 #include "SystemZGenDAGISel.inc" 379 }; 380 } // end anonymous namespace 381 382 FunctionPass *llvm::createSystemZISelDag(SystemZTargetMachine &TM, 383 CodeGenOpt::Level OptLevel) { 384 return new SystemZDAGToDAGISel(TM, OptLevel); 385 } 386 387 // Return true if Val should be selected as a displacement for an address 388 // with range DR. Here we're interested in the range of both the instruction 389 // described by DR and of any pairing instruction. 390 static bool selectDisp(SystemZAddressingMode::DispRange DR, int64_t Val) { 391 switch (DR) { 392 case SystemZAddressingMode::Disp12Only: 393 return isUInt<12>(Val); 394 395 case SystemZAddressingMode::Disp12Pair: 396 case SystemZAddressingMode::Disp20Only: 397 case SystemZAddressingMode::Disp20Pair: 398 return isInt<20>(Val); 399 400 case SystemZAddressingMode::Disp20Only128: 401 return isInt<20>(Val) && isInt<20>(Val + 8); 402 } 403 llvm_unreachable("Unhandled displacement range"); 404 } 405 406 // Change the base or index in AM to Value, where IsBase selects 407 // between the base and index. 408 static void changeComponent(SystemZAddressingMode &AM, bool IsBase, 409 SDValue Value) { 410 if (IsBase) 411 AM.Base = Value; 412 else 413 AM.Index = Value; 414 } 415 416 // The base or index of AM is equivalent to Value + ADJDYNALLOC, 417 // where IsBase selects between the base and index. Try to fold the 418 // ADJDYNALLOC into AM. 419 static bool expandAdjDynAlloc(SystemZAddressingMode &AM, bool IsBase, 420 SDValue Value) { 421 if (AM.isDynAlloc() && !AM.IncludesDynAlloc) { 422 changeComponent(AM, IsBase, Value); 423 AM.IncludesDynAlloc = true; 424 return true; 425 } 426 return false; 427 } 428 429 // The base of AM is equivalent to Base + Index. Try to use Index as 430 // the index register. 431 static bool expandIndex(SystemZAddressingMode &AM, SDValue Base, 432 SDValue Index) { 433 if (AM.hasIndexField() && !AM.Index.getNode()) { 434 AM.Base = Base; 435 AM.Index = Index; 436 return true; 437 } 438 return false; 439 } 440 441 // The base or index of AM is equivalent to Op0 + Op1, where IsBase selects 442 // between the base and index. Try to fold Op1 into AM's displacement. 443 static bool expandDisp(SystemZAddressingMode &AM, bool IsBase, 444 SDValue Op0, uint64_t Op1) { 445 // First try adjusting the displacement. 446 int64_t TestDisp = AM.Disp + Op1; 447 if (selectDisp(AM.DR, TestDisp)) { 448 changeComponent(AM, IsBase, Op0); 449 AM.Disp = TestDisp; 450 return true; 451 } 452 453 // We could consider forcing the displacement into a register and 454 // using it as an index, but it would need to be carefully tuned. 455 return false; 456 } 457 458 bool SystemZDAGToDAGISel::expandAddress(SystemZAddressingMode &AM, 459 bool IsBase) const { 460 SDValue N = IsBase ? AM.Base : AM.Index; 461 unsigned Opcode = N.getOpcode(); 462 if (Opcode == ISD::TRUNCATE) { 463 N = N.getOperand(0); 464 Opcode = N.getOpcode(); 465 } 466 if (Opcode == ISD::ADD || CurDAG->isBaseWithConstantOffset(N)) { 467 SDValue Op0 = N.getOperand(0); 468 SDValue Op1 = N.getOperand(1); 469 470 unsigned Op0Code = Op0->getOpcode(); 471 unsigned Op1Code = Op1->getOpcode(); 472 473 if (Op0Code == SystemZISD::ADJDYNALLOC) 474 return expandAdjDynAlloc(AM, IsBase, Op1); 475 if (Op1Code == SystemZISD::ADJDYNALLOC) 476 return expandAdjDynAlloc(AM, IsBase, Op0); 477 478 if (Op0Code == ISD::Constant) 479 return expandDisp(AM, IsBase, Op1, 480 cast<ConstantSDNode>(Op0)->getSExtValue()); 481 if (Op1Code == ISD::Constant) 482 return expandDisp(AM, IsBase, Op0, 483 cast<ConstantSDNode>(Op1)->getSExtValue()); 484 485 if (IsBase && expandIndex(AM, Op0, Op1)) 486 return true; 487 } 488 if (Opcode == SystemZISD::PCREL_OFFSET) { 489 SDValue Full = N.getOperand(0); 490 SDValue Base = N.getOperand(1); 491 SDValue Anchor = Base.getOperand(0); 492 uint64_t Offset = (cast<GlobalAddressSDNode>(Full)->getOffset() - 493 cast<GlobalAddressSDNode>(Anchor)->getOffset()); 494 return expandDisp(AM, IsBase, Base, Offset); 495 } 496 return false; 497 } 498 499 // Return true if an instruction with displacement range DR should be 500 // used for displacement value Val. selectDisp(DR, Val) must already hold. 501 static bool isValidDisp(SystemZAddressingMode::DispRange DR, int64_t Val) { 502 assert(selectDisp(DR, Val) && "Invalid displacement"); 503 switch (DR) { 504 case SystemZAddressingMode::Disp12Only: 505 case SystemZAddressingMode::Disp20Only: 506 case SystemZAddressingMode::Disp20Only128: 507 return true; 508 509 case SystemZAddressingMode::Disp12Pair: 510 // Use the other instruction if the displacement is too large. 511 return isUInt<12>(Val); 512 513 case SystemZAddressingMode::Disp20Pair: 514 // Use the other instruction if the displacement is small enough. 515 return !isUInt<12>(Val); 516 } 517 llvm_unreachable("Unhandled displacement range"); 518 } 519 520 // Return true if Base + Disp + Index should be performed by LA(Y). 521 static bool shouldUseLA(SDNode *Base, int64_t Disp, SDNode *Index) { 522 // Don't use LA(Y) for constants. 523 if (!Base) 524 return false; 525 526 // Always use LA(Y) for frame addresses, since we know that the destination 527 // register is almost always (perhaps always) going to be different from 528 // the frame register. 529 if (Base->getOpcode() == ISD::FrameIndex) 530 return true; 531 532 if (Disp) { 533 // Always use LA(Y) if there is a base, displacement and index. 534 if (Index) 535 return true; 536 537 // Always use LA if the displacement is small enough. It should always 538 // be no worse than AGHI (and better if it avoids a move). 539 if (isUInt<12>(Disp)) 540 return true; 541 542 // For similar reasons, always use LAY if the constant is too big for AGHI. 543 // LAY should be no worse than AGFI. 544 if (!isInt<16>(Disp)) 545 return true; 546 } else { 547 // Don't use LA for plain registers. 548 if (!Index) 549 return false; 550 551 // Don't use LA for plain addition if the index operand is only used 552 // once. It should be a natural two-operand addition in that case. 553 if (Index->hasOneUse()) 554 return false; 555 556 // Prefer addition if the second operation is sign-extended, in the 557 // hope of using AGF. 558 unsigned IndexOpcode = Index->getOpcode(); 559 if (IndexOpcode == ISD::SIGN_EXTEND || 560 IndexOpcode == ISD::SIGN_EXTEND_INREG) 561 return false; 562 } 563 564 // Don't use LA for two-operand addition if either operand is only 565 // used once. The addition instructions are better in that case. 566 if (Base->hasOneUse()) 567 return false; 568 569 return true; 570 } 571 572 // Return true if Addr is suitable for AM, updating AM if so. 573 bool SystemZDAGToDAGISel::selectAddress(SDValue Addr, 574 SystemZAddressingMode &AM) const { 575 // Start out assuming that the address will need to be loaded separately, 576 // then try to extend it as much as we can. 577 AM.Base = Addr; 578 579 // First try treating the address as a constant. 580 if (Addr.getOpcode() == ISD::Constant && 581 expandDisp(AM, true, SDValue(), 582 cast<ConstantSDNode>(Addr)->getSExtValue())) 583 ; 584 // Also see if it's a bare ADJDYNALLOC. 585 else if (Addr.getOpcode() == SystemZISD::ADJDYNALLOC && 586 expandAdjDynAlloc(AM, true, SDValue())) 587 ; 588 else 589 // Otherwise try expanding each component. 590 while (expandAddress(AM, true) || 591 (AM.Index.getNode() && expandAddress(AM, false))) 592 continue; 593 594 // Reject cases where it isn't profitable to use LA(Y). 595 if (AM.Form == SystemZAddressingMode::FormBDXLA && 596 !shouldUseLA(AM.Base.getNode(), AM.Disp, AM.Index.getNode())) 597 return false; 598 599 // Reject cases where the other instruction in a pair should be used. 600 if (!isValidDisp(AM.DR, AM.Disp)) 601 return false; 602 603 // Make sure that ADJDYNALLOC is included where necessary. 604 if (AM.isDynAlloc() && !AM.IncludesDynAlloc) 605 return false; 606 607 LLVM_DEBUG(AM.dump(CurDAG)); 608 return true; 609 } 610 611 // Insert a node into the DAG at least before Pos. This will reposition 612 // the node as needed, and will assign it a node ID that is <= Pos's ID. 613 // Note that this does *not* preserve the uniqueness of node IDs! 614 // The selection DAG must no longer depend on their uniqueness when this 615 // function is used. 616 static void insertDAGNode(SelectionDAG *DAG, SDNode *Pos, SDValue N) { 617 if (N->getNodeId() == -1 || 618 (SelectionDAGISel::getUninvalidatedNodeId(N.getNode()) > 619 SelectionDAGISel::getUninvalidatedNodeId(Pos))) { 620 DAG->RepositionNode(Pos->getIterator(), N.getNode()); 621 // Mark Node as invalid for pruning as after this it may be a successor to a 622 // selected node but otherwise be in the same position of Pos. 623 // Conservatively mark it with the same -abs(Id) to assure node id 624 // invariant is preserved. 625 N->setNodeId(Pos->getNodeId()); 626 SelectionDAGISel::InvalidateNodeId(N.getNode()); 627 } 628 } 629 630 void SystemZDAGToDAGISel::getAddressOperands(const SystemZAddressingMode &AM, 631 EVT VT, SDValue &Base, 632 SDValue &Disp) const { 633 Base = AM.Base; 634 if (!Base.getNode()) 635 // Register 0 means "no base". This is mostly useful for shifts. 636 Base = CurDAG->getRegister(0, VT); 637 else if (Base.getOpcode() == ISD::FrameIndex) { 638 // Lower a FrameIndex to a TargetFrameIndex. 639 int64_t FrameIndex = cast<FrameIndexSDNode>(Base)->getIndex(); 640 Base = CurDAG->getTargetFrameIndex(FrameIndex, VT); 641 } else if (Base.getValueType() != VT) { 642 // Truncate values from i64 to i32, for shifts. 643 assert(VT == MVT::i32 && Base.getValueType() == MVT::i64 && 644 "Unexpected truncation"); 645 SDLoc DL(Base); 646 SDValue Trunc = CurDAG->getNode(ISD::TRUNCATE, DL, VT, Base); 647 insertDAGNode(CurDAG, Base.getNode(), Trunc); 648 Base = Trunc; 649 } 650 651 // Lower the displacement to a TargetConstant. 652 Disp = CurDAG->getTargetConstant(AM.Disp, SDLoc(Base), VT); 653 } 654 655 void SystemZDAGToDAGISel::getAddressOperands(const SystemZAddressingMode &AM, 656 EVT VT, SDValue &Base, 657 SDValue &Disp, 658 SDValue &Index) const { 659 getAddressOperands(AM, VT, Base, Disp); 660 661 Index = AM.Index; 662 if (!Index.getNode()) 663 // Register 0 means "no index". 664 Index = CurDAG->getRegister(0, VT); 665 } 666 667 bool SystemZDAGToDAGISel::selectBDAddr(SystemZAddressingMode::DispRange DR, 668 SDValue Addr, SDValue &Base, 669 SDValue &Disp) const { 670 SystemZAddressingMode AM(SystemZAddressingMode::FormBD, DR); 671 if (!selectAddress(Addr, AM)) 672 return false; 673 674 getAddressOperands(AM, Addr.getValueType(), Base, Disp); 675 return true; 676 } 677 678 bool SystemZDAGToDAGISel::selectMVIAddr(SystemZAddressingMode::DispRange DR, 679 SDValue Addr, SDValue &Base, 680 SDValue &Disp) const { 681 SystemZAddressingMode AM(SystemZAddressingMode::FormBDXNormal, DR); 682 if (!selectAddress(Addr, AM) || AM.Index.getNode()) 683 return false; 684 685 getAddressOperands(AM, Addr.getValueType(), Base, Disp); 686 return true; 687 } 688 689 bool SystemZDAGToDAGISel::selectBDXAddr(SystemZAddressingMode::AddrForm Form, 690 SystemZAddressingMode::DispRange DR, 691 SDValue Addr, SDValue &Base, 692 SDValue &Disp, SDValue &Index) const { 693 SystemZAddressingMode AM(Form, DR); 694 if (!selectAddress(Addr, AM)) 695 return false; 696 697 getAddressOperands(AM, Addr.getValueType(), Base, Disp, Index); 698 return true; 699 } 700 701 bool SystemZDAGToDAGISel::selectBDVAddr12Only(SDValue Addr, SDValue Elem, 702 SDValue &Base, 703 SDValue &Disp, 704 SDValue &Index) const { 705 SDValue Regs[2]; 706 if (selectBDXAddr12Only(Addr, Regs[0], Disp, Regs[1]) && 707 Regs[0].getNode() && Regs[1].getNode()) { 708 for (unsigned int I = 0; I < 2; ++I) { 709 Base = Regs[I]; 710 Index = Regs[1 - I]; 711 // We can't tell here whether the index vector has the right type 712 // for the access; the caller needs to do that instead. 713 if (Index.getOpcode() == ISD::ZERO_EXTEND) 714 Index = Index.getOperand(0); 715 if (Index.getOpcode() == ISD::EXTRACT_VECTOR_ELT && 716 Index.getOperand(1) == Elem) { 717 Index = Index.getOperand(0); 718 return true; 719 } 720 } 721 } 722 return false; 723 } 724 725 bool SystemZDAGToDAGISel::detectOrAndInsertion(SDValue &Op, 726 uint64_t InsertMask) const { 727 // We're only interested in cases where the insertion is into some operand 728 // of Op, rather than into Op itself. The only useful case is an AND. 729 if (Op.getOpcode() != ISD::AND) 730 return false; 731 732 // We need a constant mask. 733 auto *MaskNode = dyn_cast<ConstantSDNode>(Op.getOperand(1).getNode()); 734 if (!MaskNode) 735 return false; 736 737 // It's not an insertion of Op.getOperand(0) if the two masks overlap. 738 uint64_t AndMask = MaskNode->getZExtValue(); 739 if (InsertMask & AndMask) 740 return false; 741 742 // It's only an insertion if all bits are covered or are known to be zero. 743 // The inner check covers all cases but is more expensive. 744 uint64_t Used = allOnes(Op.getValueSizeInBits()); 745 if (Used != (AndMask | InsertMask)) { 746 KnownBits Known = CurDAG->computeKnownBits(Op.getOperand(0)); 747 if (Used != (AndMask | InsertMask | Known.Zero.getZExtValue())) 748 return false; 749 } 750 751 Op = Op.getOperand(0); 752 return true; 753 } 754 755 bool SystemZDAGToDAGISel::refineRxSBGMask(RxSBGOperands &RxSBG, 756 uint64_t Mask) const { 757 const SystemZInstrInfo *TII = getInstrInfo(); 758 if (RxSBG.Rotate != 0) 759 Mask = (Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate)); 760 Mask &= RxSBG.Mask; 761 if (TII->isRxSBGMask(Mask, RxSBG.BitSize, RxSBG.Start, RxSBG.End)) { 762 RxSBG.Mask = Mask; 763 return true; 764 } 765 return false; 766 } 767 768 // Return true if any bits of (RxSBG.Input & Mask) are significant. 769 static bool maskMatters(RxSBGOperands &RxSBG, uint64_t Mask) { 770 // Rotate the mask in the same way as RxSBG.Input is rotated. 771 if (RxSBG.Rotate != 0) 772 Mask = ((Mask << RxSBG.Rotate) | (Mask >> (64 - RxSBG.Rotate))); 773 return (Mask & RxSBG.Mask) != 0; 774 } 775 776 bool SystemZDAGToDAGISel::expandRxSBG(RxSBGOperands &RxSBG) const { 777 SDValue N = RxSBG.Input; 778 unsigned Opcode = N.getOpcode(); 779 switch (Opcode) { 780 case ISD::TRUNCATE: { 781 if (RxSBG.Opcode == SystemZ::RNSBG) 782 return false; 783 uint64_t BitSize = N.getValueSizeInBits(); 784 uint64_t Mask = allOnes(BitSize); 785 if (!refineRxSBGMask(RxSBG, Mask)) 786 return false; 787 RxSBG.Input = N.getOperand(0); 788 return true; 789 } 790 case ISD::AND: { 791 if (RxSBG.Opcode == SystemZ::RNSBG) 792 return false; 793 794 auto *MaskNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode()); 795 if (!MaskNode) 796 return false; 797 798 SDValue Input = N.getOperand(0); 799 uint64_t Mask = MaskNode->getZExtValue(); 800 if (!refineRxSBGMask(RxSBG, Mask)) { 801 // If some bits of Input are already known zeros, those bits will have 802 // been removed from the mask. See if adding them back in makes the 803 // mask suitable. 804 KnownBits Known = CurDAG->computeKnownBits(Input); 805 Mask |= Known.Zero.getZExtValue(); 806 if (!refineRxSBGMask(RxSBG, Mask)) 807 return false; 808 } 809 RxSBG.Input = Input; 810 return true; 811 } 812 813 case ISD::OR: { 814 if (RxSBG.Opcode != SystemZ::RNSBG) 815 return false; 816 817 auto *MaskNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode()); 818 if (!MaskNode) 819 return false; 820 821 SDValue Input = N.getOperand(0); 822 uint64_t Mask = ~MaskNode->getZExtValue(); 823 if (!refineRxSBGMask(RxSBG, Mask)) { 824 // If some bits of Input are already known ones, those bits will have 825 // been removed from the mask. See if adding them back in makes the 826 // mask suitable. 827 KnownBits Known = CurDAG->computeKnownBits(Input); 828 Mask &= ~Known.One.getZExtValue(); 829 if (!refineRxSBGMask(RxSBG, Mask)) 830 return false; 831 } 832 RxSBG.Input = Input; 833 return true; 834 } 835 836 case ISD::ROTL: { 837 // Any 64-bit rotate left can be merged into the RxSBG. 838 if (RxSBG.BitSize != 64 || N.getValueType() != MVT::i64) 839 return false; 840 auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode()); 841 if (!CountNode) 842 return false; 843 844 RxSBG.Rotate = (RxSBG.Rotate + CountNode->getZExtValue()) & 63; 845 RxSBG.Input = N.getOperand(0); 846 return true; 847 } 848 849 case ISD::ANY_EXTEND: 850 // Bits above the extended operand are don't-care. 851 RxSBG.Input = N.getOperand(0); 852 return true; 853 854 case ISD::ZERO_EXTEND: 855 if (RxSBG.Opcode != SystemZ::RNSBG) { 856 // Restrict the mask to the extended operand. 857 unsigned InnerBitSize = N.getOperand(0).getValueSizeInBits(); 858 if (!refineRxSBGMask(RxSBG, allOnes(InnerBitSize))) 859 return false; 860 861 RxSBG.Input = N.getOperand(0); 862 return true; 863 } 864 LLVM_FALLTHROUGH; 865 866 case ISD::SIGN_EXTEND: { 867 // Check that the extension bits are don't-care (i.e. are masked out 868 // by the final mask). 869 unsigned BitSize = N.getValueSizeInBits(); 870 unsigned InnerBitSize = N.getOperand(0).getValueSizeInBits(); 871 if (maskMatters(RxSBG, allOnes(BitSize) - allOnes(InnerBitSize))) { 872 // In the case where only the sign bit is active, increase Rotate with 873 // the extension width. 874 if (RxSBG.Mask == 1 && RxSBG.Rotate == 1) 875 RxSBG.Rotate += (BitSize - InnerBitSize); 876 else 877 return false; 878 } 879 880 RxSBG.Input = N.getOperand(0); 881 return true; 882 } 883 884 case ISD::SHL: { 885 auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode()); 886 if (!CountNode) 887 return false; 888 889 uint64_t Count = CountNode->getZExtValue(); 890 unsigned BitSize = N.getValueSizeInBits(); 891 if (Count < 1 || Count >= BitSize) 892 return false; 893 894 if (RxSBG.Opcode == SystemZ::RNSBG) { 895 // Treat (shl X, count) as (rotl X, size-count) as long as the bottom 896 // count bits from RxSBG.Input are ignored. 897 if (maskMatters(RxSBG, allOnes(Count))) 898 return false; 899 } else { 900 // Treat (shl X, count) as (and (rotl X, count), ~0<<count). 901 if (!refineRxSBGMask(RxSBG, allOnes(BitSize - Count) << Count)) 902 return false; 903 } 904 905 RxSBG.Rotate = (RxSBG.Rotate + Count) & 63; 906 RxSBG.Input = N.getOperand(0); 907 return true; 908 } 909 910 case ISD::SRL: 911 case ISD::SRA: { 912 auto *CountNode = dyn_cast<ConstantSDNode>(N.getOperand(1).getNode()); 913 if (!CountNode) 914 return false; 915 916 uint64_t Count = CountNode->getZExtValue(); 917 unsigned BitSize = N.getValueSizeInBits(); 918 if (Count < 1 || Count >= BitSize) 919 return false; 920 921 if (RxSBG.Opcode == SystemZ::RNSBG || Opcode == ISD::SRA) { 922 // Treat (srl|sra X, count) as (rotl X, size-count) as long as the top 923 // count bits from RxSBG.Input are ignored. 924 if (maskMatters(RxSBG, allOnes(Count) << (BitSize - Count))) 925 return false; 926 } else { 927 // Treat (srl X, count), mask) as (and (rotl X, size-count), ~0>>count), 928 // which is similar to SLL above. 929 if (!refineRxSBGMask(RxSBG, allOnes(BitSize - Count))) 930 return false; 931 } 932 933 RxSBG.Rotate = (RxSBG.Rotate - Count) & 63; 934 RxSBG.Input = N.getOperand(0); 935 return true; 936 } 937 default: 938 return false; 939 } 940 } 941 942 SDValue SystemZDAGToDAGISel::getUNDEF(const SDLoc &DL, EVT VT) const { 943 SDNode *N = CurDAG->getMachineNode(TargetOpcode::IMPLICIT_DEF, DL, VT); 944 return SDValue(N, 0); 945 } 946 947 SDValue SystemZDAGToDAGISel::convertTo(const SDLoc &DL, EVT VT, 948 SDValue N) const { 949 if (N.getValueType() == MVT::i32 && VT == MVT::i64) 950 return CurDAG->getTargetInsertSubreg(SystemZ::subreg_l32, 951 DL, VT, getUNDEF(DL, MVT::i64), N); 952 if (N.getValueType() == MVT::i64 && VT == MVT::i32) 953 return CurDAG->getTargetExtractSubreg(SystemZ::subreg_l32, DL, VT, N); 954 assert(N.getValueType() == VT && "Unexpected value types"); 955 return N; 956 } 957 958 bool SystemZDAGToDAGISel::tryRISBGZero(SDNode *N) { 959 SDLoc DL(N); 960 EVT VT = N->getValueType(0); 961 if (!VT.isInteger() || VT.getSizeInBits() > 64) 962 return false; 963 RxSBGOperands RISBG(SystemZ::RISBG, SDValue(N, 0)); 964 unsigned Count = 0; 965 while (expandRxSBG(RISBG)) 966 // The widening or narrowing is expected to be free. 967 // Counting widening or narrowing as a saved operation will result in 968 // preferring an R*SBG over a simple shift/logical instruction. 969 if (RISBG.Input.getOpcode() != ISD::ANY_EXTEND && 970 RISBG.Input.getOpcode() != ISD::TRUNCATE) 971 Count += 1; 972 if (Count == 0) 973 return false; 974 975 // Prefer to use normal shift instructions over RISBG, since they can handle 976 // all cases and are sometimes shorter. 977 if (Count == 1 && N->getOpcode() != ISD::AND) 978 return false; 979 980 // Prefer register extensions like LLC over RISBG. Also prefer to start 981 // out with normal ANDs if one instruction would be enough. We can convert 982 // these ANDs into an RISBG later if a three-address instruction is useful. 983 if (RISBG.Rotate == 0) { 984 bool PreferAnd = false; 985 // Prefer AND for any 32-bit and-immediate operation. 986 if (VT == MVT::i32) 987 PreferAnd = true; 988 // As well as for any 64-bit operation that can be implemented via LLC(R), 989 // LLH(R), LLGT(R), or one of the and-immediate instructions. 990 else if (RISBG.Mask == 0xff || 991 RISBG.Mask == 0xffff || 992 RISBG.Mask == 0x7fffffff || 993 SystemZ::isImmLF(~RISBG.Mask) || 994 SystemZ::isImmHF(~RISBG.Mask)) 995 PreferAnd = true; 996 // And likewise for the LLZRGF instruction, which doesn't have a register 997 // to register version. 998 else if (auto *Load = dyn_cast<LoadSDNode>(RISBG.Input)) { 999 if (Load->getMemoryVT() == MVT::i32 && 1000 (Load->getExtensionType() == ISD::EXTLOAD || 1001 Load->getExtensionType() == ISD::ZEXTLOAD) && 1002 RISBG.Mask == 0xffffff00 && 1003 Subtarget->hasLoadAndZeroRightmostByte()) 1004 PreferAnd = true; 1005 } 1006 if (PreferAnd) { 1007 // Replace the current node with an AND. Note that the current node 1008 // might already be that same AND, in which case it is already CSE'd 1009 // with it, and we must not call ReplaceNode. 1010 SDValue In = convertTo(DL, VT, RISBG.Input); 1011 SDValue Mask = CurDAG->getConstant(RISBG.Mask, DL, VT); 1012 SDValue New = CurDAG->getNode(ISD::AND, DL, VT, In, Mask); 1013 if (N != New.getNode()) { 1014 insertDAGNode(CurDAG, N, Mask); 1015 insertDAGNode(CurDAG, N, New); 1016 ReplaceNode(N, New.getNode()); 1017 N = New.getNode(); 1018 } 1019 // Now, select the machine opcode to implement this operation. 1020 if (!N->isMachineOpcode()) 1021 SelectCode(N); 1022 return true; 1023 } 1024 } 1025 1026 unsigned Opcode = SystemZ::RISBG; 1027 // Prefer RISBGN if available, since it does not clobber CC. 1028 if (Subtarget->hasMiscellaneousExtensions()) 1029 Opcode = SystemZ::RISBGN; 1030 EVT OpcodeVT = MVT::i64; 1031 if (VT == MVT::i32 && Subtarget->hasHighWord() && 1032 // We can only use the 32-bit instructions if all source bits are 1033 // in the low 32 bits without wrapping, both after rotation (because 1034 // of the smaller range for Start and End) and before rotation 1035 // (because the input value is truncated). 1036 RISBG.Start >= 32 && RISBG.End >= RISBG.Start && 1037 ((RISBG.Start + RISBG.Rotate) & 63) >= 32 && 1038 ((RISBG.End + RISBG.Rotate) & 63) >= 1039 ((RISBG.Start + RISBG.Rotate) & 63)) { 1040 Opcode = SystemZ::RISBMux; 1041 OpcodeVT = MVT::i32; 1042 RISBG.Start &= 31; 1043 RISBG.End &= 31; 1044 } 1045 SDValue Ops[5] = { 1046 getUNDEF(DL, OpcodeVT), 1047 convertTo(DL, OpcodeVT, RISBG.Input), 1048 CurDAG->getTargetConstant(RISBG.Start, DL, MVT::i32), 1049 CurDAG->getTargetConstant(RISBG.End | 128, DL, MVT::i32), 1050 CurDAG->getTargetConstant(RISBG.Rotate, DL, MVT::i32) 1051 }; 1052 SDValue New = convertTo( 1053 DL, VT, SDValue(CurDAG->getMachineNode(Opcode, DL, OpcodeVT, Ops), 0)); 1054 ReplaceNode(N, New.getNode()); 1055 return true; 1056 } 1057 1058 bool SystemZDAGToDAGISel::tryRxSBG(SDNode *N, unsigned Opcode) { 1059 SDLoc DL(N); 1060 EVT VT = N->getValueType(0); 1061 if (!VT.isInteger() || VT.getSizeInBits() > 64) 1062 return false; 1063 // Try treating each operand of N as the second operand of the RxSBG 1064 // and see which goes deepest. 1065 RxSBGOperands RxSBG[] = { 1066 RxSBGOperands(Opcode, N->getOperand(0)), 1067 RxSBGOperands(Opcode, N->getOperand(1)) 1068 }; 1069 unsigned Count[] = { 0, 0 }; 1070 for (unsigned I = 0; I < 2; ++I) 1071 while (expandRxSBG(RxSBG[I])) 1072 // The widening or narrowing is expected to be free. 1073 // Counting widening or narrowing as a saved operation will result in 1074 // preferring an R*SBG over a simple shift/logical instruction. 1075 if (RxSBG[I].Input.getOpcode() != ISD::ANY_EXTEND && 1076 RxSBG[I].Input.getOpcode() != ISD::TRUNCATE) 1077 Count[I] += 1; 1078 1079 // Do nothing if neither operand is suitable. 1080 if (Count[0] == 0 && Count[1] == 0) 1081 return false; 1082 1083 // Pick the deepest second operand. 1084 unsigned I = Count[0] > Count[1] ? 0 : 1; 1085 SDValue Op0 = N->getOperand(I ^ 1); 1086 1087 // Prefer IC for character insertions from memory. 1088 if (Opcode == SystemZ::ROSBG && (RxSBG[I].Mask & 0xff) == 0) 1089 if (auto *Load = dyn_cast<LoadSDNode>(Op0.getNode())) 1090 if (Load->getMemoryVT() == MVT::i8) 1091 return false; 1092 1093 // See whether we can avoid an AND in the first operand by converting 1094 // ROSBG to RISBG. 1095 if (Opcode == SystemZ::ROSBG && detectOrAndInsertion(Op0, RxSBG[I].Mask)) { 1096 Opcode = SystemZ::RISBG; 1097 // Prefer RISBGN if available, since it does not clobber CC. 1098 if (Subtarget->hasMiscellaneousExtensions()) 1099 Opcode = SystemZ::RISBGN; 1100 } 1101 1102 SDValue Ops[5] = { 1103 convertTo(DL, MVT::i64, Op0), 1104 convertTo(DL, MVT::i64, RxSBG[I].Input), 1105 CurDAG->getTargetConstant(RxSBG[I].Start, DL, MVT::i32), 1106 CurDAG->getTargetConstant(RxSBG[I].End, DL, MVT::i32), 1107 CurDAG->getTargetConstant(RxSBG[I].Rotate, DL, MVT::i32) 1108 }; 1109 SDValue New = convertTo( 1110 DL, VT, SDValue(CurDAG->getMachineNode(Opcode, DL, MVT::i64, Ops), 0)); 1111 ReplaceNode(N, New.getNode()); 1112 return true; 1113 } 1114 1115 void SystemZDAGToDAGISel::splitLargeImmediate(unsigned Opcode, SDNode *Node, 1116 SDValue Op0, uint64_t UpperVal, 1117 uint64_t LowerVal) { 1118 EVT VT = Node->getValueType(0); 1119 SDLoc DL(Node); 1120 SDValue Upper = CurDAG->getConstant(UpperVal, DL, VT); 1121 if (Op0.getNode()) 1122 Upper = CurDAG->getNode(Opcode, DL, VT, Op0, Upper); 1123 1124 { 1125 // When we haven't passed in Op0, Upper will be a constant. In order to 1126 // prevent folding back to the large immediate in `Or = getNode(...)` we run 1127 // SelectCode first and end up with an opaque machine node. This means that 1128 // we need to use a handle to keep track of Upper in case it gets CSE'd by 1129 // SelectCode. 1130 // 1131 // Note that in the case where Op0 is passed in we could just call 1132 // SelectCode(Upper) later, along with the SelectCode(Or), and avoid needing 1133 // the handle at all, but it's fine to do it here. 1134 // 1135 // TODO: This is a pretty hacky way to do this. Can we do something that 1136 // doesn't require a two paragraph explanation? 1137 HandleSDNode Handle(Upper); 1138 SelectCode(Upper.getNode()); 1139 Upper = Handle.getValue(); 1140 } 1141 1142 SDValue Lower = CurDAG->getConstant(LowerVal, DL, VT); 1143 SDValue Or = CurDAG->getNode(Opcode, DL, VT, Upper, Lower); 1144 1145 ReplaceNode(Node, Or.getNode()); 1146 1147 SelectCode(Or.getNode()); 1148 } 1149 1150 void SystemZDAGToDAGISel::loadVectorConstant( 1151 const SystemZVectorConstantInfo &VCI, SDNode *Node) { 1152 assert((VCI.Opcode == SystemZISD::BYTE_MASK || 1153 VCI.Opcode == SystemZISD::REPLICATE || 1154 VCI.Opcode == SystemZISD::ROTATE_MASK) && 1155 "Bad opcode!"); 1156 assert(VCI.VecVT.getSizeInBits() == 128 && "Expected a vector type"); 1157 EVT VT = Node->getValueType(0); 1158 SDLoc DL(Node); 1159 SmallVector<SDValue, 2> Ops; 1160 for (unsigned OpVal : VCI.OpVals) 1161 Ops.push_back(CurDAG->getTargetConstant(OpVal, DL, MVT::i32)); 1162 SDValue Op = CurDAG->getNode(VCI.Opcode, DL, VCI.VecVT, Ops); 1163 1164 if (VCI.VecVT == VT.getSimpleVT()) 1165 ReplaceNode(Node, Op.getNode()); 1166 else if (VT.getSizeInBits() == 128) { 1167 SDValue BitCast = CurDAG->getNode(ISD::BITCAST, DL, VT, Op); 1168 ReplaceNode(Node, BitCast.getNode()); 1169 SelectCode(BitCast.getNode()); 1170 } else { // float or double 1171 unsigned SubRegIdx = 1172 (VT.getSizeInBits() == 32 ? SystemZ::subreg_h32 : SystemZ::subreg_h64); 1173 ReplaceNode( 1174 Node, CurDAG->getTargetExtractSubreg(SubRegIdx, DL, VT, Op).getNode()); 1175 } 1176 SelectCode(Op.getNode()); 1177 } 1178 1179 bool SystemZDAGToDAGISel::tryGather(SDNode *N, unsigned Opcode) { 1180 SDValue ElemV = N->getOperand(2); 1181 auto *ElemN = dyn_cast<ConstantSDNode>(ElemV); 1182 if (!ElemN) 1183 return false; 1184 1185 unsigned Elem = ElemN->getZExtValue(); 1186 EVT VT = N->getValueType(0); 1187 if (Elem >= VT.getVectorNumElements()) 1188 return false; 1189 1190 auto *Load = dyn_cast<LoadSDNode>(N->getOperand(1)); 1191 if (!Load || !Load->hasNUsesOfValue(1, 0)) 1192 return false; 1193 if (Load->getMemoryVT().getSizeInBits() != 1194 Load->getValueType(0).getSizeInBits()) 1195 return false; 1196 1197 SDValue Base, Disp, Index; 1198 if (!selectBDVAddr12Only(Load->getBasePtr(), ElemV, Base, Disp, Index) || 1199 Index.getValueType() != VT.changeVectorElementTypeToInteger()) 1200 return false; 1201 1202 SDLoc DL(Load); 1203 SDValue Ops[] = { 1204 N->getOperand(0), Base, Disp, Index, 1205 CurDAG->getTargetConstant(Elem, DL, MVT::i32), Load->getChain() 1206 }; 1207 SDNode *Res = CurDAG->getMachineNode(Opcode, DL, VT, MVT::Other, Ops); 1208 ReplaceUses(SDValue(Load, 1), SDValue(Res, 1)); 1209 ReplaceNode(N, Res); 1210 return true; 1211 } 1212 1213 bool SystemZDAGToDAGISel::tryScatter(StoreSDNode *Store, unsigned Opcode) { 1214 SDValue Value = Store->getValue(); 1215 if (Value.getOpcode() != ISD::EXTRACT_VECTOR_ELT) 1216 return false; 1217 if (Store->getMemoryVT().getSizeInBits() != Value.getValueSizeInBits()) 1218 return false; 1219 1220 SDValue ElemV = Value.getOperand(1); 1221 auto *ElemN = dyn_cast<ConstantSDNode>(ElemV); 1222 if (!ElemN) 1223 return false; 1224 1225 SDValue Vec = Value.getOperand(0); 1226 EVT VT = Vec.getValueType(); 1227 unsigned Elem = ElemN->getZExtValue(); 1228 if (Elem >= VT.getVectorNumElements()) 1229 return false; 1230 1231 SDValue Base, Disp, Index; 1232 if (!selectBDVAddr12Only(Store->getBasePtr(), ElemV, Base, Disp, Index) || 1233 Index.getValueType() != VT.changeVectorElementTypeToInteger()) 1234 return false; 1235 1236 SDLoc DL(Store); 1237 SDValue Ops[] = { 1238 Vec, Base, Disp, Index, CurDAG->getTargetConstant(Elem, DL, MVT::i32), 1239 Store->getChain() 1240 }; 1241 ReplaceNode(Store, CurDAG->getMachineNode(Opcode, DL, MVT::Other, Ops)); 1242 return true; 1243 } 1244 1245 // Check whether or not the chain ending in StoreNode is suitable for doing 1246 // the {load; op; store} to modify transformation. 1247 static bool isFusableLoadOpStorePattern(StoreSDNode *StoreNode, 1248 SDValue StoredVal, SelectionDAG *CurDAG, 1249 LoadSDNode *&LoadNode, 1250 SDValue &InputChain) { 1251 // Is the stored value result 0 of the operation? 1252 if (StoredVal.getResNo() != 0) 1253 return false; 1254 1255 // Are there other uses of the loaded value than the operation? 1256 if (!StoredVal.getNode()->hasNUsesOfValue(1, 0)) 1257 return false; 1258 1259 // Is the store non-extending and non-indexed? 1260 if (!ISD::isNormalStore(StoreNode) || StoreNode->isNonTemporal()) 1261 return false; 1262 1263 SDValue Load = StoredVal->getOperand(0); 1264 // Is the stored value a non-extending and non-indexed load? 1265 if (!ISD::isNormalLoad(Load.getNode())) 1266 return false; 1267 1268 // Return LoadNode by reference. 1269 LoadNode = cast<LoadSDNode>(Load); 1270 1271 // Is store the only read of the loaded value? 1272 if (!Load.hasOneUse()) 1273 return false; 1274 1275 // Is the address of the store the same as the load? 1276 if (LoadNode->getBasePtr() != StoreNode->getBasePtr() || 1277 LoadNode->getOffset() != StoreNode->getOffset()) 1278 return false; 1279 1280 // Check if the chain is produced by the load or is a TokenFactor with 1281 // the load output chain as an operand. Return InputChain by reference. 1282 SDValue Chain = StoreNode->getChain(); 1283 1284 bool ChainCheck = false; 1285 if (Chain == Load.getValue(1)) { 1286 ChainCheck = true; 1287 InputChain = LoadNode->getChain(); 1288 } else if (Chain.getOpcode() == ISD::TokenFactor) { 1289 SmallVector<SDValue, 4> ChainOps; 1290 SmallVector<const SDNode *, 4> LoopWorklist; 1291 SmallPtrSet<const SDNode *, 16> Visited; 1292 const unsigned int Max = 1024; 1293 for (unsigned i = 0, e = Chain.getNumOperands(); i != e; ++i) { 1294 SDValue Op = Chain.getOperand(i); 1295 if (Op == Load.getValue(1)) { 1296 ChainCheck = true; 1297 // Drop Load, but keep its chain. No cycle check necessary. 1298 ChainOps.push_back(Load.getOperand(0)); 1299 continue; 1300 } 1301 LoopWorklist.push_back(Op.getNode()); 1302 ChainOps.push_back(Op); 1303 } 1304 1305 if (ChainCheck) { 1306 // Add the other operand of StoredVal to worklist. 1307 for (SDValue Op : StoredVal->ops()) 1308 if (Op.getNode() != LoadNode) 1309 LoopWorklist.push_back(Op.getNode()); 1310 1311 // Check if Load is reachable from any of the nodes in the worklist. 1312 if (SDNode::hasPredecessorHelper(Load.getNode(), Visited, LoopWorklist, Max, 1313 true)) 1314 return false; 1315 1316 // Make a new TokenFactor with all the other input chains except 1317 // for the load. 1318 InputChain = CurDAG->getNode(ISD::TokenFactor, SDLoc(Chain), 1319 MVT::Other, ChainOps); 1320 } 1321 } 1322 if (!ChainCheck) 1323 return false; 1324 1325 return true; 1326 } 1327 1328 // Change a chain of {load; op; store} of the same value into a simple op 1329 // through memory of that value, if the uses of the modified value and its 1330 // address are suitable. 1331 // 1332 // The tablegen pattern memory operand pattern is currently not able to match 1333 // the case where the CC on the original operation are used. 1334 // 1335 // See the equivalent routine in X86ISelDAGToDAG for further comments. 1336 bool SystemZDAGToDAGISel::tryFoldLoadStoreIntoMemOperand(SDNode *Node) { 1337 StoreSDNode *StoreNode = cast<StoreSDNode>(Node); 1338 SDValue StoredVal = StoreNode->getOperand(1); 1339 unsigned Opc = StoredVal->getOpcode(); 1340 SDLoc DL(StoreNode); 1341 1342 // Before we try to select anything, make sure this is memory operand size 1343 // and opcode we can handle. Note that this must match the code below that 1344 // actually lowers the opcodes. 1345 EVT MemVT = StoreNode->getMemoryVT(); 1346 unsigned NewOpc = 0; 1347 bool NegateOperand = false; 1348 switch (Opc) { 1349 default: 1350 return false; 1351 case SystemZISD::SSUBO: 1352 NegateOperand = true; 1353 LLVM_FALLTHROUGH; 1354 case SystemZISD::SADDO: 1355 if (MemVT == MVT::i32) 1356 NewOpc = SystemZ::ASI; 1357 else if (MemVT == MVT::i64) 1358 NewOpc = SystemZ::AGSI; 1359 else 1360 return false; 1361 break; 1362 case SystemZISD::USUBO: 1363 NegateOperand = true; 1364 LLVM_FALLTHROUGH; 1365 case SystemZISD::UADDO: 1366 if (MemVT == MVT::i32) 1367 NewOpc = SystemZ::ALSI; 1368 else if (MemVT == MVT::i64) 1369 NewOpc = SystemZ::ALGSI; 1370 else 1371 return false; 1372 break; 1373 } 1374 1375 LoadSDNode *LoadNode = nullptr; 1376 SDValue InputChain; 1377 if (!isFusableLoadOpStorePattern(StoreNode, StoredVal, CurDAG, LoadNode, 1378 InputChain)) 1379 return false; 1380 1381 SDValue Operand = StoredVal.getOperand(1); 1382 auto *OperandC = dyn_cast<ConstantSDNode>(Operand); 1383 if (!OperandC) 1384 return false; 1385 auto OperandV = OperandC->getAPIntValue(); 1386 if (NegateOperand) 1387 OperandV = -OperandV; 1388 if (OperandV.getMinSignedBits() > 8) 1389 return false; 1390 Operand = CurDAG->getTargetConstant(OperandV, DL, MemVT); 1391 1392 SDValue Base, Disp; 1393 if (!selectBDAddr20Only(StoreNode->getBasePtr(), Base, Disp)) 1394 return false; 1395 1396 SDValue Ops[] = { Base, Disp, Operand, InputChain }; 1397 MachineSDNode *Result = 1398 CurDAG->getMachineNode(NewOpc, DL, MVT::i32, MVT::Other, Ops); 1399 CurDAG->setNodeMemRefs( 1400 Result, {StoreNode->getMemOperand(), LoadNode->getMemOperand()}); 1401 1402 ReplaceUses(SDValue(StoreNode, 0), SDValue(Result, 1)); 1403 ReplaceUses(SDValue(StoredVal.getNode(), 1), SDValue(Result, 0)); 1404 CurDAG->RemoveDeadNode(Node); 1405 return true; 1406 } 1407 1408 bool SystemZDAGToDAGISel::canUseBlockOperation(StoreSDNode *Store, 1409 LoadSDNode *Load) const { 1410 // Check that the two memory operands have the same size. 1411 if (Load->getMemoryVT() != Store->getMemoryVT()) 1412 return false; 1413 1414 // Volatility stops an access from being decomposed. 1415 if (Load->isVolatile() || Store->isVolatile()) 1416 return false; 1417 1418 // There's no chance of overlap if the load is invariant. 1419 if (Load->isInvariant() && Load->isDereferenceable()) 1420 return true; 1421 1422 // Otherwise we need to check whether there's an alias. 1423 const Value *V1 = Load->getMemOperand()->getValue(); 1424 const Value *V2 = Store->getMemOperand()->getValue(); 1425 if (!V1 || !V2) 1426 return false; 1427 1428 // Reject equality. 1429 uint64_t Size = Load->getMemoryVT().getStoreSize(); 1430 int64_t End1 = Load->getSrcValueOffset() + Size; 1431 int64_t End2 = Store->getSrcValueOffset() + Size; 1432 if (V1 == V2 && End1 == End2) 1433 return false; 1434 1435 return AA->isNoAlias(MemoryLocation(V1, End1, Load->getAAInfo()), 1436 MemoryLocation(V2, End2, Store->getAAInfo())); 1437 } 1438 1439 bool SystemZDAGToDAGISel::storeLoadCanUseMVC(SDNode *N) const { 1440 auto *Store = cast<StoreSDNode>(N); 1441 auto *Load = cast<LoadSDNode>(Store->getValue()); 1442 1443 // Prefer not to use MVC if either address can use ... RELATIVE LONG 1444 // instructions. 1445 uint64_t Size = Load->getMemoryVT().getStoreSize(); 1446 if (Size > 1 && Size <= 8) { 1447 // Prefer LHRL, LRL and LGRL. 1448 if (SystemZISD::isPCREL(Load->getBasePtr().getOpcode())) 1449 return false; 1450 // Prefer STHRL, STRL and STGRL. 1451 if (SystemZISD::isPCREL(Store->getBasePtr().getOpcode())) 1452 return false; 1453 } 1454 1455 return canUseBlockOperation(Store, Load); 1456 } 1457 1458 bool SystemZDAGToDAGISel::storeLoadCanUseBlockBinary(SDNode *N, 1459 unsigned I) const { 1460 auto *StoreA = cast<StoreSDNode>(N); 1461 auto *LoadA = cast<LoadSDNode>(StoreA->getValue().getOperand(1 - I)); 1462 auto *LoadB = cast<LoadSDNode>(StoreA->getValue().getOperand(I)); 1463 return !LoadA->isVolatile() && LoadA->getMemoryVT() == LoadB->getMemoryVT() && 1464 canUseBlockOperation(StoreA, LoadB); 1465 } 1466 1467 bool SystemZDAGToDAGISel::storeLoadIsAligned(SDNode *N) const { 1468 1469 auto *MemAccess = cast<LSBaseSDNode>(N); 1470 TypeSize StoreSize = MemAccess->getMemoryVT().getStoreSize(); 1471 SDValue BasePtr = MemAccess->getBasePtr(); 1472 MachineMemOperand *MMO = MemAccess->getMemOperand(); 1473 assert(MMO && "Expected a memory operand."); 1474 1475 // The memory access must have a proper alignment and no index register. 1476 if (MemAccess->getAlignment() < StoreSize || 1477 !MemAccess->getOffset().isUndef()) 1478 return false; 1479 1480 // The MMO must not have an unaligned offset. 1481 if (MMO->getOffset() % StoreSize != 0) 1482 return false; 1483 1484 // An access to GOT or the Constant Pool is aligned. 1485 if (const PseudoSourceValue *PSV = MMO->getPseudoValue()) 1486 if ((PSV->isGOT() || PSV->isConstantPool())) 1487 return true; 1488 1489 // Check the alignment of a Global Address. 1490 if (BasePtr.getNumOperands()) 1491 if (GlobalAddressSDNode *GA = 1492 dyn_cast<GlobalAddressSDNode>(BasePtr.getOperand(0))) { 1493 // The immediate offset must be aligned. 1494 if (GA->getOffset() % StoreSize != 0) 1495 return false; 1496 1497 // The alignment of the symbol itself must be at least the store size. 1498 const GlobalValue *GV = GA->getGlobal(); 1499 const DataLayout &DL = GV->getParent()->getDataLayout(); 1500 if (GV->getPointerAlignment(DL).value() < StoreSize) 1501 return false; 1502 } 1503 1504 return true; 1505 } 1506 1507 void SystemZDAGToDAGISel::Select(SDNode *Node) { 1508 // If we have a custom node, we already have selected! 1509 if (Node->isMachineOpcode()) { 1510 LLVM_DEBUG(errs() << "== "; Node->dump(CurDAG); errs() << "\n"); 1511 Node->setNodeId(-1); 1512 return; 1513 } 1514 1515 unsigned Opcode = Node->getOpcode(); 1516 switch (Opcode) { 1517 case ISD::OR: 1518 if (Node->getOperand(1).getOpcode() != ISD::Constant) 1519 if (tryRxSBG(Node, SystemZ::ROSBG)) 1520 return; 1521 goto or_xor; 1522 1523 case ISD::XOR: 1524 if (Node->getOperand(1).getOpcode() != ISD::Constant) 1525 if (tryRxSBG(Node, SystemZ::RXSBG)) 1526 return; 1527 // Fall through. 1528 or_xor: 1529 // If this is a 64-bit operation in which both 32-bit halves are nonzero, 1530 // split the operation into two. If both operands here happen to be 1531 // constant, leave this to common code to optimize. 1532 if (Node->getValueType(0) == MVT::i64 && 1533 Node->getOperand(0).getOpcode() != ISD::Constant) 1534 if (auto *Op1 = dyn_cast<ConstantSDNode>(Node->getOperand(1))) { 1535 uint64_t Val = Op1->getZExtValue(); 1536 // Don't split the operation if we can match one of the combined 1537 // logical operations provided by miscellaneous-extensions-3. 1538 if (Subtarget->hasMiscellaneousExtensions3()) { 1539 unsigned ChildOpcode = Node->getOperand(0).getOpcode(); 1540 // Check whether this expression matches NAND/NOR/NXOR. 1541 if (Val == (uint64_t)-1 && Opcode == ISD::XOR) 1542 if (ChildOpcode == ISD::AND || ChildOpcode == ISD::OR || 1543 ChildOpcode == ISD::XOR) 1544 break; 1545 // Check whether this expression matches OR-with-complement 1546 // (or matches an alternate pattern for NXOR). 1547 if (ChildOpcode == ISD::XOR) { 1548 auto Op0 = Node->getOperand(0); 1549 if (auto *Op0Op1 = dyn_cast<ConstantSDNode>(Op0->getOperand(1))) 1550 if (Op0Op1->getZExtValue() == (uint64_t)-1) 1551 break; 1552 } 1553 } 1554 if (!SystemZ::isImmLF(Val) && !SystemZ::isImmHF(Val)) { 1555 splitLargeImmediate(Opcode, Node, Node->getOperand(0), 1556 Val - uint32_t(Val), uint32_t(Val)); 1557 return; 1558 } 1559 } 1560 break; 1561 1562 case ISD::AND: 1563 if (Node->getOperand(1).getOpcode() != ISD::Constant) 1564 if (tryRxSBG(Node, SystemZ::RNSBG)) 1565 return; 1566 LLVM_FALLTHROUGH; 1567 case ISD::ROTL: 1568 case ISD::SHL: 1569 case ISD::SRL: 1570 case ISD::ZERO_EXTEND: 1571 if (tryRISBGZero(Node)) 1572 return; 1573 break; 1574 1575 case ISD::Constant: 1576 // If this is a 64-bit constant that is out of the range of LLILF, 1577 // LLIHF and LGFI, split it into two 32-bit pieces. 1578 if (Node->getValueType(0) == MVT::i64) { 1579 uint64_t Val = cast<ConstantSDNode>(Node)->getZExtValue(); 1580 if (!SystemZ::isImmLF(Val) && !SystemZ::isImmHF(Val) && !isInt<32>(Val)) { 1581 splitLargeImmediate(ISD::OR, Node, SDValue(), Val - uint32_t(Val), 1582 uint32_t(Val)); 1583 return; 1584 } 1585 } 1586 break; 1587 1588 case SystemZISD::SELECT_CCMASK: { 1589 SDValue Op0 = Node->getOperand(0); 1590 SDValue Op1 = Node->getOperand(1); 1591 // Prefer to put any load first, so that it can be matched as a 1592 // conditional load. Likewise for constants in range for LOCHI. 1593 if ((Op1.getOpcode() == ISD::LOAD && Op0.getOpcode() != ISD::LOAD) || 1594 (Subtarget->hasLoadStoreOnCond2() && 1595 Node->getValueType(0).isInteger() && 1596 Op1.getOpcode() == ISD::Constant && 1597 isInt<16>(cast<ConstantSDNode>(Op1)->getSExtValue()) && 1598 !(Op0.getOpcode() == ISD::Constant && 1599 isInt<16>(cast<ConstantSDNode>(Op0)->getSExtValue())))) { 1600 SDValue CCValid = Node->getOperand(2); 1601 SDValue CCMask = Node->getOperand(3); 1602 uint64_t ConstCCValid = 1603 cast<ConstantSDNode>(CCValid.getNode())->getZExtValue(); 1604 uint64_t ConstCCMask = 1605 cast<ConstantSDNode>(CCMask.getNode())->getZExtValue(); 1606 // Invert the condition. 1607 CCMask = CurDAG->getTargetConstant(ConstCCValid ^ ConstCCMask, 1608 SDLoc(Node), CCMask.getValueType()); 1609 SDValue Op4 = Node->getOperand(4); 1610 SDNode *UpdatedNode = 1611 CurDAG->UpdateNodeOperands(Node, Op1, Op0, CCValid, CCMask, Op4); 1612 if (UpdatedNode != Node) { 1613 // In case this node already exists then replace Node with it. 1614 ReplaceNode(Node, UpdatedNode); 1615 Node = UpdatedNode; 1616 } 1617 } 1618 break; 1619 } 1620 1621 case ISD::INSERT_VECTOR_ELT: { 1622 EVT VT = Node->getValueType(0); 1623 unsigned ElemBitSize = VT.getScalarSizeInBits(); 1624 if (ElemBitSize == 32) { 1625 if (tryGather(Node, SystemZ::VGEF)) 1626 return; 1627 } else if (ElemBitSize == 64) { 1628 if (tryGather(Node, SystemZ::VGEG)) 1629 return; 1630 } 1631 break; 1632 } 1633 1634 case ISD::BUILD_VECTOR: { 1635 auto *BVN = cast<BuildVectorSDNode>(Node); 1636 SystemZVectorConstantInfo VCI(BVN); 1637 if (VCI.isVectorConstantLegal(*Subtarget)) { 1638 loadVectorConstant(VCI, Node); 1639 return; 1640 } 1641 break; 1642 } 1643 1644 case ISD::ConstantFP: { 1645 APFloat Imm = cast<ConstantFPSDNode>(Node)->getValueAPF(); 1646 if (Imm.isZero() || Imm.isNegZero()) 1647 break; 1648 SystemZVectorConstantInfo VCI(Imm); 1649 bool Success = VCI.isVectorConstantLegal(*Subtarget); (void)Success; 1650 assert(Success && "Expected legal FP immediate"); 1651 loadVectorConstant(VCI, Node); 1652 return; 1653 } 1654 1655 case ISD::STORE: { 1656 if (tryFoldLoadStoreIntoMemOperand(Node)) 1657 return; 1658 auto *Store = cast<StoreSDNode>(Node); 1659 unsigned ElemBitSize = Store->getValue().getValueSizeInBits(); 1660 if (ElemBitSize == 32) { 1661 if (tryScatter(Store, SystemZ::VSCEF)) 1662 return; 1663 } else if (ElemBitSize == 64) { 1664 if (tryScatter(Store, SystemZ::VSCEG)) 1665 return; 1666 } 1667 break; 1668 } 1669 } 1670 1671 SelectCode(Node); 1672 } 1673 1674 bool SystemZDAGToDAGISel:: 1675 SelectInlineAsmMemoryOperand(const SDValue &Op, 1676 unsigned ConstraintID, 1677 std::vector<SDValue> &OutOps) { 1678 SystemZAddressingMode::AddrForm Form; 1679 SystemZAddressingMode::DispRange DispRange; 1680 SDValue Base, Disp, Index; 1681 1682 switch(ConstraintID) { 1683 default: 1684 llvm_unreachable("Unexpected asm memory constraint"); 1685 case InlineAsm::Constraint_i: 1686 case InlineAsm::Constraint_Q: 1687 // Accept an address with a short displacement, but no index. 1688 Form = SystemZAddressingMode::FormBD; 1689 DispRange = SystemZAddressingMode::Disp12Only; 1690 break; 1691 case InlineAsm::Constraint_R: 1692 // Accept an address with a short displacement and an index. 1693 Form = SystemZAddressingMode::FormBDXNormal; 1694 DispRange = SystemZAddressingMode::Disp12Only; 1695 break; 1696 case InlineAsm::Constraint_S: 1697 // Accept an address with a long displacement, but no index. 1698 Form = SystemZAddressingMode::FormBD; 1699 DispRange = SystemZAddressingMode::Disp20Only; 1700 break; 1701 case InlineAsm::Constraint_T: 1702 case InlineAsm::Constraint_m: 1703 case InlineAsm::Constraint_o: 1704 // Accept an address with a long displacement and an index. 1705 // m works the same as T, as this is the most general case. 1706 // We don't really have any special handling of "offsettable" 1707 // memory addresses, so just treat o the same as m. 1708 Form = SystemZAddressingMode::FormBDXNormal; 1709 DispRange = SystemZAddressingMode::Disp20Only; 1710 break; 1711 } 1712 1713 if (selectBDXAddr(Form, DispRange, Op, Base, Disp, Index)) { 1714 const TargetRegisterClass *TRC = 1715 Subtarget->getRegisterInfo()->getPointerRegClass(*MF); 1716 SDLoc DL(Base); 1717 SDValue RC = CurDAG->getTargetConstant(TRC->getID(), DL, MVT::i32); 1718 1719 // Make sure that the base address doesn't go into %r0. 1720 // If it's a TargetFrameIndex or a fixed register, we shouldn't do anything. 1721 if (Base.getOpcode() != ISD::TargetFrameIndex && 1722 Base.getOpcode() != ISD::Register) { 1723 Base = 1724 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, 1725 DL, Base.getValueType(), 1726 Base, RC), 0); 1727 } 1728 1729 // Make sure that the index register isn't assigned to %r0 either. 1730 if (Index.getOpcode() != ISD::Register) { 1731 Index = 1732 SDValue(CurDAG->getMachineNode(TargetOpcode::COPY_TO_REGCLASS, 1733 DL, Index.getValueType(), 1734 Index, RC), 0); 1735 } 1736 1737 OutOps.push_back(Base); 1738 OutOps.push_back(Disp); 1739 OutOps.push_back(Index); 1740 return false; 1741 } 1742 1743 return true; 1744 } 1745 1746 // IsProfitableToFold - Returns true if is profitable to fold the specific 1747 // operand node N of U during instruction selection that starts at Root. 1748 bool 1749 SystemZDAGToDAGISel::IsProfitableToFold(SDValue N, SDNode *U, 1750 SDNode *Root) const { 1751 // We want to avoid folding a LOAD into an ICMP node if as a result 1752 // we would be forced to spill the condition code into a GPR. 1753 if (N.getOpcode() == ISD::LOAD && U->getOpcode() == SystemZISD::ICMP) { 1754 if (!N.hasOneUse() || !U->hasOneUse()) 1755 return false; 1756 1757 // The user of the CC value will usually be a CopyToReg into the 1758 // physical CC register, which in turn is glued and chained to the 1759 // actual instruction that uses the CC value. Bail out if we have 1760 // anything else than that. 1761 SDNode *CCUser = *U->use_begin(); 1762 SDNode *CCRegUser = nullptr; 1763 if (CCUser->getOpcode() == ISD::CopyToReg || 1764 cast<RegisterSDNode>(CCUser->getOperand(1))->getReg() == SystemZ::CC) { 1765 for (auto *U : CCUser->uses()) { 1766 if (CCRegUser == nullptr) 1767 CCRegUser = U; 1768 else if (CCRegUser != U) 1769 return false; 1770 } 1771 } 1772 if (CCRegUser == nullptr) 1773 return false; 1774 1775 // If the actual instruction is a branch, the only thing that remains to be 1776 // checked is whether the CCUser chain is a predecessor of the load. 1777 if (CCRegUser->isMachineOpcode() && 1778 CCRegUser->getMachineOpcode() == SystemZ::BRC) 1779 return !N->isPredecessorOf(CCUser->getOperand(0).getNode()); 1780 1781 // Otherwise, the instruction may have multiple operands, and we need to 1782 // verify that none of them are a predecessor of the load. This is exactly 1783 // the same check that would be done by common code if the CC setter were 1784 // glued to the CC user, so simply invoke that check here. 1785 if (!IsLegalToFold(N, U, CCRegUser, OptLevel, false)) 1786 return false; 1787 } 1788 1789 return true; 1790 } 1791 1792 namespace { 1793 // Represents a sequence for extracting a 0/1 value from an IPM result: 1794 // (((X ^ XORValue) + AddValue) >> Bit) 1795 struct IPMConversion { 1796 IPMConversion(unsigned xorValue, int64_t addValue, unsigned bit) 1797 : XORValue(xorValue), AddValue(addValue), Bit(bit) {} 1798 1799 int64_t XORValue; 1800 int64_t AddValue; 1801 unsigned Bit; 1802 }; 1803 } // end anonymous namespace 1804 1805 // Return a sequence for getting a 1 from an IPM result when CC has a 1806 // value in CCMask and a 0 when CC has a value in CCValid & ~CCMask. 1807 // The handling of CC values outside CCValid doesn't matter. 1808 static IPMConversion getIPMConversion(unsigned CCValid, unsigned CCMask) { 1809 // Deal with cases where the result can be taken directly from a bit 1810 // of the IPM result. 1811 if (CCMask == (CCValid & (SystemZ::CCMASK_1 | SystemZ::CCMASK_3))) 1812 return IPMConversion(0, 0, SystemZ::IPM_CC); 1813 if (CCMask == (CCValid & (SystemZ::CCMASK_2 | SystemZ::CCMASK_3))) 1814 return IPMConversion(0, 0, SystemZ::IPM_CC + 1); 1815 1816 // Deal with cases where we can add a value to force the sign bit 1817 // to contain the right value. Putting the bit in 31 means we can 1818 // use SRL rather than RISBG(L), and also makes it easier to get a 1819 // 0/-1 value, so it has priority over the other tests below. 1820 // 1821 // These sequences rely on the fact that the upper two bits of the 1822 // IPM result are zero. 1823 uint64_t TopBit = uint64_t(1) << 31; 1824 if (CCMask == (CCValid & SystemZ::CCMASK_0)) 1825 return IPMConversion(0, -(1 << SystemZ::IPM_CC), 31); 1826 if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_1))) 1827 return IPMConversion(0, -(2 << SystemZ::IPM_CC), 31); 1828 if (CCMask == (CCValid & (SystemZ::CCMASK_0 1829 | SystemZ::CCMASK_1 1830 | SystemZ::CCMASK_2))) 1831 return IPMConversion(0, -(3 << SystemZ::IPM_CC), 31); 1832 if (CCMask == (CCValid & SystemZ::CCMASK_3)) 1833 return IPMConversion(0, TopBit - (3 << SystemZ::IPM_CC), 31); 1834 if (CCMask == (CCValid & (SystemZ::CCMASK_1 1835 | SystemZ::CCMASK_2 1836 | SystemZ::CCMASK_3))) 1837 return IPMConversion(0, TopBit - (1 << SystemZ::IPM_CC), 31); 1838 1839 // Next try inverting the value and testing a bit. 0/1 could be 1840 // handled this way too, but we dealt with that case above. 1841 if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_2))) 1842 return IPMConversion(-1, 0, SystemZ::IPM_CC); 1843 1844 // Handle cases where adding a value forces a non-sign bit to contain 1845 // the right value. 1846 if (CCMask == (CCValid & (SystemZ::CCMASK_1 | SystemZ::CCMASK_2))) 1847 return IPMConversion(0, 1 << SystemZ::IPM_CC, SystemZ::IPM_CC + 1); 1848 if (CCMask == (CCValid & (SystemZ::CCMASK_0 | SystemZ::CCMASK_3))) 1849 return IPMConversion(0, -(1 << SystemZ::IPM_CC), SystemZ::IPM_CC + 1); 1850 1851 // The remaining cases are 1, 2, 0/1/3 and 0/2/3. All these are 1852 // can be done by inverting the low CC bit and applying one of the 1853 // sign-based extractions above. 1854 if (CCMask == (CCValid & SystemZ::CCMASK_1)) 1855 return IPMConversion(1 << SystemZ::IPM_CC, -(1 << SystemZ::IPM_CC), 31); 1856 if (CCMask == (CCValid & SystemZ::CCMASK_2)) 1857 return IPMConversion(1 << SystemZ::IPM_CC, 1858 TopBit - (3 << SystemZ::IPM_CC), 31); 1859 if (CCMask == (CCValid & (SystemZ::CCMASK_0 1860 | SystemZ::CCMASK_1 1861 | SystemZ::CCMASK_3))) 1862 return IPMConversion(1 << SystemZ::IPM_CC, -(3 << SystemZ::IPM_CC), 31); 1863 if (CCMask == (CCValid & (SystemZ::CCMASK_0 1864 | SystemZ::CCMASK_2 1865 | SystemZ::CCMASK_3))) 1866 return IPMConversion(1 << SystemZ::IPM_CC, 1867 TopBit - (1 << SystemZ::IPM_CC), 31); 1868 1869 llvm_unreachable("Unexpected CC combination"); 1870 } 1871 1872 SDValue SystemZDAGToDAGISel::expandSelectBoolean(SDNode *Node) { 1873 auto *TrueOp = dyn_cast<ConstantSDNode>(Node->getOperand(0)); 1874 auto *FalseOp = dyn_cast<ConstantSDNode>(Node->getOperand(1)); 1875 if (!TrueOp || !FalseOp) 1876 return SDValue(); 1877 if (FalseOp->getZExtValue() != 0) 1878 return SDValue(); 1879 if (TrueOp->getSExtValue() != 1 && TrueOp->getSExtValue() != -1) 1880 return SDValue(); 1881 1882 auto *CCValidOp = dyn_cast<ConstantSDNode>(Node->getOperand(2)); 1883 auto *CCMaskOp = dyn_cast<ConstantSDNode>(Node->getOperand(3)); 1884 if (!CCValidOp || !CCMaskOp) 1885 return SDValue(); 1886 int CCValid = CCValidOp->getZExtValue(); 1887 int CCMask = CCMaskOp->getZExtValue(); 1888 1889 SDLoc DL(Node); 1890 SDValue CCReg = Node->getOperand(4); 1891 IPMConversion IPM = getIPMConversion(CCValid, CCMask); 1892 SDValue Result = CurDAG->getNode(SystemZISD::IPM, DL, MVT::i32, CCReg); 1893 1894 if (IPM.XORValue) 1895 Result = CurDAG->getNode(ISD::XOR, DL, MVT::i32, Result, 1896 CurDAG->getConstant(IPM.XORValue, DL, MVT::i32)); 1897 1898 if (IPM.AddValue) 1899 Result = CurDAG->getNode(ISD::ADD, DL, MVT::i32, Result, 1900 CurDAG->getConstant(IPM.AddValue, DL, MVT::i32)); 1901 1902 EVT VT = Node->getValueType(0); 1903 if (VT == MVT::i32 && IPM.Bit == 31) { 1904 unsigned ShiftOp = TrueOp->getSExtValue() == 1 ? ISD::SRL : ISD::SRA; 1905 Result = CurDAG->getNode(ShiftOp, DL, MVT::i32, Result, 1906 CurDAG->getConstant(IPM.Bit, DL, MVT::i32)); 1907 } else { 1908 if (VT != MVT::i32) 1909 Result = CurDAG->getNode(ISD::ANY_EXTEND, DL, VT, Result); 1910 1911 if (TrueOp->getSExtValue() == 1) { 1912 // The SHR/AND sequence should get optimized to an RISBG. 1913 Result = CurDAG->getNode(ISD::SRL, DL, VT, Result, 1914 CurDAG->getConstant(IPM.Bit, DL, MVT::i32)); 1915 Result = CurDAG->getNode(ISD::AND, DL, VT, Result, 1916 CurDAG->getConstant(1, DL, VT)); 1917 } else { 1918 // Sign-extend from IPM.Bit using a pair of shifts. 1919 int ShlAmt = VT.getSizeInBits() - 1 - IPM.Bit; 1920 int SraAmt = VT.getSizeInBits() - 1; 1921 Result = CurDAG->getNode(ISD::SHL, DL, VT, Result, 1922 CurDAG->getConstant(ShlAmt, DL, MVT::i32)); 1923 Result = CurDAG->getNode(ISD::SRA, DL, VT, Result, 1924 CurDAG->getConstant(SraAmt, DL, MVT::i32)); 1925 } 1926 } 1927 1928 return Result; 1929 } 1930 1931 void SystemZDAGToDAGISel::PreprocessISelDAG() { 1932 // If we have conditional immediate loads, we always prefer 1933 // using those over an IPM sequence. 1934 if (Subtarget->hasLoadStoreOnCond2()) 1935 return; 1936 1937 bool MadeChange = false; 1938 1939 for (SelectionDAG::allnodes_iterator I = CurDAG->allnodes_begin(), 1940 E = CurDAG->allnodes_end(); 1941 I != E;) { 1942 SDNode *N = &*I++; 1943 if (N->use_empty()) 1944 continue; 1945 1946 SDValue Res; 1947 switch (N->getOpcode()) { 1948 default: break; 1949 case SystemZISD::SELECT_CCMASK: 1950 Res = expandSelectBoolean(N); 1951 break; 1952 } 1953 1954 if (Res) { 1955 LLVM_DEBUG(dbgs() << "SystemZ DAG preprocessing replacing:\nOld: "); 1956 LLVM_DEBUG(N->dump(CurDAG)); 1957 LLVM_DEBUG(dbgs() << "\nNew: "); 1958 LLVM_DEBUG(Res.getNode()->dump(CurDAG)); 1959 LLVM_DEBUG(dbgs() << "\n"); 1960 1961 CurDAG->ReplaceAllUsesOfValueWith(SDValue(N, 0), Res); 1962 MadeChange = true; 1963 } 1964 } 1965 1966 if (MadeChange) 1967 CurDAG->RemoveDeadNodes(); 1968 } 1969