1 //===-- SystemZISelLowering.h - SystemZ DAG lowering interface --*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines the interfaces that SystemZ uses to lower LLVM code into a
10 // selection DAG.
11 //
12 //===----------------------------------------------------------------------===//
13
14 #ifndef LLVM_LIB_TARGET_SYSTEMZ_SYSTEMZISELLOWERING_H
15 #define LLVM_LIB_TARGET_SYSTEMZ_SYSTEMZISELLOWERING_H
16
17 #include "SystemZ.h"
18 #include "SystemZInstrInfo.h"
19 #include "llvm/CodeGen/MachineBasicBlock.h"
20 #include "llvm/CodeGen/SelectionDAG.h"
21 #include "llvm/CodeGen/TargetLowering.h"
22 #include <optional>
23
24 namespace llvm {
25 namespace SystemZISD {
26 enum NodeType : unsigned {
27 FIRST_NUMBER = ISD::BUILTIN_OP_END,
28
29 // Return with a glue operand. Operand 0 is the chain operand.
30 RET_GLUE,
31
32 // Calls a function. Operand 0 is the chain operand and operand 1
33 // is the target address. The arguments start at operand 2.
34 // There is an optional glue operand at the end.
35 CALL,
36 SIBCALL,
37
38 // TLS calls. Like regular calls, except operand 1 is the TLS symbol.
39 // (The call target is implicitly __tls_get_offset.)
40 TLS_GDCALL,
41 TLS_LDCALL,
42
43 // Wraps a TargetGlobalAddress that should be loaded using PC-relative
44 // accesses (LARL). Operand 0 is the address.
45 PCREL_WRAPPER,
46
47 // Used in cases where an offset is applied to a TargetGlobalAddress.
48 // Operand 0 is the full TargetGlobalAddress and operand 1 is a
49 // PCREL_WRAPPER for an anchor point. This is used so that we can
50 // cheaply refer to either the full address or the anchor point
51 // as a register base.
52 PCREL_OFFSET,
53
54 // Integer comparisons. There are three operands: the two values
55 // to compare, and an integer of type SystemZICMP.
56 ICMP,
57
58 // Floating-point comparisons. The two operands are the values to compare.
59 FCMP,
60
61 // Test under mask. The first operand is ANDed with the second operand
62 // and the condition codes are set on the result. The third operand is
63 // a boolean that is true if the condition codes need to distinguish
64 // between CCMASK_TM_MIXED_MSB_0 and CCMASK_TM_MIXED_MSB_1 (which the
65 // register forms do but the memory forms don't).
66 TM,
67
68 // Branches if a condition is true. Operand 0 is the chain operand;
69 // operand 1 is the 4-bit condition-code mask, with bit N in
70 // big-endian order meaning "branch if CC=N"; operand 2 is the
71 // target block and operand 3 is the flag operand.
72 BR_CCMASK,
73
74 // Selects between operand 0 and operand 1. Operand 2 is the
75 // mask of condition-code values for which operand 0 should be
76 // chosen over operand 1; it has the same form as BR_CCMASK.
77 // Operand 3 is the flag operand.
78 SELECT_CCMASK,
79
80 // Evaluates to the gap between the stack pointer and the
81 // base of the dynamically-allocatable area.
82 ADJDYNALLOC,
83
84 // For allocating stack space when using stack clash protector.
85 // Allocation is performed by block, and each block is probed.
86 PROBED_ALLOCA,
87
88 // Count number of bits set in operand 0 per byte.
89 POPCNT,
90
91 // Wrappers around the ISD opcodes of the same name. The output is GR128.
92 // Input operands may be GR64 or GR32, depending on the instruction.
93 SMUL_LOHI,
94 UMUL_LOHI,
95 SDIVREM,
96 UDIVREM,
97
98 // Add/subtract with overflow/carry. These have the same operands as
99 // the corresponding standard operations, except with the carry flag
100 // replaced by a condition code value.
101 SADDO, SSUBO, UADDO, USUBO, ADDCARRY, SUBCARRY,
102
103 // Set the condition code from a boolean value in operand 0.
104 // Operand 1 is a mask of all condition-code values that may result of this
105 // operation, operand 2 is a mask of condition-code values that may result
106 // if the boolean is true.
107 // Note that this operation is always optimized away, we will never
108 // generate any code for it.
109 GET_CCMASK,
110
111 // Use a series of MVCs to copy bytes from one memory location to another.
112 // The operands are:
113 // - the target address
114 // - the source address
115 // - the constant length
116 //
117 // This isn't a memory opcode because we'd need to attach two
118 // MachineMemOperands rather than one.
119 MVC,
120
121 // Similar to MVC, but for logic operations (AND, OR, XOR).
122 NC,
123 OC,
124 XC,
125
126 // Use CLC to compare two blocks of memory, with the same comments
127 // as for MVC.
128 CLC,
129
130 // Use MVC to set a block of memory after storing the first byte.
131 MEMSET_MVC,
132
133 // Use an MVST-based sequence to implement stpcpy().
134 STPCPY,
135
136 // Use a CLST-based sequence to implement strcmp(). The two input operands
137 // are the addresses of the strings to compare.
138 STRCMP,
139
140 // Use an SRST-based sequence to search a block of memory. The first
141 // operand is the end address, the second is the start, and the third
142 // is the character to search for. CC is set to 1 on success and 2
143 // on failure.
144 SEARCH_STRING,
145
146 // Store the CC value in bits 29 and 28 of an integer.
147 IPM,
148
149 // Transaction begin. The first operand is the chain, the second
150 // the TDB pointer, and the third the immediate control field.
151 // Returns CC value and chain.
152 TBEGIN,
153 TBEGIN_NOFLOAT,
154
155 // Transaction end. Just the chain operand. Returns CC value and chain.
156 TEND,
157
158 // Create a vector constant by filling byte N of the result with bit
159 // 15-N of the single operand.
160 BYTE_MASK,
161
162 // Create a vector constant by replicating an element-sized RISBG-style mask.
163 // The first operand specifies the starting set bit and the second operand
164 // specifies the ending set bit. Both operands count from the MSB of the
165 // element.
166 ROTATE_MASK,
167
168 // Replicate a GPR scalar value into all elements of a vector.
169 REPLICATE,
170
171 // Create a vector from two i64 GPRs.
172 JOIN_DWORDS,
173
174 // Replicate one element of a vector into all elements. The first operand
175 // is the vector and the second is the index of the element to replicate.
176 SPLAT,
177
178 // Interleave elements from the high half of operand 0 and the high half
179 // of operand 1.
180 MERGE_HIGH,
181
182 // Likewise for the low halves.
183 MERGE_LOW,
184
185 // Concatenate the vectors in the first two operands, shift them left
186 // by the third operand, and take the first half of the result.
187 SHL_DOUBLE,
188
189 // Take one element of the first v2i64 operand and the one element of
190 // the second v2i64 operand and concatenate them to form a v2i64 result.
191 // The third operand is a 4-bit value of the form 0A0B, where A and B
192 // are the element selectors for the first operand and second operands
193 // respectively.
194 PERMUTE_DWORDS,
195
196 // Perform a general vector permute on vector operands 0 and 1.
197 // Each byte of operand 2 controls the corresponding byte of the result,
198 // in the same way as a byte-level VECTOR_SHUFFLE mask.
199 PERMUTE,
200
201 // Pack vector operands 0 and 1 into a single vector with half-sized elements.
202 PACK,
203
204 // Likewise, but saturate the result and set CC. PACKS_CC does signed
205 // saturation and PACKLS_CC does unsigned saturation.
206 PACKS_CC,
207 PACKLS_CC,
208
209 // Unpack the first half of vector operand 0 into double-sized elements.
210 // UNPACK_HIGH sign-extends and UNPACKL_HIGH zero-extends.
211 UNPACK_HIGH,
212 UNPACKL_HIGH,
213
214 // Likewise for the second half.
215 UNPACK_LOW,
216 UNPACKL_LOW,
217
218 // Shift/rotate each element of vector operand 0 by the number of bits
219 // specified by scalar operand 1.
220 VSHL_BY_SCALAR,
221 VSRL_BY_SCALAR,
222 VSRA_BY_SCALAR,
223 VROTL_BY_SCALAR,
224
225 // Concatenate the vectors in the first two operands, shift them left/right
226 // bitwise by the third operand, and take the first/last half of the result.
227 SHL_DOUBLE_BIT,
228 SHR_DOUBLE_BIT,
229
230 // For each element of the output type, sum across all sub-elements of
231 // operand 0 belonging to the corresponding element, and add in the
232 // rightmost sub-element of the corresponding element of operand 1.
233 VSUM,
234
235 // Compute carry/borrow indication for add/subtract.
236 VACC, VSCBI,
237 // Add/subtract with carry/borrow.
238 VAC, VSBI,
239 // Compute carry/borrow indication for add/subtract with carry/borrow.
240 VACCC, VSBCBI,
241
242 // High-word multiply-and-add.
243 VMAH, VMALH,
244 // Widen and multiply even/odd vector elements.
245 VME, VMLE, VMO, VMLO,
246
247 // Compare integer vector operands 0 and 1 to produce the usual 0/-1
248 // vector result. VICMPE is for equality, VICMPH for "signed greater than"
249 // and VICMPHL for "unsigned greater than".
250 VICMPE,
251 VICMPH,
252 VICMPHL,
253
254 // Likewise, but also set the condition codes on the result.
255 VICMPES,
256 VICMPHS,
257 VICMPHLS,
258
259 // Compare floating-point vector operands 0 and 1 to produce the usual 0/-1
260 // vector result. VFCMPE is for "ordered and equal", VFCMPH for "ordered and
261 // greater than" and VFCMPHE for "ordered and greater than or equal to".
262 VFCMPE,
263 VFCMPH,
264 VFCMPHE,
265
266 // Likewise, but also set the condition codes on the result.
267 VFCMPES,
268 VFCMPHS,
269 VFCMPHES,
270
271 // Test floating-point data class for vectors.
272 VFTCI,
273
274 // Extend the even f32 elements of vector operand 0 to produce a vector
275 // of f64 elements.
276 VEXTEND,
277
278 // Round the f64 elements of vector operand 0 to f32s and store them in the
279 // even elements of the result.
280 VROUND,
281
282 // AND the two vector operands together and set CC based on the result.
283 VTM,
284
285 // i128 high integer comparisons.
286 SCMP128HI,
287 UCMP128HI,
288
289 // String operations that set CC as a side-effect.
290 VFAE_CC,
291 VFAEZ_CC,
292 VFEE_CC,
293 VFEEZ_CC,
294 VFENE_CC,
295 VFENEZ_CC,
296 VISTR_CC,
297 VSTRC_CC,
298 VSTRCZ_CC,
299 VSTRS_CC,
300 VSTRSZ_CC,
301
302 // Test Data Class.
303 //
304 // Operand 0: the value to test
305 // Operand 1: the bit mask
306 TDC,
307
308 // z/OS XPLINK ADA Entry
309 // Wraps a TargetGlobalAddress that should be loaded from a function's
310 // AssociatedData Area (ADA). Tha ADA is passed to the function by the
311 // caller in the XPLink ABI defined register R5.
312 // Operand 0: the GlobalValue/External Symbol
313 // Operand 1: the ADA register
314 // Operand 2: the offset (0 for the first and 8 for the second element in the
315 // function descriptor)
316 ADA_ENTRY,
317
318 // Strict variants of scalar floating-point comparisons.
319 // Quiet and signaling versions.
320 FIRST_STRICTFP_OPCODE,
321 STRICT_FCMP = FIRST_STRICTFP_OPCODE,
322 STRICT_FCMPS,
323
324 // Strict variants of vector floating-point comparisons.
325 // Quiet and signaling versions.
326 STRICT_VFCMPE,
327 STRICT_VFCMPH,
328 STRICT_VFCMPHE,
329 STRICT_VFCMPES,
330 STRICT_VFCMPHS,
331 STRICT_VFCMPHES,
332
333 // Strict variants of VEXTEND and VROUND.
334 STRICT_VEXTEND,
335 STRICT_VROUND,
336 LAST_STRICTFP_OPCODE = STRICT_VROUND,
337
338 // Wrappers around the inner loop of an 8- or 16-bit ATOMIC_SWAP or
339 // ATOMIC_LOAD_<op>.
340 //
341 // Operand 0: the address of the containing 32-bit-aligned field
342 // Operand 1: the second operand of <op>, in the high bits of an i32
343 // for everything except ATOMIC_SWAPW
344 // Operand 2: how many bits to rotate the i32 left to bring the first
345 // operand into the high bits
346 // Operand 3: the negative of operand 2, for rotating the other way
347 // Operand 4: the width of the field in bits (8 or 16)
348 FIRST_MEMORY_OPCODE,
349 ATOMIC_SWAPW = FIRST_MEMORY_OPCODE,
350 ATOMIC_LOADW_ADD,
351 ATOMIC_LOADW_SUB,
352 ATOMIC_LOADW_AND,
353 ATOMIC_LOADW_OR,
354 ATOMIC_LOADW_XOR,
355 ATOMIC_LOADW_NAND,
356 ATOMIC_LOADW_MIN,
357 ATOMIC_LOADW_MAX,
358 ATOMIC_LOADW_UMIN,
359 ATOMIC_LOADW_UMAX,
360
361 // A wrapper around the inner loop of an ATOMIC_CMP_SWAP.
362 //
363 // Operand 0: the address of the containing 32-bit-aligned field
364 // Operand 1: the compare value, in the low bits of an i32
365 // Operand 2: the swap value, in the low bits of an i32
366 // Operand 3: how many bits to rotate the i32 left to bring the first
367 // operand into the high bits
368 // Operand 4: the negative of operand 2, for rotating the other way
369 // Operand 5: the width of the field in bits (8 or 16)
370 ATOMIC_CMP_SWAPW,
371
372 // Atomic compare-and-swap returning CC value.
373 // Val, CC, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap)
374 ATOMIC_CMP_SWAP,
375
376 // 128-bit atomic load.
377 // Val, OUTCHAIN = ATOMIC_LOAD_128(INCHAIN, ptr)
378 ATOMIC_LOAD_128,
379
380 // 128-bit atomic store.
381 // OUTCHAIN = ATOMIC_STORE_128(INCHAIN, val, ptr)
382 ATOMIC_STORE_128,
383
384 // 128-bit atomic compare-and-swap.
385 // Val, CC, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap)
386 ATOMIC_CMP_SWAP_128,
387
388 // Byte swapping load/store. Same operands as regular load/store.
389 LRV, STRV,
390
391 // Element swapping load/store. Same operands as regular load/store.
392 VLER, VSTER,
393
394 // Use STORE CLOCK FAST to store current TOD clock value.
395 STCKF,
396
397 // Prefetch from the second operand using the 4-bit control code in
398 // the first operand. The code is 1 for a load prefetch and 2 for
399 // a store prefetch.
400 PREFETCH,
401 LAST_MEMORY_OPCODE = PREFETCH,
402 };
403
404 // Return true if OPCODE is some kind of PC-relative address.
isPCREL(unsigned Opcode)405 inline bool isPCREL(unsigned Opcode) {
406 return Opcode == PCREL_WRAPPER || Opcode == PCREL_OFFSET;
407 }
408 } // end namespace SystemZISD
409
410 namespace SystemZICMP {
411 // Describes whether an integer comparison needs to be signed or unsigned,
412 // or whether either type is OK.
413 enum {
414 Any,
415 UnsignedOnly,
416 SignedOnly
417 };
418 } // end namespace SystemZICMP
419
420 class SystemZSubtarget;
421
422 class SystemZTargetLowering : public TargetLowering {
423 public:
424 explicit SystemZTargetLowering(const TargetMachine &TM,
425 const SystemZSubtarget &STI);
426
427 bool useSoftFloat() const override;
428
429 // Override TargetLowering.
getScalarShiftAmountTy(const DataLayout &,EVT)430 MVT getScalarShiftAmountTy(const DataLayout &, EVT) const override {
431 return MVT::i32;
432 }
getVectorIdxWidth(const DataLayout & DL)433 unsigned getVectorIdxWidth(const DataLayout &DL) const override {
434 // Only the lower 12 bits of an element index are used, so we don't
435 // want to clobber the upper 32 bits of a GPR unnecessarily.
436 return 32;
437 }
getPreferredVectorAction(MVT VT)438 TargetLoweringBase::LegalizeTypeAction getPreferredVectorAction(MVT VT)
439 const override {
440 // Widen subvectors to the full width rather than promoting integer
441 // elements. This is better because:
442 //
443 // (a) it means that we can handle the ABI for passing and returning
444 // sub-128 vectors without having to handle them as legal types.
445 //
446 // (b) we don't have instructions to extend on load and truncate on store,
447 // so promoting the integers is less efficient.
448 //
449 // (c) there are no multiplication instructions for the widest integer
450 // type (v2i64).
451 if (VT.getScalarSizeInBits() % 8 == 0)
452 return TypeWidenVector;
453 return TargetLoweringBase::getPreferredVectorAction(VT);
454 }
455 unsigned
getNumRegisters(LLVMContext & Context,EVT VT,std::optional<MVT> RegisterVT)456 getNumRegisters(LLVMContext &Context, EVT VT,
457 std::optional<MVT> RegisterVT) const override {
458 // i128 inline assembly operand.
459 if (VT == MVT::i128 && RegisterVT && *RegisterVT == MVT::Untyped)
460 return 1;
461 return TargetLowering::getNumRegisters(Context, VT);
462 }
getRegisterTypeForCallingConv(LLVMContext & Context,CallingConv::ID CC,EVT VT)463 MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC,
464 EVT VT) const override {
465 // 128-bit single-element vector types are passed like other vectors,
466 // not like their element type.
467 if (VT.isVector() && VT.getSizeInBits() == 128 &&
468 VT.getVectorNumElements() == 1)
469 return MVT::v16i8;
470 return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
471 }
isCheapToSpeculateCtlz(Type *)472 bool isCheapToSpeculateCtlz(Type *) const override { return true; }
isCheapToSpeculateCttz(Type *)473 bool isCheapToSpeculateCttz(Type *) const override { return true; }
preferZeroCompareBranch()474 bool preferZeroCompareBranch() const override { return true; }
isMaskAndCmp0FoldingBeneficial(const Instruction & AndI)475 bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override {
476 ConstantInt* Mask = dyn_cast<ConstantInt>(AndI.getOperand(1));
477 return Mask && Mask->getValue().isIntN(16);
478 }
convertSetCCLogicToBitwiseLogic(EVT VT)479 bool convertSetCCLogicToBitwiseLogic(EVT VT) const override {
480 return VT.isScalarInteger();
481 }
482 EVT getSetCCResultType(const DataLayout &DL, LLVMContext &,
483 EVT) const override;
484 bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
485 EVT VT) const override;
486 bool isFPImmLegal(const APFloat &Imm, EVT VT,
487 bool ForCodeSize) const override;
ShouldShrinkFPConstant(EVT VT)488 bool ShouldShrinkFPConstant(EVT VT) const override {
489 // Do not shrink 64-bit FP constpool entries since LDEB is slower than
490 // LD, and having the full constant in memory enables reg/mem opcodes.
491 return VT != MVT::f64;
492 }
493 MachineBasicBlock *emitEHSjLjSetJmp(MachineInstr &MI,
494 MachineBasicBlock *MBB) const;
495
496 MachineBasicBlock *emitEHSjLjLongJmp(MachineInstr &MI,
497 MachineBasicBlock *MBB) const;
498
499 bool hasInlineStackProbe(const MachineFunction &MF) const override;
500 AtomicExpansionKind shouldCastAtomicLoadInIR(LoadInst *LI) const override;
501 AtomicExpansionKind shouldCastAtomicStoreInIR(StoreInst *SI) const override;
502 AtomicExpansionKind
503 shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const override;
504 bool isLegalICmpImmediate(int64_t Imm) const override;
505 bool isLegalAddImmediate(int64_t Imm) const override;
506 bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty,
507 unsigned AS,
508 Instruction *I = nullptr) const override;
509 bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AS, Align Alignment,
510 MachineMemOperand::Flags Flags,
511 unsigned *Fast) const override;
512 bool
513 findOptimalMemOpLowering(LLVMContext &Context, std::vector<EVT> &MemOps,
514 unsigned Limit, const MemOp &Op, unsigned DstAS,
515 unsigned SrcAS,
516 const AttributeList &FuncAttributes) const override;
517 EVT getOptimalMemOpType(LLVMContext &Context, const MemOp &Op,
518 const AttributeList &FuncAttributes) const override;
519 bool isTruncateFree(Type *, Type *) const override;
isTruncateFree(EVT,EVT)520 bool isTruncateFree(EVT, EVT) const override;
521
522 bool shouldFormOverflowOp(unsigned Opcode, EVT VT,
523 bool MathUsed) const override {
524 // Form add and sub with overflow intrinsics regardless of any extra
525 // users of the math result.
526 return VT == MVT::i32 || VT == MVT::i64;
527 }
528
shouldConsiderGEPOffsetSplit()529 bool shouldConsiderGEPOffsetSplit() const override { return true; }
530
shouldExpandCmpUsingSelects(EVT VT)531 bool shouldExpandCmpUsingSelects(EVT VT) const override { return true; }
532
533 const char *getTargetNodeName(unsigned Opcode) const override;
534 std::pair<unsigned, const TargetRegisterClass *>
535 getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
536 StringRef Constraint, MVT VT) const override;
537 TargetLowering::ConstraintType
538 getConstraintType(StringRef Constraint) const override;
539 TargetLowering::ConstraintWeight
540 getSingleConstraintMatchWeight(AsmOperandInfo &info,
541 const char *constraint) const override;
542 void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint,
543 std::vector<SDValue> &Ops,
544 SelectionDAG &DAG) const override;
545
546 InlineAsm::ConstraintCode
getInlineAsmMemConstraint(StringRef ConstraintCode)547 getInlineAsmMemConstraint(StringRef ConstraintCode) const override {
548 if (ConstraintCode.size() == 1) {
549 switch(ConstraintCode[0]) {
550 default:
551 break;
552 case 'o':
553 return InlineAsm::ConstraintCode::o;
554 case 'Q':
555 return InlineAsm::ConstraintCode::Q;
556 case 'R':
557 return InlineAsm::ConstraintCode::R;
558 case 'S':
559 return InlineAsm::ConstraintCode::S;
560 case 'T':
561 return InlineAsm::ConstraintCode::T;
562 }
563 } else if (ConstraintCode.size() == 2 && ConstraintCode[0] == 'Z') {
564 switch (ConstraintCode[1]) {
565 default:
566 break;
567 case 'Q':
568 return InlineAsm::ConstraintCode::ZQ;
569 case 'R':
570 return InlineAsm::ConstraintCode::ZR;
571 case 'S':
572 return InlineAsm::ConstraintCode::ZS;
573 case 'T':
574 return InlineAsm::ConstraintCode::ZT;
575 }
576 }
577 return TargetLowering::getInlineAsmMemConstraint(ConstraintCode);
578 }
579
580 Register getRegisterByName(const char *RegName, LLT VT,
581 const MachineFunction &MF) const override;
582
583 /// If a physical register, this returns the register that receives the
584 /// exception address on entry to an EH pad.
585 Register
586 getExceptionPointerRegister(const Constant *PersonalityFn) const override;
587
588 /// If a physical register, this returns the register that receives the
589 /// exception typeid on entry to a landing pad.
590 Register
591 getExceptionSelectorRegister(const Constant *PersonalityFn) const override;
592
593 /// Override to support customized stack guard loading.
useLoadStackGuardNode(const Module & M)594 bool useLoadStackGuardNode(const Module &M) const override { return true; }
insertSSPDeclarations(Module & M)595 void insertSSPDeclarations(Module &M) const override {
596 }
597
598 MachineBasicBlock *
599 EmitInstrWithCustomInserter(MachineInstr &MI,
600 MachineBasicBlock *BB) const override;
601 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
602 void LowerOperationWrapper(SDNode *N, SmallVectorImpl<SDValue> &Results,
603 SelectionDAG &DAG) const override;
604 void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue>&Results,
605 SelectionDAG &DAG) const override;
606 const MCPhysReg *getScratchRegisters(CallingConv::ID CC) const override;
607 bool allowTruncateForTailCall(Type *, Type *) const override;
608 bool mayBeEmittedAsTailCall(const CallInst *CI) const override;
609 bool splitValueIntoRegisterParts(
610 SelectionDAG & DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
611 unsigned NumParts, MVT PartVT, std::optional<CallingConv::ID> CC)
612 const override;
613 SDValue joinRegisterPartsIntoValue(
614 SelectionDAG & DAG, const SDLoc &DL, const SDValue *Parts,
615 unsigned NumParts, MVT PartVT, EVT ValueVT,
616 std::optional<CallingConv::ID> CC) const override;
617 SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv,
618 bool isVarArg,
619 const SmallVectorImpl<ISD::InputArg> &Ins,
620 const SDLoc &DL, SelectionDAG &DAG,
621 SmallVectorImpl<SDValue> &InVals) const override;
622 SDValue LowerCall(CallLoweringInfo &CLI,
623 SmallVectorImpl<SDValue> &InVals) const override;
624
625 std::pair<SDValue, SDValue>
626 makeExternalCall(SDValue Chain, SelectionDAG &DAG, const char *CalleeName,
627 EVT RetVT, ArrayRef<SDValue> Ops, CallingConv::ID CallConv,
628 bool IsSigned, SDLoc DL, bool DoesNotReturn,
629 bool IsReturnValueUsed) const;
630
631 SDValue useLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, MVT VT, SDValue Arg,
632 SDLoc DL, SDValue Chain, bool IsStrict) const;
633
634 bool CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF,
635 bool isVarArg,
636 const SmallVectorImpl<ISD::OutputArg> &Outs,
637 LLVMContext &Context,
638 const Type *RetTy) const override;
639 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
640 const SmallVectorImpl<ISD::OutputArg> &Outs,
641 const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL,
642 SelectionDAG &DAG) const override;
643 SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
644
645 /// Determine which of the bits specified in Mask are known to be either
646 /// zero or one and return them in the KnownZero/KnownOne bitsets.
647 void computeKnownBitsForTargetNode(const SDValue Op,
648 KnownBits &Known,
649 const APInt &DemandedElts,
650 const SelectionDAG &DAG,
651 unsigned Depth = 0) const override;
652
653 /// Determine the number of bits in the operation that are sign bits.
654 unsigned ComputeNumSignBitsForTargetNode(SDValue Op,
655 const APInt &DemandedElts,
656 const SelectionDAG &DAG,
657 unsigned Depth) const override;
658
659 bool isGuaranteedNotToBeUndefOrPoisonForTargetNode(
660 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
661 bool PoisonOnly, unsigned Depth) const override;
662
getExtendForAtomicOps()663 ISD::NodeType getExtendForAtomicOps() const override {
664 return ISD::ANY_EXTEND;
665 }
getExtendForAtomicCmpSwapArg()666 ISD::NodeType getExtendForAtomicCmpSwapArg() const override {
667 return ISD::ZERO_EXTEND;
668 }
669
supportSwiftError()670 bool supportSwiftError() const override {
671 return true;
672 }
673
674 unsigned getStackProbeSize(const MachineFunction &MF) const;
675 bool hasAndNot(SDValue Y) const override;
676
677 private:
678 const SystemZSubtarget &Subtarget;
679
680 // Implement LowerOperation for individual opcodes.
681 SDValue getVectorCmp(SelectionDAG &DAG, unsigned Opcode,
682 const SDLoc &DL, EVT VT,
683 SDValue CmpOp0, SDValue CmpOp1, SDValue Chain) const;
684 SDValue lowerVectorSETCC(SelectionDAG &DAG, const SDLoc &DL,
685 EVT VT, ISD::CondCode CC,
686 SDValue CmpOp0, SDValue CmpOp1,
687 SDValue Chain = SDValue(),
688 bool IsSignaling = false) const;
689 SDValue lowerSETCC(SDValue Op, SelectionDAG &DAG) const;
690 SDValue lowerSTRICT_FSETCC(SDValue Op, SelectionDAG &DAG,
691 bool IsSignaling) const;
692 SDValue lowerBR_CC(SDValue Op, SelectionDAG &DAG) const;
693 SDValue lowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const;
694 SDValue lowerGlobalAddress(GlobalAddressSDNode *Node,
695 SelectionDAG &DAG) const;
696 SDValue lowerTLSGetOffset(GlobalAddressSDNode *Node,
697 SelectionDAG &DAG, unsigned Opcode,
698 SDValue GOTOffset) const;
699 SDValue lowerThreadPointer(const SDLoc &DL, SelectionDAG &DAG) const;
700 SDValue lowerGlobalTLSAddress(GlobalAddressSDNode *Node,
701 SelectionDAG &DAG) const;
702 SDValue lowerBlockAddress(BlockAddressSDNode *Node,
703 SelectionDAG &DAG) const;
704 SDValue lowerJumpTable(JumpTableSDNode *JT, SelectionDAG &DAG) const;
705 SDValue lowerConstantPool(ConstantPoolSDNode *CP, SelectionDAG &DAG) const;
706 SDValue lowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const;
707 SDValue lowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
708 SDValue lowerVASTART(SDValue Op, SelectionDAG &DAG) const;
709 SDValue lowerVASTART_ELF(SDValue Op, SelectionDAG &DAG) const;
710 SDValue lowerVASTART_XPLINK(SDValue Op, SelectionDAG &DAG) const;
711 SDValue lowerVACOPY(SDValue Op, SelectionDAG &DAG) const;
712 SDValue lowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const;
713 SDValue lowerDYNAMIC_STACKALLOC_ELF(SDValue Op, SelectionDAG &DAG) const;
714 SDValue lowerDYNAMIC_STACKALLOC_XPLINK(SDValue Op, SelectionDAG &DAG) const;
715 SDValue lowerGET_DYNAMIC_AREA_OFFSET(SDValue Op, SelectionDAG &DAG) const;
716 SDValue lowerMULH(SDValue Op, SelectionDAG &DAG, unsigned Opcode) const;
717 SDValue lowerSMUL_LOHI(SDValue Op, SelectionDAG &DAG) const;
718 SDValue lowerUMUL_LOHI(SDValue Op, SelectionDAG &DAG) const;
719 SDValue lowerSDIVREM(SDValue Op, SelectionDAG &DAG) const;
720 SDValue lowerUDIVREM(SDValue Op, SelectionDAG &DAG) const;
721 SDValue lowerXALUO(SDValue Op, SelectionDAG &DAG) const;
722 SDValue lowerUADDSUBO_CARRY(SDValue Op, SelectionDAG &DAG) const;
723 SDValue lowerBITCAST(SDValue Op, SelectionDAG &DAG) const;
724 SDValue lowerOR(SDValue Op, SelectionDAG &DAG) const;
725 SDValue lowerCTPOP(SDValue Op, SelectionDAG &DAG) const;
726 SDValue lowerVECREDUCE_ADD(SDValue Op, SelectionDAG &DAG) const;
727 SDValue lowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG) const;
728 SDValue lowerATOMIC_LOAD(SDValue Op, SelectionDAG &DAG) const;
729 SDValue lowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG) const;
730 SDValue lowerATOMIC_LDST_I128(SDValue Op, SelectionDAG &DAG) const;
731 SDValue lowerATOMIC_LOAD_OP(SDValue Op, SelectionDAG &DAG,
732 unsigned Opcode) const;
733 SDValue lowerATOMIC_LOAD_SUB(SDValue Op, SelectionDAG &DAG) const;
734 SDValue lowerATOMIC_CMP_SWAP(SDValue Op, SelectionDAG &DAG) const;
735 SDValue lowerSTACKSAVE(SDValue Op, SelectionDAG &DAG) const;
736 SDValue lowerSTACKRESTORE(SDValue Op, SelectionDAG &DAG) const;
737 SDValue lowerPREFETCH(SDValue Op, SelectionDAG &DAG) const;
738 SDValue lowerINTRINSIC_W_CHAIN(SDValue Op, SelectionDAG &DAG) const;
739 SDValue lowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
740 bool isVectorElementLoad(SDValue Op) const;
741 SDValue buildVector(SelectionDAG &DAG, const SDLoc &DL, EVT VT,
742 SmallVectorImpl<SDValue> &Elems) const;
743 SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
744 SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const;
745 SDValue lowerSCALAR_TO_VECTOR(SDValue Op, SelectionDAG &DAG) const;
746 SDValue lowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
747 SDValue lowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
748 SDValue lowerSIGN_EXTEND_VECTOR_INREG(SDValue Op, SelectionDAG &DAG) const;
749 SDValue lowerZERO_EXTEND_VECTOR_INREG(SDValue Op, SelectionDAG &DAG) const;
750 SDValue lowerShift(SDValue Op, SelectionDAG &DAG, unsigned ByScalar) const;
751 SDValue lowerFSHL(SDValue Op, SelectionDAG &DAG) const;
752 SDValue lowerFSHR(SDValue Op, SelectionDAG &DAG) const;
753 SDValue lowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const;
754 SDValue lower_FP_TO_INT(SDValue Op, SelectionDAG &DAG) const;
755 SDValue lower_INT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
756 SDValue lowerLoadF16(SDValue Op, SelectionDAG &DAG) const;
757 SDValue lowerStoreF16(SDValue Op, SelectionDAG &DAG) const;
758
759 SDValue lowerIS_FPCLASS(SDValue Op, SelectionDAG &DAG) const;
760 SDValue lowerGET_ROUNDING(SDValue Op, SelectionDAG &DAG) const;
761 SDValue lowerREADCYCLECOUNTER(SDValue Op, SelectionDAG &DAG) const;
762
763 bool canTreatAsByteVector(EVT VT) const;
764 SDValue combineExtract(const SDLoc &DL, EVT ElemVT, EVT VecVT, SDValue OrigOp,
765 unsigned Index, DAGCombinerInfo &DCI,
766 bool Force) const;
767 SDValue combineTruncateExtract(const SDLoc &DL, EVT TruncVT, SDValue Op,
768 DAGCombinerInfo &DCI) const;
769 SDValue combineZERO_EXTEND(SDNode *N, DAGCombinerInfo &DCI) const;
770 SDValue combineSIGN_EXTEND(SDNode *N, DAGCombinerInfo &DCI) const;
771 SDValue combineSIGN_EXTEND_INREG(SDNode *N, DAGCombinerInfo &DCI) const;
772 SDValue combineMERGE(SDNode *N, DAGCombinerInfo &DCI) const;
773 bool canLoadStoreByteSwapped(EVT VT) const;
774 SDValue combineLOAD(SDNode *N, DAGCombinerInfo &DCI) const;
775 SDValue combineSTORE(SDNode *N, DAGCombinerInfo &DCI) const;
776 SDValue combineVECTOR_SHUFFLE(SDNode *N, DAGCombinerInfo &DCI) const;
777 SDValue combineEXTRACT_VECTOR_ELT(SDNode *N, DAGCombinerInfo &DCI) const;
778 SDValue combineJOIN_DWORDS(SDNode *N, DAGCombinerInfo &DCI) const;
779 SDValue combineFP_ROUND(SDNode *N, DAGCombinerInfo &DCI) const;
780 SDValue combineFP_EXTEND(SDNode *N, DAGCombinerInfo &DCI) const;
781 SDValue combineINT_TO_FP(SDNode *N, DAGCombinerInfo &DCI) const;
782 SDValue combineFCOPYSIGN(SDNode *N, DAGCombinerInfo &DCI) const;
783 SDValue combineBSWAP(SDNode *N, DAGCombinerInfo &DCI) const;
784 SDValue combineSETCC(SDNode *N, DAGCombinerInfo &DCI) const;
785 SDValue combineBR_CCMASK(SDNode *N, DAGCombinerInfo &DCI) const;
786 SDValue combineSELECT_CCMASK(SDNode *N, DAGCombinerInfo &DCI) const;
787 SDValue combineGET_CCMASK(SDNode *N, DAGCombinerInfo &DCI) const;
788 SDValue combineShiftToMulAddHigh(SDNode *N, DAGCombinerInfo &DCI) const;
789 SDValue combineMUL(SDNode *N, DAGCombinerInfo &DCI) const;
790 SDValue combineIntDIVREM(SDNode *N, DAGCombinerInfo &DCI) const;
791 SDValue combineINTRINSIC(SDNode *N, DAGCombinerInfo &DCI) const;
792
793 SDValue unwrapAddress(SDValue N) const override;
794
795 // If the last instruction before MBBI in MBB was some form of COMPARE,
796 // try to replace it with a COMPARE AND BRANCH just before MBBI.
797 // CCMask and Target are the BRC-like operands for the branch.
798 // Return true if the change was made.
799 bool convertPrevCompareToBranch(MachineBasicBlock *MBB,
800 MachineBasicBlock::iterator MBBI,
801 unsigned CCMask,
802 MachineBasicBlock *Target) const;
803
804 // Implement EmitInstrWithCustomInserter for individual operation types.
805 MachineBasicBlock *emitAdjCallStack(MachineInstr &MI,
806 MachineBasicBlock *BB) const;
807 MachineBasicBlock *emitSelect(MachineInstr &MI, MachineBasicBlock *BB) const;
808 MachineBasicBlock *emitCondStore(MachineInstr &MI, MachineBasicBlock *BB,
809 unsigned StoreOpcode, unsigned STOCOpcode,
810 bool Invert) const;
811 MachineBasicBlock *emitICmp128Hi(MachineInstr &MI, MachineBasicBlock *BB,
812 bool Unsigned) const;
813 MachineBasicBlock *emitPair128(MachineInstr &MI,
814 MachineBasicBlock *MBB) const;
815 MachineBasicBlock *emitExt128(MachineInstr &MI, MachineBasicBlock *MBB,
816 bool ClearEven) const;
817 MachineBasicBlock *emitAtomicLoadBinary(MachineInstr &MI,
818 MachineBasicBlock *BB,
819 unsigned BinOpcode,
820 bool Invert = false) const;
821 MachineBasicBlock *emitAtomicLoadMinMax(MachineInstr &MI,
822 MachineBasicBlock *MBB,
823 unsigned CompareOpcode,
824 unsigned KeepOldMask) const;
825 MachineBasicBlock *emitAtomicCmpSwapW(MachineInstr &MI,
826 MachineBasicBlock *BB) const;
827 MachineBasicBlock *emitMemMemWrapper(MachineInstr &MI, MachineBasicBlock *BB,
828 unsigned Opcode,
829 bool IsMemset = false) const;
830 MachineBasicBlock *emitStringWrapper(MachineInstr &MI, MachineBasicBlock *BB,
831 unsigned Opcode) const;
832 MachineBasicBlock *emitTransactionBegin(MachineInstr &MI,
833 MachineBasicBlock *MBB,
834 unsigned Opcode, bool NoFloat) const;
835 MachineBasicBlock *emitLoadAndTestCmp0(MachineInstr &MI,
836 MachineBasicBlock *MBB,
837 unsigned Opcode) const;
838 MachineBasicBlock *emitProbedAlloca(MachineInstr &MI,
839 MachineBasicBlock *MBB) const;
840
841 SDValue getBackchainAddress(SDValue SP, SelectionDAG &DAG) const;
842
843 MachineMemOperand::Flags
844 getTargetMMOFlags(const Instruction &I) const override;
845 const TargetRegisterClass *getRepRegClassFor(MVT VT) const override;
846
847 private:
848 bool isInternal(const Function *Fn) const;
849 mutable std::map<const Function *, bool> IsInternalCache;
850 void verifyNarrowIntegerArgs_Call(const SmallVectorImpl<ISD::OutputArg> &Outs,
851 const Function *F, SDValue Callee) const;
852 void verifyNarrowIntegerArgs_Ret(const SmallVectorImpl<ISD::OutputArg> &Outs,
853 const Function *F) const;
854 bool
855 verifyNarrowIntegerArgs(const SmallVectorImpl<ISD::OutputArg> &Outs) const;
856
857 public:
858 };
859
860 struct SystemZVectorConstantInfo {
861 private:
862 APInt IntBits; // The 128 bits as an integer.
863 APInt SplatBits; // Smallest splat value.
864 APInt SplatUndef; // Bits correspoding to undef operands of the BVN.
865 unsigned SplatBitSize = 0;
866 bool isFP128 = false;
867 public:
868 unsigned Opcode = 0;
869 SmallVector<unsigned, 2> OpVals;
870 MVT VecVT;
871 SystemZVectorConstantInfo(APInt IntImm);
SystemZVectorConstantInfoSystemZVectorConstantInfo872 SystemZVectorConstantInfo(APFloat FPImm)
873 : SystemZVectorConstantInfo(FPImm.bitcastToAPInt()) {
874 isFP128 = (&FPImm.getSemantics() == &APFloat::IEEEquad());
875 }
876 SystemZVectorConstantInfo(BuildVectorSDNode *BVN);
877 bool isVectorConstantLegal(const SystemZSubtarget &Subtarget);
878 };
879
880 } // end namespace llvm
881
882 #endif
883