xref: /freebsd/contrib/llvm-project/llvm/lib/Target/SystemZ/SystemZTargetTransformInfo.cpp (revision ec0ea6efa1ad229d75c394c1a9b9cac33af2b1d3)
1 //===-- SystemZTargetTransformInfo.cpp - SystemZ-specific TTI -------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements a TargetTransformInfo analysis pass specific to the
10 // SystemZ target machine. It uses the target's detailed information to provide
11 // more precise answers to certain TTI queries, while letting the target
12 // independent and default TTI implementations handle the rest.
13 //
14 //===----------------------------------------------------------------------===//
15 
16 #include "SystemZTargetTransformInfo.h"
17 #include "llvm/Analysis/TargetTransformInfo.h"
18 #include "llvm/CodeGen/BasicTTIImpl.h"
19 #include "llvm/CodeGen/CostTable.h"
20 #include "llvm/CodeGen/TargetLowering.h"
21 #include "llvm/IR/IntrinsicInst.h"
22 #include "llvm/Support/Debug.h"
23 using namespace llvm;
24 
25 #define DEBUG_TYPE "systemztti"
26 
27 //===----------------------------------------------------------------------===//
28 //
29 // SystemZ cost model.
30 //
31 //===----------------------------------------------------------------------===//
32 
33 InstructionCost SystemZTTIImpl::getIntImmCost(const APInt &Imm, Type *Ty,
34                                               TTI::TargetCostKind CostKind) {
35   assert(Ty->isIntegerTy());
36 
37   unsigned BitSize = Ty->getPrimitiveSizeInBits();
38   // There is no cost model for constants with a bit size of 0. Return TCC_Free
39   // here, so that constant hoisting will ignore this constant.
40   if (BitSize == 0)
41     return TTI::TCC_Free;
42   // No cost model for operations on integers larger than 64 bit implemented yet.
43   if (BitSize > 64)
44     return TTI::TCC_Free;
45 
46   if (Imm == 0)
47     return TTI::TCC_Free;
48 
49   if (Imm.getBitWidth() <= 64) {
50     // Constants loaded via lgfi.
51     if (isInt<32>(Imm.getSExtValue()))
52       return TTI::TCC_Basic;
53     // Constants loaded via llilf.
54     if (isUInt<32>(Imm.getZExtValue()))
55       return TTI::TCC_Basic;
56     // Constants loaded via llihf:
57     if ((Imm.getZExtValue() & 0xffffffff) == 0)
58       return TTI::TCC_Basic;
59 
60     return 2 * TTI::TCC_Basic;
61   }
62 
63   return 4 * TTI::TCC_Basic;
64 }
65 
66 InstructionCost SystemZTTIImpl::getIntImmCostInst(unsigned Opcode, unsigned Idx,
67                                                   const APInt &Imm, Type *Ty,
68                                                   TTI::TargetCostKind CostKind,
69                                                   Instruction *Inst) {
70   assert(Ty->isIntegerTy());
71 
72   unsigned BitSize = Ty->getPrimitiveSizeInBits();
73   // There is no cost model for constants with a bit size of 0. Return TCC_Free
74   // here, so that constant hoisting will ignore this constant.
75   if (BitSize == 0)
76     return TTI::TCC_Free;
77   // No cost model for operations on integers larger than 64 bit implemented yet.
78   if (BitSize > 64)
79     return TTI::TCC_Free;
80 
81   switch (Opcode) {
82   default:
83     return TTI::TCC_Free;
84   case Instruction::GetElementPtr:
85     // Always hoist the base address of a GetElementPtr. This prevents the
86     // creation of new constants for every base constant that gets constant
87     // folded with the offset.
88     if (Idx == 0)
89       return 2 * TTI::TCC_Basic;
90     return TTI::TCC_Free;
91   case Instruction::Store:
92     if (Idx == 0 && Imm.getBitWidth() <= 64) {
93       // Any 8-bit immediate store can by implemented via mvi.
94       if (BitSize == 8)
95         return TTI::TCC_Free;
96       // 16-bit immediate values can be stored via mvhhi/mvhi/mvghi.
97       if (isInt<16>(Imm.getSExtValue()))
98         return TTI::TCC_Free;
99     }
100     break;
101   case Instruction::ICmp:
102     if (Idx == 1 && Imm.getBitWidth() <= 64) {
103       // Comparisons against signed 32-bit immediates implemented via cgfi.
104       if (isInt<32>(Imm.getSExtValue()))
105         return TTI::TCC_Free;
106       // Comparisons against unsigned 32-bit immediates implemented via clgfi.
107       if (isUInt<32>(Imm.getZExtValue()))
108         return TTI::TCC_Free;
109     }
110     break;
111   case Instruction::Add:
112   case Instruction::Sub:
113     if (Idx == 1 && Imm.getBitWidth() <= 64) {
114       // We use algfi/slgfi to add/subtract 32-bit unsigned immediates.
115       if (isUInt<32>(Imm.getZExtValue()))
116         return TTI::TCC_Free;
117       // Or their negation, by swapping addition vs. subtraction.
118       if (isUInt<32>(-Imm.getSExtValue()))
119         return TTI::TCC_Free;
120     }
121     break;
122   case Instruction::Mul:
123     if (Idx == 1 && Imm.getBitWidth() <= 64) {
124       // We use msgfi to multiply by 32-bit signed immediates.
125       if (isInt<32>(Imm.getSExtValue()))
126         return TTI::TCC_Free;
127     }
128     break;
129   case Instruction::Or:
130   case Instruction::Xor:
131     if (Idx == 1 && Imm.getBitWidth() <= 64) {
132       // Masks supported by oilf/xilf.
133       if (isUInt<32>(Imm.getZExtValue()))
134         return TTI::TCC_Free;
135       // Masks supported by oihf/xihf.
136       if ((Imm.getZExtValue() & 0xffffffff) == 0)
137         return TTI::TCC_Free;
138     }
139     break;
140   case Instruction::And:
141     if (Idx == 1 && Imm.getBitWidth() <= 64) {
142       // Any 32-bit AND operation can by implemented via nilf.
143       if (BitSize <= 32)
144         return TTI::TCC_Free;
145       // 64-bit masks supported by nilf.
146       if (isUInt<32>(~Imm.getZExtValue()))
147         return TTI::TCC_Free;
148       // 64-bit masks supported by nilh.
149       if ((Imm.getZExtValue() & 0xffffffff) == 0xffffffff)
150         return TTI::TCC_Free;
151       // Some 64-bit AND operations can be implemented via risbg.
152       const SystemZInstrInfo *TII = ST->getInstrInfo();
153       unsigned Start, End;
154       if (TII->isRxSBGMask(Imm.getZExtValue(), BitSize, Start, End))
155         return TTI::TCC_Free;
156     }
157     break;
158   case Instruction::Shl:
159   case Instruction::LShr:
160   case Instruction::AShr:
161     // Always return TCC_Free for the shift value of a shift instruction.
162     if (Idx == 1)
163       return TTI::TCC_Free;
164     break;
165   case Instruction::UDiv:
166   case Instruction::SDiv:
167   case Instruction::URem:
168   case Instruction::SRem:
169   case Instruction::Trunc:
170   case Instruction::ZExt:
171   case Instruction::SExt:
172   case Instruction::IntToPtr:
173   case Instruction::PtrToInt:
174   case Instruction::BitCast:
175   case Instruction::PHI:
176   case Instruction::Call:
177   case Instruction::Select:
178   case Instruction::Ret:
179   case Instruction::Load:
180     break;
181   }
182 
183   return SystemZTTIImpl::getIntImmCost(Imm, Ty, CostKind);
184 }
185 
186 InstructionCost
187 SystemZTTIImpl::getIntImmCostIntrin(Intrinsic::ID IID, unsigned Idx,
188                                     const APInt &Imm, Type *Ty,
189                                     TTI::TargetCostKind CostKind) {
190   assert(Ty->isIntegerTy());
191 
192   unsigned BitSize = Ty->getPrimitiveSizeInBits();
193   // There is no cost model for constants with a bit size of 0. Return TCC_Free
194   // here, so that constant hoisting will ignore this constant.
195   if (BitSize == 0)
196     return TTI::TCC_Free;
197   // No cost model for operations on integers larger than 64 bit implemented yet.
198   if (BitSize > 64)
199     return TTI::TCC_Free;
200 
201   switch (IID) {
202   default:
203     return TTI::TCC_Free;
204   case Intrinsic::sadd_with_overflow:
205   case Intrinsic::uadd_with_overflow:
206   case Intrinsic::ssub_with_overflow:
207   case Intrinsic::usub_with_overflow:
208     // These get expanded to include a normal addition/subtraction.
209     if (Idx == 1 && Imm.getBitWidth() <= 64) {
210       if (isUInt<32>(Imm.getZExtValue()))
211         return TTI::TCC_Free;
212       if (isUInt<32>(-Imm.getSExtValue()))
213         return TTI::TCC_Free;
214     }
215     break;
216   case Intrinsic::smul_with_overflow:
217   case Intrinsic::umul_with_overflow:
218     // These get expanded to include a normal multiplication.
219     if (Idx == 1 && Imm.getBitWidth() <= 64) {
220       if (isInt<32>(Imm.getSExtValue()))
221         return TTI::TCC_Free;
222     }
223     break;
224   case Intrinsic::experimental_stackmap:
225     if ((Idx < 2) || (Imm.getBitWidth() <= 64 && isInt<64>(Imm.getSExtValue())))
226       return TTI::TCC_Free;
227     break;
228   case Intrinsic::experimental_patchpoint_void:
229   case Intrinsic::experimental_patchpoint_i64:
230     if ((Idx < 4) || (Imm.getBitWidth() <= 64 && isInt<64>(Imm.getSExtValue())))
231       return TTI::TCC_Free;
232     break;
233   }
234   return SystemZTTIImpl::getIntImmCost(Imm, Ty, CostKind);
235 }
236 
237 TargetTransformInfo::PopcntSupportKind
238 SystemZTTIImpl::getPopcntSupport(unsigned TyWidth) {
239   assert(isPowerOf2_32(TyWidth) && "Type width must be power of 2");
240   if (ST->hasPopulationCount() && TyWidth <= 64)
241     return TTI::PSK_FastHardware;
242   return TTI::PSK_Software;
243 }
244 
245 void SystemZTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
246                                              TTI::UnrollingPreferences &UP) {
247   // Find out if L contains a call, what the machine instruction count
248   // estimate is, and how many stores there are.
249   bool HasCall = false;
250   InstructionCost NumStores = 0;
251   for (auto &BB : L->blocks())
252     for (auto &I : *BB) {
253       if (isa<CallInst>(&I) || isa<InvokeInst>(&I)) {
254         if (const Function *F = cast<CallBase>(I).getCalledFunction()) {
255           if (isLoweredToCall(F))
256             HasCall = true;
257           if (F->getIntrinsicID() == Intrinsic::memcpy ||
258               F->getIntrinsicID() == Intrinsic::memset)
259             NumStores++;
260         } else { // indirect call.
261           HasCall = true;
262         }
263       }
264       if (isa<StoreInst>(&I)) {
265         Type *MemAccessTy = I.getOperand(0)->getType();
266         NumStores += getMemoryOpCost(Instruction::Store, MemAccessTy, None, 0,
267                                      TTI::TCK_RecipThroughput);
268       }
269     }
270 
271   // The z13 processor will run out of store tags if too many stores
272   // are fed into it too quickly. Therefore make sure there are not
273   // too many stores in the resulting unrolled loop.
274   unsigned const NumStoresVal = *NumStores.getValue();
275   unsigned const Max = (NumStoresVal ? (12 / NumStoresVal) : UINT_MAX);
276 
277   if (HasCall) {
278     // Only allow full unrolling if loop has any calls.
279     UP.FullUnrollMaxCount = Max;
280     UP.MaxCount = 1;
281     return;
282   }
283 
284   UP.MaxCount = Max;
285   if (UP.MaxCount <= 1)
286     return;
287 
288   // Allow partial and runtime trip count unrolling.
289   UP.Partial = UP.Runtime = true;
290 
291   UP.PartialThreshold = 75;
292   UP.DefaultUnrollRuntimeCount = 4;
293 
294   // Allow expensive instructions in the pre-header of the loop.
295   UP.AllowExpensiveTripCount = true;
296 
297   UP.Force = true;
298 }
299 
300 void SystemZTTIImpl::getPeelingPreferences(Loop *L, ScalarEvolution &SE,
301                                            TTI::PeelingPreferences &PP) {
302   BaseT::getPeelingPreferences(L, SE, PP);
303 }
304 
305 bool SystemZTTIImpl::isLSRCostLess(TargetTransformInfo::LSRCost &C1,
306                                    TargetTransformInfo::LSRCost &C2) {
307   // SystemZ specific: check instruction count (first), and don't care about
308   // ImmCost, since offsets are checked explicitly.
309   return std::tie(C1.Insns, C1.NumRegs, C1.AddRecCost,
310                   C1.NumIVMuls, C1.NumBaseAdds,
311                   C1.ScaleCost, C1.SetupCost) <
312     std::tie(C2.Insns, C2.NumRegs, C2.AddRecCost,
313              C2.NumIVMuls, C2.NumBaseAdds,
314              C2.ScaleCost, C2.SetupCost);
315 }
316 
317 unsigned SystemZTTIImpl::getNumberOfRegisters(unsigned ClassID) const {
318   bool Vector = (ClassID == 1);
319   if (!Vector)
320     // Discount the stack pointer.  Also leave out %r0, since it can't
321     // be used in an address.
322     return 14;
323   if (ST->hasVector())
324     return 32;
325   return 0;
326 }
327 
328 TypeSize
329 SystemZTTIImpl::getRegisterBitWidth(TargetTransformInfo::RegisterKind K) const {
330   switch (K) {
331   case TargetTransformInfo::RGK_Scalar:
332     return TypeSize::getFixed(64);
333   case TargetTransformInfo::RGK_FixedWidthVector:
334     return TypeSize::getFixed(ST->hasVector() ? 128 : 0);
335   case TargetTransformInfo::RGK_ScalableVector:
336     return TypeSize::getScalable(0);
337   }
338 
339   llvm_unreachable("Unsupported register kind");
340 }
341 
342 unsigned SystemZTTIImpl::getMinPrefetchStride(unsigned NumMemAccesses,
343                                               unsigned NumStridedMemAccesses,
344                                               unsigned NumPrefetches,
345                                               bool HasCall) const {
346   // Don't prefetch a loop with many far apart accesses.
347   if (NumPrefetches > 16)
348     return UINT_MAX;
349 
350   // Emit prefetch instructions for smaller strides in cases where we think
351   // the hardware prefetcher might not be able to keep up.
352   if (NumStridedMemAccesses > 32 && !HasCall &&
353       (NumMemAccesses - NumStridedMemAccesses) * 32 <= NumStridedMemAccesses)
354     return 1;
355 
356   return ST->hasMiscellaneousExtensions3() ? 8192 : 2048;
357 }
358 
359 bool SystemZTTIImpl::hasDivRemOp(Type *DataType, bool IsSigned) {
360   EVT VT = TLI->getValueType(DL, DataType);
361   return (VT.isScalarInteger() && TLI->isTypeLegal(VT));
362 }
363 
364 // Return the bit size for the scalar type or vector element
365 // type. getScalarSizeInBits() returns 0 for a pointer type.
366 static unsigned getScalarSizeInBits(Type *Ty) {
367   unsigned Size =
368     (Ty->isPtrOrPtrVectorTy() ? 64U : Ty->getScalarSizeInBits());
369   assert(Size > 0 && "Element must have non-zero size.");
370   return Size;
371 }
372 
373 // getNumberOfParts() calls getTypeLegalizationCost() which splits the vector
374 // type until it is legal. This would e.g. return 4 for <6 x i64>, instead of
375 // 3.
376 static unsigned getNumVectorRegs(Type *Ty) {
377   auto *VTy = cast<FixedVectorType>(Ty);
378   unsigned WideBits = getScalarSizeInBits(Ty) * VTy->getNumElements();
379   assert(WideBits > 0 && "Could not compute size of vector");
380   return ((WideBits % 128U) ? ((WideBits / 128U) + 1) : (WideBits / 128U));
381 }
382 
383 InstructionCost SystemZTTIImpl::getArithmeticInstrCost(
384     unsigned Opcode, Type *Ty, TTI::TargetCostKind CostKind,
385     TTI::OperandValueKind Op1Info, TTI::OperandValueKind Op2Info,
386     TTI::OperandValueProperties Opd1PropInfo,
387     TTI::OperandValueProperties Opd2PropInfo, ArrayRef<const Value *> Args,
388     const Instruction *CxtI) {
389 
390   // TODO: Handle more cost kinds.
391   if (CostKind != TTI::TCK_RecipThroughput)
392     return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info,
393                                          Op2Info, Opd1PropInfo,
394                                          Opd2PropInfo, Args, CxtI);
395 
396   // TODO: return a good value for BB-VECTORIZER that includes the
397   // immediate loads, which we do not want to count for the loop
398   // vectorizer, since they are hopefully hoisted out of the loop. This
399   // would require a new parameter 'InLoop', but not sure if constant
400   // args are common enough to motivate this.
401 
402   unsigned ScalarBits = Ty->getScalarSizeInBits();
403 
404   // There are thre cases of division and remainder: Dividing with a register
405   // needs a divide instruction. A divisor which is a power of two constant
406   // can be implemented with a sequence of shifts. Any other constant needs a
407   // multiply and shifts.
408   const unsigned DivInstrCost = 20;
409   const unsigned DivMulSeqCost = 10;
410   const unsigned SDivPow2Cost = 4;
411 
412   bool SignedDivRem =
413       Opcode == Instruction::SDiv || Opcode == Instruction::SRem;
414   bool UnsignedDivRem =
415       Opcode == Instruction::UDiv || Opcode == Instruction::URem;
416 
417   // Check for a constant divisor.
418   bool DivRemConst = false;
419   bool DivRemConstPow2 = false;
420   if ((SignedDivRem || UnsignedDivRem) && Args.size() == 2) {
421     if (const Constant *C = dyn_cast<Constant>(Args[1])) {
422       const ConstantInt *CVal =
423           (C->getType()->isVectorTy()
424                ? dyn_cast_or_null<const ConstantInt>(C->getSplatValue())
425                : dyn_cast<const ConstantInt>(C));
426       if (CVal != nullptr &&
427           (CVal->getValue().isPowerOf2() || (-CVal->getValue()).isPowerOf2()))
428         DivRemConstPow2 = true;
429       else
430         DivRemConst = true;
431     }
432   }
433 
434   if (!Ty->isVectorTy()) {
435     // These FP operations are supported with a dedicated instruction for
436     // float, double and fp128 (base implementation assumes float generally
437     // costs 2).
438     if (Opcode == Instruction::FAdd || Opcode == Instruction::FSub ||
439         Opcode == Instruction::FMul || Opcode == Instruction::FDiv)
440       return 1;
441 
442     // There is no native support for FRem.
443     if (Opcode == Instruction::FRem)
444       return LIBCALL_COST;
445 
446     // Give discount for some combined logical operations if supported.
447     if (Args.size() == 2 && ST->hasMiscellaneousExtensions3()) {
448       if (Opcode == Instruction::Xor) {
449         for (const Value *A : Args) {
450           if (const Instruction *I = dyn_cast<Instruction>(A))
451             if (I->hasOneUse() &&
452                 (I->getOpcode() == Instruction::And ||
453                  I->getOpcode() == Instruction::Or ||
454                  I->getOpcode() == Instruction::Xor))
455               return 0;
456         }
457       }
458       else if (Opcode == Instruction::Or || Opcode == Instruction::And) {
459         for (const Value *A : Args) {
460           if (const Instruction *I = dyn_cast<Instruction>(A))
461             if (I->hasOneUse() && I->getOpcode() == Instruction::Xor)
462               return 0;
463         }
464       }
465     }
466 
467     // Or requires one instruction, although it has custom handling for i64.
468     if (Opcode == Instruction::Or)
469       return 1;
470 
471     if (Opcode == Instruction::Xor && ScalarBits == 1) {
472       if (ST->hasLoadStoreOnCond2())
473         return 5; // 2 * (li 0; loc 1); xor
474       return 7; // 2 * ipm sequences ; xor ; shift ; compare
475     }
476 
477     if (DivRemConstPow2)
478       return (SignedDivRem ? SDivPow2Cost : 1);
479     if (DivRemConst)
480       return DivMulSeqCost;
481     if (SignedDivRem || UnsignedDivRem)
482       return DivInstrCost;
483   }
484   else if (ST->hasVector()) {
485     auto *VTy = cast<FixedVectorType>(Ty);
486     unsigned VF = VTy->getNumElements();
487     unsigned NumVectors = getNumVectorRegs(Ty);
488 
489     // These vector operations are custom handled, but are still supported
490     // with one instruction per vector, regardless of element size.
491     if (Opcode == Instruction::Shl || Opcode == Instruction::LShr ||
492         Opcode == Instruction::AShr) {
493       return NumVectors;
494     }
495 
496     if (DivRemConstPow2)
497       return (NumVectors * (SignedDivRem ? SDivPow2Cost : 1));
498     if (DivRemConst) {
499       SmallVector<Type *> Tys(Args.size(), Ty);
500       return VF * DivMulSeqCost + getScalarizationOverhead(VTy, Args, Tys);
501     }
502     if ((SignedDivRem || UnsignedDivRem) && VF > 4)
503       // Temporary hack: disable high vectorization factors with integer
504       // division/remainder, which will get scalarized and handled with
505       // GR128 registers. The mischeduler is not clever enough to avoid
506       // spilling yet.
507       return 1000;
508 
509     // These FP operations are supported with a single vector instruction for
510     // double (base implementation assumes float generally costs 2). For
511     // FP128, the scalar cost is 1, and there is no overhead since the values
512     // are already in scalar registers.
513     if (Opcode == Instruction::FAdd || Opcode == Instruction::FSub ||
514         Opcode == Instruction::FMul || Opcode == Instruction::FDiv) {
515       switch (ScalarBits) {
516       case 32: {
517         // The vector enhancements facility 1 provides v4f32 instructions.
518         if (ST->hasVectorEnhancements1())
519           return NumVectors;
520         // Return the cost of multiple scalar invocation plus the cost of
521         // inserting and extracting the values.
522         InstructionCost ScalarCost =
523             getArithmeticInstrCost(Opcode, Ty->getScalarType(), CostKind);
524         SmallVector<Type *> Tys(Args.size(), Ty);
525         InstructionCost Cost =
526             (VF * ScalarCost) + getScalarizationOverhead(VTy, Args, Tys);
527         // FIXME: VF 2 for these FP operations are currently just as
528         // expensive as for VF 4.
529         if (VF == 2)
530           Cost *= 2;
531         return Cost;
532       }
533       case 64:
534       case 128:
535         return NumVectors;
536       default:
537         break;
538       }
539     }
540 
541     // There is no native support for FRem.
542     if (Opcode == Instruction::FRem) {
543       SmallVector<Type *> Tys(Args.size(), Ty);
544       InstructionCost Cost =
545           (VF * LIBCALL_COST) + getScalarizationOverhead(VTy, Args, Tys);
546       // FIXME: VF 2 for float is currently just as expensive as for VF 4.
547       if (VF == 2 && ScalarBits == 32)
548         Cost *= 2;
549       return Cost;
550     }
551   }
552 
553   // Fallback to the default implementation.
554   return BaseT::getArithmeticInstrCost(Opcode, Ty, CostKind, Op1Info, Op2Info,
555                                        Opd1PropInfo, Opd2PropInfo, Args, CxtI);
556 }
557 
558 InstructionCost SystemZTTIImpl::getShuffleCost(TTI::ShuffleKind Kind,
559                                                VectorType *Tp,
560                                                ArrayRef<int> Mask, int Index,
561                                                VectorType *SubTp) {
562   Kind = improveShuffleKindFromMask(Kind, Mask);
563   if (ST->hasVector()) {
564     unsigned NumVectors = getNumVectorRegs(Tp);
565 
566     // TODO: Since fp32 is expanded, the shuffle cost should always be 0.
567 
568     // FP128 values are always in scalar registers, so there is no work
569     // involved with a shuffle, except for broadcast. In that case register
570     // moves are done with a single instruction per element.
571     if (Tp->getScalarType()->isFP128Ty())
572       return (Kind == TargetTransformInfo::SK_Broadcast ? NumVectors - 1 : 0);
573 
574     switch (Kind) {
575     case  TargetTransformInfo::SK_ExtractSubvector:
576       // ExtractSubvector Index indicates start offset.
577 
578       // Extracting a subvector from first index is a noop.
579       return (Index == 0 ? 0 : NumVectors);
580 
581     case TargetTransformInfo::SK_Broadcast:
582       // Loop vectorizer calls here to figure out the extra cost of
583       // broadcasting a loaded value to all elements of a vector. Since vlrep
584       // loads and replicates with a single instruction, adjust the returned
585       // value.
586       return NumVectors - 1;
587 
588     default:
589 
590       // SystemZ supports single instruction permutation / replication.
591       return NumVectors;
592     }
593   }
594 
595   return BaseT::getShuffleCost(Kind, Tp, Mask, Index, SubTp);
596 }
597 
598 // Return the log2 difference of the element sizes of the two vector types.
599 static unsigned getElSizeLog2Diff(Type *Ty0, Type *Ty1) {
600   unsigned Bits0 = Ty0->getScalarSizeInBits();
601   unsigned Bits1 = Ty1->getScalarSizeInBits();
602 
603   if (Bits1 >  Bits0)
604     return (Log2_32(Bits1) - Log2_32(Bits0));
605 
606   return (Log2_32(Bits0) - Log2_32(Bits1));
607 }
608 
609 // Return the number of instructions needed to truncate SrcTy to DstTy.
610 unsigned SystemZTTIImpl::
611 getVectorTruncCost(Type *SrcTy, Type *DstTy) {
612   assert (SrcTy->isVectorTy() && DstTy->isVectorTy());
613   assert(SrcTy->getPrimitiveSizeInBits().getFixedSize() >
614              DstTy->getPrimitiveSizeInBits().getFixedSize() &&
615          "Packing must reduce size of vector type.");
616   assert(cast<FixedVectorType>(SrcTy)->getNumElements() ==
617              cast<FixedVectorType>(DstTy)->getNumElements() &&
618          "Packing should not change number of elements.");
619 
620   // TODO: Since fp32 is expanded, the extract cost should always be 0.
621 
622   unsigned NumParts = getNumVectorRegs(SrcTy);
623   if (NumParts <= 2)
624     // Up to 2 vector registers can be truncated efficiently with pack or
625     // permute. The latter requires an immediate mask to be loaded, which
626     // typically gets hoisted out of a loop.  TODO: return a good value for
627     // BB-VECTORIZER that includes the immediate loads, which we do not want
628     // to count for the loop vectorizer.
629     return 1;
630 
631   unsigned Cost = 0;
632   unsigned Log2Diff = getElSizeLog2Diff(SrcTy, DstTy);
633   unsigned VF = cast<FixedVectorType>(SrcTy)->getNumElements();
634   for (unsigned P = 0; P < Log2Diff; ++P) {
635     if (NumParts > 1)
636       NumParts /= 2;
637     Cost += NumParts;
638   }
639 
640   // Currently, a general mix of permutes and pack instructions is output by
641   // isel, which follow the cost computation above except for this case which
642   // is one instruction less:
643   if (VF == 8 && SrcTy->getScalarSizeInBits() == 64 &&
644       DstTy->getScalarSizeInBits() == 8)
645     Cost--;
646 
647   return Cost;
648 }
649 
650 // Return the cost of converting a vector bitmask produced by a compare
651 // (SrcTy), to the type of the select or extend instruction (DstTy).
652 unsigned SystemZTTIImpl::
653 getVectorBitmaskConversionCost(Type *SrcTy, Type *DstTy) {
654   assert (SrcTy->isVectorTy() && DstTy->isVectorTy() &&
655           "Should only be called with vector types.");
656 
657   unsigned PackCost = 0;
658   unsigned SrcScalarBits = SrcTy->getScalarSizeInBits();
659   unsigned DstScalarBits = DstTy->getScalarSizeInBits();
660   unsigned Log2Diff = getElSizeLog2Diff(SrcTy, DstTy);
661   if (SrcScalarBits > DstScalarBits)
662     // The bitmask will be truncated.
663     PackCost = getVectorTruncCost(SrcTy, DstTy);
664   else if (SrcScalarBits < DstScalarBits) {
665     unsigned DstNumParts = getNumVectorRegs(DstTy);
666     // Each vector select needs its part of the bitmask unpacked.
667     PackCost = Log2Diff * DstNumParts;
668     // Extra cost for moving part of mask before unpacking.
669     PackCost += DstNumParts - 1;
670   }
671 
672   return PackCost;
673 }
674 
675 // Return the type of the compared operands. This is needed to compute the
676 // cost for a Select / ZExt or SExt instruction.
677 static Type *getCmpOpsType(const Instruction *I, unsigned VF = 1) {
678   Type *OpTy = nullptr;
679   if (CmpInst *CI = dyn_cast<CmpInst>(I->getOperand(0)))
680     OpTy = CI->getOperand(0)->getType();
681   else if (Instruction *LogicI = dyn_cast<Instruction>(I->getOperand(0)))
682     if (LogicI->getNumOperands() == 2)
683       if (CmpInst *CI0 = dyn_cast<CmpInst>(LogicI->getOperand(0)))
684         if (isa<CmpInst>(LogicI->getOperand(1)))
685           OpTy = CI0->getOperand(0)->getType();
686 
687   if (OpTy != nullptr) {
688     if (VF == 1) {
689       assert (!OpTy->isVectorTy() && "Expected scalar type");
690       return OpTy;
691     }
692     // Return the potentially vectorized type based on 'I' and 'VF'.  'I' may
693     // be either scalar or already vectorized with a same or lesser VF.
694     Type *ElTy = OpTy->getScalarType();
695     return FixedVectorType::get(ElTy, VF);
696   }
697 
698   return nullptr;
699 }
700 
701 // Get the cost of converting a boolean vector to a vector with same width
702 // and element size as Dst, plus the cost of zero extending if needed.
703 unsigned SystemZTTIImpl::
704 getBoolVecToIntConversionCost(unsigned Opcode, Type *Dst,
705                               const Instruction *I) {
706   auto *DstVTy = cast<FixedVectorType>(Dst);
707   unsigned VF = DstVTy->getNumElements();
708   unsigned Cost = 0;
709   // If we know what the widths of the compared operands, get any cost of
710   // converting it to match Dst. Otherwise assume same widths.
711   Type *CmpOpTy = ((I != nullptr) ? getCmpOpsType(I, VF) : nullptr);
712   if (CmpOpTy != nullptr)
713     Cost = getVectorBitmaskConversionCost(CmpOpTy, Dst);
714   if (Opcode == Instruction::ZExt || Opcode == Instruction::UIToFP)
715     // One 'vn' per dst vector with an immediate mask.
716     Cost += getNumVectorRegs(Dst);
717   return Cost;
718 }
719 
720 InstructionCost SystemZTTIImpl::getCastInstrCost(unsigned Opcode, Type *Dst,
721                                                  Type *Src,
722                                                  TTI::CastContextHint CCH,
723                                                  TTI::TargetCostKind CostKind,
724                                                  const Instruction *I) {
725   // FIXME: Can the logic below also be used for these cost kinds?
726   if (CostKind == TTI::TCK_CodeSize || CostKind == TTI::TCK_SizeAndLatency) {
727     auto BaseCost = BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
728     return BaseCost == 0 ? BaseCost : 1;
729   }
730 
731   unsigned DstScalarBits = Dst->getScalarSizeInBits();
732   unsigned SrcScalarBits = Src->getScalarSizeInBits();
733 
734   if (!Src->isVectorTy()) {
735     assert (!Dst->isVectorTy());
736 
737     if (Opcode == Instruction::SIToFP || Opcode == Instruction::UIToFP) {
738       if (SrcScalarBits >= 32 ||
739           (I != nullptr && isa<LoadInst>(I->getOperand(0))))
740         return 1;
741       return SrcScalarBits > 1 ? 2 /*i8/i16 extend*/ : 5 /*branch seq.*/;
742     }
743 
744     if ((Opcode == Instruction::ZExt || Opcode == Instruction::SExt) &&
745         Src->isIntegerTy(1)) {
746       if (ST->hasLoadStoreOnCond2())
747         return 2; // li 0; loc 1
748 
749       // This should be extension of a compare i1 result, which is done with
750       // ipm and a varying sequence of instructions.
751       unsigned Cost = 0;
752       if (Opcode == Instruction::SExt)
753         Cost = (DstScalarBits < 64 ? 3 : 4);
754       if (Opcode == Instruction::ZExt)
755         Cost = 3;
756       Type *CmpOpTy = ((I != nullptr) ? getCmpOpsType(I) : nullptr);
757       if (CmpOpTy != nullptr && CmpOpTy->isFloatingPointTy())
758         // If operands of an fp-type was compared, this costs +1.
759         Cost++;
760       return Cost;
761     }
762   }
763   else if (ST->hasVector()) {
764     // Vector to scalar cast.
765     auto *SrcVecTy = cast<FixedVectorType>(Src);
766     auto *DstVecTy = dyn_cast<FixedVectorType>(Dst);
767     if (!DstVecTy) {
768       // TODO: tune vector-to-scalar cast.
769       return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
770     }
771     unsigned VF = SrcVecTy->getNumElements();
772     unsigned NumDstVectors = getNumVectorRegs(Dst);
773     unsigned NumSrcVectors = getNumVectorRegs(Src);
774 
775     if (Opcode == Instruction::Trunc) {
776       if (Src->getScalarSizeInBits() == Dst->getScalarSizeInBits())
777         return 0; // Check for NOOP conversions.
778       return getVectorTruncCost(Src, Dst);
779     }
780 
781     if (Opcode == Instruction::ZExt || Opcode == Instruction::SExt) {
782       if (SrcScalarBits >= 8) {
783         // ZExt/SExt will be handled with one unpack per doubling of width.
784         unsigned NumUnpacks = getElSizeLog2Diff(Src, Dst);
785 
786         // For types that spans multiple vector registers, some additional
787         // instructions are used to setup the unpacking.
788         unsigned NumSrcVectorOps =
789           (NumUnpacks > 1 ? (NumDstVectors - NumSrcVectors)
790                           : (NumDstVectors / 2));
791 
792         return (NumUnpacks * NumDstVectors) + NumSrcVectorOps;
793       }
794       else if (SrcScalarBits == 1)
795         return getBoolVecToIntConversionCost(Opcode, Dst, I);
796     }
797 
798     if (Opcode == Instruction::SIToFP || Opcode == Instruction::UIToFP ||
799         Opcode == Instruction::FPToSI || Opcode == Instruction::FPToUI) {
800       // TODO: Fix base implementation which could simplify things a bit here
801       // (seems to miss on differentiating on scalar/vector types).
802 
803       // Only 64 bit vector conversions are natively supported before z15.
804       if (DstScalarBits == 64 || ST->hasVectorEnhancements2()) {
805         if (SrcScalarBits == DstScalarBits)
806           return NumDstVectors;
807 
808         if (SrcScalarBits == 1)
809           return getBoolVecToIntConversionCost(Opcode, Dst, I) + NumDstVectors;
810       }
811 
812       // Return the cost of multiple scalar invocation plus the cost of
813       // inserting and extracting the values. Base implementation does not
814       // realize float->int gets scalarized.
815       InstructionCost ScalarCost = getCastInstrCost(
816           Opcode, Dst->getScalarType(), Src->getScalarType(), CCH, CostKind);
817       InstructionCost TotCost = VF * ScalarCost;
818       bool NeedsInserts = true, NeedsExtracts = true;
819       // FP128 registers do not get inserted or extracted.
820       if (DstScalarBits == 128 &&
821           (Opcode == Instruction::SIToFP || Opcode == Instruction::UIToFP))
822         NeedsInserts = false;
823       if (SrcScalarBits == 128 &&
824           (Opcode == Instruction::FPToSI || Opcode == Instruction::FPToUI))
825         NeedsExtracts = false;
826 
827       TotCost += getScalarizationOverhead(SrcVecTy, false, NeedsExtracts);
828       TotCost += getScalarizationOverhead(DstVecTy, NeedsInserts, false);
829 
830       // FIXME: VF 2 for float<->i32 is currently just as expensive as for VF 4.
831       if (VF == 2 && SrcScalarBits == 32 && DstScalarBits == 32)
832         TotCost *= 2;
833 
834       return TotCost;
835     }
836 
837     if (Opcode == Instruction::FPTrunc) {
838       if (SrcScalarBits == 128)  // fp128 -> double/float + inserts of elements.
839         return VF /*ldxbr/lexbr*/ +
840                getScalarizationOverhead(DstVecTy, true, false);
841       else // double -> float
842         return VF / 2 /*vledb*/ + std::max(1U, VF / 4 /*vperm*/);
843     }
844 
845     if (Opcode == Instruction::FPExt) {
846       if (SrcScalarBits == 32 && DstScalarBits == 64) {
847         // float -> double is very rare and currently unoptimized. Instead of
848         // using vldeb, which can do two at a time, all conversions are
849         // scalarized.
850         return VF * 2;
851       }
852       // -> fp128.  VF * lxdb/lxeb + extraction of elements.
853       return VF + getScalarizationOverhead(SrcVecTy, false, true);
854     }
855   }
856 
857   return BaseT::getCastInstrCost(Opcode, Dst, Src, CCH, CostKind, I);
858 }
859 
860 // Scalar i8 / i16 operations will typically be made after first extending
861 // the operands to i32.
862 static unsigned getOperandsExtensionCost(const Instruction *I) {
863   unsigned ExtCost = 0;
864   for (Value *Op : I->operands())
865     // A load of i8 or i16 sign/zero extends to i32.
866     if (!isa<LoadInst>(Op) && !isa<ConstantInt>(Op))
867       ExtCost++;
868 
869   return ExtCost;
870 }
871 
872 InstructionCost SystemZTTIImpl::getCmpSelInstrCost(unsigned Opcode, Type *ValTy,
873                                                    Type *CondTy,
874                                                    CmpInst::Predicate VecPred,
875                                                    TTI::TargetCostKind CostKind,
876                                                    const Instruction *I) {
877   if (CostKind != TTI::TCK_RecipThroughput)
878     return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind);
879 
880   if (!ValTy->isVectorTy()) {
881     switch (Opcode) {
882     case Instruction::ICmp: {
883       // A loaded value compared with 0 with multiple users becomes Load and
884       // Test. The load is then not foldable, so return 0 cost for the ICmp.
885       unsigned ScalarBits = ValTy->getScalarSizeInBits();
886       if (I != nullptr && ScalarBits >= 32)
887         if (LoadInst *Ld = dyn_cast<LoadInst>(I->getOperand(0)))
888           if (const ConstantInt *C = dyn_cast<ConstantInt>(I->getOperand(1)))
889             if (!Ld->hasOneUse() && Ld->getParent() == I->getParent() &&
890                 C->isZero())
891               return 0;
892 
893       unsigned Cost = 1;
894       if (ValTy->isIntegerTy() && ValTy->getScalarSizeInBits() <= 16)
895         Cost += (I != nullptr ? getOperandsExtensionCost(I) : 2);
896       return Cost;
897     }
898     case Instruction::Select:
899       if (ValTy->isFloatingPointTy())
900         return 4; // No load on condition for FP - costs a conditional jump.
901       return 1; // Load On Condition / Select Register.
902     }
903   }
904   else if (ST->hasVector()) {
905     unsigned VF = cast<FixedVectorType>(ValTy)->getNumElements();
906 
907     // Called with a compare instruction.
908     if (Opcode == Instruction::ICmp || Opcode == Instruction::FCmp) {
909       unsigned PredicateExtraCost = 0;
910       if (I != nullptr) {
911         // Some predicates cost one or two extra instructions.
912         switch (cast<CmpInst>(I)->getPredicate()) {
913         case CmpInst::Predicate::ICMP_NE:
914         case CmpInst::Predicate::ICMP_UGE:
915         case CmpInst::Predicate::ICMP_ULE:
916         case CmpInst::Predicate::ICMP_SGE:
917         case CmpInst::Predicate::ICMP_SLE:
918           PredicateExtraCost = 1;
919           break;
920         case CmpInst::Predicate::FCMP_ONE:
921         case CmpInst::Predicate::FCMP_ORD:
922         case CmpInst::Predicate::FCMP_UEQ:
923         case CmpInst::Predicate::FCMP_UNO:
924           PredicateExtraCost = 2;
925           break;
926         default:
927           break;
928         }
929       }
930 
931       // Float is handled with 2*vmr[lh]f + 2*vldeb + vfchdb for each pair of
932       // floats.  FIXME: <2 x float> generates same code as <4 x float>.
933       unsigned CmpCostPerVector = (ValTy->getScalarType()->isFloatTy() ? 10 : 1);
934       unsigned NumVecs_cmp = getNumVectorRegs(ValTy);
935 
936       unsigned Cost = (NumVecs_cmp * (CmpCostPerVector + PredicateExtraCost));
937       return Cost;
938     }
939     else { // Called with a select instruction.
940       assert (Opcode == Instruction::Select);
941 
942       // We can figure out the extra cost of packing / unpacking if the
943       // instruction was passed and the compare instruction is found.
944       unsigned PackCost = 0;
945       Type *CmpOpTy = ((I != nullptr) ? getCmpOpsType(I, VF) : nullptr);
946       if (CmpOpTy != nullptr)
947         PackCost =
948           getVectorBitmaskConversionCost(CmpOpTy, ValTy);
949 
950       return getNumVectorRegs(ValTy) /*vsel*/ + PackCost;
951     }
952   }
953 
954   return BaseT::getCmpSelInstrCost(Opcode, ValTy, CondTy, VecPred, CostKind);
955 }
956 
957 InstructionCost SystemZTTIImpl::getVectorInstrCost(unsigned Opcode, Type *Val,
958                                                    unsigned Index) {
959   // vlvgp will insert two grs into a vector register, so only count half the
960   // number of instructions.
961   if (Opcode == Instruction::InsertElement && Val->isIntOrIntVectorTy(64))
962     return ((Index % 2 == 0) ? 1 : 0);
963 
964   if (Opcode == Instruction::ExtractElement) {
965     int Cost = ((getScalarSizeInBits(Val) == 1) ? 2 /*+test-under-mask*/ : 1);
966 
967     // Give a slight penalty for moving out of vector pipeline to FXU unit.
968     if (Index == 0 && Val->isIntOrIntVectorTy())
969       Cost += 1;
970 
971     return Cost;
972   }
973 
974   return BaseT::getVectorInstrCost(Opcode, Val, Index);
975 }
976 
977 // Check if a load may be folded as a memory operand in its user.
978 bool SystemZTTIImpl::
979 isFoldableLoad(const LoadInst *Ld, const Instruction *&FoldedValue) {
980   if (!Ld->hasOneUse())
981     return false;
982   FoldedValue = Ld;
983   const Instruction *UserI = cast<Instruction>(*Ld->user_begin());
984   unsigned LoadedBits = getScalarSizeInBits(Ld->getType());
985   unsigned TruncBits = 0;
986   unsigned SExtBits = 0;
987   unsigned ZExtBits = 0;
988   if (UserI->hasOneUse()) {
989     unsigned UserBits = UserI->getType()->getScalarSizeInBits();
990     if (isa<TruncInst>(UserI))
991       TruncBits = UserBits;
992     else if (isa<SExtInst>(UserI))
993       SExtBits = UserBits;
994     else if (isa<ZExtInst>(UserI))
995       ZExtBits = UserBits;
996   }
997   if (TruncBits || SExtBits || ZExtBits) {
998     FoldedValue = UserI;
999     UserI = cast<Instruction>(*UserI->user_begin());
1000     // Load (single use) -> trunc/extend (single use) -> UserI
1001   }
1002   if ((UserI->getOpcode() == Instruction::Sub ||
1003        UserI->getOpcode() == Instruction::SDiv ||
1004        UserI->getOpcode() == Instruction::UDiv) &&
1005       UserI->getOperand(1) != FoldedValue)
1006     return false; // Not commutative, only RHS foldable.
1007   // LoadOrTruncBits holds the number of effectively loaded bits, but 0 if an
1008   // extension was made of the load.
1009   unsigned LoadOrTruncBits =
1010       ((SExtBits || ZExtBits) ? 0 : (TruncBits ? TruncBits : LoadedBits));
1011   switch (UserI->getOpcode()) {
1012   case Instruction::Add: // SE: 16->32, 16/32->64, z14:16->64. ZE: 32->64
1013   case Instruction::Sub:
1014   case Instruction::ICmp:
1015     if (LoadedBits == 32 && ZExtBits == 64)
1016       return true;
1017     LLVM_FALLTHROUGH;
1018   case Instruction::Mul: // SE: 16->32, 32->64, z14:16->64
1019     if (UserI->getOpcode() != Instruction::ICmp) {
1020       if (LoadedBits == 16 &&
1021           (SExtBits == 32 ||
1022            (SExtBits == 64 && ST->hasMiscellaneousExtensions2())))
1023         return true;
1024       if (LoadOrTruncBits == 16)
1025         return true;
1026     }
1027     LLVM_FALLTHROUGH;
1028   case Instruction::SDiv:// SE: 32->64
1029     if (LoadedBits == 32 && SExtBits == 64)
1030       return true;
1031     LLVM_FALLTHROUGH;
1032   case Instruction::UDiv:
1033   case Instruction::And:
1034   case Instruction::Or:
1035   case Instruction::Xor:
1036     // This also makes sense for float operations, but disabled for now due
1037     // to regressions.
1038     // case Instruction::FCmp:
1039     // case Instruction::FAdd:
1040     // case Instruction::FSub:
1041     // case Instruction::FMul:
1042     // case Instruction::FDiv:
1043 
1044     // All possible extensions of memory checked above.
1045 
1046     // Comparison between memory and immediate.
1047     if (UserI->getOpcode() == Instruction::ICmp)
1048       if (ConstantInt *CI = dyn_cast<ConstantInt>(UserI->getOperand(1)))
1049         if (CI->getValue().isIntN(16))
1050           return true;
1051     return (LoadOrTruncBits == 32 || LoadOrTruncBits == 64);
1052     break;
1053   }
1054   return false;
1055 }
1056 
1057 static bool isBswapIntrinsicCall(const Value *V) {
1058   if (const Instruction *I = dyn_cast<Instruction>(V))
1059     if (auto *CI = dyn_cast<CallInst>(I))
1060       if (auto *F = CI->getCalledFunction())
1061         if (F->getIntrinsicID() == Intrinsic::bswap)
1062           return true;
1063   return false;
1064 }
1065 
1066 InstructionCost SystemZTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
1067                                                 MaybeAlign Alignment,
1068                                                 unsigned AddressSpace,
1069                                                 TTI::TargetCostKind CostKind,
1070                                                 const Instruction *I) {
1071   assert(!Src->isVoidTy() && "Invalid type");
1072 
1073   // TODO: Handle other cost kinds.
1074   if (CostKind != TTI::TCK_RecipThroughput)
1075     return 1;
1076 
1077   if (!Src->isVectorTy() && Opcode == Instruction::Load && I != nullptr) {
1078     // Store the load or its truncated or extended value in FoldedValue.
1079     const Instruction *FoldedValue = nullptr;
1080     if (isFoldableLoad(cast<LoadInst>(I), FoldedValue)) {
1081       const Instruction *UserI = cast<Instruction>(*FoldedValue->user_begin());
1082       assert (UserI->getNumOperands() == 2 && "Expected a binop.");
1083 
1084       // UserI can't fold two loads, so in that case return 0 cost only
1085       // half of the time.
1086       for (unsigned i = 0; i < 2; ++i) {
1087         if (UserI->getOperand(i) == FoldedValue)
1088           continue;
1089 
1090         if (Instruction *OtherOp = dyn_cast<Instruction>(UserI->getOperand(i))){
1091           LoadInst *OtherLoad = dyn_cast<LoadInst>(OtherOp);
1092           if (!OtherLoad &&
1093               (isa<TruncInst>(OtherOp) || isa<SExtInst>(OtherOp) ||
1094                isa<ZExtInst>(OtherOp)))
1095             OtherLoad = dyn_cast<LoadInst>(OtherOp->getOperand(0));
1096           if (OtherLoad && isFoldableLoad(OtherLoad, FoldedValue/*dummy*/))
1097             return i == 0; // Both operands foldable.
1098         }
1099       }
1100 
1101       return 0; // Only I is foldable in user.
1102     }
1103   }
1104 
1105   unsigned NumOps =
1106     (Src->isVectorTy() ? getNumVectorRegs(Src) : getNumberOfParts(Src));
1107 
1108   // Store/Load reversed saves one instruction.
1109   if (((!Src->isVectorTy() && NumOps == 1) || ST->hasVectorEnhancements2()) &&
1110       I != nullptr) {
1111     if (Opcode == Instruction::Load && I->hasOneUse()) {
1112       const Instruction *LdUser = cast<Instruction>(*I->user_begin());
1113       // In case of load -> bswap -> store, return normal cost for the load.
1114       if (isBswapIntrinsicCall(LdUser) &&
1115           (!LdUser->hasOneUse() || !isa<StoreInst>(*LdUser->user_begin())))
1116         return 0;
1117     }
1118     else if (const StoreInst *SI = dyn_cast<StoreInst>(I)) {
1119       const Value *StoredVal = SI->getValueOperand();
1120       if (StoredVal->hasOneUse() && isBswapIntrinsicCall(StoredVal))
1121         return 0;
1122     }
1123   }
1124 
1125   if (Src->getScalarSizeInBits() == 128)
1126     // 128 bit scalars are held in a pair of two 64 bit registers.
1127     NumOps *= 2;
1128 
1129   return  NumOps;
1130 }
1131 
1132 // The generic implementation of getInterleavedMemoryOpCost() is based on
1133 // adding costs of the memory operations plus all the extracts and inserts
1134 // needed for using / defining the vector operands. The SystemZ version does
1135 // roughly the same but bases the computations on vector permutations
1136 // instead.
1137 InstructionCost SystemZTTIImpl::getInterleavedMemoryOpCost(
1138     unsigned Opcode, Type *VecTy, unsigned Factor, ArrayRef<unsigned> Indices,
1139     Align Alignment, unsigned AddressSpace, TTI::TargetCostKind CostKind,
1140     bool UseMaskForCond, bool UseMaskForGaps) {
1141   if (UseMaskForCond || UseMaskForGaps)
1142     return BaseT::getInterleavedMemoryOpCost(Opcode, VecTy, Factor, Indices,
1143                                              Alignment, AddressSpace, CostKind,
1144                                              UseMaskForCond, UseMaskForGaps);
1145   assert(isa<VectorType>(VecTy) &&
1146          "Expect a vector type for interleaved memory op");
1147 
1148   unsigned NumElts = cast<FixedVectorType>(VecTy)->getNumElements();
1149   assert(Factor > 1 && NumElts % Factor == 0 && "Invalid interleave factor");
1150   unsigned VF = NumElts / Factor;
1151   unsigned NumEltsPerVecReg = (128U / getScalarSizeInBits(VecTy));
1152   unsigned NumVectorMemOps = getNumVectorRegs(VecTy);
1153   unsigned NumPermutes = 0;
1154 
1155   if (Opcode == Instruction::Load) {
1156     // Loading interleave groups may have gaps, which may mean fewer
1157     // loads. Find out how many vectors will be loaded in total, and in how
1158     // many of them each value will be in.
1159     BitVector UsedInsts(NumVectorMemOps, false);
1160     std::vector<BitVector> ValueVecs(Factor, BitVector(NumVectorMemOps, false));
1161     for (unsigned Index : Indices)
1162       for (unsigned Elt = 0; Elt < VF; ++Elt) {
1163         unsigned Vec = (Index + Elt * Factor) / NumEltsPerVecReg;
1164         UsedInsts.set(Vec);
1165         ValueVecs[Index].set(Vec);
1166       }
1167     NumVectorMemOps = UsedInsts.count();
1168 
1169     for (unsigned Index : Indices) {
1170       // Estimate that each loaded source vector containing this Index
1171       // requires one operation, except that vperm can handle two input
1172       // registers first time for each dst vector.
1173       unsigned NumSrcVecs = ValueVecs[Index].count();
1174       unsigned NumDstVecs = divideCeil(VF * getScalarSizeInBits(VecTy), 128U);
1175       assert (NumSrcVecs >= NumDstVecs && "Expected at least as many sources");
1176       NumPermutes += std::max(1U, NumSrcVecs - NumDstVecs);
1177     }
1178   } else {
1179     // Estimate the permutes for each stored vector as the smaller of the
1180     // number of elements and the number of source vectors. Subtract one per
1181     // dst vector for vperm (S.A.).
1182     unsigned NumSrcVecs = std::min(NumEltsPerVecReg, Factor);
1183     unsigned NumDstVecs = NumVectorMemOps;
1184     assert (NumSrcVecs > 1 && "Expected at least two source vectors.");
1185     NumPermutes += (NumDstVecs * NumSrcVecs) - NumDstVecs;
1186   }
1187 
1188   // Cost of load/store operations and the permutations needed.
1189   return NumVectorMemOps + NumPermutes;
1190 }
1191 
1192 static int getVectorIntrinsicInstrCost(Intrinsic::ID ID, Type *RetTy) {
1193   if (RetTy->isVectorTy() && ID == Intrinsic::bswap)
1194     return getNumVectorRegs(RetTy); // VPERM
1195   return -1;
1196 }
1197 
1198 InstructionCost
1199 SystemZTTIImpl::getIntrinsicInstrCost(const IntrinsicCostAttributes &ICA,
1200                                       TTI::TargetCostKind CostKind) {
1201   InstructionCost Cost =
1202       getVectorIntrinsicInstrCost(ICA.getID(), ICA.getReturnType());
1203   if (Cost != -1)
1204     return Cost;
1205   return BaseT::getIntrinsicInstrCost(ICA, CostKind);
1206 }
1207