1 //===-- RISCVInterleavedAccess.cpp - RISC-V Interleaved Access Transform --===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Functions and callbacks related to the InterleavedAccessPass.
10 //
11 //===----------------------------------------------------------------------===//
12
13 #include "RISCV.h"
14 #include "RISCVISelLowering.h"
15 #include "RISCVSubtarget.h"
16 #include "llvm/Analysis/ValueTracking.h"
17 #include "llvm/CodeGen/ValueTypes.h"
18 #include "llvm/IR/IRBuilder.h"
19 #include "llvm/IR/Instructions.h"
20 #include "llvm/IR/IntrinsicsRISCV.h"
21 #include "llvm/IR/Module.h"
22 #include "llvm/IR/PatternMatch.h"
23
24 using namespace llvm;
25
isLegalInterleavedAccessType(VectorType * VTy,unsigned Factor,Align Alignment,unsigned AddrSpace,const DataLayout & DL) const26 bool RISCVTargetLowering::isLegalInterleavedAccessType(
27 VectorType *VTy, unsigned Factor, Align Alignment, unsigned AddrSpace,
28 const DataLayout &DL) const {
29 EVT VT = getValueType(DL, VTy);
30 // Don't lower vlseg/vsseg for vector types that can't be split.
31 if (!isTypeLegal(VT))
32 return false;
33
34 if (!isLegalElementTypeForRVV(VT.getScalarType()) ||
35 !allowsMemoryAccessForAlignment(VTy->getContext(), DL, VT, AddrSpace,
36 Alignment))
37 return false;
38
39 MVT ContainerVT = VT.getSimpleVT();
40
41 if (auto *FVTy = dyn_cast<FixedVectorType>(VTy)) {
42 if (!Subtarget.useRVVForFixedLengthVectors())
43 return false;
44 // Sometimes the interleaved access pass picks up splats as interleaves of
45 // one element. Don't lower these.
46 if (FVTy->getNumElements() < 2)
47 return false;
48
49 ContainerVT = getContainerForFixedLengthVector(VT.getSimpleVT());
50 }
51
52 // Need to make sure that EMUL * NFIELDS ≤ 8
53 auto [LMUL, Fractional] = RISCVVType::decodeVLMUL(getLMUL(ContainerVT));
54 if (Fractional)
55 return true;
56 return Factor * LMUL <= 8;
57 }
58
59 static const Intrinsic::ID FixedVlsegIntrIds[] = {
60 Intrinsic::riscv_seg2_load_mask, Intrinsic::riscv_seg3_load_mask,
61 Intrinsic::riscv_seg4_load_mask, Intrinsic::riscv_seg5_load_mask,
62 Intrinsic::riscv_seg6_load_mask, Intrinsic::riscv_seg7_load_mask,
63 Intrinsic::riscv_seg8_load_mask};
64
65 static const Intrinsic::ID ScalableVlsegIntrIds[] = {
66 Intrinsic::riscv_vlseg2_mask, Intrinsic::riscv_vlseg3_mask,
67 Intrinsic::riscv_vlseg4_mask, Intrinsic::riscv_vlseg5_mask,
68 Intrinsic::riscv_vlseg6_mask, Intrinsic::riscv_vlseg7_mask,
69 Intrinsic::riscv_vlseg8_mask};
70
71 /// Lower an interleaved load into a vlsegN intrinsic.
72 ///
73 /// E.g. Lower an interleaved load (Factor = 2):
74 /// %wide.vec = load <8 x i32>, <8 x i32>* %ptr
75 /// %v0 = shuffle %wide.vec, undef, <0, 2, 4, 6> ; Extract even elements
76 /// %v1 = shuffle %wide.vec, undef, <1, 3, 5, 7> ; Extract odd elements
77 ///
78 /// Into:
79 /// %ld2 = { <4 x i32>, <4 x i32> } call llvm.riscv.seg2.load.v4i32.p0.i64(
80 /// %ptr, i64 4)
81 /// %vec0 = extractelement { <4 x i32>, <4 x i32> } %ld2, i32 0
82 /// %vec1 = extractelement { <4 x i32>, <4 x i32> } %ld2, i32 1
lowerInterleavedLoad(LoadInst * LI,ArrayRef<ShuffleVectorInst * > Shuffles,ArrayRef<unsigned> Indices,unsigned Factor) const83 bool RISCVTargetLowering::lowerInterleavedLoad(
84 LoadInst *LI, ArrayRef<ShuffleVectorInst *> Shuffles,
85 ArrayRef<unsigned> Indices, unsigned Factor) const {
86 assert(Indices.size() == Shuffles.size());
87
88 IRBuilder<> Builder(LI);
89
90 const DataLayout &DL = LI->getDataLayout();
91
92 auto *VTy = cast<FixedVectorType>(Shuffles[0]->getType());
93 if (!isLegalInterleavedAccessType(VTy, Factor, LI->getAlign(),
94 LI->getPointerAddressSpace(), DL))
95 return false;
96
97 auto *PtrTy = LI->getPointerOperandType();
98 auto *XLenTy = Type::getIntNTy(LI->getContext(), Subtarget.getXLen());
99
100 // If the segment load is going to be performed segment at a time anyways
101 // and there's only one element used, use a strided load instead. This
102 // will be equally fast, and create less vector register pressure.
103 if (Indices.size() == 1 && !Subtarget.hasOptimizedSegmentLoadStore(Factor)) {
104 unsigned ScalarSizeInBytes = DL.getTypeStoreSize(VTy->getElementType());
105 Value *Stride = ConstantInt::get(XLenTy, Factor * ScalarSizeInBytes);
106 Value *Offset = ConstantInt::get(XLenTy, Indices[0] * ScalarSizeInBytes);
107 Value *BasePtr = Builder.CreatePtrAdd(LI->getPointerOperand(), Offset);
108 Value *Mask = Builder.getAllOnesMask(VTy->getElementCount());
109 Value *VL = Builder.getInt32(VTy->getNumElements());
110
111 CallInst *CI =
112 Builder.CreateIntrinsic(Intrinsic::experimental_vp_strided_load,
113 {VTy, BasePtr->getType(), Stride->getType()},
114 {BasePtr, Stride, Mask, VL});
115 CI->addParamAttr(
116 0, Attribute::getWithAlignment(CI->getContext(), LI->getAlign()));
117 Shuffles[0]->replaceAllUsesWith(CI);
118 return true;
119 };
120
121 Value *VL = ConstantInt::get(XLenTy, VTy->getNumElements());
122 Value *Mask = Builder.getAllOnesMask(VTy->getElementCount());
123 CallInst *VlsegN = Builder.CreateIntrinsic(
124 FixedVlsegIntrIds[Factor - 2], {VTy, PtrTy, XLenTy},
125 {LI->getPointerOperand(), Mask, VL});
126
127 for (unsigned i = 0; i < Shuffles.size(); i++) {
128 Value *SubVec = Builder.CreateExtractValue(VlsegN, Indices[i]);
129 Shuffles[i]->replaceAllUsesWith(SubVec);
130 }
131
132 return true;
133 }
134
135 static const Intrinsic::ID FixedVssegIntrIds[] = {
136 Intrinsic::riscv_seg2_store_mask, Intrinsic::riscv_seg3_store_mask,
137 Intrinsic::riscv_seg4_store_mask, Intrinsic::riscv_seg5_store_mask,
138 Intrinsic::riscv_seg6_store_mask, Intrinsic::riscv_seg7_store_mask,
139 Intrinsic::riscv_seg8_store_mask};
140
141 static const Intrinsic::ID ScalableVssegIntrIds[] = {
142 Intrinsic::riscv_vsseg2_mask, Intrinsic::riscv_vsseg3_mask,
143 Intrinsic::riscv_vsseg4_mask, Intrinsic::riscv_vsseg5_mask,
144 Intrinsic::riscv_vsseg6_mask, Intrinsic::riscv_vsseg7_mask,
145 Intrinsic::riscv_vsseg8_mask};
146
147 /// Lower an interleaved store into a vssegN intrinsic.
148 ///
149 /// E.g. Lower an interleaved store (Factor = 3):
150 /// %i.vec = shuffle <8 x i32> %v0, <8 x i32> %v1,
151 /// <0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11>
152 /// store <12 x i32> %i.vec, <12 x i32>* %ptr
153 ///
154 /// Into:
155 /// %sub.v0 = shuffle <8 x i32> %v0, <8 x i32> v1, <0, 1, 2, 3>
156 /// %sub.v1 = shuffle <8 x i32> %v0, <8 x i32> v1, <4, 5, 6, 7>
157 /// %sub.v2 = shuffle <8 x i32> %v0, <8 x i32> v1, <8, 9, 10, 11>
158 /// call void llvm.riscv.seg3.store.v4i32.p0.i64(%sub.v0, %sub.v1, %sub.v2,
159 /// %ptr, i32 4)
160 ///
161 /// Note that the new shufflevectors will be removed and we'll only generate one
162 /// vsseg3 instruction in CodeGen.
lowerInterleavedStore(StoreInst * SI,ShuffleVectorInst * SVI,unsigned Factor) const163 bool RISCVTargetLowering::lowerInterleavedStore(StoreInst *SI,
164 ShuffleVectorInst *SVI,
165 unsigned Factor) const {
166 IRBuilder<> Builder(SI);
167 const DataLayout &DL = SI->getDataLayout();
168 auto Mask = SVI->getShuffleMask();
169 auto *ShuffleVTy = cast<FixedVectorType>(SVI->getType());
170 // Given SVI : <n*factor x ty>, then VTy : <n x ty>
171 auto *VTy = FixedVectorType::get(ShuffleVTy->getElementType(),
172 ShuffleVTy->getNumElements() / Factor);
173 if (!isLegalInterleavedAccessType(VTy, Factor, SI->getAlign(),
174 SI->getPointerAddressSpace(), DL))
175 return false;
176
177 auto *PtrTy = SI->getPointerOperandType();
178 auto *XLenTy = Type::getIntNTy(SI->getContext(), Subtarget.getXLen());
179
180 unsigned Index;
181 // If the segment store only has one active lane (i.e. the interleave is
182 // just a spread shuffle), we can use a strided store instead. This will
183 // be equally fast, and create less vector register pressure.
184 if (!Subtarget.hasOptimizedSegmentLoadStore(Factor) &&
185 isSpreadMask(Mask, Factor, Index)) {
186 unsigned ScalarSizeInBytes =
187 DL.getTypeStoreSize(ShuffleVTy->getElementType());
188 Value *Data = SVI->getOperand(0);
189 auto *DataVTy = cast<FixedVectorType>(Data->getType());
190 Value *Stride = ConstantInt::get(XLenTy, Factor * ScalarSizeInBytes);
191 Value *Offset = ConstantInt::get(XLenTy, Index * ScalarSizeInBytes);
192 Value *BasePtr = Builder.CreatePtrAdd(SI->getPointerOperand(), Offset);
193 Value *Mask = Builder.getAllOnesMask(DataVTy->getElementCount());
194 Value *VL = Builder.getInt32(VTy->getNumElements());
195
196 CallInst *CI = Builder.CreateIntrinsic(
197 Intrinsic::experimental_vp_strided_store,
198 {Data->getType(), BasePtr->getType(), Stride->getType()},
199 {Data, BasePtr, Stride, Mask, VL});
200 CI->addParamAttr(
201 1, Attribute::getWithAlignment(CI->getContext(), SI->getAlign()));
202
203 return true;
204 }
205
206 Function *VssegNFunc = Intrinsic::getOrInsertDeclaration(
207 SI->getModule(), FixedVssegIntrIds[Factor - 2], {VTy, PtrTy, XLenTy});
208
209 SmallVector<Value *, 10> Ops;
210 SmallVector<int, 16> NewShuffleMask;
211
212 for (unsigned i = 0; i < Factor; i++) {
213 // Collect shuffle mask for this lane.
214 for (unsigned j = 0; j < VTy->getNumElements(); j++)
215 NewShuffleMask.push_back(Mask[i + Factor * j]);
216
217 Value *Shuffle = Builder.CreateShuffleVector(
218 SVI->getOperand(0), SVI->getOperand(1), NewShuffleMask);
219 Ops.push_back(Shuffle);
220
221 NewShuffleMask.clear();
222 }
223 // This VL should be OK (should be executable in one vsseg instruction,
224 // potentially under larger LMULs) because we checked that the fixed vector
225 // type fits in isLegalInterleavedAccessType
226 Value *VL = ConstantInt::get(XLenTy, VTy->getNumElements());
227 Value *StoreMask = Builder.getAllOnesMask(VTy->getElementCount());
228 Ops.append({SI->getPointerOperand(), StoreMask, VL});
229
230 Builder.CreateCall(VssegNFunc, Ops);
231
232 return true;
233 }
234
lowerDeinterleaveIntrinsicToLoad(LoadInst * LI,ArrayRef<Value * > DeinterleaveValues) const235 bool RISCVTargetLowering::lowerDeinterleaveIntrinsicToLoad(
236 LoadInst *LI, ArrayRef<Value *> DeinterleaveValues) const {
237 const unsigned Factor = DeinterleaveValues.size();
238 if (Factor > 8)
239 return false;
240
241 assert(LI->isSimple());
242 IRBuilder<> Builder(LI);
243
244 Value *FirstActive =
245 *llvm::find_if(DeinterleaveValues, [](Value *V) { return V != nullptr; });
246 VectorType *ResVTy = cast<VectorType>(FirstActive->getType());
247
248 const DataLayout &DL = LI->getDataLayout();
249
250 if (!isLegalInterleavedAccessType(ResVTy, Factor, LI->getAlign(),
251 LI->getPointerAddressSpace(), DL))
252 return false;
253
254 Value *Return;
255 Type *PtrTy = LI->getPointerOperandType();
256 Type *XLenTy = Type::getIntNTy(LI->getContext(), Subtarget.getXLen());
257
258 if (auto *FVTy = dyn_cast<FixedVectorType>(ResVTy)) {
259 Value *VL = ConstantInt::get(XLenTy, FVTy->getNumElements());
260 Value *Mask = Builder.getAllOnesMask(FVTy->getElementCount());
261 Return = Builder.CreateIntrinsic(FixedVlsegIntrIds[Factor - 2],
262 {ResVTy, PtrTy, XLenTy},
263 {LI->getPointerOperand(), Mask, VL});
264 } else {
265 static const Intrinsic::ID IntrIds[] = {
266 Intrinsic::riscv_vlseg2, Intrinsic::riscv_vlseg3,
267 Intrinsic::riscv_vlseg4, Intrinsic::riscv_vlseg5,
268 Intrinsic::riscv_vlseg6, Intrinsic::riscv_vlseg7,
269 Intrinsic::riscv_vlseg8};
270
271 unsigned SEW = DL.getTypeSizeInBits(ResVTy->getElementType());
272 unsigned NumElts = ResVTy->getElementCount().getKnownMinValue();
273 Type *VecTupTy = TargetExtType::get(
274 LI->getContext(), "riscv.vector.tuple",
275 ScalableVectorType::get(Type::getInt8Ty(LI->getContext()),
276 NumElts * SEW / 8),
277 Factor);
278
279 Value *VL = Constant::getAllOnesValue(XLenTy);
280
281 Value *Vlseg = Builder.CreateIntrinsic(
282 IntrIds[Factor - 2], {VecTupTy, PtrTy, XLenTy},
283 {PoisonValue::get(VecTupTy), LI->getPointerOperand(), VL,
284 ConstantInt::get(XLenTy, Log2_64(SEW))});
285
286 SmallVector<Type *, 2> AggrTypes{Factor, ResVTy};
287 Return = PoisonValue::get(StructType::get(LI->getContext(), AggrTypes));
288 for (unsigned i = 0; i < Factor; ++i) {
289 Value *VecExtract = Builder.CreateIntrinsic(
290 Intrinsic::riscv_tuple_extract, {ResVTy, VecTupTy},
291 {Vlseg, Builder.getInt32(i)});
292 Return = Builder.CreateInsertValue(Return, VecExtract, i);
293 }
294 }
295
296 for (auto [Idx, DIV] : enumerate(DeinterleaveValues)) {
297 if (!DIV)
298 continue;
299 // We have to create a brand new ExtractValue to replace each
300 // of these old ExtractValue instructions.
301 Value *NewEV =
302 Builder.CreateExtractValue(Return, {static_cast<unsigned>(Idx)});
303 DIV->replaceAllUsesWith(NewEV);
304 }
305
306 return true;
307 }
308
lowerInterleaveIntrinsicToStore(StoreInst * SI,ArrayRef<Value * > InterleaveValues) const309 bool RISCVTargetLowering::lowerInterleaveIntrinsicToStore(
310 StoreInst *SI, ArrayRef<Value *> InterleaveValues) const {
311 unsigned Factor = InterleaveValues.size();
312 if (Factor > 8)
313 return false;
314
315 assert(SI->isSimple());
316 IRBuilder<> Builder(SI);
317
318 auto *InVTy = cast<VectorType>(InterleaveValues[0]->getType());
319 auto *PtrTy = SI->getPointerOperandType();
320 const DataLayout &DL = SI->getDataLayout();
321
322 if (!isLegalInterleavedAccessType(InVTy, Factor, SI->getAlign(),
323 SI->getPointerAddressSpace(), DL))
324 return false;
325
326 Type *XLenTy = Type::getIntNTy(SI->getContext(), Subtarget.getXLen());
327
328 if (auto *FVTy = dyn_cast<FixedVectorType>(InVTy)) {
329 Function *VssegNFunc = Intrinsic::getOrInsertDeclaration(
330 SI->getModule(), FixedVssegIntrIds[Factor - 2], {InVTy, PtrTy, XLenTy});
331
332 SmallVector<Value *, 10> Ops(InterleaveValues);
333 Value *VL = ConstantInt::get(XLenTy, FVTy->getNumElements());
334 Value *Mask = Builder.getAllOnesMask(FVTy->getElementCount());
335 Ops.append({SI->getPointerOperand(), Mask, VL});
336
337 Builder.CreateCall(VssegNFunc, Ops);
338 } else {
339 static const Intrinsic::ID IntrIds[] = {
340 Intrinsic::riscv_vsseg2, Intrinsic::riscv_vsseg3,
341 Intrinsic::riscv_vsseg4, Intrinsic::riscv_vsseg5,
342 Intrinsic::riscv_vsseg6, Intrinsic::riscv_vsseg7,
343 Intrinsic::riscv_vsseg8};
344
345 unsigned SEW = DL.getTypeSizeInBits(InVTy->getElementType());
346 unsigned NumElts = InVTy->getElementCount().getKnownMinValue();
347 Type *VecTupTy = TargetExtType::get(
348 SI->getContext(), "riscv.vector.tuple",
349 ScalableVectorType::get(Type::getInt8Ty(SI->getContext()),
350 NumElts * SEW / 8),
351 Factor);
352
353 Function *VssegNFunc = Intrinsic::getOrInsertDeclaration(
354 SI->getModule(), IntrIds[Factor - 2], {VecTupTy, PtrTy, XLenTy});
355
356 Value *VL = Constant::getAllOnesValue(XLenTy);
357
358 Value *StoredVal = PoisonValue::get(VecTupTy);
359 for (unsigned i = 0; i < Factor; ++i)
360 StoredVal = Builder.CreateIntrinsic(
361 Intrinsic::riscv_tuple_insert, {VecTupTy, InVTy},
362 {StoredVal, InterleaveValues[i], Builder.getInt32(i)});
363
364 Builder.CreateCall(VssegNFunc, {StoredVal, SI->getPointerOperand(), VL,
365 ConstantInt::get(XLenTy, Log2_64(SEW))});
366 }
367
368 return true;
369 }
370
isMultipleOfN(const Value * V,const DataLayout & DL,unsigned N)371 static bool isMultipleOfN(const Value *V, const DataLayout &DL, unsigned N) {
372 assert(N);
373 if (N == 1)
374 return true;
375
376 using namespace PatternMatch;
377 // Right now we're only recognizing the simplest pattern.
378 uint64_t C;
379 if (match(V, m_CombineOr(m_ConstantInt(C),
380 m_c_Mul(m_Value(), m_ConstantInt(C)))) &&
381 C && C % N == 0)
382 return true;
383
384 if (isPowerOf2_32(N)) {
385 KnownBits KB = llvm::computeKnownBits(V, DL);
386 return KB.countMinTrailingZeros() >= Log2_32(N);
387 }
388
389 return false;
390 }
391
392 /// Lower an interleaved vp.load into a vlsegN intrinsic.
393 ///
394 /// E.g. Lower an interleaved vp.load (Factor = 2):
395 /// %l = call <vscale x 64 x i8> @llvm.vp.load.nxv64i8.p0(ptr %ptr,
396 /// %mask,
397 /// i32 %wide.rvl)
398 /// %dl = tail call { <vscale x 32 x i8>, <vscale x 32 x i8> }
399 /// @llvm.vector.deinterleave2.nxv64i8(
400 /// <vscale x 64 x i8> %l)
401 /// %r0 = extractvalue { <vscale x 32 x i8>, <vscale x 32 x i8> } %dl, 0
402 /// %r1 = extractvalue { <vscale x 32 x i8>, <vscale x 32 x i8> } %dl, 1
403 ///
404 /// Into:
405 /// %rvl = udiv %wide.rvl, 2
406 /// %sl = call { <vscale x 32 x i8>, <vscale x 32 x i8> }
407 /// @llvm.riscv.vlseg2.mask.nxv32i8.i64(<vscale x 32 x i8> undef,
408 /// <vscale x 32 x i8> undef,
409 /// ptr %ptr,
410 /// %mask,
411 /// i64 %rvl,
412 /// i64 1)
413 /// %r0 = extractvalue { <vscale x 32 x i8>, <vscale x 32 x i8> } %sl, 0
414 /// %r1 = extractvalue { <vscale x 32 x i8>, <vscale x 32 x i8> } %sl, 1
415 ///
416 /// NOTE: the deinterleave2 intrinsic won't be touched and is expected to be
417 /// removed by the caller
418 /// TODO: We probably can loosen the dependency on matching extractvalue when
419 /// dealing with factor of 2 (extractvalue is still required for most of other
420 /// factors though).
lowerInterleavedVPLoad(VPIntrinsic * Load,Value * Mask,ArrayRef<Value * > DeinterleaveResults) const421 bool RISCVTargetLowering::lowerInterleavedVPLoad(
422 VPIntrinsic *Load, Value *Mask,
423 ArrayRef<Value *> DeinterleaveResults) const {
424 const unsigned Factor = DeinterleaveResults.size();
425 assert(Mask && "Expect a valid mask");
426 assert(Load->getIntrinsicID() == Intrinsic::vp_load &&
427 "Unexpected intrinsic");
428
429 Value *FirstActive = *llvm::find_if(DeinterleaveResults,
430 [](Value *V) { return V != nullptr; });
431 VectorType *VTy = cast<VectorType>(FirstActive->getType());
432
433 auto &DL = Load->getModule()->getDataLayout();
434 Align Alignment = Load->getParamAlign(0).value_or(
435 DL.getABITypeAlign(VTy->getElementType()));
436 if (!isLegalInterleavedAccessType(
437 VTy, Factor, Alignment,
438 Load->getArgOperand(0)->getType()->getPointerAddressSpace(), DL))
439 return false;
440
441 IRBuilder<> Builder(Load);
442
443 Value *WideEVL = Load->getVectorLengthParam();
444 // Conservatively check if EVL is a multiple of factor, otherwise some
445 // (trailing) elements might be lost after the transformation.
446 if (!isMultipleOfN(WideEVL, Load->getDataLayout(), Factor))
447 return false;
448
449 auto *PtrTy = Load->getArgOperand(0)->getType();
450 auto *XLenTy = Type::getIntNTy(Load->getContext(), Subtarget.getXLen());
451 Value *EVL = Builder.CreateZExt(
452 Builder.CreateUDiv(WideEVL, ConstantInt::get(WideEVL->getType(), Factor)),
453 XLenTy);
454
455 Value *Return = nullptr;
456 if (auto *FVTy = dyn_cast<FixedVectorType>(VTy)) {
457 Return = Builder.CreateIntrinsic(FixedVlsegIntrIds[Factor - 2],
458 {FVTy, PtrTy, XLenTy},
459 {Load->getArgOperand(0), Mask, EVL});
460 } else {
461 unsigned SEW = DL.getTypeSizeInBits(VTy->getElementType());
462 unsigned NumElts = VTy->getElementCount().getKnownMinValue();
463 Type *VecTupTy = TargetExtType::get(
464 Load->getContext(), "riscv.vector.tuple",
465 ScalableVectorType::get(Type::getInt8Ty(Load->getContext()),
466 NumElts * SEW / 8),
467 Factor);
468
469 Value *PoisonVal = PoisonValue::get(VecTupTy);
470
471 Function *VlsegNFunc = Intrinsic::getOrInsertDeclaration(
472 Load->getModule(), ScalableVlsegIntrIds[Factor - 2],
473 {VecTupTy, PtrTy, Mask->getType(), EVL->getType()});
474
475 Value *Operands[] = {
476 PoisonVal,
477 Load->getArgOperand(0),
478 Mask,
479 EVL,
480 ConstantInt::get(XLenTy,
481 RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC),
482 ConstantInt::get(XLenTy, Log2_64(SEW))};
483
484 CallInst *VlsegN = Builder.CreateCall(VlsegNFunc, Operands);
485
486 SmallVector<Type *, 8> AggrTypes{Factor, VTy};
487 Return = PoisonValue::get(StructType::get(Load->getContext(), AggrTypes));
488 Function *VecExtractFunc = Intrinsic::getOrInsertDeclaration(
489 Load->getModule(), Intrinsic::riscv_tuple_extract, {VTy, VecTupTy});
490 for (unsigned i = 0; i < Factor; ++i) {
491 Value *VecExtract =
492 Builder.CreateCall(VecExtractFunc, {VlsegN, Builder.getInt32(i)});
493 Return = Builder.CreateInsertValue(Return, VecExtract, i);
494 }
495 }
496
497 for (auto [Idx, DIO] : enumerate(DeinterleaveResults)) {
498 if (!DIO)
499 continue;
500 // We have to create a brand new ExtractValue to replace each
501 // of these old ExtractValue instructions.
502 Value *NewEV =
503 Builder.CreateExtractValue(Return, {static_cast<unsigned>(Idx)});
504 DIO->replaceAllUsesWith(NewEV);
505 }
506
507 return true;
508 }
509
510 /// Lower an interleaved vp.store into a vssegN intrinsic.
511 ///
512 /// E.g. Lower an interleaved vp.store (Factor = 2):
513 ///
514 /// %is = tail call <vscale x 64 x i8>
515 /// @llvm.vector.interleave2.nxv64i8(
516 /// <vscale x 32 x i8> %load0,
517 /// <vscale x 32 x i8> %load1
518 /// %wide.rvl = shl nuw nsw i32 %rvl, 1
519 /// tail call void @llvm.vp.store.nxv64i8.p0(
520 /// <vscale x 64 x i8> %is, ptr %ptr,
521 /// %mask,
522 /// i32 %wide.rvl)
523 ///
524 /// Into:
525 /// call void @llvm.riscv.vsseg2.mask.nxv32i8.i64(
526 /// <vscale x 32 x i8> %load1,
527 /// <vscale x 32 x i8> %load2, ptr %ptr,
528 /// %mask,
529 /// i64 %rvl)
lowerInterleavedVPStore(VPIntrinsic * Store,Value * Mask,ArrayRef<Value * > InterleaveOperands) const530 bool RISCVTargetLowering::lowerInterleavedVPStore(
531 VPIntrinsic *Store, Value *Mask,
532 ArrayRef<Value *> InterleaveOperands) const {
533 assert(Mask && "Expect a valid mask");
534 assert(Store->getIntrinsicID() == Intrinsic::vp_store &&
535 "Unexpected intrinsic");
536
537 const unsigned Factor = InterleaveOperands.size();
538
539 auto *VTy = dyn_cast<VectorType>(InterleaveOperands[0]->getType());
540 if (!VTy)
541 return false;
542
543 const DataLayout &DL = Store->getDataLayout();
544 Align Alignment = Store->getParamAlign(1).value_or(
545 DL.getABITypeAlign(VTy->getElementType()));
546 if (!isLegalInterleavedAccessType(
547 VTy, Factor, Alignment,
548 Store->getArgOperand(1)->getType()->getPointerAddressSpace(), DL))
549 return false;
550
551 IRBuilder<> Builder(Store);
552 Value *WideEVL = Store->getArgOperand(3);
553 // Conservatively check if EVL is a multiple of factor, otherwise some
554 // (trailing) elements might be lost after the transformation.
555 if (!isMultipleOfN(WideEVL, Store->getDataLayout(), Factor))
556 return false;
557
558 auto *PtrTy = Store->getArgOperand(1)->getType();
559 auto *XLenTy = Type::getIntNTy(Store->getContext(), Subtarget.getXLen());
560 Value *EVL = Builder.CreateZExt(
561 Builder.CreateUDiv(WideEVL, ConstantInt::get(WideEVL->getType(), Factor)),
562 XLenTy);
563
564 if (auto *FVTy = dyn_cast<FixedVectorType>(VTy)) {
565 SmallVector<Value *, 8> Operands(InterleaveOperands);
566 Operands.append({Store->getArgOperand(1), Mask, EVL});
567 Builder.CreateIntrinsic(FixedVssegIntrIds[Factor - 2],
568 {FVTy, PtrTy, XLenTy}, Operands);
569 return true;
570 }
571
572 unsigned SEW = DL.getTypeSizeInBits(VTy->getElementType());
573 unsigned NumElts = VTy->getElementCount().getKnownMinValue();
574 Type *VecTupTy = TargetExtType::get(
575 Store->getContext(), "riscv.vector.tuple",
576 ScalableVectorType::get(Type::getInt8Ty(Store->getContext()),
577 NumElts * SEW / 8),
578 Factor);
579
580 Function *VecInsertFunc = Intrinsic::getOrInsertDeclaration(
581 Store->getModule(), Intrinsic::riscv_tuple_insert, {VecTupTy, VTy});
582 Value *StoredVal = PoisonValue::get(VecTupTy);
583 for (unsigned i = 0; i < Factor; ++i)
584 StoredVal = Builder.CreateCall(
585 VecInsertFunc, {StoredVal, InterleaveOperands[i], Builder.getInt32(i)});
586
587 Function *VssegNFunc = Intrinsic::getOrInsertDeclaration(
588 Store->getModule(), ScalableVssegIntrIds[Factor - 2],
589 {VecTupTy, PtrTy, Mask->getType(), EVL->getType()});
590
591 Value *Operands[] = {StoredVal, Store->getArgOperand(1), Mask, EVL,
592 ConstantInt::get(XLenTy, Log2_64(SEW))};
593
594 Builder.CreateCall(VssegNFunc, Operands);
595 return true;
596 }
597