1 //===- VFABIDemangler.cpp - Vector Function ABI demangler -----------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #include "llvm/IR/VFABIDemangler.h"
10 #include "llvm/ADT/SetVector.h"
11 #include "llvm/ADT/SmallString.h"
12 #include "llvm/ADT/StringSwitch.h"
13 #include "llvm/IR/Module.h"
14 #include "llvm/IR/VectorTypeUtils.h"
15 #include "llvm/Support/Debug.h"
16 #include "llvm/Support/raw_ostream.h"
17 #include <limits>
18
19 using namespace llvm;
20
21 #define DEBUG_TYPE "vfabi-demangler"
22
23 namespace {
24 /// Utilities for the Vector Function ABI name parser.
25
26 /// Return types for the parser functions.
27 enum class ParseRet {
28 OK, // Found.
29 None, // Not found.
30 Error // Syntax error.
31 };
32
33 /// Extracts the `<isa>` information from the mangled string, and
34 /// sets the `ISA` accordingly. If successful, the <isa> token is removed
35 /// from the input string `MangledName`.
tryParseISA(StringRef & MangledName,VFISAKind & ISA)36 static ParseRet tryParseISA(StringRef &MangledName, VFISAKind &ISA) {
37 if (MangledName.empty())
38 return ParseRet::Error;
39
40 if (MangledName.consume_front(VFABI::_LLVM_)) {
41 ISA = VFISAKind::LLVM;
42 } else {
43 ISA = StringSwitch<VFISAKind>(MangledName.take_front(1))
44 .Case("n", VFISAKind::AdvancedSIMD)
45 .Case("s", VFISAKind::SVE)
46 .Case("r", VFISAKind::RVV)
47 .Case("b", VFISAKind::SSE)
48 .Case("c", VFISAKind::AVX)
49 .Case("d", VFISAKind::AVX2)
50 .Case("e", VFISAKind::AVX512)
51 .Default(VFISAKind::Unknown);
52 MangledName = MangledName.drop_front(1);
53 }
54
55 return ParseRet::OK;
56 }
57
58 /// Extracts the `<mask>` information from the mangled string, and
59 /// sets `IsMasked` accordingly. If successful, the <mask> token is removed
60 /// from the input string `MangledName`.
tryParseMask(StringRef & MangledName,bool & IsMasked)61 static ParseRet tryParseMask(StringRef &MangledName, bool &IsMasked) {
62 if (MangledName.consume_front("M")) {
63 IsMasked = true;
64 return ParseRet::OK;
65 }
66
67 if (MangledName.consume_front("N")) {
68 IsMasked = false;
69 return ParseRet::OK;
70 }
71
72 return ParseRet::Error;
73 }
74
75 /// Extract the `<vlen>` information from the mangled string, and
76 /// sets `ParsedVF` accordingly. A `<vlen> == "x"` token is interpreted as a
77 /// scalable vector length and the boolean is set to true, otherwise a nonzero
78 /// unsigned integer will be directly used as a VF. On success, the `<vlen>`
79 /// token is removed from the input string `ParseString`.
tryParseVLEN(StringRef & ParseString,VFISAKind ISA,std::pair<unsigned,bool> & ParsedVF)80 static ParseRet tryParseVLEN(StringRef &ParseString, VFISAKind ISA,
81 std::pair<unsigned, bool> &ParsedVF) {
82 if (ParseString.consume_front("x")) {
83 // SVE is the only scalable ISA currently supported.
84 if (ISA != VFISAKind::SVE && ISA != VFISAKind::RVV) {
85 LLVM_DEBUG(dbgs() << "Vector function variant declared with scalable VF "
86 << "but ISA supported for SVE and RVV only\n");
87 return ParseRet::Error;
88 }
89 // We can't determine the VF of a scalable vector by looking at the vlen
90 // string (just 'x'), so say we successfully parsed it but return a 'true'
91 // for the scalable field with an invalid VF field so that we know to look
92 // up the actual VF based on element types from the parameters or return.
93 ParsedVF = {0, true};
94 return ParseRet::OK;
95 }
96
97 unsigned VF = 0;
98 if (ParseString.consumeInteger(10, VF))
99 return ParseRet::Error;
100
101 // The token `0` is invalid for VLEN.
102 if (VF == 0)
103 return ParseRet::Error;
104
105 ParsedVF = {VF, false};
106 return ParseRet::OK;
107 }
108
109 /// The function looks for the following strings at the beginning of
110 /// the input string `ParseString`:
111 ///
112 /// <token> <number>
113 ///
114 /// On success, it removes the parsed parameter from `ParseString`,
115 /// sets `PKind` to the correspondent enum value, sets `Pos` to
116 /// <number>, and return success. On a syntax error, it return a
117 /// parsing error. If nothing is parsed, it returns std::nullopt.
118 ///
119 /// The function expects <token> to be one of "ls", "Rs", "Us" or
120 /// "Ls".
tryParseLinearTokenWithRuntimeStep(StringRef & ParseString,VFParamKind & PKind,int & Pos,const StringRef Token)121 static ParseRet tryParseLinearTokenWithRuntimeStep(StringRef &ParseString,
122 VFParamKind &PKind, int &Pos,
123 const StringRef Token) {
124 if (ParseString.consume_front(Token)) {
125 PKind = VFABI::getVFParamKindFromString(Token);
126 if (ParseString.consumeInteger(10, Pos))
127 return ParseRet::Error;
128 return ParseRet::OK;
129 }
130
131 return ParseRet::None;
132 }
133
134 /// The function looks for the following string at the beginning of
135 /// the input string `ParseString`:
136 ///
137 /// <token> <number>
138 ///
139 /// <token> is one of "ls", "Rs", "Us" or "Ls".
140 ///
141 /// On success, it removes the parsed parameter from `ParseString`,
142 /// sets `PKind` to the correspondent enum value, sets `StepOrPos` to
143 /// <number>, and return success. On a syntax error, it return a
144 /// parsing error. If nothing is parsed, it returns std::nullopt.
tryParseLinearWithRuntimeStep(StringRef & ParseString,VFParamKind & PKind,int & StepOrPos)145 static ParseRet tryParseLinearWithRuntimeStep(StringRef &ParseString,
146 VFParamKind &PKind,
147 int &StepOrPos) {
148 ParseRet Ret;
149
150 // "ls" <RuntimeStepPos>
151 Ret = tryParseLinearTokenWithRuntimeStep(ParseString, PKind, StepOrPos, "ls");
152 if (Ret != ParseRet::None)
153 return Ret;
154
155 // "Rs" <RuntimeStepPos>
156 Ret = tryParseLinearTokenWithRuntimeStep(ParseString, PKind, StepOrPos, "Rs");
157 if (Ret != ParseRet::None)
158 return Ret;
159
160 // "Ls" <RuntimeStepPos>
161 Ret = tryParseLinearTokenWithRuntimeStep(ParseString, PKind, StepOrPos, "Ls");
162 if (Ret != ParseRet::None)
163 return Ret;
164
165 // "Us" <RuntimeStepPos>
166 Ret = tryParseLinearTokenWithRuntimeStep(ParseString, PKind, StepOrPos, "Us");
167 if (Ret != ParseRet::None)
168 return Ret;
169
170 return ParseRet::None;
171 }
172
173 /// The function looks for the following strings at the beginning of
174 /// the input string `ParseString`:
175 ///
176 /// <token> {"n"} <number>
177 ///
178 /// On success, it removes the parsed parameter from `ParseString`,
179 /// sets `PKind` to the correspondent enum value, sets `LinearStep` to
180 /// <number>, and return success. On a syntax error, it return a
181 /// parsing error. If nothing is parsed, it returns std::nullopt.
182 ///
183 /// The function expects <token> to be one of "l", "R", "U" or
184 /// "L".
tryParseCompileTimeLinearToken(StringRef & ParseString,VFParamKind & PKind,int & LinearStep,const StringRef Token)185 static ParseRet tryParseCompileTimeLinearToken(StringRef &ParseString,
186 VFParamKind &PKind,
187 int &LinearStep,
188 const StringRef Token) {
189 if (ParseString.consume_front(Token)) {
190 PKind = VFABI::getVFParamKindFromString(Token);
191 const bool Negate = ParseString.consume_front("n");
192 if (ParseString.consumeInteger(10, LinearStep))
193 LinearStep = 1;
194 if (Negate)
195 LinearStep *= -1;
196 return ParseRet::OK;
197 }
198
199 return ParseRet::None;
200 }
201
202 /// The function looks for the following strings at the beginning of
203 /// the input string `ParseString`:
204 ///
205 /// ["l" | "R" | "U" | "L"] {"n"} <number>
206 ///
207 /// On success, it removes the parsed parameter from `ParseString`,
208 /// sets `PKind` to the correspondent enum value, sets `LinearStep` to
209 /// <number>, and return success. On a syntax error, it return a
210 /// parsing error. If nothing is parsed, it returns std::nullopt.
tryParseLinearWithCompileTimeStep(StringRef & ParseString,VFParamKind & PKind,int & StepOrPos)211 static ParseRet tryParseLinearWithCompileTimeStep(StringRef &ParseString,
212 VFParamKind &PKind,
213 int &StepOrPos) {
214 // "l" {"n"} <CompileTimeStep>
215 if (tryParseCompileTimeLinearToken(ParseString, PKind, StepOrPos, "l") ==
216 ParseRet::OK)
217 return ParseRet::OK;
218
219 // "R" {"n"} <CompileTimeStep>
220 if (tryParseCompileTimeLinearToken(ParseString, PKind, StepOrPos, "R") ==
221 ParseRet::OK)
222 return ParseRet::OK;
223
224 // "L" {"n"} <CompileTimeStep>
225 if (tryParseCompileTimeLinearToken(ParseString, PKind, StepOrPos, "L") ==
226 ParseRet::OK)
227 return ParseRet::OK;
228
229 // "U" {"n"} <CompileTimeStep>
230 if (tryParseCompileTimeLinearToken(ParseString, PKind, StepOrPos, "U") ==
231 ParseRet::OK)
232 return ParseRet::OK;
233
234 return ParseRet::None;
235 }
236
237 /// Looks into the <parameters> part of the mangled name in search
238 /// for valid paramaters at the beginning of the string
239 /// `ParseString`.
240 ///
241 /// On success, it removes the parsed parameter from `ParseString`,
242 /// sets `PKind` to the correspondent enum value, sets `StepOrPos`
243 /// accordingly, and return success. On a syntax error, it return a
244 /// parsing error. If nothing is parsed, it returns std::nullopt.
tryParseParameter(StringRef & ParseString,VFParamKind & PKind,int & StepOrPos)245 static ParseRet tryParseParameter(StringRef &ParseString, VFParamKind &PKind,
246 int &StepOrPos) {
247 if (ParseString.consume_front("v")) {
248 PKind = VFParamKind::Vector;
249 StepOrPos = 0;
250 return ParseRet::OK;
251 }
252
253 if (ParseString.consume_front("u")) {
254 PKind = VFParamKind::OMP_Uniform;
255 StepOrPos = 0;
256 return ParseRet::OK;
257 }
258
259 const ParseRet HasLinearRuntime =
260 tryParseLinearWithRuntimeStep(ParseString, PKind, StepOrPos);
261 if (HasLinearRuntime != ParseRet::None)
262 return HasLinearRuntime;
263
264 const ParseRet HasLinearCompileTime =
265 tryParseLinearWithCompileTimeStep(ParseString, PKind, StepOrPos);
266 if (HasLinearCompileTime != ParseRet::None)
267 return HasLinearCompileTime;
268
269 return ParseRet::None;
270 }
271
272 /// Looks into the <parameters> part of the mangled name in search
273 /// of a valid 'aligned' clause. The function should be invoked
274 /// after parsing a parameter via `tryParseParameter`.
275 ///
276 /// On success, it removes the parsed parameter from `ParseString`,
277 /// sets `PKind` to the correspondent enum value, sets `StepOrPos`
278 /// accordingly, and return success. On a syntax error, it return a
279 /// parsing error. If nothing is parsed, it returns std::nullopt.
tryParseAlign(StringRef & ParseString,Align & Alignment)280 static ParseRet tryParseAlign(StringRef &ParseString, Align &Alignment) {
281 uint64_t Val;
282 // "a" <number>
283 if (ParseString.consume_front("a")) {
284 if (ParseString.consumeInteger(10, Val))
285 return ParseRet::Error;
286
287 if (!isPowerOf2_64(Val))
288 return ParseRet::Error;
289
290 Alignment = Align(Val);
291
292 return ParseRet::OK;
293 }
294
295 return ParseRet::None;
296 }
297
298 // Returns the 'natural' VF for a given scalar element type, based on the
299 // current architecture.
300 //
301 // For SVE (currently the only scalable architecture with a defined name
302 // mangling), we assume a minimum vector size of 128b and return a VF based on
303 // the number of elements of the given type which would fit in such a vector.
getElementCountForTy(const VFISAKind ISA,const Type * Ty)304 static std::optional<ElementCount> getElementCountForTy(const VFISAKind ISA,
305 const Type *Ty) {
306 assert((ISA == VFISAKind::SVE || ISA == VFISAKind::RVV) &&
307 "Scalable VF decoding only implemented for SVE and RVV\n");
308
309 if (Ty->isIntegerTy(64) || Ty->isDoubleTy() || Ty->isPointerTy())
310 return ElementCount::getScalable(2);
311 if (Ty->isIntegerTy(32) || Ty->isFloatTy())
312 return ElementCount::getScalable(4);
313 if (Ty->isIntegerTy(16) || Ty->is16bitFPTy())
314 return ElementCount::getScalable(8);
315 if (Ty->isIntegerTy(8))
316 return ElementCount::getScalable(16);
317
318 return std::nullopt;
319 }
320
321 // Extract the VectorizationFactor from a given function signature, based
322 // on the widest scalar element types that will become vector parameters.
323 static std::optional<ElementCount>
getScalableECFromSignature(const FunctionType * Signature,const VFISAKind ISA,const SmallVectorImpl<VFParameter> & Params)324 getScalableECFromSignature(const FunctionType *Signature, const VFISAKind ISA,
325 const SmallVectorImpl<VFParameter> &Params) {
326 // Start with a very wide EC and drop when we find smaller ECs based on type.
327 ElementCount MinEC =
328 ElementCount::getScalable(std::numeric_limits<unsigned int>::max());
329 for (auto &Param : Params) {
330 // Only vector parameters are used when determining the VF; uniform or
331 // linear are left as scalars, so do not affect VF.
332 if (Param.ParamKind == VFParamKind::Vector) {
333 Type *PTy = Signature->getParamType(Param.ParamPos);
334
335 std::optional<ElementCount> EC = getElementCountForTy(ISA, PTy);
336 // If we have an unknown scalar element type we can't find a reasonable
337 // VF.
338 if (!EC)
339 return std::nullopt;
340
341 // Find the smallest VF, based on the widest scalar type.
342 if (ElementCount::isKnownLT(*EC, MinEC))
343 MinEC = *EC;
344 }
345 }
346
347 // Also check the return type if not void.
348 Type *RetTy = Signature->getReturnType();
349 if (!RetTy->isVoidTy()) {
350 // If the return type is a struct, only allow unpacked struct literals.
351 StructType *StructTy = dyn_cast<StructType>(RetTy);
352 if (StructTy && !isUnpackedStructLiteral(StructTy))
353 return std::nullopt;
354
355 for (Type *RetTy : getContainedTypes(RetTy)) {
356 std::optional<ElementCount> ReturnEC = getElementCountForTy(ISA, RetTy);
357 // If we have an unknown scalar element type we can't find a reasonable
358 // VF.
359 if (!ReturnEC)
360 return std::nullopt;
361 if (ElementCount::isKnownLT(*ReturnEC, MinEC))
362 MinEC = *ReturnEC;
363 }
364 }
365
366 // The SVE Vector function call ABI bases the VF on the widest element types
367 // present, and vector arguments containing types of that width are always
368 // considered to be packed. Arguments with narrower elements are considered
369 // to be unpacked.
370 if (MinEC.getKnownMinValue() < std::numeric_limits<unsigned int>::max())
371 return MinEC;
372
373 return std::nullopt;
374 }
375 } // namespace
376
377 // Format of the ABI name:
378 // _ZGV<isa><mask><vlen><parameters>_<scalarname>[(<redirection>)]
tryDemangleForVFABI(StringRef MangledName,const FunctionType * FTy)379 std::optional<VFInfo> VFABI::tryDemangleForVFABI(StringRef MangledName,
380 const FunctionType *FTy) {
381 const StringRef OriginalName = MangledName;
382 // Assume there is no custom name <redirection>, and therefore the
383 // vector name consists of
384 // _ZGV<isa><mask><vlen><parameters>_<scalarname>.
385 StringRef VectorName = MangledName;
386
387 // Parse the fixed size part of the mangled name
388 if (!MangledName.consume_front("_ZGV"))
389 return std::nullopt;
390
391 // Extract ISA. An unknow ISA is also supported, so we accept all
392 // values.
393 VFISAKind ISA;
394 if (tryParseISA(MangledName, ISA) != ParseRet::OK)
395 return std::nullopt;
396
397 // Extract <mask>.
398 bool IsMasked;
399 if (tryParseMask(MangledName, IsMasked) != ParseRet::OK)
400 return std::nullopt;
401
402 // Parse the variable size, starting from <vlen>.
403 std::pair<unsigned, bool> ParsedVF;
404 if (tryParseVLEN(MangledName, ISA, ParsedVF) != ParseRet::OK)
405 return std::nullopt;
406
407 // Parse the <parameters>.
408 ParseRet ParamFound;
409 SmallVector<VFParameter, 8> Parameters;
410 do {
411 const unsigned ParameterPos = Parameters.size();
412 VFParamKind PKind;
413 int StepOrPos;
414 ParamFound = tryParseParameter(MangledName, PKind, StepOrPos);
415
416 // Bail off if there is a parsing error in the parsing of the parameter.
417 if (ParamFound == ParseRet::Error)
418 return std::nullopt;
419
420 if (ParamFound == ParseRet::OK) {
421 Align Alignment;
422 // Look for the alignment token "a <number>".
423 const ParseRet AlignFound = tryParseAlign(MangledName, Alignment);
424 // Bail off if there is a syntax error in the align token.
425 if (AlignFound == ParseRet::Error)
426 return std::nullopt;
427
428 // Add the parameter.
429 Parameters.push_back({ParameterPos, PKind, StepOrPos, Alignment});
430 }
431 } while (ParamFound == ParseRet::OK);
432
433 // A valid MangledName must have at least one valid entry in the
434 // <parameters>.
435 if (Parameters.empty())
436 return std::nullopt;
437
438 // If the number of arguments of the scalar function does not match the
439 // vector variant we have just demangled then reject the mapping.
440 if (Parameters.size() != FTy->getNumParams())
441 return std::nullopt;
442
443 // Figure out the number of lanes in vectors for this function variant. This
444 // is easy for fixed length, as the vlen encoding just gives us the value
445 // directly. However, if the vlen mangling indicated that this function
446 // variant expects scalable vectors we need to work it out based on the
447 // demangled parameter types and the scalar function signature.
448 std::optional<ElementCount> EC;
449 if (ParsedVF.second) {
450 EC = getScalableECFromSignature(FTy, ISA, Parameters);
451 if (!EC)
452 return std::nullopt;
453 } else
454 EC = ElementCount::getFixed(ParsedVF.first);
455
456 // Check for the <scalarname> and the optional <redirection>, which
457 // are separated from the prefix with "_"
458 if (!MangledName.consume_front("_"))
459 return std::nullopt;
460
461 // The rest of the string must be in the format:
462 // <scalarname>[(<redirection>)]
463 const StringRef ScalarName =
464 MangledName.take_while([](char In) { return In != '('; });
465
466 if (ScalarName.empty())
467 return std::nullopt;
468
469 // Reduce MangledName to [(<redirection>)].
470 MangledName = MangledName.ltrim(ScalarName);
471 // Find the optional custom name redirection.
472 if (MangledName.consume_front("(")) {
473 if (!MangledName.consume_back(")"))
474 return std::nullopt;
475 // Update the vector variant with the one specified by the user.
476 VectorName = MangledName;
477 // If the vector name is missing, bail out.
478 if (VectorName.empty())
479 return std::nullopt;
480 }
481
482 // LLVM internal mapping via the TargetLibraryInfo (TLI) must be
483 // redirected to an existing name.
484 if (ISA == VFISAKind::LLVM && VectorName == OriginalName)
485 return std::nullopt;
486
487 // When <mask> is "M", we need to add a parameter that is used as
488 // global predicate for the function.
489 if (IsMasked) {
490 const unsigned Pos = Parameters.size();
491 Parameters.push_back({Pos, VFParamKind::GlobalPredicate});
492 }
493
494 // Asserts for parameters of type `VFParamKind::GlobalPredicate`, as
495 // prescribed by the Vector Function ABI specifications supported by
496 // this parser:
497 // 1. Uniqueness.
498 // 2. Must be the last in the parameter list.
499 const auto NGlobalPreds =
500 llvm::count_if(Parameters, [](const VFParameter &PK) {
501 return PK.ParamKind == VFParamKind::GlobalPredicate;
502 });
503 assert(NGlobalPreds < 2 && "Cannot have more than one global predicate.");
504 if (NGlobalPreds)
505 assert(Parameters.back().ParamKind == VFParamKind::GlobalPredicate &&
506 "The global predicate must be the last parameter");
507
508 const VFShape Shape({*EC, Parameters});
509 return VFInfo({Shape, std::string(ScalarName), std::string(VectorName), ISA});
510 }
511
getVFParamKindFromString(const StringRef Token)512 VFParamKind VFABI::getVFParamKindFromString(const StringRef Token) {
513 const VFParamKind ParamKind = StringSwitch<VFParamKind>(Token)
514 .Case("v", VFParamKind::Vector)
515 .Case("l", VFParamKind::OMP_Linear)
516 .Case("R", VFParamKind::OMP_LinearRef)
517 .Case("L", VFParamKind::OMP_LinearVal)
518 .Case("U", VFParamKind::OMP_LinearUVal)
519 .Case("ls", VFParamKind::OMP_LinearPos)
520 .Case("Ls", VFParamKind::OMP_LinearValPos)
521 .Case("Rs", VFParamKind::OMP_LinearRefPos)
522 .Case("Us", VFParamKind::OMP_LinearUValPos)
523 .Case("u", VFParamKind::OMP_Uniform)
524 .Default(VFParamKind::Unknown);
525
526 if (ParamKind != VFParamKind::Unknown)
527 return ParamKind;
528
529 // This function should never be invoked with an invalid input.
530 llvm_unreachable("This fuction should be invoken only on parameters"
531 " that have a textual representation in the mangled name"
532 " of the Vector Function ABI");
533 }
534
getVectorVariantNames(const CallInst & CI,SmallVectorImpl<std::string> & VariantMappings)535 void VFABI::getVectorVariantNames(
536 const CallInst &CI, SmallVectorImpl<std::string> &VariantMappings) {
537 const StringRef S = CI.getFnAttr(VFABI::MappingsAttrName).getValueAsString();
538 if (S.empty())
539 return;
540
541 SmallVector<StringRef, 8> ListAttr;
542 S.split(ListAttr, ",");
543
544 for (const auto &S : SetVector<StringRef>(llvm::from_range, ListAttr)) {
545 std::optional<VFInfo> Info =
546 VFABI::tryDemangleForVFABI(S, CI.getFunctionType());
547 if (Info && CI.getModule()->getFunction(Info->VectorName)) {
548 LLVM_DEBUG(dbgs() << "VFABI: Adding mapping '" << S << "' for " << CI
549 << "\n");
550 VariantMappings.push_back(std::string(S));
551 } else
552 LLVM_DEBUG(dbgs() << "VFABI: Invalid mapping '" << S << "'\n");
553 }
554 }
555
createFunctionType(const VFInfo & Info,const FunctionType * ScalarFTy)556 FunctionType *VFABI::createFunctionType(const VFInfo &Info,
557 const FunctionType *ScalarFTy) {
558 // Create vector parameter types
559 SmallVector<Type *, 8> VecTypes;
560 ElementCount VF = Info.Shape.VF;
561 int ScalarParamIndex = 0;
562 for (auto VFParam : Info.Shape.Parameters) {
563 if (VFParam.ParamKind == VFParamKind::GlobalPredicate) {
564 VectorType *MaskTy =
565 VectorType::get(Type::getInt1Ty(ScalarFTy->getContext()), VF);
566 VecTypes.push_back(MaskTy);
567 continue;
568 }
569
570 Type *OperandTy = ScalarFTy->getParamType(ScalarParamIndex++);
571 if (VFParam.ParamKind == VFParamKind::Vector)
572 OperandTy = VectorType::get(OperandTy, VF);
573 VecTypes.push_back(OperandTy);
574 }
575
576 auto *RetTy = ScalarFTy->getReturnType();
577 if (!RetTy->isVoidTy())
578 RetTy = toVectorizedTy(RetTy, VF);
579 return FunctionType::get(RetTy, VecTypes, false);
580 }
581
setVectorVariantNames(CallInst * CI,ArrayRef<std::string> VariantMappings)582 void VFABI::setVectorVariantNames(CallInst *CI,
583 ArrayRef<std::string> VariantMappings) {
584 if (VariantMappings.empty())
585 return;
586
587 SmallString<256> Buffer;
588 llvm::raw_svector_ostream Out(Buffer);
589 for (const std::string &VariantMapping : VariantMappings)
590 Out << VariantMapping << ",";
591 // Get rid of the trailing ','.
592 assert(!Buffer.str().empty() && "Must have at least one char.");
593 Buffer.pop_back();
594
595 Module *M = CI->getModule();
596 #ifndef NDEBUG
597 for (const std::string &VariantMapping : VariantMappings) {
598 LLVM_DEBUG(dbgs() << "VFABI: adding mapping '" << VariantMapping << "'\n");
599 std::optional<VFInfo> VI =
600 VFABI::tryDemangleForVFABI(VariantMapping, CI->getFunctionType());
601 assert(VI && "Cannot add an invalid VFABI name.");
602 assert(M->getNamedValue(VI->VectorName) &&
603 "Cannot add variant to attribute: "
604 "vector function declaration is missing.");
605 }
606 #endif
607 CI->addFnAttr(
608 Attribute::get(M->getContext(), MappingsAttrName, Buffer.str()));
609 }
610
hasValidParameterList() const611 bool VFShape::hasValidParameterList() const {
612 for (unsigned Pos = 0, NumParams = Parameters.size(); Pos < NumParams;
613 ++Pos) {
614 assert(Parameters[Pos].ParamPos == Pos && "Broken parameter list.");
615
616 switch (Parameters[Pos].ParamKind) {
617 default: // Nothing to check.
618 break;
619 case VFParamKind::OMP_Linear:
620 case VFParamKind::OMP_LinearRef:
621 case VFParamKind::OMP_LinearVal:
622 case VFParamKind::OMP_LinearUVal:
623 // Compile time linear steps must be non-zero.
624 if (Parameters[Pos].LinearStepOrPos == 0)
625 return false;
626 break;
627 case VFParamKind::OMP_LinearPos:
628 case VFParamKind::OMP_LinearRefPos:
629 case VFParamKind::OMP_LinearValPos:
630 case VFParamKind::OMP_LinearUValPos:
631 // The runtime linear step must be referring to some other
632 // parameters in the signature.
633 if (Parameters[Pos].LinearStepOrPos >= int(NumParams))
634 return false;
635 // The linear step parameter must be marked as uniform.
636 if (Parameters[Parameters[Pos].LinearStepOrPos].ParamKind !=
637 VFParamKind::OMP_Uniform)
638 return false;
639 // The linear step parameter can't point at itself.
640 if (Parameters[Pos].LinearStepOrPos == int(Pos))
641 return false;
642 break;
643 case VFParamKind::GlobalPredicate:
644 // The global predicate must be the unique. Can be placed anywhere in the
645 // signature.
646 for (unsigned NextPos = Pos + 1; NextPos < NumParams; ++NextPos)
647 if (Parameters[NextPos].ParamKind == VFParamKind::GlobalPredicate)
648 return false;
649 break;
650 }
651 }
652 return true;
653 }
654