1 //===- IntrinsicEmitter.cpp - Generate intrinsic information --------------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This tablegen backend emits information about intrinsic functions. 10 // 11 //===----------------------------------------------------------------------===// 12 13 #include "CodeGenIntrinsics.h" 14 #include "CodeGenTarget.h" 15 #include "SequenceToOffsetTable.h" 16 #include "TableGenBackends.h" 17 #include "llvm/ADT/StringExtras.h" 18 #include "llvm/Support/CommandLine.h" 19 #include "llvm/TableGen/Error.h" 20 #include "llvm/TableGen/Record.h" 21 #include "llvm/TableGen/StringToOffsetTable.h" 22 #include "llvm/TableGen/TableGenBackend.h" 23 #include <algorithm> 24 using namespace llvm; 25 26 cl::OptionCategory GenIntrinsicCat("Options for -gen-intrinsic-enums"); 27 cl::opt<std::string> 28 IntrinsicPrefix("intrinsic-prefix", 29 cl::desc("Generate intrinsics with this target prefix"), 30 cl::value_desc("target prefix"), cl::cat(GenIntrinsicCat)); 31 32 namespace { 33 class IntrinsicEmitter { 34 RecordKeeper &Records; 35 36 public: 37 IntrinsicEmitter(RecordKeeper &R) : Records(R) {} 38 39 void run(raw_ostream &OS, bool Enums); 40 41 void EmitEnumInfo(const CodeGenIntrinsicTable &Ints, raw_ostream &OS); 42 void EmitTargetInfo(const CodeGenIntrinsicTable &Ints, raw_ostream &OS); 43 void EmitIntrinsicToNameTable(const CodeGenIntrinsicTable &Ints, 44 raw_ostream &OS); 45 void EmitIntrinsicToOverloadTable(const CodeGenIntrinsicTable &Ints, 46 raw_ostream &OS); 47 void EmitGenerator(const CodeGenIntrinsicTable &Ints, raw_ostream &OS); 48 void EmitAttributes(const CodeGenIntrinsicTable &Ints, raw_ostream &OS); 49 void EmitIntrinsicToBuiltinMap(const CodeGenIntrinsicTable &Ints, bool IsClang, 50 raw_ostream &OS); 51 }; 52 } // End anonymous namespace 53 54 //===----------------------------------------------------------------------===// 55 // IntrinsicEmitter Implementation 56 //===----------------------------------------------------------------------===// 57 58 void IntrinsicEmitter::run(raw_ostream &OS, bool Enums) { 59 emitSourceFileHeader("Intrinsic Function Source Fragment", OS); 60 61 CodeGenIntrinsicTable Ints(Records); 62 63 if (Enums) { 64 // Emit the enum information. 65 EmitEnumInfo(Ints, OS); 66 } else { 67 // Emit the target metadata. 68 EmitTargetInfo(Ints, OS); 69 70 // Emit the intrinsic ID -> name table. 71 EmitIntrinsicToNameTable(Ints, OS); 72 73 // Emit the intrinsic ID -> overload table. 74 EmitIntrinsicToOverloadTable(Ints, OS); 75 76 // Emit the intrinsic declaration generator. 77 EmitGenerator(Ints, OS); 78 79 // Emit the intrinsic parameter attributes. 80 EmitAttributes(Ints, OS); 81 82 // Emit code to translate GCC builtins into LLVM intrinsics. 83 EmitIntrinsicToBuiltinMap(Ints, true, OS); 84 85 // Emit code to translate MS builtins into LLVM intrinsics. 86 EmitIntrinsicToBuiltinMap(Ints, false, OS); 87 } 88 } 89 90 void IntrinsicEmitter::EmitEnumInfo(const CodeGenIntrinsicTable &Ints, 91 raw_ostream &OS) { 92 // Find the TargetSet for which to generate enums. There will be an initial 93 // set with an empty target prefix which will include target independent 94 // intrinsics like dbg.value. 95 const CodeGenIntrinsicTable::TargetSet *Set = nullptr; 96 for (const auto &Target : Ints.Targets) { 97 if (Target.Name == IntrinsicPrefix) { 98 Set = &Target; 99 break; 100 } 101 } 102 if (!Set) { 103 std::vector<std::string> KnownTargets; 104 for (const auto &Target : Ints.Targets) 105 if (!Target.Name.empty()) 106 KnownTargets.push_back(Target.Name); 107 PrintFatalError("tried to generate intrinsics for unknown target " + 108 IntrinsicPrefix + 109 "\nKnown targets are: " + join(KnownTargets, ", ") + "\n"); 110 } 111 112 // Generate a complete header for target specific intrinsics. 113 if (!IntrinsicPrefix.empty()) { 114 std::string UpperPrefix = StringRef(IntrinsicPrefix).upper(); 115 OS << "#ifndef LLVM_IR_INTRINSIC_" << UpperPrefix << "_ENUMS_H\n"; 116 OS << "#define LLVM_IR_INTRINSIC_" << UpperPrefix << "_ENUMS_H\n\n"; 117 OS << "namespace llvm {\n"; 118 OS << "namespace Intrinsic {\n"; 119 OS << "enum " << UpperPrefix << "Intrinsics : unsigned {\n"; 120 } 121 122 OS << "// Enum values for intrinsics\n"; 123 for (unsigned i = Set->Offset, e = Set->Offset + Set->Count; i != e; ++i) { 124 OS << " " << Ints[i].EnumName; 125 126 // Assign a value to the first intrinsic in this target set so that all 127 // intrinsic ids are distinct. 128 if (i == Set->Offset) 129 OS << " = " << (Set->Offset + 1); 130 131 OS << ", "; 132 if (Ints[i].EnumName.size() < 40) 133 OS.indent(40 - Ints[i].EnumName.size()); 134 OS << " // " << Ints[i].Name << "\n"; 135 } 136 137 // Emit num_intrinsics into the target neutral enum. 138 if (IntrinsicPrefix.empty()) { 139 OS << " num_intrinsics = " << (Ints.size() + 1) << "\n"; 140 } else { 141 OS << "}; // enum\n"; 142 OS << "} // namespace Intrinsic\n"; 143 OS << "} // namespace llvm\n\n"; 144 OS << "#endif\n"; 145 } 146 } 147 148 void IntrinsicEmitter::EmitTargetInfo(const CodeGenIntrinsicTable &Ints, 149 raw_ostream &OS) { 150 OS << "// Target mapping\n"; 151 OS << "#ifdef GET_INTRINSIC_TARGET_DATA\n"; 152 OS << "struct IntrinsicTargetInfo {\n" 153 << " llvm::StringLiteral Name;\n" 154 << " size_t Offset;\n" 155 << " size_t Count;\n" 156 << "};\n"; 157 OS << "static constexpr IntrinsicTargetInfo TargetInfos[] = {\n"; 158 for (auto Target : Ints.Targets) 159 OS << " {llvm::StringLiteral(\"" << Target.Name << "\"), " << Target.Offset 160 << ", " << Target.Count << "},\n"; 161 OS << "};\n"; 162 OS << "#endif\n\n"; 163 } 164 165 void IntrinsicEmitter::EmitIntrinsicToNameTable( 166 const CodeGenIntrinsicTable &Ints, raw_ostream &OS) { 167 OS << "// Intrinsic ID to name table\n"; 168 OS << "#ifdef GET_INTRINSIC_NAME_TABLE\n"; 169 OS << " // Note that entry #0 is the invalid intrinsic!\n"; 170 for (unsigned i = 0, e = Ints.size(); i != e; ++i) 171 OS << " \"" << Ints[i].Name << "\",\n"; 172 OS << "#endif\n\n"; 173 } 174 175 void IntrinsicEmitter::EmitIntrinsicToOverloadTable( 176 const CodeGenIntrinsicTable &Ints, raw_ostream &OS) { 177 OS << "// Intrinsic ID to overload bitset\n"; 178 OS << "#ifdef GET_INTRINSIC_OVERLOAD_TABLE\n"; 179 OS << "static const uint8_t OTable[] = {\n"; 180 OS << " 0"; 181 for (unsigned i = 0, e = Ints.size(); i != e; ++i) { 182 // Add one to the index so we emit a null bit for the invalid #0 intrinsic. 183 if ((i+1)%8 == 0) 184 OS << ",\n 0"; 185 if (Ints[i].isOverloaded) 186 OS << " | (1<<" << (i+1)%8 << ')'; 187 } 188 OS << "\n};\n\n"; 189 // OTable contains a true bit at the position if the intrinsic is overloaded. 190 OS << "return (OTable[id/8] & (1 << (id%8))) != 0;\n"; 191 OS << "#endif\n\n"; 192 } 193 194 195 // NOTE: This must be kept in synch with the copy in lib/IR/Function.cpp! 196 enum IIT_Info { 197 // Common values should be encoded with 0-15. 198 IIT_Done = 0, 199 IIT_I1 = 1, 200 IIT_I8 = 2, 201 IIT_I16 = 3, 202 IIT_I32 = 4, 203 IIT_I64 = 5, 204 IIT_F16 = 6, 205 IIT_F32 = 7, 206 IIT_F64 = 8, 207 IIT_V2 = 9, 208 IIT_V4 = 10, 209 IIT_V8 = 11, 210 IIT_V16 = 12, 211 IIT_V32 = 13, 212 IIT_PTR = 14, 213 IIT_ARG = 15, 214 215 // Values from 16+ are only encodable with the inefficient encoding. 216 IIT_V64 = 16, 217 IIT_MMX = 17, 218 IIT_TOKEN = 18, 219 IIT_METADATA = 19, 220 IIT_EMPTYSTRUCT = 20, 221 IIT_STRUCT2 = 21, 222 IIT_STRUCT3 = 22, 223 IIT_STRUCT4 = 23, 224 IIT_STRUCT5 = 24, 225 IIT_EXTEND_ARG = 25, 226 IIT_TRUNC_ARG = 26, 227 IIT_ANYPTR = 27, 228 IIT_V1 = 28, 229 IIT_VARARG = 29, 230 IIT_HALF_VEC_ARG = 30, 231 IIT_SAME_VEC_WIDTH_ARG = 31, 232 IIT_PTR_TO_ARG = 32, 233 IIT_PTR_TO_ELT = 33, 234 IIT_VEC_OF_ANYPTRS_TO_ELT = 34, 235 IIT_I128 = 35, 236 IIT_V512 = 36, 237 IIT_V1024 = 37, 238 IIT_STRUCT6 = 38, 239 IIT_STRUCT7 = 39, 240 IIT_STRUCT8 = 40, 241 IIT_F128 = 41, 242 IIT_VEC_ELEMENT = 42, 243 IIT_SCALABLE_VEC = 43, 244 IIT_SUBDIVIDE2_ARG = 44, 245 IIT_SUBDIVIDE4_ARG = 45, 246 IIT_VEC_OF_BITCASTS_TO_INT = 46, 247 IIT_V128 = 47, 248 IIT_BF16 = 48, 249 IIT_STRUCT9 = 49, 250 IIT_V256 = 50, 251 IIT_AMX = 51, 252 IIT_PPCF128 = 52, 253 IIT_V3 = 53, 254 IIT_EXTERNREF = 54, 255 IIT_FUNCREF = 55, 256 IIT_ANYPTR_TO_ELT = 56, 257 IIT_I2 = 57, 258 IIT_I4 = 58, 259 }; 260 261 static void EncodeFixedValueType(MVT::SimpleValueType VT, 262 std::vector<unsigned char> &Sig) { 263 // clang-format off 264 if (MVT(VT).isInteger()) { 265 unsigned BitWidth = MVT(VT).getFixedSizeInBits(); 266 switch (BitWidth) { 267 default: PrintFatalError("unhandled integer type width in intrinsic!"); 268 case 1: return Sig.push_back(IIT_I1); 269 case 2: return Sig.push_back(IIT_I2); 270 case 4: return Sig.push_back(IIT_I4); 271 case 8: return Sig.push_back(IIT_I8); 272 case 16: return Sig.push_back(IIT_I16); 273 case 32: return Sig.push_back(IIT_I32); 274 case 64: return Sig.push_back(IIT_I64); 275 case 128: return Sig.push_back(IIT_I128); 276 } 277 } 278 279 switch (VT) { 280 default: PrintFatalError("unhandled MVT in intrinsic!"); 281 case MVT::f16: return Sig.push_back(IIT_F16); 282 case MVT::bf16: return Sig.push_back(IIT_BF16); 283 case MVT::f32: return Sig.push_back(IIT_F32); 284 case MVT::f64: return Sig.push_back(IIT_F64); 285 case MVT::f128: return Sig.push_back(IIT_F128); 286 case MVT::ppcf128: return Sig.push_back(IIT_PPCF128); 287 case MVT::token: return Sig.push_back(IIT_TOKEN); 288 case MVT::Metadata: return Sig.push_back(IIT_METADATA); 289 case MVT::x86mmx: return Sig.push_back(IIT_MMX); 290 case MVT::x86amx: return Sig.push_back(IIT_AMX); 291 // MVT::OtherVT is used to mean the empty struct type here. 292 case MVT::Other: return Sig.push_back(IIT_EMPTYSTRUCT); 293 // MVT::isVoid is used to represent varargs here. 294 case MVT::isVoid: return Sig.push_back(IIT_VARARG); 295 case MVT::externref: 296 return Sig.push_back(IIT_EXTERNREF); 297 case MVT::funcref: 298 return Sig.push_back(IIT_FUNCREF); 299 } 300 // clang-format on 301 } 302 303 #if defined(_MSC_VER) && !defined(__clang__) 304 #pragma optimize("",off) // MSVC 2015 optimizer can't deal with this function. 305 #endif 306 307 static void EncodeFixedType(Record *R, std::vector<unsigned char> &ArgCodes, 308 unsigned &NextArgCode, 309 std::vector<unsigned char> &Sig, 310 ArrayRef<unsigned char> Mapping) { 311 312 if (R->isSubClassOf("LLVMMatchType")) { 313 unsigned Number = Mapping[R->getValueAsInt("Number")]; 314 assert(Number < ArgCodes.size() && "Invalid matching number!"); 315 if (R->isSubClassOf("LLVMExtendedType")) 316 Sig.push_back(IIT_EXTEND_ARG); 317 else if (R->isSubClassOf("LLVMTruncatedType")) 318 Sig.push_back(IIT_TRUNC_ARG); 319 else if (R->isSubClassOf("LLVMHalfElementsVectorType")) 320 Sig.push_back(IIT_HALF_VEC_ARG); 321 else if (R->isSubClassOf("LLVMScalarOrSameVectorWidth")) { 322 Sig.push_back(IIT_SAME_VEC_WIDTH_ARG); 323 Sig.push_back((Number << 3) | ArgCodes[Number]); 324 MVT::SimpleValueType VT = getValueType(R->getValueAsDef("ElTy")); 325 EncodeFixedValueType(VT, Sig); 326 return; 327 } 328 else if (R->isSubClassOf("LLVMPointerTo")) 329 Sig.push_back(IIT_PTR_TO_ARG); 330 else if (R->isSubClassOf("LLVMVectorOfAnyPointersToElt")) { 331 Sig.push_back(IIT_VEC_OF_ANYPTRS_TO_ELT); 332 // Encode overloaded ArgNo 333 Sig.push_back(NextArgCode++); 334 // Encode LLVMMatchType<Number> ArgNo 335 Sig.push_back(Number); 336 return; 337 } else if (R->isSubClassOf("LLVMAnyPointerToElt")) { 338 Sig.push_back(IIT_ANYPTR_TO_ELT); 339 // Encode overloaded ArgNo 340 Sig.push_back(NextArgCode++); 341 // Encode LLVMMatchType<Number> ArgNo 342 Sig.push_back(Number); 343 return; 344 } else if (R->isSubClassOf("LLVMPointerToElt")) 345 Sig.push_back(IIT_PTR_TO_ELT); 346 else if (R->isSubClassOf("LLVMVectorElementType")) 347 Sig.push_back(IIT_VEC_ELEMENT); 348 else if (R->isSubClassOf("LLVMSubdivide2VectorType")) 349 Sig.push_back(IIT_SUBDIVIDE2_ARG); 350 else if (R->isSubClassOf("LLVMSubdivide4VectorType")) 351 Sig.push_back(IIT_SUBDIVIDE4_ARG); 352 else if (R->isSubClassOf("LLVMVectorOfBitcastsToInt")) 353 Sig.push_back(IIT_VEC_OF_BITCASTS_TO_INT); 354 else 355 Sig.push_back(IIT_ARG); 356 return Sig.push_back((Number << 3) | 7 /*IITDescriptor::AK_MatchType*/); 357 } 358 359 MVT::SimpleValueType VT = getValueType(R->getValueAsDef("VT")); 360 361 unsigned Tmp = 0; 362 switch (VT) { 363 default: break; 364 case MVT::iPTRAny: ++Tmp; [[fallthrough]]; 365 case MVT::vAny: ++Tmp; [[fallthrough]]; 366 case MVT::fAny: ++Tmp; [[fallthrough]]; 367 case MVT::iAny: ++Tmp; [[fallthrough]]; 368 case MVT::Any: { 369 // If this is an "any" valuetype, then the type is the type of the next 370 // type in the list specified to getIntrinsic(). 371 Sig.push_back(IIT_ARG); 372 373 // Figure out what arg # this is consuming, and remember what kind it was. 374 assert(NextArgCode < ArgCodes.size() && ArgCodes[NextArgCode] == Tmp && 375 "Invalid or no ArgCode associated with overloaded VT!"); 376 unsigned ArgNo = NextArgCode++; 377 378 // Encode what sort of argument it must be in the low 3 bits of the ArgNo. 379 return Sig.push_back((ArgNo << 3) | Tmp); 380 } 381 382 case MVT::iPTR: { 383 unsigned AddrSpace = 0; 384 if (R->isSubClassOf("LLVMQualPointerType")) { 385 AddrSpace = R->getValueAsInt("AddrSpace"); 386 assert(AddrSpace < 256 && "Address space exceeds 255"); 387 } 388 if (AddrSpace) { 389 Sig.push_back(IIT_ANYPTR); 390 Sig.push_back(AddrSpace); 391 } else { 392 Sig.push_back(IIT_PTR); 393 } 394 return EncodeFixedType(R->getValueAsDef("ElTy"), ArgCodes, NextArgCode, Sig, 395 Mapping); 396 } 397 } 398 399 if (MVT(VT).isVector()) { 400 MVT VVT = VT; 401 if (VVT.isScalableVector()) 402 Sig.push_back(IIT_SCALABLE_VEC); 403 switch (VVT.getVectorMinNumElements()) { 404 default: PrintFatalError("unhandled vector type width in intrinsic!"); 405 case 1: Sig.push_back(IIT_V1); break; 406 case 2: Sig.push_back(IIT_V2); break; 407 case 3: Sig.push_back(IIT_V3); break; 408 case 4: Sig.push_back(IIT_V4); break; 409 case 8: Sig.push_back(IIT_V8); break; 410 case 16: Sig.push_back(IIT_V16); break; 411 case 32: Sig.push_back(IIT_V32); break; 412 case 64: Sig.push_back(IIT_V64); break; 413 case 128: Sig.push_back(IIT_V128); break; 414 case 256: Sig.push_back(IIT_V256); break; 415 case 512: Sig.push_back(IIT_V512); break; 416 case 1024: Sig.push_back(IIT_V1024); break; 417 } 418 419 return EncodeFixedValueType(VVT.getVectorElementType().SimpleTy, Sig); 420 } 421 422 EncodeFixedValueType(VT, Sig); 423 } 424 425 static void UpdateArgCodes(Record *R, std::vector<unsigned char> &ArgCodes, 426 unsigned int &NumInserted, 427 SmallVectorImpl<unsigned char> &Mapping) { 428 if (R->isSubClassOf("LLVMMatchType")) { 429 if (R->isSubClassOf("LLVMVectorOfAnyPointersToElt")) { 430 ArgCodes.push_back(3 /*vAny*/); 431 ++NumInserted; 432 } else if (R->isSubClassOf("LLVMAnyPointerToElt")) { 433 ArgCodes.push_back(4 /*iPTRAny*/); 434 ++NumInserted; 435 } 436 return; 437 } 438 439 unsigned Tmp = 0; 440 switch (getValueType(R->getValueAsDef("VT"))) { 441 default: break; 442 case MVT::iPTR: 443 UpdateArgCodes(R->getValueAsDef("ElTy"), ArgCodes, NumInserted, Mapping); 444 break; 445 case MVT::iPTRAny: 446 ++Tmp; 447 [[fallthrough]]; 448 case MVT::vAny: 449 ++Tmp; 450 [[fallthrough]]; 451 case MVT::fAny: 452 ++Tmp; 453 [[fallthrough]]; 454 case MVT::iAny: 455 ++Tmp; 456 [[fallthrough]]; 457 case MVT::Any: 458 unsigned OriginalIdx = ArgCodes.size() - NumInserted; 459 assert(OriginalIdx >= Mapping.size()); 460 Mapping.resize(OriginalIdx+1); 461 Mapping[OriginalIdx] = ArgCodes.size(); 462 ArgCodes.push_back(Tmp); 463 break; 464 } 465 } 466 467 #if defined(_MSC_VER) && !defined(__clang__) 468 #pragma optimize("",on) 469 #endif 470 471 /// ComputeFixedEncoding - If we can encode the type signature for this 472 /// intrinsic into 32 bits, return it. If not, return ~0U. 473 static void ComputeFixedEncoding(const CodeGenIntrinsic &Int, 474 std::vector<unsigned char> &TypeSig) { 475 std::vector<unsigned char> ArgCodes; 476 477 // Add codes for any overloaded result VTs. 478 unsigned int NumInserted = 0; 479 SmallVector<unsigned char, 8> ArgMapping; 480 for (unsigned i = 0, e = Int.IS.RetVTs.size(); i != e; ++i) 481 UpdateArgCodes(Int.IS.RetTypeDefs[i], ArgCodes, NumInserted, ArgMapping); 482 483 // Add codes for any overloaded operand VTs. 484 for (unsigned i = 0, e = Int.IS.ParamTypeDefs.size(); i != e; ++i) 485 UpdateArgCodes(Int.IS.ParamTypeDefs[i], ArgCodes, NumInserted, ArgMapping); 486 487 unsigned NextArgCode = 0; 488 if (Int.IS.RetVTs.empty()) 489 TypeSig.push_back(IIT_Done); 490 else if (Int.IS.RetVTs.size() == 1 && 491 Int.IS.RetVTs[0] == MVT::isVoid) 492 TypeSig.push_back(IIT_Done); 493 else { 494 switch (Int.IS.RetVTs.size()) { 495 case 1: break; 496 case 2: TypeSig.push_back(IIT_STRUCT2); break; 497 case 3: TypeSig.push_back(IIT_STRUCT3); break; 498 case 4: TypeSig.push_back(IIT_STRUCT4); break; 499 case 5: TypeSig.push_back(IIT_STRUCT5); break; 500 case 6: TypeSig.push_back(IIT_STRUCT6); break; 501 case 7: TypeSig.push_back(IIT_STRUCT7); break; 502 case 8: TypeSig.push_back(IIT_STRUCT8); break; 503 case 9: TypeSig.push_back(IIT_STRUCT9); break; 504 default: llvm_unreachable("Unhandled case in struct"); 505 } 506 507 for (unsigned i = 0, e = Int.IS.RetVTs.size(); i != e; ++i) 508 EncodeFixedType(Int.IS.RetTypeDefs[i], ArgCodes, NextArgCode, TypeSig, 509 ArgMapping); 510 } 511 512 for (unsigned i = 0, e = Int.IS.ParamTypeDefs.size(); i != e; ++i) 513 EncodeFixedType(Int.IS.ParamTypeDefs[i], ArgCodes, NextArgCode, TypeSig, 514 ArgMapping); 515 } 516 517 static void printIITEntry(raw_ostream &OS, unsigned char X) { 518 OS << (unsigned)X; 519 } 520 521 void IntrinsicEmitter::EmitGenerator(const CodeGenIntrinsicTable &Ints, 522 raw_ostream &OS) { 523 // If we can compute a 32-bit fixed encoding for this intrinsic, do so and 524 // capture it in this vector, otherwise store a ~0U. 525 std::vector<unsigned> FixedEncodings; 526 527 SequenceToOffsetTable<std::vector<unsigned char> > LongEncodingTable; 528 529 std::vector<unsigned char> TypeSig; 530 531 // Compute the unique argument type info. 532 for (unsigned i = 0, e = Ints.size(); i != e; ++i) { 533 // Get the signature for the intrinsic. 534 TypeSig.clear(); 535 ComputeFixedEncoding(Ints[i], TypeSig); 536 537 // Check to see if we can encode it into a 32-bit word. We can only encode 538 // 8 nibbles into a 32-bit word. 539 if (TypeSig.size() <= 8) { 540 bool Failed = false; 541 unsigned Result = 0; 542 for (unsigned i = 0, e = TypeSig.size(); i != e; ++i) { 543 // If we had an unencodable argument, bail out. 544 if (TypeSig[i] > 15) { 545 Failed = true; 546 break; 547 } 548 Result = (Result << 4) | TypeSig[e-i-1]; 549 } 550 551 // If this could be encoded into a 31-bit word, return it. 552 if (!Failed && (Result >> 31) == 0) { 553 FixedEncodings.push_back(Result); 554 continue; 555 } 556 } 557 558 // Otherwise, we're going to unique the sequence into the 559 // LongEncodingTable, and use its offset in the 32-bit table instead. 560 LongEncodingTable.add(TypeSig); 561 562 // This is a placehold that we'll replace after the table is laid out. 563 FixedEncodings.push_back(~0U); 564 } 565 566 LongEncodingTable.layout(); 567 568 OS << "// Global intrinsic function declaration type table.\n"; 569 OS << "#ifdef GET_INTRINSIC_GENERATOR_GLOBAL\n"; 570 571 OS << "static const unsigned IIT_Table[] = {\n "; 572 573 for (unsigned i = 0, e = FixedEncodings.size(); i != e; ++i) { 574 if ((i & 7) == 7) 575 OS << "\n "; 576 577 // If the entry fit in the table, just emit it. 578 if (FixedEncodings[i] != ~0U) { 579 OS << "0x" << Twine::utohexstr(FixedEncodings[i]) << ", "; 580 continue; 581 } 582 583 TypeSig.clear(); 584 ComputeFixedEncoding(Ints[i], TypeSig); 585 586 587 // Otherwise, emit the offset into the long encoding table. We emit it this 588 // way so that it is easier to read the offset in the .def file. 589 OS << "(1U<<31) | " << LongEncodingTable.get(TypeSig) << ", "; 590 } 591 592 OS << "0\n};\n\n"; 593 594 // Emit the shared table of register lists. 595 OS << "static const unsigned char IIT_LongEncodingTable[] = {\n"; 596 if (!LongEncodingTable.empty()) 597 LongEncodingTable.emit(OS, printIITEntry); 598 OS << " 255\n};\n\n"; 599 600 OS << "#endif\n\n"; // End of GET_INTRINSIC_GENERATOR_GLOBAL 601 } 602 603 namespace { 604 std::optional<bool> compareFnAttributes(const CodeGenIntrinsic *L, 605 const CodeGenIntrinsic *R) { 606 // Sort throwing intrinsics after non-throwing intrinsics. 607 if (L->canThrow != R->canThrow) 608 return R->canThrow; 609 610 if (L->isNoDuplicate != R->isNoDuplicate) 611 return R->isNoDuplicate; 612 613 if (L->isNoMerge != R->isNoMerge) 614 return R->isNoMerge; 615 616 if (L->isNoReturn != R->isNoReturn) 617 return R->isNoReturn; 618 619 if (L->isNoCallback != R->isNoCallback) 620 return R->isNoCallback; 621 622 if (L->isNoSync != R->isNoSync) 623 return R->isNoSync; 624 625 if (L->isNoFree != R->isNoFree) 626 return R->isNoFree; 627 628 if (L->isWillReturn != R->isWillReturn) 629 return R->isWillReturn; 630 631 if (L->isCold != R->isCold) 632 return R->isCold; 633 634 if (L->isConvergent != R->isConvergent) 635 return R->isConvergent; 636 637 if (L->isSpeculatable != R->isSpeculatable) 638 return R->isSpeculatable; 639 640 if (L->hasSideEffects != R->hasSideEffects) 641 return R->hasSideEffects; 642 643 // Try to order by readonly/readnone attribute. 644 uint32_t LK = L->ME.toIntValue(); 645 uint32_t RK = R->ME.toIntValue(); 646 if (LK != RK) return (LK > RK); 647 648 return std::nullopt; 649 } 650 651 struct FnAttributeComparator { 652 bool operator()(const CodeGenIntrinsic *L, const CodeGenIntrinsic *R) const { 653 return compareFnAttributes(L, R).value_or(false); 654 } 655 }; 656 657 struct AttributeComparator { 658 bool operator()(const CodeGenIntrinsic *L, const CodeGenIntrinsic *R) const { 659 if (std::optional<bool> Res = compareFnAttributes(L, R)) 660 return *Res; 661 662 // Order by argument attributes. 663 // This is reliable because each side is already sorted internally. 664 return (L->ArgumentAttributes < R->ArgumentAttributes); 665 } 666 }; 667 } // End anonymous namespace 668 669 /// EmitAttributes - This emits the Intrinsic::getAttributes method. 670 void IntrinsicEmitter::EmitAttributes(const CodeGenIntrinsicTable &Ints, 671 raw_ostream &OS) { 672 OS << "// Add parameter attributes that are not common to all intrinsics.\n"; 673 OS << "#ifdef GET_INTRINSIC_ATTRIBUTES\n"; 674 675 // Compute unique argument attribute sets. 676 std::map<SmallVector<CodeGenIntrinsic::ArgAttribute, 0>, unsigned> 677 UniqArgAttributes; 678 OS << "static AttributeSet getIntrinsicArgAttributeSet(" 679 << "LLVMContext &C, unsigned ID) {\n" 680 << " switch (ID) {\n" 681 << " default: llvm_unreachable(\"Invalid attribute set number\");\n"; 682 for (const CodeGenIntrinsic &Int : Ints) { 683 for (auto &Attrs : Int.ArgumentAttributes) { 684 if (Attrs.empty()) 685 continue; 686 687 unsigned ID = UniqArgAttributes.size(); 688 if (!UniqArgAttributes.try_emplace(Attrs, ID).second) 689 continue; 690 691 assert(is_sorted(Attrs) && 692 "Argument attributes are not sorted"); 693 694 OS << " case " << ID << ":\n"; 695 OS << " return AttributeSet::get(C, {\n"; 696 for (const CodeGenIntrinsic::ArgAttribute &Attr : Attrs) { 697 switch (Attr.Kind) { 698 case CodeGenIntrinsic::NoCapture: 699 OS << " Attribute::get(C, Attribute::NoCapture),\n"; 700 break; 701 case CodeGenIntrinsic::NoAlias: 702 OS << " Attribute::get(C, Attribute::NoAlias),\n"; 703 break; 704 case CodeGenIntrinsic::NoUndef: 705 OS << " Attribute::get(C, Attribute::NoUndef),\n"; 706 break; 707 case CodeGenIntrinsic::NonNull: 708 OS << " Attribute::get(C, Attribute::NonNull),\n"; 709 break; 710 case CodeGenIntrinsic::Returned: 711 OS << " Attribute::get(C, Attribute::Returned),\n"; 712 break; 713 case CodeGenIntrinsic::ReadOnly: 714 OS << " Attribute::get(C, Attribute::ReadOnly),\n"; 715 break; 716 case CodeGenIntrinsic::WriteOnly: 717 OS << " Attribute::get(C, Attribute::WriteOnly),\n"; 718 break; 719 case CodeGenIntrinsic::ReadNone: 720 OS << " Attribute::get(C, Attribute::ReadNone),\n"; 721 break; 722 case CodeGenIntrinsic::ImmArg: 723 OS << " Attribute::get(C, Attribute::ImmArg),\n"; 724 break; 725 case CodeGenIntrinsic::Alignment: 726 OS << " Attribute::get(C, Attribute::Alignment, " 727 << Attr.Value << "),\n"; 728 break; 729 } 730 } 731 OS << " });\n"; 732 } 733 } 734 OS << " }\n"; 735 OS << "}\n\n"; 736 737 // Compute unique function attribute sets. 738 std::map<const CodeGenIntrinsic*, unsigned, FnAttributeComparator> 739 UniqFnAttributes; 740 OS << "static AttributeSet getIntrinsicFnAttributeSet(" 741 << "LLVMContext &C, unsigned ID) {\n" 742 << " switch (ID) {\n" 743 << " default: llvm_unreachable(\"Invalid attribute set number\");\n"; 744 for (const CodeGenIntrinsic &Intrinsic : Ints) { 745 unsigned ID = UniqFnAttributes.size(); 746 if (!UniqFnAttributes.try_emplace(&Intrinsic, ID).second) 747 continue; 748 749 OS << " case " << ID << ":\n" 750 << " return AttributeSet::get(C, {\n"; 751 if (!Intrinsic.canThrow) 752 OS << " Attribute::get(C, Attribute::NoUnwind),\n"; 753 if (Intrinsic.isNoReturn) 754 OS << " Attribute::get(C, Attribute::NoReturn),\n"; 755 if (Intrinsic.isNoCallback) 756 OS << " Attribute::get(C, Attribute::NoCallback),\n"; 757 if (Intrinsic.isNoSync) 758 OS << " Attribute::get(C, Attribute::NoSync),\n"; 759 if (Intrinsic.isNoFree) 760 OS << " Attribute::get(C, Attribute::NoFree),\n"; 761 if (Intrinsic.isWillReturn) 762 OS << " Attribute::get(C, Attribute::WillReturn),\n"; 763 if (Intrinsic.isCold) 764 OS << " Attribute::get(C, Attribute::Cold),\n"; 765 if (Intrinsic.isNoDuplicate) 766 OS << " Attribute::get(C, Attribute::NoDuplicate),\n"; 767 if (Intrinsic.isNoMerge) 768 OS << " Attribute::get(C, Attribute::NoMerge),\n"; 769 if (Intrinsic.isConvergent) 770 OS << " Attribute::get(C, Attribute::Convergent),\n"; 771 if (Intrinsic.isSpeculatable) 772 OS << " Attribute::get(C, Attribute::Speculatable),\n"; 773 774 MemoryEffects ME = Intrinsic.ME; 775 // TODO: IntrHasSideEffects should affect not only readnone intrinsics. 776 if (ME.doesNotAccessMemory() && Intrinsic.hasSideEffects) 777 ME = MemoryEffects::unknown(); 778 if (ME != MemoryEffects::unknown()) { 779 OS << " Attribute::getWithMemoryEffects(C, " 780 << "MemoryEffects::createFromIntValue(" << ME.toIntValue() << ")),\n"; 781 } 782 OS << " });\n"; 783 } 784 OS << " }\n"; 785 OS << "}\n\n"; 786 OS << "AttributeList Intrinsic::getAttributes(LLVMContext &C, ID id) {\n"; 787 788 // Compute the maximum number of attribute arguments and the map 789 typedef std::map<const CodeGenIntrinsic*, unsigned, 790 AttributeComparator> UniqAttrMapTy; 791 UniqAttrMapTy UniqAttributes; 792 unsigned maxArgAttrs = 0; 793 unsigned AttrNum = 0; 794 for (unsigned i = 0, e = Ints.size(); i != e; ++i) { 795 const CodeGenIntrinsic &intrinsic = Ints[i]; 796 maxArgAttrs = 797 std::max(maxArgAttrs, unsigned(intrinsic.ArgumentAttributes.size())); 798 unsigned &N = UniqAttributes[&intrinsic]; 799 if (N) continue; 800 N = ++AttrNum; 801 assert(N < 65536 && "Too many unique attributes for table!"); 802 } 803 804 // Emit an array of AttributeList. Most intrinsics will have at least one 805 // entry, for the function itself (index ~1), which is usually nounwind. 806 OS << " static const uint16_t IntrinsicsToAttributesMap[] = {\n"; 807 808 for (unsigned i = 0, e = Ints.size(); i != e; ++i) { 809 const CodeGenIntrinsic &intrinsic = Ints[i]; 810 811 OS << " " << UniqAttributes[&intrinsic] << ", // " 812 << intrinsic.Name << "\n"; 813 } 814 OS << " };\n\n"; 815 816 OS << " std::pair<unsigned, AttributeSet> AS[" << maxArgAttrs + 1 << "];\n"; 817 OS << " unsigned NumAttrs = 0;\n"; 818 OS << " if (id != 0) {\n"; 819 OS << " switch(IntrinsicsToAttributesMap[id - 1]) {\n"; 820 OS << " default: llvm_unreachable(\"Invalid attribute number\");\n"; 821 for (auto UniqAttribute : UniqAttributes) { 822 OS << " case " << UniqAttribute.second << ": {\n"; 823 824 const CodeGenIntrinsic &Intrinsic = *(UniqAttribute.first); 825 826 // Keep track of the number of attributes we're writing out. 827 unsigned numAttrs = 0; 828 829 for (const auto &[AttrIdx, Attrs] : 830 enumerate(Intrinsic.ArgumentAttributes)) { 831 if (Attrs.empty()) 832 continue; 833 834 unsigned ID = UniqArgAttributes.find(Attrs)->second; 835 OS << " AS[" << numAttrs++ << "] = {" << AttrIdx 836 << ", getIntrinsicArgAttributeSet(C, " << ID << ")};\n"; 837 } 838 839 if (!Intrinsic.canThrow || 840 (Intrinsic.ME != MemoryEffects::unknown() && 841 !Intrinsic.hasSideEffects) || 842 Intrinsic.isNoReturn || Intrinsic.isNoCallback || Intrinsic.isNoSync || 843 Intrinsic.isNoFree || Intrinsic.isWillReturn || Intrinsic.isCold || 844 Intrinsic.isNoDuplicate || Intrinsic.isNoMerge || 845 Intrinsic.isConvergent || Intrinsic.isSpeculatable) { 846 unsigned ID = UniqFnAttributes.find(&Intrinsic)->second; 847 OS << " AS[" << numAttrs++ << "] = {AttributeList::FunctionIndex, " 848 << "getIntrinsicFnAttributeSet(C, " << ID << ")};\n"; 849 } 850 851 if (numAttrs) { 852 OS << " NumAttrs = " << numAttrs << ";\n"; 853 OS << " break;\n"; 854 OS << " }\n"; 855 } else { 856 OS << " return AttributeList();\n"; 857 OS << " }\n"; 858 } 859 } 860 861 OS << " }\n"; 862 OS << " }\n"; 863 OS << " return AttributeList::get(C, ArrayRef(AS, NumAttrs));\n"; 864 OS << "}\n"; 865 OS << "#endif // GET_INTRINSIC_ATTRIBUTES\n\n"; 866 } 867 868 void IntrinsicEmitter::EmitIntrinsicToBuiltinMap( 869 const CodeGenIntrinsicTable &Ints, bool IsClang, raw_ostream &OS) { 870 StringRef CompilerName = (IsClang ? "Clang" : "MS"); 871 StringRef UpperCompilerName = (IsClang ? "CLANG" : "MS"); 872 typedef std::map<std::string, std::map<std::string, std::string>> BIMTy; 873 BIMTy BuiltinMap; 874 StringToOffsetTable Table; 875 for (unsigned i = 0, e = Ints.size(); i != e; ++i) { 876 const std::string &BuiltinName = 877 IsClang ? Ints[i].ClangBuiltinName : Ints[i].MSBuiltinName; 878 if (!BuiltinName.empty()) { 879 // Get the map for this target prefix. 880 std::map<std::string, std::string> &BIM = 881 BuiltinMap[Ints[i].TargetPrefix]; 882 883 if (!BIM.insert(std::make_pair(BuiltinName, Ints[i].EnumName)).second) 884 PrintFatalError(Ints[i].TheDef->getLoc(), 885 "Intrinsic '" + Ints[i].TheDef->getName() + 886 "': duplicate " + CompilerName + " builtin name!"); 887 Table.GetOrAddStringOffset(BuiltinName); 888 } 889 } 890 891 OS << "// Get the LLVM intrinsic that corresponds to a builtin.\n"; 892 OS << "// This is used by the C front-end. The builtin name is passed\n"; 893 OS << "// in as BuiltinName, and a target prefix (e.g. 'ppc') is passed\n"; 894 OS << "// in as TargetPrefix. The result is assigned to 'IntrinsicID'.\n"; 895 OS << "#ifdef GET_LLVM_INTRINSIC_FOR_" << UpperCompilerName << "_BUILTIN\n"; 896 897 OS << "Intrinsic::ID Intrinsic::getIntrinsicFor" << CompilerName 898 << "Builtin(const char " 899 << "*TargetPrefixStr, StringRef BuiltinNameStr) {\n"; 900 901 if (Table.Empty()) { 902 OS << " return Intrinsic::not_intrinsic;\n"; 903 OS << "}\n"; 904 OS << "#endif\n\n"; 905 return; 906 } 907 908 OS << " static const char BuiltinNames[] = {\n"; 909 Table.EmitCharArray(OS); 910 OS << " };\n\n"; 911 912 OS << " struct BuiltinEntry {\n"; 913 OS << " Intrinsic::ID IntrinID;\n"; 914 OS << " unsigned StrTabOffset;\n"; 915 OS << " const char *getName() const {\n"; 916 OS << " return &BuiltinNames[StrTabOffset];\n"; 917 OS << " }\n"; 918 OS << " bool operator<(StringRef RHS) const {\n"; 919 OS << " return strncmp(getName(), RHS.data(), RHS.size()) < 0;\n"; 920 OS << " }\n"; 921 OS << " };\n"; 922 923 OS << " StringRef TargetPrefix(TargetPrefixStr);\n\n"; 924 925 // Note: this could emit significantly better code if we cared. 926 for (auto &I : BuiltinMap) { 927 OS << " "; 928 if (!I.first.empty()) 929 OS << "if (TargetPrefix == \"" << I.first << "\") "; 930 else 931 OS << "/* Target Independent Builtins */ "; 932 OS << "{\n"; 933 934 // Emit the comparisons for this target prefix. 935 OS << " static const BuiltinEntry " << I.first << "Names[] = {\n"; 936 for (const auto &P : I.second) { 937 OS << " {Intrinsic::" << P.second << ", " 938 << Table.GetOrAddStringOffset(P.first) << "}, // " << P.first << "\n"; 939 } 940 OS << " };\n"; 941 OS << " auto I = std::lower_bound(std::begin(" << I.first << "Names),\n"; 942 OS << " std::end(" << I.first << "Names),\n"; 943 OS << " BuiltinNameStr);\n"; 944 OS << " if (I != std::end(" << I.first << "Names) &&\n"; 945 OS << " I->getName() == BuiltinNameStr)\n"; 946 OS << " return I->IntrinID;\n"; 947 OS << " }\n"; 948 } 949 OS << " return "; 950 OS << "Intrinsic::not_intrinsic;\n"; 951 OS << "}\n"; 952 OS << "#endif\n\n"; 953 } 954 955 void llvm::EmitIntrinsicEnums(RecordKeeper &RK, raw_ostream &OS) { 956 IntrinsicEmitter(RK).run(OS, /*Enums=*/true); 957 } 958 959 void llvm::EmitIntrinsicImpl(RecordKeeper &RK, raw_ostream &OS) { 960 IntrinsicEmitter(RK).run(OS, /*Enums=*/false); 961 } 962