1 //===------ BPFAbstractMemberAccess.cpp - Abstracting Member Accesses -----===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This pass abstracted struct/union member accesses in order to support 10 // compile-once run-everywhere (CO-RE). The CO-RE intends to compile the program 11 // which can run on different kernels. In particular, if bpf program tries to 12 // access a particular kernel data structure member, the details of the 13 // intermediate member access will be remembered so bpf loader can do 14 // necessary adjustment right before program loading. 15 // 16 // For example, 17 // 18 // struct s { 19 // int a; 20 // int b; 21 // }; 22 // struct t { 23 // struct s c; 24 // int d; 25 // }; 26 // struct t e; 27 // 28 // For the member access e.c.b, the compiler will generate code 29 // &e + 4 30 // 31 // The compile-once run-everywhere instead generates the following code 32 // r = 4 33 // &e + r 34 // The "4" in "r = 4" can be changed based on a particular kernel version. 35 // For example, on a particular kernel version, if struct s is changed to 36 // 37 // struct s { 38 // int new_field; 39 // int a; 40 // int b; 41 // } 42 // 43 // By repeating the member access on the host, the bpf loader can 44 // adjust "r = 4" as "r = 8". 45 // 46 // This feature relies on the following three intrinsic calls: 47 // addr = preserve_array_access_index(base, dimension, index) 48 // addr = preserve_union_access_index(base, di_index) 49 // !llvm.preserve.access.index <union_ditype> 50 // addr = preserve_struct_access_index(base, gep_index, di_index) 51 // !llvm.preserve.access.index <struct_ditype> 52 // 53 // Bitfield member access needs special attention. User cannot take the 54 // address of a bitfield acceess. To facilitate kernel verifier 55 // for easy bitfield code optimization, a new clang intrinsic is introduced: 56 // uint32_t __builtin_preserve_field_info(member_access, info_kind) 57 // In IR, a chain with two (or more) intrinsic calls will be generated: 58 // ... 59 // addr = preserve_struct_access_index(base, 1, 1) !struct s 60 // uint32_t result = bpf_preserve_field_info(addr, info_kind) 61 // 62 // Suppose the info_kind is FIELD_SIGNEDNESS, 63 // The above two IR intrinsics will be replaced with 64 // a relocatable insn: 65 // signness = /* signness of member_access */ 66 // and signness can be changed by bpf loader based on the 67 // types on the host. 68 // 69 // User can also test whether a field exists or not with 70 // uint32_t result = bpf_preserve_field_info(member_access, FIELD_EXISTENCE) 71 // The field will be always available (result = 1) during initial 72 // compilation, but bpf loader can patch with the correct value 73 // on the target host where the member_access may or may not be available 74 // 75 //===----------------------------------------------------------------------===// 76 77 #include "BPF.h" 78 #include "BPFCORE.h" 79 #include "BPFTargetMachine.h" 80 #include "llvm/BinaryFormat/Dwarf.h" 81 #include "llvm/IR/DebugInfoMetadata.h" 82 #include "llvm/IR/GlobalVariable.h" 83 #include "llvm/IR/Instruction.h" 84 #include "llvm/IR/Instructions.h" 85 #include "llvm/IR/IntrinsicsBPF.h" 86 #include "llvm/IR/Module.h" 87 #include "llvm/IR/PassManager.h" 88 #include "llvm/IR/Type.h" 89 #include "llvm/IR/User.h" 90 #include "llvm/IR/Value.h" 91 #include "llvm/Pass.h" 92 #include "llvm/Transforms/Utils/BasicBlockUtils.h" 93 #include <stack> 94 95 #define DEBUG_TYPE "bpf-abstract-member-access" 96 97 namespace llvm { 98 constexpr StringRef BPFCoreSharedInfo::AmaAttr; 99 uint32_t BPFCoreSharedInfo::SeqNum; 100 101 Instruction *BPFCoreSharedInfo::insertPassThrough(Module *M, BasicBlock *BB, 102 Instruction *Input, 103 Instruction *Before) { 104 Function *Fn = Intrinsic::getDeclaration( 105 M, Intrinsic::bpf_passthrough, {Input->getType(), Input->getType()}); 106 Constant *SeqNumVal = ConstantInt::get(Type::getInt32Ty(BB->getContext()), 107 BPFCoreSharedInfo::SeqNum++); 108 109 auto *NewInst = CallInst::Create(Fn, {SeqNumVal, Input}); 110 NewInst->insertBefore(Before); 111 return NewInst; 112 } 113 } // namespace llvm 114 115 using namespace llvm; 116 117 namespace { 118 class BPFAbstractMemberAccess final { 119 public: 120 BPFAbstractMemberAccess(BPFTargetMachine *TM) : TM(TM) {} 121 122 bool run(Function &F); 123 124 struct CallInfo { 125 uint32_t Kind; 126 uint32_t AccessIndex; 127 MaybeAlign RecordAlignment; 128 MDNode *Metadata; 129 WeakTrackingVH Base; 130 }; 131 typedef std::stack<std::pair<CallInst *, CallInfo>> CallInfoStack; 132 133 private: 134 enum : uint32_t { 135 BPFPreserveArrayAI = 1, 136 BPFPreserveUnionAI = 2, 137 BPFPreserveStructAI = 3, 138 BPFPreserveFieldInfoAI = 4, 139 }; 140 141 TargetMachine *TM; 142 const DataLayout *DL = nullptr; 143 Module *M = nullptr; 144 145 static std::map<std::string, GlobalVariable *> GEPGlobals; 146 // A map to link preserve_*_access_index intrinsic calls. 147 std::map<CallInst *, std::pair<CallInst *, CallInfo>> AIChain; 148 // A map to hold all the base preserve_*_access_index intrinsic calls. 149 // The base call is not an input of any other preserve_* 150 // intrinsics. 151 std::map<CallInst *, CallInfo> BaseAICalls; 152 // A map to hold <AnonRecord, TypeDef> relationships 153 std::map<DICompositeType *, DIDerivedType *> AnonRecords; 154 155 void CheckAnonRecordType(DIDerivedType *ParentTy, DIType *Ty); 156 void CheckCompositeType(DIDerivedType *ParentTy, DICompositeType *CTy); 157 void CheckDerivedType(DIDerivedType *ParentTy, DIDerivedType *DTy); 158 void ResetMetadata(struct CallInfo &CInfo); 159 160 bool doTransformation(Function &F); 161 162 void traceAICall(CallInst *Call, CallInfo &ParentInfo); 163 void traceBitCast(BitCastInst *BitCast, CallInst *Parent, 164 CallInfo &ParentInfo); 165 void traceGEP(GetElementPtrInst *GEP, CallInst *Parent, 166 CallInfo &ParentInfo); 167 void collectAICallChains(Function &F); 168 169 bool IsPreserveDIAccessIndexCall(const CallInst *Call, CallInfo &Cinfo); 170 bool IsValidAIChain(const MDNode *ParentMeta, uint32_t ParentAI, 171 const MDNode *ChildMeta); 172 bool removePreserveAccessIndexIntrinsic(Function &F); 173 void replaceWithGEP(std::vector<CallInst *> &CallList, 174 uint32_t NumOfZerosIndex, uint32_t DIIndex); 175 bool HasPreserveFieldInfoCall(CallInfoStack &CallStack); 176 void GetStorageBitRange(DIDerivedType *MemberTy, Align RecordAlignment, 177 uint32_t &StartBitOffset, uint32_t &EndBitOffset); 178 uint32_t GetFieldInfo(uint32_t InfoKind, DICompositeType *CTy, 179 uint32_t AccessIndex, uint32_t PatchImm, 180 MaybeAlign RecordAlignment); 181 182 Value *computeBaseAndAccessKey(CallInst *Call, CallInfo &CInfo, 183 std::string &AccessKey, MDNode *&BaseMeta); 184 MDNode *computeAccessKey(CallInst *Call, CallInfo &CInfo, 185 std::string &AccessKey, bool &IsInt32Ret); 186 uint64_t getConstant(const Value *IndexValue); 187 bool transformGEPChain(CallInst *Call, CallInfo &CInfo); 188 }; 189 190 std::map<std::string, GlobalVariable *> BPFAbstractMemberAccess::GEPGlobals; 191 } // End anonymous namespace 192 193 bool BPFAbstractMemberAccess::run(Function &F) { 194 LLVM_DEBUG(dbgs() << "********** Abstract Member Accesses **********\n"); 195 196 M = F.getParent(); 197 if (!M) 198 return false; 199 200 // Bail out if no debug info. 201 if (M->debug_compile_units().empty()) 202 return false; 203 204 // For each argument/return/local_variable type, trace the type 205 // pattern like '[derived_type]* [composite_type]' to check 206 // and remember (anon record -> typedef) relations where the 207 // anon record is defined as 208 // typedef [const/volatile/restrict]* [anon record] 209 DISubprogram *SP = F.getSubprogram(); 210 if (SP && SP->isDefinition()) { 211 for (DIType *Ty: SP->getType()->getTypeArray()) 212 CheckAnonRecordType(nullptr, Ty); 213 for (const DINode *DN : SP->getRetainedNodes()) { 214 if (const auto *DV = dyn_cast<DILocalVariable>(DN)) 215 CheckAnonRecordType(nullptr, DV->getType()); 216 } 217 } 218 219 DL = &M->getDataLayout(); 220 return doTransformation(F); 221 } 222 223 void BPFAbstractMemberAccess::ResetMetadata(struct CallInfo &CInfo) { 224 if (auto Ty = dyn_cast<DICompositeType>(CInfo.Metadata)) { 225 if (AnonRecords.find(Ty) != AnonRecords.end()) { 226 if (AnonRecords[Ty] != nullptr) 227 CInfo.Metadata = AnonRecords[Ty]; 228 } 229 } 230 } 231 232 void BPFAbstractMemberAccess::CheckCompositeType(DIDerivedType *ParentTy, 233 DICompositeType *CTy) { 234 if (!CTy->getName().empty() || !ParentTy || 235 ParentTy->getTag() != dwarf::DW_TAG_typedef) 236 return; 237 238 if (AnonRecords.find(CTy) == AnonRecords.end()) { 239 AnonRecords[CTy] = ParentTy; 240 return; 241 } 242 243 // Two or more typedef's may point to the same anon record. 244 // If this is the case, set the typedef DIType to be nullptr 245 // to indicate the duplication case. 246 DIDerivedType *CurrTy = AnonRecords[CTy]; 247 if (CurrTy == ParentTy) 248 return; 249 AnonRecords[CTy] = nullptr; 250 } 251 252 void BPFAbstractMemberAccess::CheckDerivedType(DIDerivedType *ParentTy, 253 DIDerivedType *DTy) { 254 DIType *BaseType = DTy->getBaseType(); 255 if (!BaseType) 256 return; 257 258 unsigned Tag = DTy->getTag(); 259 if (Tag == dwarf::DW_TAG_pointer_type) 260 CheckAnonRecordType(nullptr, BaseType); 261 else if (Tag == dwarf::DW_TAG_typedef) 262 CheckAnonRecordType(DTy, BaseType); 263 else 264 CheckAnonRecordType(ParentTy, BaseType); 265 } 266 267 void BPFAbstractMemberAccess::CheckAnonRecordType(DIDerivedType *ParentTy, 268 DIType *Ty) { 269 if (!Ty) 270 return; 271 272 if (auto *CTy = dyn_cast<DICompositeType>(Ty)) 273 return CheckCompositeType(ParentTy, CTy); 274 else if (auto *DTy = dyn_cast<DIDerivedType>(Ty)) 275 return CheckDerivedType(ParentTy, DTy); 276 } 277 278 static bool SkipDIDerivedTag(unsigned Tag, bool skipTypedef) { 279 if (Tag != dwarf::DW_TAG_typedef && Tag != dwarf::DW_TAG_const_type && 280 Tag != dwarf::DW_TAG_volatile_type && 281 Tag != dwarf::DW_TAG_restrict_type && 282 Tag != dwarf::DW_TAG_member) 283 return false; 284 if (Tag == dwarf::DW_TAG_typedef && !skipTypedef) 285 return false; 286 return true; 287 } 288 289 static DIType * stripQualifiers(DIType *Ty, bool skipTypedef = true) { 290 while (auto *DTy = dyn_cast<DIDerivedType>(Ty)) { 291 if (!SkipDIDerivedTag(DTy->getTag(), skipTypedef)) 292 break; 293 Ty = DTy->getBaseType(); 294 } 295 return Ty; 296 } 297 298 static const DIType * stripQualifiers(const DIType *Ty) { 299 while (auto *DTy = dyn_cast<DIDerivedType>(Ty)) { 300 if (!SkipDIDerivedTag(DTy->getTag(), true)) 301 break; 302 Ty = DTy->getBaseType(); 303 } 304 return Ty; 305 } 306 307 static uint32_t calcArraySize(const DICompositeType *CTy, uint32_t StartDim) { 308 DINodeArray Elements = CTy->getElements(); 309 uint32_t DimSize = 1; 310 for (uint32_t I = StartDim; I < Elements.size(); ++I) { 311 if (auto *Element = dyn_cast_or_null<DINode>(Elements[I])) 312 if (Element->getTag() == dwarf::DW_TAG_subrange_type) { 313 const DISubrange *SR = cast<DISubrange>(Element); 314 auto *CI = SR->getCount().dyn_cast<ConstantInt *>(); 315 DimSize *= CI->getSExtValue(); 316 } 317 } 318 319 return DimSize; 320 } 321 322 static Type *getBaseElementType(const CallInst *Call) { 323 // Element type is stored in an elementtype() attribute on the first param. 324 return Call->getParamElementType(0); 325 } 326 327 /// Check whether a call is a preserve_*_access_index intrinsic call or not. 328 bool BPFAbstractMemberAccess::IsPreserveDIAccessIndexCall(const CallInst *Call, 329 CallInfo &CInfo) { 330 if (!Call) 331 return false; 332 333 const auto *GV = dyn_cast<GlobalValue>(Call->getCalledOperand()); 334 if (!GV) 335 return false; 336 if (GV->getName().startswith("llvm.preserve.array.access.index")) { 337 CInfo.Kind = BPFPreserveArrayAI; 338 CInfo.Metadata = Call->getMetadata(LLVMContext::MD_preserve_access_index); 339 if (!CInfo.Metadata) 340 report_fatal_error("Missing metadata for llvm.preserve.array.access.index intrinsic"); 341 CInfo.AccessIndex = getConstant(Call->getArgOperand(2)); 342 CInfo.Base = Call->getArgOperand(0); 343 CInfo.RecordAlignment = DL->getABITypeAlign(getBaseElementType(Call)); 344 return true; 345 } 346 if (GV->getName().startswith("llvm.preserve.union.access.index")) { 347 CInfo.Kind = BPFPreserveUnionAI; 348 CInfo.Metadata = Call->getMetadata(LLVMContext::MD_preserve_access_index); 349 if (!CInfo.Metadata) 350 report_fatal_error("Missing metadata for llvm.preserve.union.access.index intrinsic"); 351 ResetMetadata(CInfo); 352 CInfo.AccessIndex = getConstant(Call->getArgOperand(1)); 353 CInfo.Base = Call->getArgOperand(0); 354 return true; 355 } 356 if (GV->getName().startswith("llvm.preserve.struct.access.index")) { 357 CInfo.Kind = BPFPreserveStructAI; 358 CInfo.Metadata = Call->getMetadata(LLVMContext::MD_preserve_access_index); 359 if (!CInfo.Metadata) 360 report_fatal_error("Missing metadata for llvm.preserve.struct.access.index intrinsic"); 361 ResetMetadata(CInfo); 362 CInfo.AccessIndex = getConstant(Call->getArgOperand(2)); 363 CInfo.Base = Call->getArgOperand(0); 364 CInfo.RecordAlignment = DL->getABITypeAlign(getBaseElementType(Call)); 365 return true; 366 } 367 if (GV->getName().startswith("llvm.bpf.preserve.field.info")) { 368 CInfo.Kind = BPFPreserveFieldInfoAI; 369 CInfo.Metadata = nullptr; 370 // Check validity of info_kind as clang did not check this. 371 uint64_t InfoKind = getConstant(Call->getArgOperand(1)); 372 if (InfoKind >= BPFCoreSharedInfo::MAX_FIELD_RELOC_KIND) 373 report_fatal_error("Incorrect info_kind for llvm.bpf.preserve.field.info intrinsic"); 374 CInfo.AccessIndex = InfoKind; 375 return true; 376 } 377 if (GV->getName().startswith("llvm.bpf.preserve.type.info")) { 378 CInfo.Kind = BPFPreserveFieldInfoAI; 379 CInfo.Metadata = Call->getMetadata(LLVMContext::MD_preserve_access_index); 380 if (!CInfo.Metadata) 381 report_fatal_error("Missing metadata for llvm.preserve.type.info intrinsic"); 382 uint64_t Flag = getConstant(Call->getArgOperand(1)); 383 if (Flag >= BPFCoreSharedInfo::MAX_PRESERVE_TYPE_INFO_FLAG) 384 report_fatal_error("Incorrect flag for llvm.bpf.preserve.type.info intrinsic"); 385 if (Flag == BPFCoreSharedInfo::PRESERVE_TYPE_INFO_EXISTENCE) 386 CInfo.AccessIndex = BPFCoreSharedInfo::TYPE_EXISTENCE; 387 else if (Flag == BPFCoreSharedInfo::PRESERVE_TYPE_INFO_MATCH) 388 CInfo.AccessIndex = BPFCoreSharedInfo::TYPE_MATCH; 389 else 390 CInfo.AccessIndex = BPFCoreSharedInfo::TYPE_SIZE; 391 return true; 392 } 393 if (GV->getName().startswith("llvm.bpf.preserve.enum.value")) { 394 CInfo.Kind = BPFPreserveFieldInfoAI; 395 CInfo.Metadata = Call->getMetadata(LLVMContext::MD_preserve_access_index); 396 if (!CInfo.Metadata) 397 report_fatal_error("Missing metadata for llvm.preserve.enum.value intrinsic"); 398 uint64_t Flag = getConstant(Call->getArgOperand(2)); 399 if (Flag >= BPFCoreSharedInfo::MAX_PRESERVE_ENUM_VALUE_FLAG) 400 report_fatal_error("Incorrect flag for llvm.bpf.preserve.enum.value intrinsic"); 401 if (Flag == BPFCoreSharedInfo::PRESERVE_ENUM_VALUE_EXISTENCE) 402 CInfo.AccessIndex = BPFCoreSharedInfo::ENUM_VALUE_EXISTENCE; 403 else 404 CInfo.AccessIndex = BPFCoreSharedInfo::ENUM_VALUE; 405 return true; 406 } 407 408 return false; 409 } 410 411 void BPFAbstractMemberAccess::replaceWithGEP(std::vector<CallInst *> &CallList, 412 uint32_t DimensionIndex, 413 uint32_t GEPIndex) { 414 for (auto *Call : CallList) { 415 uint32_t Dimension = 1; 416 if (DimensionIndex > 0) 417 Dimension = getConstant(Call->getArgOperand(DimensionIndex)); 418 419 Constant *Zero = 420 ConstantInt::get(Type::getInt32Ty(Call->getParent()->getContext()), 0); 421 SmallVector<Value *, 4> IdxList; 422 for (unsigned I = 0; I < Dimension; ++I) 423 IdxList.push_back(Zero); 424 IdxList.push_back(Call->getArgOperand(GEPIndex)); 425 426 auto *GEP = GetElementPtrInst::CreateInBounds( 427 getBaseElementType(Call), Call->getArgOperand(0), IdxList, "", Call); 428 Call->replaceAllUsesWith(GEP); 429 Call->eraseFromParent(); 430 } 431 } 432 433 bool BPFAbstractMemberAccess::removePreserveAccessIndexIntrinsic(Function &F) { 434 std::vector<CallInst *> PreserveArrayIndexCalls; 435 std::vector<CallInst *> PreserveUnionIndexCalls; 436 std::vector<CallInst *> PreserveStructIndexCalls; 437 bool Found = false; 438 439 for (auto &BB : F) 440 for (auto &I : BB) { 441 auto *Call = dyn_cast<CallInst>(&I); 442 CallInfo CInfo; 443 if (!IsPreserveDIAccessIndexCall(Call, CInfo)) 444 continue; 445 446 Found = true; 447 if (CInfo.Kind == BPFPreserveArrayAI) 448 PreserveArrayIndexCalls.push_back(Call); 449 else if (CInfo.Kind == BPFPreserveUnionAI) 450 PreserveUnionIndexCalls.push_back(Call); 451 else 452 PreserveStructIndexCalls.push_back(Call); 453 } 454 455 // do the following transformation: 456 // . addr = preserve_array_access_index(base, dimension, index) 457 // is transformed to 458 // addr = GEP(base, dimenion's zero's, index) 459 // . addr = preserve_union_access_index(base, di_index) 460 // is transformed to 461 // addr = base, i.e., all usages of "addr" are replaced by "base". 462 // . addr = preserve_struct_access_index(base, gep_index, di_index) 463 // is transformed to 464 // addr = GEP(base, 0, gep_index) 465 replaceWithGEP(PreserveArrayIndexCalls, 1, 2); 466 replaceWithGEP(PreserveStructIndexCalls, 0, 1); 467 for (auto *Call : PreserveUnionIndexCalls) { 468 Call->replaceAllUsesWith(Call->getArgOperand(0)); 469 Call->eraseFromParent(); 470 } 471 472 return Found; 473 } 474 475 /// Check whether the access index chain is valid. We check 476 /// here because there may be type casts between two 477 /// access indexes. We want to ensure memory access still valid. 478 bool BPFAbstractMemberAccess::IsValidAIChain(const MDNode *ParentType, 479 uint32_t ParentAI, 480 const MDNode *ChildType) { 481 if (!ChildType) 482 return true; // preserve_field_info, no type comparison needed. 483 484 const DIType *PType = stripQualifiers(cast<DIType>(ParentType)); 485 const DIType *CType = stripQualifiers(cast<DIType>(ChildType)); 486 487 // Child is a derived/pointer type, which is due to type casting. 488 // Pointer type cannot be in the middle of chain. 489 if (isa<DIDerivedType>(CType)) 490 return false; 491 492 // Parent is a pointer type. 493 if (const auto *PtrTy = dyn_cast<DIDerivedType>(PType)) { 494 if (PtrTy->getTag() != dwarf::DW_TAG_pointer_type) 495 return false; 496 return stripQualifiers(PtrTy->getBaseType()) == CType; 497 } 498 499 // Otherwise, struct/union/array types 500 const auto *PTy = dyn_cast<DICompositeType>(PType); 501 const auto *CTy = dyn_cast<DICompositeType>(CType); 502 assert(PTy && CTy && "ParentType or ChildType is null or not composite"); 503 504 uint32_t PTyTag = PTy->getTag(); 505 assert(PTyTag == dwarf::DW_TAG_array_type || 506 PTyTag == dwarf::DW_TAG_structure_type || 507 PTyTag == dwarf::DW_TAG_union_type); 508 509 uint32_t CTyTag = CTy->getTag(); 510 assert(CTyTag == dwarf::DW_TAG_array_type || 511 CTyTag == dwarf::DW_TAG_structure_type || 512 CTyTag == dwarf::DW_TAG_union_type); 513 514 // Multi dimensional arrays, base element should be the same 515 if (PTyTag == dwarf::DW_TAG_array_type && PTyTag == CTyTag) 516 return PTy->getBaseType() == CTy->getBaseType(); 517 518 DIType *Ty; 519 if (PTyTag == dwarf::DW_TAG_array_type) 520 Ty = PTy->getBaseType(); 521 else 522 Ty = dyn_cast<DIType>(PTy->getElements()[ParentAI]); 523 524 return dyn_cast<DICompositeType>(stripQualifiers(Ty)) == CTy; 525 } 526 527 void BPFAbstractMemberAccess::traceAICall(CallInst *Call, 528 CallInfo &ParentInfo) { 529 for (User *U : Call->users()) { 530 Instruction *Inst = dyn_cast<Instruction>(U); 531 if (!Inst) 532 continue; 533 534 if (auto *BI = dyn_cast<BitCastInst>(Inst)) { 535 traceBitCast(BI, Call, ParentInfo); 536 } else if (auto *CI = dyn_cast<CallInst>(Inst)) { 537 CallInfo ChildInfo; 538 539 if (IsPreserveDIAccessIndexCall(CI, ChildInfo) && 540 IsValidAIChain(ParentInfo.Metadata, ParentInfo.AccessIndex, 541 ChildInfo.Metadata)) { 542 AIChain[CI] = std::make_pair(Call, ParentInfo); 543 traceAICall(CI, ChildInfo); 544 } else { 545 BaseAICalls[Call] = ParentInfo; 546 } 547 } else if (auto *GI = dyn_cast<GetElementPtrInst>(Inst)) { 548 if (GI->hasAllZeroIndices()) 549 traceGEP(GI, Call, ParentInfo); 550 else 551 BaseAICalls[Call] = ParentInfo; 552 } else { 553 BaseAICalls[Call] = ParentInfo; 554 } 555 } 556 } 557 558 void BPFAbstractMemberAccess::traceBitCast(BitCastInst *BitCast, 559 CallInst *Parent, 560 CallInfo &ParentInfo) { 561 for (User *U : BitCast->users()) { 562 Instruction *Inst = dyn_cast<Instruction>(U); 563 if (!Inst) 564 continue; 565 566 if (auto *BI = dyn_cast<BitCastInst>(Inst)) { 567 traceBitCast(BI, Parent, ParentInfo); 568 } else if (auto *CI = dyn_cast<CallInst>(Inst)) { 569 CallInfo ChildInfo; 570 if (IsPreserveDIAccessIndexCall(CI, ChildInfo) && 571 IsValidAIChain(ParentInfo.Metadata, ParentInfo.AccessIndex, 572 ChildInfo.Metadata)) { 573 AIChain[CI] = std::make_pair(Parent, ParentInfo); 574 traceAICall(CI, ChildInfo); 575 } else { 576 BaseAICalls[Parent] = ParentInfo; 577 } 578 } else if (auto *GI = dyn_cast<GetElementPtrInst>(Inst)) { 579 if (GI->hasAllZeroIndices()) 580 traceGEP(GI, Parent, ParentInfo); 581 else 582 BaseAICalls[Parent] = ParentInfo; 583 } else { 584 BaseAICalls[Parent] = ParentInfo; 585 } 586 } 587 } 588 589 void BPFAbstractMemberAccess::traceGEP(GetElementPtrInst *GEP, CallInst *Parent, 590 CallInfo &ParentInfo) { 591 for (User *U : GEP->users()) { 592 Instruction *Inst = dyn_cast<Instruction>(U); 593 if (!Inst) 594 continue; 595 596 if (auto *BI = dyn_cast<BitCastInst>(Inst)) { 597 traceBitCast(BI, Parent, ParentInfo); 598 } else if (auto *CI = dyn_cast<CallInst>(Inst)) { 599 CallInfo ChildInfo; 600 if (IsPreserveDIAccessIndexCall(CI, ChildInfo) && 601 IsValidAIChain(ParentInfo.Metadata, ParentInfo.AccessIndex, 602 ChildInfo.Metadata)) { 603 AIChain[CI] = std::make_pair(Parent, ParentInfo); 604 traceAICall(CI, ChildInfo); 605 } else { 606 BaseAICalls[Parent] = ParentInfo; 607 } 608 } else if (auto *GI = dyn_cast<GetElementPtrInst>(Inst)) { 609 if (GI->hasAllZeroIndices()) 610 traceGEP(GI, Parent, ParentInfo); 611 else 612 BaseAICalls[Parent] = ParentInfo; 613 } else { 614 BaseAICalls[Parent] = ParentInfo; 615 } 616 } 617 } 618 619 void BPFAbstractMemberAccess::collectAICallChains(Function &F) { 620 AIChain.clear(); 621 BaseAICalls.clear(); 622 623 for (auto &BB : F) 624 for (auto &I : BB) { 625 CallInfo CInfo; 626 auto *Call = dyn_cast<CallInst>(&I); 627 if (!IsPreserveDIAccessIndexCall(Call, CInfo) || 628 AIChain.find(Call) != AIChain.end()) 629 continue; 630 631 traceAICall(Call, CInfo); 632 } 633 } 634 635 uint64_t BPFAbstractMemberAccess::getConstant(const Value *IndexValue) { 636 const ConstantInt *CV = dyn_cast<ConstantInt>(IndexValue); 637 assert(CV); 638 return CV->getValue().getZExtValue(); 639 } 640 641 /// Get the start and the end of storage offset for \p MemberTy. 642 void BPFAbstractMemberAccess::GetStorageBitRange(DIDerivedType *MemberTy, 643 Align RecordAlignment, 644 uint32_t &StartBitOffset, 645 uint32_t &EndBitOffset) { 646 uint32_t MemberBitSize = MemberTy->getSizeInBits(); 647 uint32_t MemberBitOffset = MemberTy->getOffsetInBits(); 648 649 if (RecordAlignment > 8) { 650 // If the Bits are within an aligned 8-byte, set the RecordAlignment 651 // to 8, other report the fatal error. 652 if (MemberBitOffset / 64 != (MemberBitOffset + MemberBitSize) / 64) 653 report_fatal_error("Unsupported field expression for llvm.bpf.preserve.field.info, " 654 "requiring too big alignment"); 655 RecordAlignment = Align(8); 656 } 657 658 uint32_t AlignBits = RecordAlignment.value() * 8; 659 if (MemberBitSize > AlignBits) 660 report_fatal_error("Unsupported field expression for llvm.bpf.preserve.field.info, " 661 "bitfield size greater than record alignment"); 662 663 StartBitOffset = MemberBitOffset & ~(AlignBits - 1); 664 if ((StartBitOffset + AlignBits) < (MemberBitOffset + MemberBitSize)) 665 report_fatal_error("Unsupported field expression for llvm.bpf.preserve.field.info, " 666 "cross alignment boundary"); 667 EndBitOffset = StartBitOffset + AlignBits; 668 } 669 670 uint32_t BPFAbstractMemberAccess::GetFieldInfo(uint32_t InfoKind, 671 DICompositeType *CTy, 672 uint32_t AccessIndex, 673 uint32_t PatchImm, 674 MaybeAlign RecordAlignment) { 675 if (InfoKind == BPFCoreSharedInfo::FIELD_EXISTENCE) 676 return 1; 677 678 uint32_t Tag = CTy->getTag(); 679 if (InfoKind == BPFCoreSharedInfo::FIELD_BYTE_OFFSET) { 680 if (Tag == dwarf::DW_TAG_array_type) { 681 auto *EltTy = stripQualifiers(CTy->getBaseType()); 682 PatchImm += AccessIndex * calcArraySize(CTy, 1) * 683 (EltTy->getSizeInBits() >> 3); 684 } else if (Tag == dwarf::DW_TAG_structure_type) { 685 auto *MemberTy = cast<DIDerivedType>(CTy->getElements()[AccessIndex]); 686 if (!MemberTy->isBitField()) { 687 PatchImm += MemberTy->getOffsetInBits() >> 3; 688 } else { 689 unsigned SBitOffset, NextSBitOffset; 690 GetStorageBitRange(MemberTy, *RecordAlignment, SBitOffset, 691 NextSBitOffset); 692 PatchImm += SBitOffset >> 3; 693 } 694 } 695 return PatchImm; 696 } 697 698 if (InfoKind == BPFCoreSharedInfo::FIELD_BYTE_SIZE) { 699 if (Tag == dwarf::DW_TAG_array_type) { 700 auto *EltTy = stripQualifiers(CTy->getBaseType()); 701 return calcArraySize(CTy, 1) * (EltTy->getSizeInBits() >> 3); 702 } else { 703 auto *MemberTy = cast<DIDerivedType>(CTy->getElements()[AccessIndex]); 704 uint32_t SizeInBits = MemberTy->getSizeInBits(); 705 if (!MemberTy->isBitField()) 706 return SizeInBits >> 3; 707 708 unsigned SBitOffset, NextSBitOffset; 709 GetStorageBitRange(MemberTy, *RecordAlignment, SBitOffset, 710 NextSBitOffset); 711 SizeInBits = NextSBitOffset - SBitOffset; 712 if (SizeInBits & (SizeInBits - 1)) 713 report_fatal_error("Unsupported field expression for llvm.bpf.preserve.field.info"); 714 return SizeInBits >> 3; 715 } 716 } 717 718 if (InfoKind == BPFCoreSharedInfo::FIELD_SIGNEDNESS) { 719 const DIType *BaseTy; 720 if (Tag == dwarf::DW_TAG_array_type) { 721 // Signedness only checked when final array elements are accessed. 722 if (CTy->getElements().size() != 1) 723 report_fatal_error("Invalid array expression for llvm.bpf.preserve.field.info"); 724 BaseTy = stripQualifiers(CTy->getBaseType()); 725 } else { 726 auto *MemberTy = cast<DIDerivedType>(CTy->getElements()[AccessIndex]); 727 BaseTy = stripQualifiers(MemberTy->getBaseType()); 728 } 729 730 // Only basic types and enum types have signedness. 731 const auto *BTy = dyn_cast<DIBasicType>(BaseTy); 732 while (!BTy) { 733 const auto *CompTy = dyn_cast<DICompositeType>(BaseTy); 734 // Report an error if the field expression does not have signedness. 735 if (!CompTy || CompTy->getTag() != dwarf::DW_TAG_enumeration_type) 736 report_fatal_error("Invalid field expression for llvm.bpf.preserve.field.info"); 737 BaseTy = stripQualifiers(CompTy->getBaseType()); 738 BTy = dyn_cast<DIBasicType>(BaseTy); 739 } 740 uint32_t Encoding = BTy->getEncoding(); 741 return (Encoding == dwarf::DW_ATE_signed || Encoding == dwarf::DW_ATE_signed_char); 742 } 743 744 if (InfoKind == BPFCoreSharedInfo::FIELD_LSHIFT_U64) { 745 // The value is loaded into a value with FIELD_BYTE_SIZE size, 746 // and then zero or sign extended to U64. 747 // FIELD_LSHIFT_U64 and FIELD_RSHIFT_U64 are operations 748 // to extract the original value. 749 const Triple &Triple = TM->getTargetTriple(); 750 DIDerivedType *MemberTy = nullptr; 751 bool IsBitField = false; 752 uint32_t SizeInBits; 753 754 if (Tag == dwarf::DW_TAG_array_type) { 755 auto *EltTy = stripQualifiers(CTy->getBaseType()); 756 SizeInBits = calcArraySize(CTy, 1) * EltTy->getSizeInBits(); 757 } else { 758 MemberTy = cast<DIDerivedType>(CTy->getElements()[AccessIndex]); 759 SizeInBits = MemberTy->getSizeInBits(); 760 IsBitField = MemberTy->isBitField(); 761 } 762 763 if (!IsBitField) { 764 if (SizeInBits > 64) 765 report_fatal_error("too big field size for llvm.bpf.preserve.field.info"); 766 return 64 - SizeInBits; 767 } 768 769 unsigned SBitOffset, NextSBitOffset; 770 GetStorageBitRange(MemberTy, *RecordAlignment, SBitOffset, NextSBitOffset); 771 if (NextSBitOffset - SBitOffset > 64) 772 report_fatal_error("too big field size for llvm.bpf.preserve.field.info"); 773 774 unsigned OffsetInBits = MemberTy->getOffsetInBits(); 775 if (Triple.getArch() == Triple::bpfel) 776 return SBitOffset + 64 - OffsetInBits - SizeInBits; 777 else 778 return OffsetInBits + 64 - NextSBitOffset; 779 } 780 781 if (InfoKind == BPFCoreSharedInfo::FIELD_RSHIFT_U64) { 782 DIDerivedType *MemberTy = nullptr; 783 bool IsBitField = false; 784 uint32_t SizeInBits; 785 if (Tag == dwarf::DW_TAG_array_type) { 786 auto *EltTy = stripQualifiers(CTy->getBaseType()); 787 SizeInBits = calcArraySize(CTy, 1) * EltTy->getSizeInBits(); 788 } else { 789 MemberTy = cast<DIDerivedType>(CTy->getElements()[AccessIndex]); 790 SizeInBits = MemberTy->getSizeInBits(); 791 IsBitField = MemberTy->isBitField(); 792 } 793 794 if (!IsBitField) { 795 if (SizeInBits > 64) 796 report_fatal_error("too big field size for llvm.bpf.preserve.field.info"); 797 return 64 - SizeInBits; 798 } 799 800 unsigned SBitOffset, NextSBitOffset; 801 GetStorageBitRange(MemberTy, *RecordAlignment, SBitOffset, NextSBitOffset); 802 if (NextSBitOffset - SBitOffset > 64) 803 report_fatal_error("too big field size for llvm.bpf.preserve.field.info"); 804 805 return 64 - SizeInBits; 806 } 807 808 llvm_unreachable("Unknown llvm.bpf.preserve.field.info info kind"); 809 } 810 811 bool BPFAbstractMemberAccess::HasPreserveFieldInfoCall(CallInfoStack &CallStack) { 812 // This is called in error return path, no need to maintain CallStack. 813 while (CallStack.size()) { 814 auto StackElem = CallStack.top(); 815 if (StackElem.second.Kind == BPFPreserveFieldInfoAI) 816 return true; 817 CallStack.pop(); 818 } 819 return false; 820 } 821 822 /// Compute the base of the whole preserve_* intrinsics chains, i.e., the base 823 /// pointer of the first preserve_*_access_index call, and construct the access 824 /// string, which will be the name of a global variable. 825 Value *BPFAbstractMemberAccess::computeBaseAndAccessKey(CallInst *Call, 826 CallInfo &CInfo, 827 std::string &AccessKey, 828 MDNode *&TypeMeta) { 829 Value *Base = nullptr; 830 std::string TypeName; 831 CallInfoStack CallStack; 832 833 // Put the access chain into a stack with the top as the head of the chain. 834 while (Call) { 835 CallStack.push(std::make_pair(Call, CInfo)); 836 CInfo = AIChain[Call].second; 837 Call = AIChain[Call].first; 838 } 839 840 // The access offset from the base of the head of chain is also 841 // calculated here as all debuginfo types are available. 842 843 // Get type name and calculate the first index. 844 // We only want to get type name from typedef, structure or union. 845 // If user wants a relocation like 846 // int *p; ... __builtin_preserve_access_index(&p[4]) ... 847 // or 848 // int a[10][20]; ... __builtin_preserve_access_index(&a[2][3]) ... 849 // we will skip them. 850 uint32_t FirstIndex = 0; 851 uint32_t PatchImm = 0; // AccessOffset or the requested field info 852 uint32_t InfoKind = BPFCoreSharedInfo::FIELD_BYTE_OFFSET; 853 while (CallStack.size()) { 854 auto StackElem = CallStack.top(); 855 Call = StackElem.first; 856 CInfo = StackElem.second; 857 858 if (!Base) 859 Base = CInfo.Base; 860 861 DIType *PossibleTypeDef = stripQualifiers(cast<DIType>(CInfo.Metadata), 862 false); 863 DIType *Ty = stripQualifiers(PossibleTypeDef); 864 if (CInfo.Kind == BPFPreserveUnionAI || 865 CInfo.Kind == BPFPreserveStructAI) { 866 // struct or union type. If the typedef is in the metadata, always 867 // use the typedef. 868 TypeName = std::string(PossibleTypeDef->getName()); 869 TypeMeta = PossibleTypeDef; 870 PatchImm += FirstIndex * (Ty->getSizeInBits() >> 3); 871 break; 872 } 873 874 assert(CInfo.Kind == BPFPreserveArrayAI); 875 876 // Array entries will always be consumed for accumulative initial index. 877 CallStack.pop(); 878 879 // BPFPreserveArrayAI 880 uint64_t AccessIndex = CInfo.AccessIndex; 881 882 DIType *BaseTy = nullptr; 883 bool CheckElemType = false; 884 if (const auto *CTy = dyn_cast<DICompositeType>(Ty)) { 885 // array type 886 assert(CTy->getTag() == dwarf::DW_TAG_array_type); 887 888 889 FirstIndex += AccessIndex * calcArraySize(CTy, 1); 890 BaseTy = stripQualifiers(CTy->getBaseType()); 891 CheckElemType = CTy->getElements().size() == 1; 892 } else { 893 // pointer type 894 auto *DTy = cast<DIDerivedType>(Ty); 895 assert(DTy->getTag() == dwarf::DW_TAG_pointer_type); 896 897 BaseTy = stripQualifiers(DTy->getBaseType()); 898 CTy = dyn_cast<DICompositeType>(BaseTy); 899 if (!CTy) { 900 CheckElemType = true; 901 } else if (CTy->getTag() != dwarf::DW_TAG_array_type) { 902 FirstIndex += AccessIndex; 903 CheckElemType = true; 904 } else { 905 FirstIndex += AccessIndex * calcArraySize(CTy, 0); 906 } 907 } 908 909 if (CheckElemType) { 910 auto *CTy = dyn_cast<DICompositeType>(BaseTy); 911 if (!CTy) { 912 if (HasPreserveFieldInfoCall(CallStack)) 913 report_fatal_error("Invalid field access for llvm.preserve.field.info intrinsic"); 914 return nullptr; 915 } 916 917 unsigned CTag = CTy->getTag(); 918 if (CTag == dwarf::DW_TAG_structure_type || CTag == dwarf::DW_TAG_union_type) { 919 TypeName = std::string(CTy->getName()); 920 } else { 921 if (HasPreserveFieldInfoCall(CallStack)) 922 report_fatal_error("Invalid field access for llvm.preserve.field.info intrinsic"); 923 return nullptr; 924 } 925 TypeMeta = CTy; 926 PatchImm += FirstIndex * (CTy->getSizeInBits() >> 3); 927 break; 928 } 929 } 930 assert(TypeName.size()); 931 AccessKey += std::to_string(FirstIndex); 932 933 // Traverse the rest of access chain to complete offset calculation 934 // and access key construction. 935 while (CallStack.size()) { 936 auto StackElem = CallStack.top(); 937 CInfo = StackElem.second; 938 CallStack.pop(); 939 940 if (CInfo.Kind == BPFPreserveFieldInfoAI) { 941 InfoKind = CInfo.AccessIndex; 942 if (InfoKind == BPFCoreSharedInfo::FIELD_EXISTENCE) 943 PatchImm = 1; 944 break; 945 } 946 947 // If the next Call (the top of the stack) is a BPFPreserveFieldInfoAI, 948 // the action will be extracting field info. 949 if (CallStack.size()) { 950 auto StackElem2 = CallStack.top(); 951 CallInfo CInfo2 = StackElem2.second; 952 if (CInfo2.Kind == BPFPreserveFieldInfoAI) { 953 InfoKind = CInfo2.AccessIndex; 954 assert(CallStack.size() == 1); 955 } 956 } 957 958 // Access Index 959 uint64_t AccessIndex = CInfo.AccessIndex; 960 AccessKey += ":" + std::to_string(AccessIndex); 961 962 MDNode *MDN = CInfo.Metadata; 963 // At this stage, it cannot be pointer type. 964 auto *CTy = cast<DICompositeType>(stripQualifiers(cast<DIType>(MDN))); 965 PatchImm = GetFieldInfo(InfoKind, CTy, AccessIndex, PatchImm, 966 CInfo.RecordAlignment); 967 } 968 969 // Access key is the 970 // "llvm." + type name + ":" + reloc type + ":" + patched imm + "$" + 971 // access string, 972 // uniquely identifying one relocation. 973 // The prefix "llvm." indicates this is a temporary global, which should 974 // not be emitted to ELF file. 975 AccessKey = "llvm." + TypeName + ":" + std::to_string(InfoKind) + ":" + 976 std::to_string(PatchImm) + "$" + AccessKey; 977 978 return Base; 979 } 980 981 MDNode *BPFAbstractMemberAccess::computeAccessKey(CallInst *Call, 982 CallInfo &CInfo, 983 std::string &AccessKey, 984 bool &IsInt32Ret) { 985 DIType *Ty = stripQualifiers(cast<DIType>(CInfo.Metadata), false); 986 assert(!Ty->getName().empty()); 987 988 int64_t PatchImm; 989 std::string AccessStr("0"); 990 if (CInfo.AccessIndex == BPFCoreSharedInfo::TYPE_EXISTENCE || 991 CInfo.AccessIndex == BPFCoreSharedInfo::TYPE_MATCH) { 992 PatchImm = 1; 993 } else if (CInfo.AccessIndex == BPFCoreSharedInfo::TYPE_SIZE) { 994 // typedef debuginfo type has size 0, get the eventual base type. 995 DIType *BaseTy = stripQualifiers(Ty, true); 996 PatchImm = BaseTy->getSizeInBits() / 8; 997 } else { 998 // ENUM_VALUE_EXISTENCE and ENUM_VALUE 999 IsInt32Ret = false; 1000 1001 // The argument could be a global variable or a getelementptr with base to 1002 // a global variable depending on whether the clang option `opaque-options` 1003 // is set or not. 1004 const GlobalVariable *GV = 1005 cast<GlobalVariable>(Call->getArgOperand(1)->stripPointerCasts()); 1006 assert(GV->hasInitializer()); 1007 const ConstantDataArray *DA = cast<ConstantDataArray>(GV->getInitializer()); 1008 assert(DA->isString()); 1009 StringRef ValueStr = DA->getAsString(); 1010 1011 // ValueStr format: <EnumeratorStr>:<Value> 1012 size_t Separator = ValueStr.find_first_of(':'); 1013 StringRef EnumeratorStr = ValueStr.substr(0, Separator); 1014 1015 // Find enumerator index in the debuginfo 1016 DIType *BaseTy = stripQualifiers(Ty, true); 1017 const auto *CTy = cast<DICompositeType>(BaseTy); 1018 assert(CTy->getTag() == dwarf::DW_TAG_enumeration_type); 1019 int EnumIndex = 0; 1020 for (const auto Element : CTy->getElements()) { 1021 const auto *Enum = cast<DIEnumerator>(Element); 1022 if (Enum->getName() == EnumeratorStr) { 1023 AccessStr = std::to_string(EnumIndex); 1024 break; 1025 } 1026 EnumIndex++; 1027 } 1028 1029 if (CInfo.AccessIndex == BPFCoreSharedInfo::ENUM_VALUE) { 1030 StringRef EValueStr = ValueStr.substr(Separator + 1); 1031 PatchImm = std::stoll(std::string(EValueStr)); 1032 } else { 1033 PatchImm = 1; 1034 } 1035 } 1036 1037 AccessKey = "llvm." + Ty->getName().str() + ":" + 1038 std::to_string(CInfo.AccessIndex) + std::string(":") + 1039 std::to_string(PatchImm) + std::string("$") + AccessStr; 1040 1041 return Ty; 1042 } 1043 1044 /// Call/Kind is the base preserve_*_access_index() call. Attempts to do 1045 /// transformation to a chain of relocable GEPs. 1046 bool BPFAbstractMemberAccess::transformGEPChain(CallInst *Call, 1047 CallInfo &CInfo) { 1048 std::string AccessKey; 1049 MDNode *TypeMeta; 1050 Value *Base = nullptr; 1051 bool IsInt32Ret; 1052 1053 IsInt32Ret = CInfo.Kind == BPFPreserveFieldInfoAI; 1054 if (CInfo.Kind == BPFPreserveFieldInfoAI && CInfo.Metadata) { 1055 TypeMeta = computeAccessKey(Call, CInfo, AccessKey, IsInt32Ret); 1056 } else { 1057 Base = computeBaseAndAccessKey(Call, CInfo, AccessKey, TypeMeta); 1058 if (!Base) 1059 return false; 1060 } 1061 1062 BasicBlock *BB = Call->getParent(); 1063 GlobalVariable *GV; 1064 1065 if (GEPGlobals.find(AccessKey) == GEPGlobals.end()) { 1066 IntegerType *VarType; 1067 if (IsInt32Ret) 1068 VarType = Type::getInt32Ty(BB->getContext()); // 32bit return value 1069 else 1070 VarType = Type::getInt64Ty(BB->getContext()); // 64bit ptr or enum value 1071 1072 GV = new GlobalVariable(*M, VarType, false, GlobalVariable::ExternalLinkage, 1073 nullptr, AccessKey); 1074 GV->addAttribute(BPFCoreSharedInfo::AmaAttr); 1075 GV->setMetadata(LLVMContext::MD_preserve_access_index, TypeMeta); 1076 GEPGlobals[AccessKey] = GV; 1077 } else { 1078 GV = GEPGlobals[AccessKey]; 1079 } 1080 1081 if (CInfo.Kind == BPFPreserveFieldInfoAI) { 1082 // Load the global variable which represents the returned field info. 1083 LoadInst *LDInst; 1084 if (IsInt32Ret) 1085 LDInst = new LoadInst(Type::getInt32Ty(BB->getContext()), GV, "", Call); 1086 else 1087 LDInst = new LoadInst(Type::getInt64Ty(BB->getContext()), GV, "", Call); 1088 1089 Instruction *PassThroughInst = 1090 BPFCoreSharedInfo::insertPassThrough(M, BB, LDInst, Call); 1091 Call->replaceAllUsesWith(PassThroughInst); 1092 Call->eraseFromParent(); 1093 return true; 1094 } 1095 1096 // For any original GEP Call and Base %2 like 1097 // %4 = bitcast %struct.net_device** %dev1 to i64* 1098 // it is transformed to: 1099 // %6 = load llvm.sk_buff:0:50$0:0:0:2:0 1100 // %7 = bitcast %struct.sk_buff* %2 to i8* 1101 // %8 = getelementptr i8, i8* %7, %6 1102 // %9 = bitcast i8* %8 to i64* 1103 // using %9 instead of %4 1104 // The original Call inst is removed. 1105 1106 // Load the global variable. 1107 auto *LDInst = new LoadInst(Type::getInt64Ty(BB->getContext()), GV, "", Call); 1108 1109 // Generate a BitCast 1110 auto *BCInst = new BitCastInst(Base, Type::getInt8PtrTy(BB->getContext())); 1111 BCInst->insertBefore(Call); 1112 1113 // Generate a GetElementPtr 1114 auto *GEP = GetElementPtrInst::Create(Type::getInt8Ty(BB->getContext()), 1115 BCInst, LDInst); 1116 GEP->insertBefore(Call); 1117 1118 // Generate a BitCast 1119 auto *BCInst2 = new BitCastInst(GEP, Call->getType()); 1120 BCInst2->insertBefore(Call); 1121 1122 // For the following code, 1123 // Block0: 1124 // ... 1125 // if (...) goto Block1 else ... 1126 // Block1: 1127 // %6 = load llvm.sk_buff:0:50$0:0:0:2:0 1128 // %7 = bitcast %struct.sk_buff* %2 to i8* 1129 // %8 = getelementptr i8, i8* %7, %6 1130 // ... 1131 // goto CommonExit 1132 // Block2: 1133 // ... 1134 // if (...) goto Block3 else ... 1135 // Block3: 1136 // %6 = load llvm.bpf_map:0:40$0:0:0:2:0 1137 // %7 = bitcast %struct.sk_buff* %2 to i8* 1138 // %8 = getelementptr i8, i8* %7, %6 1139 // ... 1140 // goto CommonExit 1141 // CommonExit 1142 // SimplifyCFG may generate: 1143 // Block0: 1144 // ... 1145 // if (...) goto Block_Common else ... 1146 // Block2: 1147 // ... 1148 // if (...) goto Block_Common else ... 1149 // Block_Common: 1150 // PHI = [llvm.sk_buff:0:50$0:0:0:2:0, llvm.bpf_map:0:40$0:0:0:2:0] 1151 // %6 = load PHI 1152 // %7 = bitcast %struct.sk_buff* %2 to i8* 1153 // %8 = getelementptr i8, i8* %7, %6 1154 // ... 1155 // goto CommonExit 1156 // For the above code, we cannot perform proper relocation since 1157 // "load PHI" has two possible relocations. 1158 // 1159 // To prevent above tail merging, we use __builtin_bpf_passthrough() 1160 // where one of its parameters is a seq_num. Since two 1161 // __builtin_bpf_passthrough() funcs will always have different seq_num, 1162 // tail merging cannot happen. The __builtin_bpf_passthrough() will be 1163 // removed in the beginning of Target IR passes. 1164 // 1165 // This approach is also used in other places when global var 1166 // representing a relocation is used. 1167 Instruction *PassThroughInst = 1168 BPFCoreSharedInfo::insertPassThrough(M, BB, BCInst2, Call); 1169 Call->replaceAllUsesWith(PassThroughInst); 1170 Call->eraseFromParent(); 1171 1172 return true; 1173 } 1174 1175 bool BPFAbstractMemberAccess::doTransformation(Function &F) { 1176 bool Transformed = false; 1177 1178 // Collect PreserveDIAccessIndex Intrinsic call chains. 1179 // The call chains will be used to generate the access 1180 // patterns similar to GEP. 1181 collectAICallChains(F); 1182 1183 for (auto &C : BaseAICalls) 1184 Transformed = transformGEPChain(C.first, C.second) || Transformed; 1185 1186 return removePreserveAccessIndexIntrinsic(F) || Transformed; 1187 } 1188 1189 PreservedAnalyses 1190 BPFAbstractMemberAccessPass::run(Function &F, FunctionAnalysisManager &AM) { 1191 return BPFAbstractMemberAccess(TM).run(F) ? PreservedAnalyses::none() 1192 : PreservedAnalyses::all(); 1193 } 1194