1 //===- HWAddressSanitizer.cpp - detector of uninitialized reads -------===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 /// \file 10 /// This file is a part of HWAddressSanitizer, an address sanity checker 11 /// based on tagged addressing. 12 //===----------------------------------------------------------------------===// 13 14 #include "llvm/Transforms/Instrumentation/HWAddressSanitizer.h" 15 #include "llvm/ADT/MapVector.h" 16 #include "llvm/ADT/SmallVector.h" 17 #include "llvm/ADT/StringExtras.h" 18 #include "llvm/ADT/StringRef.h" 19 #include "llvm/ADT/Triple.h" 20 #include "llvm/BinaryFormat/ELF.h" 21 #include "llvm/IR/Attributes.h" 22 #include "llvm/IR/BasicBlock.h" 23 #include "llvm/IR/Constant.h" 24 #include "llvm/IR/Constants.h" 25 #include "llvm/IR/DataLayout.h" 26 #include "llvm/IR/DebugInfoMetadata.h" 27 #include "llvm/IR/DerivedTypes.h" 28 #include "llvm/IR/Function.h" 29 #include "llvm/IR/IRBuilder.h" 30 #include "llvm/IR/InlineAsm.h" 31 #include "llvm/IR/InstVisitor.h" 32 #include "llvm/IR/Instruction.h" 33 #include "llvm/IR/Instructions.h" 34 #include "llvm/IR/IntrinsicInst.h" 35 #include "llvm/IR/Intrinsics.h" 36 #include "llvm/IR/LLVMContext.h" 37 #include "llvm/IR/MDBuilder.h" 38 #include "llvm/IR/Module.h" 39 #include "llvm/IR/Type.h" 40 #include "llvm/IR/Value.h" 41 #include "llvm/InitializePasses.h" 42 #include "llvm/Pass.h" 43 #include "llvm/Support/Casting.h" 44 #include "llvm/Support/CommandLine.h" 45 #include "llvm/Support/Debug.h" 46 #include "llvm/Support/raw_ostream.h" 47 #include "llvm/Transforms/Instrumentation.h" 48 #include "llvm/Transforms/Utils/BasicBlockUtils.h" 49 #include "llvm/Transforms/Utils/ModuleUtils.h" 50 #include "llvm/Transforms/Utils/PromoteMemToReg.h" 51 #include <sstream> 52 53 using namespace llvm; 54 55 #define DEBUG_TYPE "hwasan" 56 57 static const char *const kHwasanModuleCtorName = "hwasan.module_ctor"; 58 static const char *const kHwasanNoteName = "hwasan.note"; 59 static const char *const kHwasanInitName = "__hwasan_init"; 60 static const char *const kHwasanPersonalityThunkName = 61 "__hwasan_personality_thunk"; 62 63 static const char *const kHwasanShadowMemoryDynamicAddress = 64 "__hwasan_shadow_memory_dynamic_address"; 65 66 // Accesses sizes are powers of two: 1, 2, 4, 8, 16. 67 static const size_t kNumberOfAccessSizes = 5; 68 69 static const size_t kDefaultShadowScale = 4; 70 static const uint64_t kDynamicShadowSentinel = 71 std::numeric_limits<uint64_t>::max(); 72 static const unsigned kPointerTagShift = 56; 73 74 static const unsigned kShadowBaseAlignment = 32; 75 76 static cl::opt<std::string> ClMemoryAccessCallbackPrefix( 77 "hwasan-memory-access-callback-prefix", 78 cl::desc("Prefix for memory access callbacks"), cl::Hidden, 79 cl::init("__hwasan_")); 80 81 static cl::opt<bool> 82 ClInstrumentWithCalls("hwasan-instrument-with-calls", 83 cl::desc("instrument reads and writes with callbacks"), 84 cl::Hidden, cl::init(false)); 85 86 static cl::opt<bool> ClInstrumentReads("hwasan-instrument-reads", 87 cl::desc("instrument read instructions"), 88 cl::Hidden, cl::init(true)); 89 90 static cl::opt<bool> ClInstrumentWrites( 91 "hwasan-instrument-writes", cl::desc("instrument write instructions"), 92 cl::Hidden, cl::init(true)); 93 94 static cl::opt<bool> ClInstrumentAtomics( 95 "hwasan-instrument-atomics", 96 cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden, 97 cl::init(true)); 98 99 static cl::opt<bool> ClRecover( 100 "hwasan-recover", 101 cl::desc("Enable recovery mode (continue-after-error)."), 102 cl::Hidden, cl::init(false)); 103 104 static cl::opt<bool> ClInstrumentStack("hwasan-instrument-stack", 105 cl::desc("instrument stack (allocas)"), 106 cl::Hidden, cl::init(true)); 107 108 static cl::opt<bool> ClUARRetagToZero( 109 "hwasan-uar-retag-to-zero", 110 cl::desc("Clear alloca tags before returning from the function to allow " 111 "non-instrumented and instrumented function calls mix. When set " 112 "to false, allocas are retagged before returning from the " 113 "function to detect use after return."), 114 cl::Hidden, cl::init(true)); 115 116 static cl::opt<bool> ClGenerateTagsWithCalls( 117 "hwasan-generate-tags-with-calls", 118 cl::desc("generate new tags with runtime library calls"), cl::Hidden, 119 cl::init(false)); 120 121 static cl::opt<bool> ClGlobals("hwasan-globals", cl::desc("Instrument globals"), 122 cl::Hidden, cl::init(false)); 123 124 static cl::opt<int> ClMatchAllTag( 125 "hwasan-match-all-tag", 126 cl::desc("don't report bad accesses via pointers with this tag"), 127 cl::Hidden, cl::init(-1)); 128 129 static cl::opt<bool> ClEnableKhwasan( 130 "hwasan-kernel", 131 cl::desc("Enable KernelHWAddressSanitizer instrumentation"), 132 cl::Hidden, cl::init(false)); 133 134 // These flags allow to change the shadow mapping and control how shadow memory 135 // is accessed. The shadow mapping looks like: 136 // Shadow = (Mem >> scale) + offset 137 138 static cl::opt<uint64_t> 139 ClMappingOffset("hwasan-mapping-offset", 140 cl::desc("HWASan shadow mapping offset [EXPERIMENTAL]"), 141 cl::Hidden, cl::init(0)); 142 143 static cl::opt<bool> 144 ClWithIfunc("hwasan-with-ifunc", 145 cl::desc("Access dynamic shadow through an ifunc global on " 146 "platforms that support this"), 147 cl::Hidden, cl::init(false)); 148 149 static cl::opt<bool> ClWithTls( 150 "hwasan-with-tls", 151 cl::desc("Access dynamic shadow through an thread-local pointer on " 152 "platforms that support this"), 153 cl::Hidden, cl::init(true)); 154 155 static cl::opt<bool> 156 ClRecordStackHistory("hwasan-record-stack-history", 157 cl::desc("Record stack frames with tagged allocations " 158 "in a thread-local ring buffer"), 159 cl::Hidden, cl::init(true)); 160 static cl::opt<bool> 161 ClInstrumentMemIntrinsics("hwasan-instrument-mem-intrinsics", 162 cl::desc("instrument memory intrinsics"), 163 cl::Hidden, cl::init(true)); 164 165 static cl::opt<bool> 166 ClInstrumentLandingPads("hwasan-instrument-landing-pads", 167 cl::desc("instrument landing pads"), cl::Hidden, 168 cl::init(false), cl::ZeroOrMore); 169 170 static cl::opt<bool> ClUseShortGranules( 171 "hwasan-use-short-granules", 172 cl::desc("use short granules in allocas and outlined checks"), cl::Hidden, 173 cl::init(false), cl::ZeroOrMore); 174 175 static cl::opt<bool> ClInstrumentPersonalityFunctions( 176 "hwasan-instrument-personality-functions", 177 cl::desc("instrument personality functions"), cl::Hidden, cl::init(false), 178 cl::ZeroOrMore); 179 180 static cl::opt<bool> ClInlineAllChecks("hwasan-inline-all-checks", 181 cl::desc("inline all checks"), 182 cl::Hidden, cl::init(false)); 183 184 namespace { 185 186 /// An instrumentation pass implementing detection of addressability bugs 187 /// using tagged pointers. 188 class HWAddressSanitizer { 189 public: 190 explicit HWAddressSanitizer(Module &M, bool CompileKernel = false, 191 bool Recover = false) : M(M) { 192 this->Recover = ClRecover.getNumOccurrences() > 0 ? ClRecover : Recover; 193 this->CompileKernel = ClEnableKhwasan.getNumOccurrences() > 0 ? 194 ClEnableKhwasan : CompileKernel; 195 196 initializeModule(); 197 } 198 199 bool sanitizeFunction(Function &F); 200 void initializeModule(); 201 202 void initializeCallbacks(Module &M); 203 204 Value *getDynamicShadowIfunc(IRBuilder<> &IRB); 205 Value *getDynamicShadowNonTls(IRBuilder<> &IRB); 206 207 void untagPointerOperand(Instruction *I, Value *Addr); 208 Value *shadowBase(); 209 Value *memToShadow(Value *Shadow, IRBuilder<> &IRB); 210 void instrumentMemAccessInline(Value *Ptr, bool IsWrite, 211 unsigned AccessSizeIndex, 212 Instruction *InsertBefore); 213 void instrumentMemIntrinsic(MemIntrinsic *MI); 214 bool instrumentMemAccess(Instruction *I); 215 Value *isInterestingMemoryAccess(Instruction *I, bool *IsWrite, 216 uint64_t *TypeSize, unsigned *Alignment, 217 Value **MaybeMask); 218 219 bool isInterestingAlloca(const AllocaInst &AI); 220 bool tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, Value *Tag, size_t Size); 221 Value *tagPointer(IRBuilder<> &IRB, Type *Ty, Value *PtrLong, Value *Tag); 222 Value *untagPointer(IRBuilder<> &IRB, Value *PtrLong); 223 bool instrumentStack( 224 SmallVectorImpl<AllocaInst *> &Allocas, 225 DenseMap<AllocaInst *, std::vector<DbgVariableIntrinsic *>> &AllocaDbgMap, 226 SmallVectorImpl<Instruction *> &RetVec, Value *StackTag); 227 Value *readRegister(IRBuilder<> &IRB, StringRef Name); 228 bool instrumentLandingPads(SmallVectorImpl<Instruction *> &RetVec); 229 Value *getNextTagWithCall(IRBuilder<> &IRB); 230 Value *getStackBaseTag(IRBuilder<> &IRB); 231 Value *getAllocaTag(IRBuilder<> &IRB, Value *StackTag, AllocaInst *AI, 232 unsigned AllocaNo); 233 Value *getUARTag(IRBuilder<> &IRB, Value *StackTag); 234 235 Value *getHwasanThreadSlotPtr(IRBuilder<> &IRB, Type *Ty); 236 void emitPrologue(IRBuilder<> &IRB, bool WithFrameRecord); 237 238 void instrumentGlobal(GlobalVariable *GV, uint8_t Tag); 239 void instrumentGlobals(); 240 241 void instrumentPersonalityFunctions(); 242 243 private: 244 LLVMContext *C; 245 Module &M; 246 Triple TargetTriple; 247 FunctionCallee HWAsanMemmove, HWAsanMemcpy, HWAsanMemset; 248 FunctionCallee HWAsanHandleVfork; 249 250 /// This struct defines the shadow mapping using the rule: 251 /// shadow = (mem >> Scale) + Offset. 252 /// If InGlobal is true, then 253 /// extern char __hwasan_shadow[]; 254 /// shadow = (mem >> Scale) + &__hwasan_shadow 255 /// If InTls is true, then 256 /// extern char *__hwasan_tls; 257 /// shadow = (mem>>Scale) + align_up(__hwasan_shadow, kShadowBaseAlignment) 258 struct ShadowMapping { 259 int Scale; 260 uint64_t Offset; 261 bool InGlobal; 262 bool InTls; 263 264 void init(Triple &TargetTriple); 265 unsigned getObjectAlignment() const { return 1U << Scale; } 266 }; 267 ShadowMapping Mapping; 268 269 Type *VoidTy = Type::getVoidTy(M.getContext()); 270 Type *IntptrTy; 271 Type *Int8PtrTy; 272 Type *Int8Ty; 273 Type *Int32Ty; 274 Type *Int64Ty = Type::getInt64Ty(M.getContext()); 275 276 bool CompileKernel; 277 bool Recover; 278 bool UseShortGranules; 279 bool InstrumentLandingPads; 280 281 Function *HwasanCtorFunction; 282 283 FunctionCallee HwasanMemoryAccessCallback[2][kNumberOfAccessSizes]; 284 FunctionCallee HwasanMemoryAccessCallbackSized[2]; 285 286 FunctionCallee HwasanTagMemoryFunc; 287 FunctionCallee HwasanGenerateTagFunc; 288 289 Constant *ShadowGlobal; 290 291 Value *LocalDynamicShadow = nullptr; 292 Value *StackBaseTag = nullptr; 293 GlobalValue *ThreadPtrGlobal = nullptr; 294 }; 295 296 class HWAddressSanitizerLegacyPass : public FunctionPass { 297 public: 298 // Pass identification, replacement for typeid. 299 static char ID; 300 301 explicit HWAddressSanitizerLegacyPass(bool CompileKernel = false, 302 bool Recover = false) 303 : FunctionPass(ID), CompileKernel(CompileKernel), Recover(Recover) {} 304 305 StringRef getPassName() const override { return "HWAddressSanitizer"; } 306 307 bool doInitialization(Module &M) override { 308 HWASan = std::make_unique<HWAddressSanitizer>(M, CompileKernel, Recover); 309 return true; 310 } 311 312 bool runOnFunction(Function &F) override { 313 return HWASan->sanitizeFunction(F); 314 } 315 316 bool doFinalization(Module &M) override { 317 HWASan.reset(); 318 return false; 319 } 320 321 private: 322 std::unique_ptr<HWAddressSanitizer> HWASan; 323 bool CompileKernel; 324 bool Recover; 325 }; 326 327 } // end anonymous namespace 328 329 char HWAddressSanitizerLegacyPass::ID = 0; 330 331 INITIALIZE_PASS_BEGIN( 332 HWAddressSanitizerLegacyPass, "hwasan", 333 "HWAddressSanitizer: detect memory bugs using tagged addressing.", false, 334 false) 335 INITIALIZE_PASS_END( 336 HWAddressSanitizerLegacyPass, "hwasan", 337 "HWAddressSanitizer: detect memory bugs using tagged addressing.", false, 338 false) 339 340 FunctionPass *llvm::createHWAddressSanitizerLegacyPassPass(bool CompileKernel, 341 bool Recover) { 342 assert(!CompileKernel || Recover); 343 return new HWAddressSanitizerLegacyPass(CompileKernel, Recover); 344 } 345 346 HWAddressSanitizerPass::HWAddressSanitizerPass(bool CompileKernel, bool Recover) 347 : CompileKernel(CompileKernel), Recover(Recover) {} 348 349 PreservedAnalyses HWAddressSanitizerPass::run(Module &M, 350 ModuleAnalysisManager &MAM) { 351 HWAddressSanitizer HWASan(M, CompileKernel, Recover); 352 bool Modified = false; 353 for (Function &F : M) 354 Modified |= HWASan.sanitizeFunction(F); 355 if (Modified) 356 return PreservedAnalyses::none(); 357 return PreservedAnalyses::all(); 358 } 359 360 /// Module-level initialization. 361 /// 362 /// inserts a call to __hwasan_init to the module's constructor list. 363 void HWAddressSanitizer::initializeModule() { 364 LLVM_DEBUG(dbgs() << "Init " << M.getName() << "\n"); 365 auto &DL = M.getDataLayout(); 366 367 TargetTriple = Triple(M.getTargetTriple()); 368 369 Mapping.init(TargetTriple); 370 371 C = &(M.getContext()); 372 IRBuilder<> IRB(*C); 373 IntptrTy = IRB.getIntPtrTy(DL); 374 Int8PtrTy = IRB.getInt8PtrTy(); 375 Int8Ty = IRB.getInt8Ty(); 376 Int32Ty = IRB.getInt32Ty(); 377 378 HwasanCtorFunction = nullptr; 379 380 // Older versions of Android do not have the required runtime support for 381 // short granules, global or personality function instrumentation. On other 382 // platforms we currently require using the latest version of the runtime. 383 bool NewRuntime = 384 !TargetTriple.isAndroid() || !TargetTriple.isAndroidVersionLT(30); 385 386 UseShortGranules = 387 ClUseShortGranules.getNumOccurrences() ? ClUseShortGranules : NewRuntime; 388 389 // If we don't have personality function support, fall back to landing pads. 390 InstrumentLandingPads = ClInstrumentLandingPads.getNumOccurrences() 391 ? ClInstrumentLandingPads 392 : !NewRuntime; 393 394 if (!CompileKernel) { 395 std::tie(HwasanCtorFunction, std::ignore) = 396 getOrCreateSanitizerCtorAndInitFunctions( 397 M, kHwasanModuleCtorName, kHwasanInitName, 398 /*InitArgTypes=*/{}, 399 /*InitArgs=*/{}, 400 // This callback is invoked when the functions are created the first 401 // time. Hook them into the global ctors list in that case: 402 [&](Function *Ctor, FunctionCallee) { 403 Comdat *CtorComdat = M.getOrInsertComdat(kHwasanModuleCtorName); 404 Ctor->setComdat(CtorComdat); 405 appendToGlobalCtors(M, Ctor, 0, Ctor); 406 }); 407 408 bool InstrumentGlobals = 409 ClGlobals.getNumOccurrences() ? ClGlobals : NewRuntime; 410 if (InstrumentGlobals) 411 instrumentGlobals(); 412 413 bool InstrumentPersonalityFunctions = 414 ClInstrumentPersonalityFunctions.getNumOccurrences() 415 ? ClInstrumentPersonalityFunctions 416 : NewRuntime; 417 if (InstrumentPersonalityFunctions) 418 instrumentPersonalityFunctions(); 419 } 420 421 if (!TargetTriple.isAndroid()) { 422 Constant *C = M.getOrInsertGlobal("__hwasan_tls", IntptrTy, [&] { 423 auto *GV = new GlobalVariable(M, IntptrTy, /*isConstant=*/false, 424 GlobalValue::ExternalLinkage, nullptr, 425 "__hwasan_tls", nullptr, 426 GlobalVariable::InitialExecTLSModel); 427 appendToCompilerUsed(M, GV); 428 return GV; 429 }); 430 ThreadPtrGlobal = cast<GlobalVariable>(C); 431 } 432 } 433 434 void HWAddressSanitizer::initializeCallbacks(Module &M) { 435 IRBuilder<> IRB(*C); 436 for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) { 437 const std::string TypeStr = AccessIsWrite ? "store" : "load"; 438 const std::string EndingStr = Recover ? "_noabort" : ""; 439 440 HwasanMemoryAccessCallbackSized[AccessIsWrite] = M.getOrInsertFunction( 441 ClMemoryAccessCallbackPrefix + TypeStr + "N" + EndingStr, 442 FunctionType::get(IRB.getVoidTy(), {IntptrTy, IntptrTy}, false)); 443 444 for (size_t AccessSizeIndex = 0; AccessSizeIndex < kNumberOfAccessSizes; 445 AccessSizeIndex++) { 446 HwasanMemoryAccessCallback[AccessIsWrite][AccessSizeIndex] = 447 M.getOrInsertFunction( 448 ClMemoryAccessCallbackPrefix + TypeStr + 449 itostr(1ULL << AccessSizeIndex) + EndingStr, 450 FunctionType::get(IRB.getVoidTy(), {IntptrTy}, false)); 451 } 452 } 453 454 HwasanTagMemoryFunc = M.getOrInsertFunction( 455 "__hwasan_tag_memory", IRB.getVoidTy(), Int8PtrTy, Int8Ty, IntptrTy); 456 HwasanGenerateTagFunc = 457 M.getOrInsertFunction("__hwasan_generate_tag", Int8Ty); 458 459 ShadowGlobal = M.getOrInsertGlobal("__hwasan_shadow", 460 ArrayType::get(IRB.getInt8Ty(), 0)); 461 462 const std::string MemIntrinCallbackPrefix = 463 CompileKernel ? std::string("") : ClMemoryAccessCallbackPrefix; 464 HWAsanMemmove = M.getOrInsertFunction(MemIntrinCallbackPrefix + "memmove", 465 IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), 466 IRB.getInt8PtrTy(), IntptrTy); 467 HWAsanMemcpy = M.getOrInsertFunction(MemIntrinCallbackPrefix + "memcpy", 468 IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), 469 IRB.getInt8PtrTy(), IntptrTy); 470 HWAsanMemset = M.getOrInsertFunction(MemIntrinCallbackPrefix + "memset", 471 IRB.getInt8PtrTy(), IRB.getInt8PtrTy(), 472 IRB.getInt32Ty(), IntptrTy); 473 474 HWAsanHandleVfork = 475 M.getOrInsertFunction("__hwasan_handle_vfork", IRB.getVoidTy(), IntptrTy); 476 } 477 478 Value *HWAddressSanitizer::getDynamicShadowIfunc(IRBuilder<> &IRB) { 479 // An empty inline asm with input reg == output reg. 480 // An opaque no-op cast, basically. 481 InlineAsm *Asm = InlineAsm::get( 482 FunctionType::get(Int8PtrTy, {ShadowGlobal->getType()}, false), 483 StringRef(""), StringRef("=r,0"), 484 /*hasSideEffects=*/false); 485 return IRB.CreateCall(Asm, {ShadowGlobal}, ".hwasan.shadow"); 486 } 487 488 Value *HWAddressSanitizer::getDynamicShadowNonTls(IRBuilder<> &IRB) { 489 // Generate code only when dynamic addressing is needed. 490 if (Mapping.Offset != kDynamicShadowSentinel) 491 return nullptr; 492 493 if (Mapping.InGlobal) { 494 return getDynamicShadowIfunc(IRB); 495 } else { 496 Value *GlobalDynamicAddress = 497 IRB.GetInsertBlock()->getParent()->getParent()->getOrInsertGlobal( 498 kHwasanShadowMemoryDynamicAddress, Int8PtrTy); 499 return IRB.CreateLoad(Int8PtrTy, GlobalDynamicAddress); 500 } 501 } 502 503 Value *HWAddressSanitizer::isInterestingMemoryAccess(Instruction *I, 504 bool *IsWrite, 505 uint64_t *TypeSize, 506 unsigned *Alignment, 507 Value **MaybeMask) { 508 // Skip memory accesses inserted by another instrumentation. 509 if (I->hasMetadata("nosanitize")) return nullptr; 510 511 // Do not instrument the load fetching the dynamic shadow address. 512 if (LocalDynamicShadow == I) 513 return nullptr; 514 515 Value *PtrOperand = nullptr; 516 const DataLayout &DL = I->getModule()->getDataLayout(); 517 if (LoadInst *LI = dyn_cast<LoadInst>(I)) { 518 if (!ClInstrumentReads) return nullptr; 519 *IsWrite = false; 520 *TypeSize = DL.getTypeStoreSizeInBits(LI->getType()); 521 *Alignment = LI->getAlignment(); 522 PtrOperand = LI->getPointerOperand(); 523 } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) { 524 if (!ClInstrumentWrites) return nullptr; 525 *IsWrite = true; 526 *TypeSize = DL.getTypeStoreSizeInBits(SI->getValueOperand()->getType()); 527 *Alignment = SI->getAlignment(); 528 PtrOperand = SI->getPointerOperand(); 529 } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I)) { 530 if (!ClInstrumentAtomics) return nullptr; 531 *IsWrite = true; 532 *TypeSize = DL.getTypeStoreSizeInBits(RMW->getValOperand()->getType()); 533 *Alignment = 0; 534 PtrOperand = RMW->getPointerOperand(); 535 } else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I)) { 536 if (!ClInstrumentAtomics) return nullptr; 537 *IsWrite = true; 538 *TypeSize = DL.getTypeStoreSizeInBits(XCHG->getCompareOperand()->getType()); 539 *Alignment = 0; 540 PtrOperand = XCHG->getPointerOperand(); 541 } 542 543 if (PtrOperand) { 544 // Do not instrument accesses from different address spaces; we cannot deal 545 // with them. 546 Type *PtrTy = cast<PointerType>(PtrOperand->getType()->getScalarType()); 547 if (PtrTy->getPointerAddressSpace() != 0) 548 return nullptr; 549 550 // Ignore swifterror addresses. 551 // swifterror memory addresses are mem2reg promoted by instruction 552 // selection. As such they cannot have regular uses like an instrumentation 553 // function and it makes no sense to track them as memory. 554 if (PtrOperand->isSwiftError()) 555 return nullptr; 556 } 557 558 return PtrOperand; 559 } 560 561 static unsigned getPointerOperandIndex(Instruction *I) { 562 if (LoadInst *LI = dyn_cast<LoadInst>(I)) 563 return LI->getPointerOperandIndex(); 564 if (StoreInst *SI = dyn_cast<StoreInst>(I)) 565 return SI->getPointerOperandIndex(); 566 if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I)) 567 return RMW->getPointerOperandIndex(); 568 if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I)) 569 return XCHG->getPointerOperandIndex(); 570 report_fatal_error("Unexpected instruction"); 571 return -1; 572 } 573 574 static size_t TypeSizeToSizeIndex(uint32_t TypeSize) { 575 size_t Res = countTrailingZeros(TypeSize / 8); 576 assert(Res < kNumberOfAccessSizes); 577 return Res; 578 } 579 580 void HWAddressSanitizer::untagPointerOperand(Instruction *I, Value *Addr) { 581 if (TargetTriple.isAArch64()) 582 return; 583 584 IRBuilder<> IRB(I); 585 Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy); 586 Value *UntaggedPtr = 587 IRB.CreateIntToPtr(untagPointer(IRB, AddrLong), Addr->getType()); 588 I->setOperand(getPointerOperandIndex(I), UntaggedPtr); 589 } 590 591 Value *HWAddressSanitizer::shadowBase() { 592 if (LocalDynamicShadow) 593 return LocalDynamicShadow; 594 return ConstantExpr::getIntToPtr(ConstantInt::get(IntptrTy, Mapping.Offset), 595 Int8PtrTy); 596 } 597 598 Value *HWAddressSanitizer::memToShadow(Value *Mem, IRBuilder<> &IRB) { 599 // Mem >> Scale 600 Value *Shadow = IRB.CreateLShr(Mem, Mapping.Scale); 601 if (Mapping.Offset == 0) 602 return IRB.CreateIntToPtr(Shadow, Int8PtrTy); 603 // (Mem >> Scale) + Offset 604 return IRB.CreateGEP(Int8Ty, shadowBase(), Shadow); 605 } 606 607 void HWAddressSanitizer::instrumentMemAccessInline(Value *Ptr, bool IsWrite, 608 unsigned AccessSizeIndex, 609 Instruction *InsertBefore) { 610 const int64_t AccessInfo = Recover * 0x20 + IsWrite * 0x10 + AccessSizeIndex; 611 IRBuilder<> IRB(InsertBefore); 612 613 if (!ClInlineAllChecks && TargetTriple.isAArch64() && 614 TargetTriple.isOSBinFormatELF() && !Recover) { 615 Module *M = IRB.GetInsertBlock()->getParent()->getParent(); 616 Ptr = IRB.CreateBitCast(Ptr, Int8PtrTy); 617 IRB.CreateCall(Intrinsic::getDeclaration( 618 M, UseShortGranules 619 ? Intrinsic::hwasan_check_memaccess_shortgranules 620 : Intrinsic::hwasan_check_memaccess), 621 {shadowBase(), Ptr, ConstantInt::get(Int32Ty, AccessInfo)}); 622 return; 623 } 624 625 Value *PtrLong = IRB.CreatePointerCast(Ptr, IntptrTy); 626 Value *PtrTag = IRB.CreateTrunc(IRB.CreateLShr(PtrLong, kPointerTagShift), 627 IRB.getInt8Ty()); 628 Value *AddrLong = untagPointer(IRB, PtrLong); 629 Value *Shadow = memToShadow(AddrLong, IRB); 630 Value *MemTag = IRB.CreateLoad(Int8Ty, Shadow); 631 Value *TagMismatch = IRB.CreateICmpNE(PtrTag, MemTag); 632 633 int matchAllTag = ClMatchAllTag.getNumOccurrences() > 0 ? 634 ClMatchAllTag : (CompileKernel ? 0xFF : -1); 635 if (matchAllTag != -1) { 636 Value *TagNotIgnored = IRB.CreateICmpNE(PtrTag, 637 ConstantInt::get(PtrTag->getType(), matchAllTag)); 638 TagMismatch = IRB.CreateAnd(TagMismatch, TagNotIgnored); 639 } 640 641 Instruction *CheckTerm = 642 SplitBlockAndInsertIfThen(TagMismatch, InsertBefore, false, 643 MDBuilder(*C).createBranchWeights(1, 100000)); 644 645 IRB.SetInsertPoint(CheckTerm); 646 Value *OutOfShortGranuleTagRange = 647 IRB.CreateICmpUGT(MemTag, ConstantInt::get(Int8Ty, 15)); 648 Instruction *CheckFailTerm = 649 SplitBlockAndInsertIfThen(OutOfShortGranuleTagRange, CheckTerm, !Recover, 650 MDBuilder(*C).createBranchWeights(1, 100000)); 651 652 IRB.SetInsertPoint(CheckTerm); 653 Value *PtrLowBits = IRB.CreateTrunc(IRB.CreateAnd(PtrLong, 15), Int8Ty); 654 PtrLowBits = IRB.CreateAdd( 655 PtrLowBits, ConstantInt::get(Int8Ty, (1 << AccessSizeIndex) - 1)); 656 Value *PtrLowBitsOOB = IRB.CreateICmpUGE(PtrLowBits, MemTag); 657 SplitBlockAndInsertIfThen(PtrLowBitsOOB, CheckTerm, false, 658 MDBuilder(*C).createBranchWeights(1, 100000), 659 nullptr, nullptr, CheckFailTerm->getParent()); 660 661 IRB.SetInsertPoint(CheckTerm); 662 Value *InlineTagAddr = IRB.CreateOr(AddrLong, 15); 663 InlineTagAddr = IRB.CreateIntToPtr(InlineTagAddr, Int8PtrTy); 664 Value *InlineTag = IRB.CreateLoad(Int8Ty, InlineTagAddr); 665 Value *InlineTagMismatch = IRB.CreateICmpNE(PtrTag, InlineTag); 666 SplitBlockAndInsertIfThen(InlineTagMismatch, CheckTerm, false, 667 MDBuilder(*C).createBranchWeights(1, 100000), 668 nullptr, nullptr, CheckFailTerm->getParent()); 669 670 IRB.SetInsertPoint(CheckFailTerm); 671 InlineAsm *Asm; 672 switch (TargetTriple.getArch()) { 673 case Triple::x86_64: 674 // The signal handler will find the data address in rdi. 675 Asm = InlineAsm::get( 676 FunctionType::get(IRB.getVoidTy(), {PtrLong->getType()}, false), 677 "int3\nnopl " + itostr(0x40 + AccessInfo) + "(%rax)", 678 "{rdi}", 679 /*hasSideEffects=*/true); 680 break; 681 case Triple::aarch64: 682 case Triple::aarch64_be: 683 // The signal handler will find the data address in x0. 684 Asm = InlineAsm::get( 685 FunctionType::get(IRB.getVoidTy(), {PtrLong->getType()}, false), 686 "brk #" + itostr(0x900 + AccessInfo), 687 "{x0}", 688 /*hasSideEffects=*/true); 689 break; 690 default: 691 report_fatal_error("unsupported architecture"); 692 } 693 IRB.CreateCall(Asm, PtrLong); 694 if (Recover) 695 cast<BranchInst>(CheckFailTerm)->setSuccessor(0, CheckTerm->getParent()); 696 } 697 698 void HWAddressSanitizer::instrumentMemIntrinsic(MemIntrinsic *MI) { 699 IRBuilder<> IRB(MI); 700 if (isa<MemTransferInst>(MI)) { 701 IRB.CreateCall( 702 isa<MemMoveInst>(MI) ? HWAsanMemmove : HWAsanMemcpy, 703 {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()), 704 IRB.CreatePointerCast(MI->getOperand(1), IRB.getInt8PtrTy()), 705 IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)}); 706 } else if (isa<MemSetInst>(MI)) { 707 IRB.CreateCall( 708 HWAsanMemset, 709 {IRB.CreatePointerCast(MI->getOperand(0), IRB.getInt8PtrTy()), 710 IRB.CreateIntCast(MI->getOperand(1), IRB.getInt32Ty(), false), 711 IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)}); 712 } 713 MI->eraseFromParent(); 714 } 715 716 bool HWAddressSanitizer::instrumentMemAccess(Instruction *I) { 717 LLVM_DEBUG(dbgs() << "Instrumenting: " << *I << "\n"); 718 bool IsWrite = false; 719 unsigned Alignment = 0; 720 uint64_t TypeSize = 0; 721 Value *MaybeMask = nullptr; 722 723 if (ClInstrumentMemIntrinsics && isa<MemIntrinsic>(I)) { 724 instrumentMemIntrinsic(cast<MemIntrinsic>(I)); 725 return true; 726 } 727 728 Value *Addr = 729 isInterestingMemoryAccess(I, &IsWrite, &TypeSize, &Alignment, &MaybeMask); 730 731 if (!Addr) 732 return false; 733 734 if (MaybeMask) 735 return false; //FIXME 736 737 IRBuilder<> IRB(I); 738 if (isPowerOf2_64(TypeSize) && 739 (TypeSize / 8 <= (1UL << (kNumberOfAccessSizes - 1))) && 740 (Alignment >= (1UL << Mapping.Scale) || Alignment == 0 || 741 Alignment >= TypeSize / 8)) { 742 size_t AccessSizeIndex = TypeSizeToSizeIndex(TypeSize); 743 if (ClInstrumentWithCalls) { 744 IRB.CreateCall(HwasanMemoryAccessCallback[IsWrite][AccessSizeIndex], 745 IRB.CreatePointerCast(Addr, IntptrTy)); 746 } else { 747 instrumentMemAccessInline(Addr, IsWrite, AccessSizeIndex, I); 748 } 749 } else { 750 IRB.CreateCall(HwasanMemoryAccessCallbackSized[IsWrite], 751 {IRB.CreatePointerCast(Addr, IntptrTy), 752 ConstantInt::get(IntptrTy, TypeSize / 8)}); 753 } 754 untagPointerOperand(I, Addr); 755 756 return true; 757 } 758 759 static uint64_t getAllocaSizeInBytes(const AllocaInst &AI) { 760 uint64_t ArraySize = 1; 761 if (AI.isArrayAllocation()) { 762 const ConstantInt *CI = dyn_cast<ConstantInt>(AI.getArraySize()); 763 assert(CI && "non-constant array size"); 764 ArraySize = CI->getZExtValue(); 765 } 766 Type *Ty = AI.getAllocatedType(); 767 uint64_t SizeInBytes = AI.getModule()->getDataLayout().getTypeAllocSize(Ty); 768 return SizeInBytes * ArraySize; 769 } 770 771 bool HWAddressSanitizer::tagAlloca(IRBuilder<> &IRB, AllocaInst *AI, 772 Value *Tag, size_t Size) { 773 size_t AlignedSize = alignTo(Size, Mapping.getObjectAlignment()); 774 if (!UseShortGranules) 775 Size = AlignedSize; 776 777 Value *JustTag = IRB.CreateTrunc(Tag, IRB.getInt8Ty()); 778 if (ClInstrumentWithCalls) { 779 IRB.CreateCall(HwasanTagMemoryFunc, 780 {IRB.CreatePointerCast(AI, Int8PtrTy), JustTag, 781 ConstantInt::get(IntptrTy, AlignedSize)}); 782 } else { 783 size_t ShadowSize = Size >> Mapping.Scale; 784 Value *ShadowPtr = memToShadow(IRB.CreatePointerCast(AI, IntptrTy), IRB); 785 // If this memset is not inlined, it will be intercepted in the hwasan 786 // runtime library. That's OK, because the interceptor skips the checks if 787 // the address is in the shadow region. 788 // FIXME: the interceptor is not as fast as real memset. Consider lowering 789 // llvm.memset right here into either a sequence of stores, or a call to 790 // hwasan_tag_memory. 791 if (ShadowSize) 792 IRB.CreateMemSet(ShadowPtr, JustTag, ShadowSize, Align::None()); 793 if (Size != AlignedSize) { 794 IRB.CreateStore( 795 ConstantInt::get(Int8Ty, Size % Mapping.getObjectAlignment()), 796 IRB.CreateConstGEP1_32(Int8Ty, ShadowPtr, ShadowSize)); 797 IRB.CreateStore(JustTag, IRB.CreateConstGEP1_32( 798 Int8Ty, IRB.CreateBitCast(AI, Int8PtrTy), 799 AlignedSize - 1)); 800 } 801 } 802 return true; 803 } 804 805 static unsigned RetagMask(unsigned AllocaNo) { 806 // A list of 8-bit numbers that have at most one run of non-zero bits. 807 // x = x ^ (mask << 56) can be encoded as a single armv8 instruction for these 808 // masks. 809 // The list does not include the value 255, which is used for UAR. 810 // 811 // Because we are more likely to use earlier elements of this list than later 812 // ones, it is sorted in increasing order of probability of collision with a 813 // mask allocated (temporally) nearby. The program that generated this list 814 // can be found at: 815 // https://github.com/google/sanitizers/blob/master/hwaddress-sanitizer/sort_masks.py 816 static unsigned FastMasks[] = {0, 128, 64, 192, 32, 96, 224, 112, 240, 817 48, 16, 120, 248, 56, 24, 8, 124, 252, 818 60, 28, 12, 4, 126, 254, 62, 30, 14, 819 6, 2, 127, 63, 31, 15, 7, 3, 1}; 820 return FastMasks[AllocaNo % (sizeof(FastMasks) / sizeof(FastMasks[0]))]; 821 } 822 823 Value *HWAddressSanitizer::getNextTagWithCall(IRBuilder<> &IRB) { 824 return IRB.CreateZExt(IRB.CreateCall(HwasanGenerateTagFunc), IntptrTy); 825 } 826 827 Value *HWAddressSanitizer::getStackBaseTag(IRBuilder<> &IRB) { 828 if (ClGenerateTagsWithCalls) 829 return getNextTagWithCall(IRB); 830 if (StackBaseTag) 831 return StackBaseTag; 832 // FIXME: use addressofreturnaddress (but implement it in aarch64 backend 833 // first). 834 Module *M = IRB.GetInsertBlock()->getParent()->getParent(); 835 auto GetStackPointerFn = Intrinsic::getDeclaration( 836 M, Intrinsic::frameaddress, 837 IRB.getInt8PtrTy(M->getDataLayout().getAllocaAddrSpace())); 838 Value *StackPointer = IRB.CreateCall( 839 GetStackPointerFn, {Constant::getNullValue(IRB.getInt32Ty())}); 840 841 // Extract some entropy from the stack pointer for the tags. 842 // Take bits 20..28 (ASLR entropy) and xor with bits 0..8 (these differ 843 // between functions). 844 Value *StackPointerLong = IRB.CreatePointerCast(StackPointer, IntptrTy); 845 Value *StackTag = 846 IRB.CreateXor(StackPointerLong, IRB.CreateLShr(StackPointerLong, 20), 847 "hwasan.stack.base.tag"); 848 return StackTag; 849 } 850 851 Value *HWAddressSanitizer::getAllocaTag(IRBuilder<> &IRB, Value *StackTag, 852 AllocaInst *AI, unsigned AllocaNo) { 853 if (ClGenerateTagsWithCalls) 854 return getNextTagWithCall(IRB); 855 return IRB.CreateXor(StackTag, 856 ConstantInt::get(IntptrTy, RetagMask(AllocaNo))); 857 } 858 859 Value *HWAddressSanitizer::getUARTag(IRBuilder<> &IRB, Value *StackTag) { 860 if (ClUARRetagToZero) 861 return ConstantInt::get(IntptrTy, 0); 862 if (ClGenerateTagsWithCalls) 863 return getNextTagWithCall(IRB); 864 return IRB.CreateXor(StackTag, ConstantInt::get(IntptrTy, 0xFFU)); 865 } 866 867 // Add a tag to an address. 868 Value *HWAddressSanitizer::tagPointer(IRBuilder<> &IRB, Type *Ty, 869 Value *PtrLong, Value *Tag) { 870 Value *TaggedPtrLong; 871 if (CompileKernel) { 872 // Kernel addresses have 0xFF in the most significant byte. 873 Value *ShiftedTag = IRB.CreateOr( 874 IRB.CreateShl(Tag, kPointerTagShift), 875 ConstantInt::get(IntptrTy, (1ULL << kPointerTagShift) - 1)); 876 TaggedPtrLong = IRB.CreateAnd(PtrLong, ShiftedTag); 877 } else { 878 // Userspace can simply do OR (tag << 56); 879 Value *ShiftedTag = IRB.CreateShl(Tag, kPointerTagShift); 880 TaggedPtrLong = IRB.CreateOr(PtrLong, ShiftedTag); 881 } 882 return IRB.CreateIntToPtr(TaggedPtrLong, Ty); 883 } 884 885 // Remove tag from an address. 886 Value *HWAddressSanitizer::untagPointer(IRBuilder<> &IRB, Value *PtrLong) { 887 Value *UntaggedPtrLong; 888 if (CompileKernel) { 889 // Kernel addresses have 0xFF in the most significant byte. 890 UntaggedPtrLong = IRB.CreateOr(PtrLong, 891 ConstantInt::get(PtrLong->getType(), 0xFFULL << kPointerTagShift)); 892 } else { 893 // Userspace addresses have 0x00. 894 UntaggedPtrLong = IRB.CreateAnd(PtrLong, 895 ConstantInt::get(PtrLong->getType(), ~(0xFFULL << kPointerTagShift))); 896 } 897 return UntaggedPtrLong; 898 } 899 900 Value *HWAddressSanitizer::getHwasanThreadSlotPtr(IRBuilder<> &IRB, Type *Ty) { 901 Module *M = IRB.GetInsertBlock()->getParent()->getParent(); 902 if (TargetTriple.isAArch64() && TargetTriple.isAndroid()) { 903 // Android provides a fixed TLS slot for sanitizers. See TLS_SLOT_SANITIZER 904 // in Bionic's libc/private/bionic_tls.h. 905 Function *ThreadPointerFunc = 906 Intrinsic::getDeclaration(M, Intrinsic::thread_pointer); 907 Value *SlotPtr = IRB.CreatePointerCast( 908 IRB.CreateConstGEP1_32(IRB.getInt8Ty(), 909 IRB.CreateCall(ThreadPointerFunc), 0x30), 910 Ty->getPointerTo(0)); 911 return SlotPtr; 912 } 913 if (ThreadPtrGlobal) 914 return ThreadPtrGlobal; 915 916 917 return nullptr; 918 } 919 920 void HWAddressSanitizer::emitPrologue(IRBuilder<> &IRB, bool WithFrameRecord) { 921 if (!Mapping.InTls) { 922 LocalDynamicShadow = getDynamicShadowNonTls(IRB); 923 return; 924 } 925 926 if (!WithFrameRecord && TargetTriple.isAndroid()) { 927 LocalDynamicShadow = getDynamicShadowIfunc(IRB); 928 return; 929 } 930 931 Value *SlotPtr = getHwasanThreadSlotPtr(IRB, IntptrTy); 932 assert(SlotPtr); 933 934 Value *ThreadLong = IRB.CreateLoad(IntptrTy, SlotPtr); 935 // Extract the address field from ThreadLong. Unnecessary on AArch64 with TBI. 936 Value *ThreadLongMaybeUntagged = 937 TargetTriple.isAArch64() ? ThreadLong : untagPointer(IRB, ThreadLong); 938 939 if (WithFrameRecord) { 940 Function *F = IRB.GetInsertBlock()->getParent(); 941 StackBaseTag = IRB.CreateAShr(ThreadLong, 3); 942 943 // Prepare ring buffer data. 944 Value *PC; 945 if (TargetTriple.getArch() == Triple::aarch64) 946 PC = readRegister(IRB, "pc"); 947 else 948 PC = IRB.CreatePtrToInt(F, IntptrTy); 949 Module *M = F->getParent(); 950 auto GetStackPointerFn = Intrinsic::getDeclaration( 951 M, Intrinsic::frameaddress, 952 IRB.getInt8PtrTy(M->getDataLayout().getAllocaAddrSpace())); 953 Value *SP = IRB.CreatePtrToInt( 954 IRB.CreateCall(GetStackPointerFn, 955 {Constant::getNullValue(IRB.getInt32Ty())}), 956 IntptrTy); 957 // Mix SP and PC. 958 // Assumptions: 959 // PC is 0x0000PPPPPPPPPPPP (48 bits are meaningful, others are zero) 960 // SP is 0xsssssssssssSSSS0 (4 lower bits are zero) 961 // We only really need ~20 lower non-zero bits (SSSS), so we mix like this: 962 // 0xSSSSPPPPPPPPPPPP 963 SP = IRB.CreateShl(SP, 44); 964 965 // Store data to ring buffer. 966 Value *RecordPtr = 967 IRB.CreateIntToPtr(ThreadLongMaybeUntagged, IntptrTy->getPointerTo(0)); 968 IRB.CreateStore(IRB.CreateOr(PC, SP), RecordPtr); 969 970 // Update the ring buffer. Top byte of ThreadLong defines the size of the 971 // buffer in pages, it must be a power of two, and the start of the buffer 972 // must be aligned by twice that much. Therefore wrap around of the ring 973 // buffer is simply Addr &= ~((ThreadLong >> 56) << 12). 974 // The use of AShr instead of LShr is due to 975 // https://bugs.llvm.org/show_bug.cgi?id=39030 976 // Runtime library makes sure not to use the highest bit. 977 Value *WrapMask = IRB.CreateXor( 978 IRB.CreateShl(IRB.CreateAShr(ThreadLong, 56), 12, "", true, true), 979 ConstantInt::get(IntptrTy, (uint64_t)-1)); 980 Value *ThreadLongNew = IRB.CreateAnd( 981 IRB.CreateAdd(ThreadLong, ConstantInt::get(IntptrTy, 8)), WrapMask); 982 IRB.CreateStore(ThreadLongNew, SlotPtr); 983 } 984 985 // Get shadow base address by aligning RecordPtr up. 986 // Note: this is not correct if the pointer is already aligned. 987 // Runtime library will make sure this never happens. 988 LocalDynamicShadow = IRB.CreateAdd( 989 IRB.CreateOr( 990 ThreadLongMaybeUntagged, 991 ConstantInt::get(IntptrTy, (1ULL << kShadowBaseAlignment) - 1)), 992 ConstantInt::get(IntptrTy, 1), "hwasan.shadow"); 993 LocalDynamicShadow = IRB.CreateIntToPtr(LocalDynamicShadow, Int8PtrTy); 994 } 995 996 Value *HWAddressSanitizer::readRegister(IRBuilder<> &IRB, StringRef Name) { 997 Module *M = IRB.GetInsertBlock()->getParent()->getParent(); 998 Function *ReadRegister = 999 Intrinsic::getDeclaration(M, Intrinsic::read_register, IntptrTy); 1000 MDNode *MD = MDNode::get(*C, {MDString::get(*C, Name)}); 1001 Value *Args[] = {MetadataAsValue::get(*C, MD)}; 1002 return IRB.CreateCall(ReadRegister, Args); 1003 } 1004 1005 bool HWAddressSanitizer::instrumentLandingPads( 1006 SmallVectorImpl<Instruction *> &LandingPadVec) { 1007 for (auto *LP : LandingPadVec) { 1008 IRBuilder<> IRB(LP->getNextNode()); 1009 IRB.CreateCall( 1010 HWAsanHandleVfork, 1011 {readRegister(IRB, (TargetTriple.getArch() == Triple::x86_64) ? "rsp" 1012 : "sp")}); 1013 } 1014 return true; 1015 } 1016 1017 bool HWAddressSanitizer::instrumentStack( 1018 SmallVectorImpl<AllocaInst *> &Allocas, 1019 DenseMap<AllocaInst *, std::vector<DbgVariableIntrinsic *>> &AllocaDbgMap, 1020 SmallVectorImpl<Instruction *> &RetVec, Value *StackTag) { 1021 // Ideally, we want to calculate tagged stack base pointer, and rewrite all 1022 // alloca addresses using that. Unfortunately, offsets are not known yet 1023 // (unless we use ASan-style mega-alloca). Instead we keep the base tag in a 1024 // temp, shift-OR it into each alloca address and xor with the retag mask. 1025 // This generates one extra instruction per alloca use. 1026 for (unsigned N = 0; N < Allocas.size(); ++N) { 1027 auto *AI = Allocas[N]; 1028 IRBuilder<> IRB(AI->getNextNode()); 1029 1030 // Replace uses of the alloca with tagged address. 1031 Value *Tag = getAllocaTag(IRB, StackTag, AI, N); 1032 Value *AILong = IRB.CreatePointerCast(AI, IntptrTy); 1033 Value *Replacement = tagPointer(IRB, AI->getType(), AILong, Tag); 1034 std::string Name = 1035 AI->hasName() ? AI->getName().str() : "alloca." + itostr(N); 1036 Replacement->setName(Name + ".hwasan"); 1037 1038 AI->replaceUsesWithIf(Replacement, 1039 [AILong](Use &U) { return U.getUser() != AILong; }); 1040 1041 for (auto *DDI : AllocaDbgMap.lookup(AI)) { 1042 // Prepend "tag_offset, N" to the dwarf expression. 1043 // Tag offset logically applies to the alloca pointer, and it makes sense 1044 // to put it at the beginning of the expression. 1045 SmallVector<uint64_t, 8> NewOps = {dwarf::DW_OP_LLVM_tag_offset, 1046 RetagMask(N)}; 1047 DDI->setArgOperand( 1048 2, MetadataAsValue::get(*C, DIExpression::prependOpcodes( 1049 DDI->getExpression(), NewOps))); 1050 } 1051 1052 size_t Size = getAllocaSizeInBytes(*AI); 1053 tagAlloca(IRB, AI, Tag, Size); 1054 1055 for (auto RI : RetVec) { 1056 IRB.SetInsertPoint(RI); 1057 1058 // Re-tag alloca memory with the special UAR tag. 1059 Value *Tag = getUARTag(IRB, StackTag); 1060 tagAlloca(IRB, AI, Tag, alignTo(Size, Mapping.getObjectAlignment())); 1061 } 1062 } 1063 1064 return true; 1065 } 1066 1067 bool HWAddressSanitizer::isInterestingAlloca(const AllocaInst &AI) { 1068 return (AI.getAllocatedType()->isSized() && 1069 // FIXME: instrument dynamic allocas, too 1070 AI.isStaticAlloca() && 1071 // alloca() may be called with 0 size, ignore it. 1072 getAllocaSizeInBytes(AI) > 0 && 1073 // We are only interested in allocas not promotable to registers. 1074 // Promotable allocas are common under -O0. 1075 !isAllocaPromotable(&AI) && 1076 // inalloca allocas are not treated as static, and we don't want 1077 // dynamic alloca instrumentation for them as well. 1078 !AI.isUsedWithInAlloca() && 1079 // swifterror allocas are register promoted by ISel 1080 !AI.isSwiftError()); 1081 } 1082 1083 bool HWAddressSanitizer::sanitizeFunction(Function &F) { 1084 if (&F == HwasanCtorFunction) 1085 return false; 1086 1087 if (!F.hasFnAttribute(Attribute::SanitizeHWAddress)) 1088 return false; 1089 1090 LLVM_DEBUG(dbgs() << "Function: " << F.getName() << "\n"); 1091 1092 SmallVector<Instruction*, 16> ToInstrument; 1093 SmallVector<AllocaInst*, 8> AllocasToInstrument; 1094 SmallVector<Instruction*, 8> RetVec; 1095 SmallVector<Instruction*, 8> LandingPadVec; 1096 DenseMap<AllocaInst *, std::vector<DbgVariableIntrinsic *>> AllocaDbgMap; 1097 for (auto &BB : F) { 1098 for (auto &Inst : BB) { 1099 if (ClInstrumentStack) 1100 if (AllocaInst *AI = dyn_cast<AllocaInst>(&Inst)) { 1101 if (isInterestingAlloca(*AI)) 1102 AllocasToInstrument.push_back(AI); 1103 continue; 1104 } 1105 1106 if (isa<ReturnInst>(Inst) || isa<ResumeInst>(Inst) || 1107 isa<CleanupReturnInst>(Inst)) 1108 RetVec.push_back(&Inst); 1109 1110 if (auto *DDI = dyn_cast<DbgVariableIntrinsic>(&Inst)) 1111 if (auto *Alloca = 1112 dyn_cast_or_null<AllocaInst>(DDI->getVariableLocation())) 1113 AllocaDbgMap[Alloca].push_back(DDI); 1114 1115 if (InstrumentLandingPads && isa<LandingPadInst>(Inst)) 1116 LandingPadVec.push_back(&Inst); 1117 1118 Value *MaybeMask = nullptr; 1119 bool IsWrite; 1120 unsigned Alignment; 1121 uint64_t TypeSize; 1122 Value *Addr = isInterestingMemoryAccess(&Inst, &IsWrite, &TypeSize, 1123 &Alignment, &MaybeMask); 1124 if (Addr || isa<MemIntrinsic>(Inst)) 1125 ToInstrument.push_back(&Inst); 1126 } 1127 } 1128 1129 initializeCallbacks(*F.getParent()); 1130 1131 if (!LandingPadVec.empty()) 1132 instrumentLandingPads(LandingPadVec); 1133 1134 if (AllocasToInstrument.empty() && F.hasPersonalityFn() && 1135 F.getPersonalityFn()->getName() == kHwasanPersonalityThunkName) { 1136 // __hwasan_personality_thunk is a no-op for functions without an 1137 // instrumented stack, so we can drop it. 1138 F.setPersonalityFn(nullptr); 1139 } 1140 1141 if (AllocasToInstrument.empty() && ToInstrument.empty()) 1142 return false; 1143 1144 assert(!LocalDynamicShadow); 1145 1146 Instruction *InsertPt = &*F.getEntryBlock().begin(); 1147 IRBuilder<> EntryIRB(InsertPt); 1148 emitPrologue(EntryIRB, 1149 /*WithFrameRecord*/ ClRecordStackHistory && 1150 !AllocasToInstrument.empty()); 1151 1152 bool Changed = false; 1153 if (!AllocasToInstrument.empty()) { 1154 Value *StackTag = 1155 ClGenerateTagsWithCalls ? nullptr : getStackBaseTag(EntryIRB); 1156 Changed |= instrumentStack(AllocasToInstrument, AllocaDbgMap, RetVec, 1157 StackTag); 1158 } 1159 1160 // Pad and align each of the allocas that we instrumented to stop small 1161 // uninteresting allocas from hiding in instrumented alloca's padding and so 1162 // that we have enough space to store real tags for short granules. 1163 DenseMap<AllocaInst *, AllocaInst *> AllocaToPaddedAllocaMap; 1164 for (AllocaInst *AI : AllocasToInstrument) { 1165 uint64_t Size = getAllocaSizeInBytes(*AI); 1166 uint64_t AlignedSize = alignTo(Size, Mapping.getObjectAlignment()); 1167 AI->setAlignment( 1168 MaybeAlign(std::max(AI->getAlignment(), Mapping.getObjectAlignment()))); 1169 if (Size != AlignedSize) { 1170 Type *AllocatedType = AI->getAllocatedType(); 1171 if (AI->isArrayAllocation()) { 1172 uint64_t ArraySize = 1173 cast<ConstantInt>(AI->getArraySize())->getZExtValue(); 1174 AllocatedType = ArrayType::get(AllocatedType, ArraySize); 1175 } 1176 Type *TypeWithPadding = StructType::get( 1177 AllocatedType, ArrayType::get(Int8Ty, AlignedSize - Size)); 1178 auto *NewAI = new AllocaInst( 1179 TypeWithPadding, AI->getType()->getAddressSpace(), nullptr, "", AI); 1180 NewAI->takeName(AI); 1181 NewAI->setAlignment(MaybeAlign(AI->getAlignment())); 1182 NewAI->setUsedWithInAlloca(AI->isUsedWithInAlloca()); 1183 NewAI->setSwiftError(AI->isSwiftError()); 1184 NewAI->copyMetadata(*AI); 1185 auto *Bitcast = new BitCastInst(NewAI, AI->getType(), "", AI); 1186 AI->replaceAllUsesWith(Bitcast); 1187 AllocaToPaddedAllocaMap[AI] = NewAI; 1188 } 1189 } 1190 1191 if (!AllocaToPaddedAllocaMap.empty()) { 1192 for (auto &BB : F) 1193 for (auto &Inst : BB) 1194 if (auto *DVI = dyn_cast<DbgVariableIntrinsic>(&Inst)) 1195 if (auto *AI = 1196 dyn_cast_or_null<AllocaInst>(DVI->getVariableLocation())) 1197 if (auto *NewAI = AllocaToPaddedAllocaMap.lookup(AI)) 1198 DVI->setArgOperand( 1199 0, MetadataAsValue::get(*C, LocalAsMetadata::get(NewAI))); 1200 for (auto &P : AllocaToPaddedAllocaMap) 1201 P.first->eraseFromParent(); 1202 } 1203 1204 // If we split the entry block, move any allocas that were originally in the 1205 // entry block back into the entry block so that they aren't treated as 1206 // dynamic allocas. 1207 if (EntryIRB.GetInsertBlock() != &F.getEntryBlock()) { 1208 InsertPt = &*F.getEntryBlock().begin(); 1209 for (auto II = EntryIRB.GetInsertBlock()->begin(), 1210 IE = EntryIRB.GetInsertBlock()->end(); 1211 II != IE;) { 1212 Instruction *I = &*II++; 1213 if (auto *AI = dyn_cast<AllocaInst>(I)) 1214 if (isa<ConstantInt>(AI->getArraySize())) 1215 I->moveBefore(InsertPt); 1216 } 1217 } 1218 1219 for (auto Inst : ToInstrument) 1220 Changed |= instrumentMemAccess(Inst); 1221 1222 LocalDynamicShadow = nullptr; 1223 StackBaseTag = nullptr; 1224 1225 return Changed; 1226 } 1227 1228 void HWAddressSanitizer::instrumentGlobal(GlobalVariable *GV, uint8_t Tag) { 1229 Constant *Initializer = GV->getInitializer(); 1230 uint64_t SizeInBytes = 1231 M.getDataLayout().getTypeAllocSize(Initializer->getType()); 1232 uint64_t NewSize = alignTo(SizeInBytes, Mapping.getObjectAlignment()); 1233 if (SizeInBytes != NewSize) { 1234 // Pad the initializer out to the next multiple of 16 bytes and add the 1235 // required short granule tag. 1236 std::vector<uint8_t> Init(NewSize - SizeInBytes, 0); 1237 Init.back() = Tag; 1238 Constant *Padding = ConstantDataArray::get(*C, Init); 1239 Initializer = ConstantStruct::getAnon({Initializer, Padding}); 1240 } 1241 1242 auto *NewGV = new GlobalVariable(M, Initializer->getType(), GV->isConstant(), 1243 GlobalValue::ExternalLinkage, Initializer, 1244 GV->getName() + ".hwasan"); 1245 NewGV->copyAttributesFrom(GV); 1246 NewGV->setLinkage(GlobalValue::PrivateLinkage); 1247 NewGV->copyMetadata(GV, 0); 1248 NewGV->setAlignment( 1249 MaybeAlign(std::max(GV->getAlignment(), Mapping.getObjectAlignment()))); 1250 1251 // It is invalid to ICF two globals that have different tags. In the case 1252 // where the size of the global is a multiple of the tag granularity the 1253 // contents of the globals may be the same but the tags (i.e. symbol values) 1254 // may be different, and the symbols are not considered during ICF. In the 1255 // case where the size is not a multiple of the granularity, the short granule 1256 // tags would discriminate two globals with different tags, but there would 1257 // otherwise be nothing stopping such a global from being incorrectly ICF'd 1258 // with an uninstrumented (i.e. tag 0) global that happened to have the short 1259 // granule tag in the last byte. 1260 NewGV->setUnnamedAddr(GlobalValue::UnnamedAddr::None); 1261 1262 // Descriptor format (assuming little-endian): 1263 // bytes 0-3: relative address of global 1264 // bytes 4-6: size of global (16MB ought to be enough for anyone, but in case 1265 // it isn't, we create multiple descriptors) 1266 // byte 7: tag 1267 auto *DescriptorTy = StructType::get(Int32Ty, Int32Ty); 1268 const uint64_t MaxDescriptorSize = 0xfffff0; 1269 for (uint64_t DescriptorPos = 0; DescriptorPos < SizeInBytes; 1270 DescriptorPos += MaxDescriptorSize) { 1271 auto *Descriptor = 1272 new GlobalVariable(M, DescriptorTy, true, GlobalValue::PrivateLinkage, 1273 nullptr, GV->getName() + ".hwasan.descriptor"); 1274 auto *GVRelPtr = ConstantExpr::getTrunc( 1275 ConstantExpr::getAdd( 1276 ConstantExpr::getSub( 1277 ConstantExpr::getPtrToInt(NewGV, Int64Ty), 1278 ConstantExpr::getPtrToInt(Descriptor, Int64Ty)), 1279 ConstantInt::get(Int64Ty, DescriptorPos)), 1280 Int32Ty); 1281 uint32_t Size = std::min(SizeInBytes - DescriptorPos, MaxDescriptorSize); 1282 auto *SizeAndTag = ConstantInt::get(Int32Ty, Size | (uint32_t(Tag) << 24)); 1283 Descriptor->setComdat(NewGV->getComdat()); 1284 Descriptor->setInitializer(ConstantStruct::getAnon({GVRelPtr, SizeAndTag})); 1285 Descriptor->setSection("hwasan_globals"); 1286 Descriptor->setMetadata(LLVMContext::MD_associated, 1287 MDNode::get(*C, ValueAsMetadata::get(NewGV))); 1288 appendToCompilerUsed(M, Descriptor); 1289 } 1290 1291 Constant *Aliasee = ConstantExpr::getIntToPtr( 1292 ConstantExpr::getAdd( 1293 ConstantExpr::getPtrToInt(NewGV, Int64Ty), 1294 ConstantInt::get(Int64Ty, uint64_t(Tag) << kPointerTagShift)), 1295 GV->getType()); 1296 auto *Alias = GlobalAlias::create(GV->getValueType(), GV->getAddressSpace(), 1297 GV->getLinkage(), "", Aliasee, &M); 1298 Alias->setVisibility(GV->getVisibility()); 1299 Alias->takeName(GV); 1300 GV->replaceAllUsesWith(Alias); 1301 GV->eraseFromParent(); 1302 } 1303 1304 void HWAddressSanitizer::instrumentGlobals() { 1305 // Start by creating a note that contains pointers to the list of global 1306 // descriptors. Adding a note to the output file will cause the linker to 1307 // create a PT_NOTE program header pointing to the note that we can use to 1308 // find the descriptor list starting from the program headers. A function 1309 // provided by the runtime initializes the shadow memory for the globals by 1310 // accessing the descriptor list via the note. The dynamic loader needs to 1311 // call this function whenever a library is loaded. 1312 // 1313 // The reason why we use a note for this instead of a more conventional 1314 // approach of having a global constructor pass a descriptor list pointer to 1315 // the runtime is because of an order of initialization problem. With 1316 // constructors we can encounter the following problematic scenario: 1317 // 1318 // 1) library A depends on library B and also interposes one of B's symbols 1319 // 2) B's constructors are called before A's (as required for correctness) 1320 // 3) during construction, B accesses one of its "own" globals (actually 1321 // interposed by A) and triggers a HWASAN failure due to the initialization 1322 // for A not having happened yet 1323 // 1324 // Even without interposition it is possible to run into similar situations in 1325 // cases where two libraries mutually depend on each other. 1326 // 1327 // We only need one note per binary, so put everything for the note in a 1328 // comdat. 1329 Comdat *NoteComdat = M.getOrInsertComdat(kHwasanNoteName); 1330 1331 Type *Int8Arr0Ty = ArrayType::get(Int8Ty, 0); 1332 auto Start = 1333 new GlobalVariable(M, Int8Arr0Ty, true, GlobalVariable::ExternalLinkage, 1334 nullptr, "__start_hwasan_globals"); 1335 Start->setVisibility(GlobalValue::HiddenVisibility); 1336 Start->setDSOLocal(true); 1337 auto Stop = 1338 new GlobalVariable(M, Int8Arr0Ty, true, GlobalVariable::ExternalLinkage, 1339 nullptr, "__stop_hwasan_globals"); 1340 Stop->setVisibility(GlobalValue::HiddenVisibility); 1341 Stop->setDSOLocal(true); 1342 1343 // Null-terminated so actually 8 bytes, which are required in order to align 1344 // the note properly. 1345 auto *Name = ConstantDataArray::get(*C, "LLVM\0\0\0"); 1346 1347 auto *NoteTy = StructType::get(Int32Ty, Int32Ty, Int32Ty, Name->getType(), 1348 Int32Ty, Int32Ty); 1349 auto *Note = 1350 new GlobalVariable(M, NoteTy, /*isConstantGlobal=*/true, 1351 GlobalValue::PrivateLinkage, nullptr, kHwasanNoteName); 1352 Note->setSection(".note.hwasan.globals"); 1353 Note->setComdat(NoteComdat); 1354 Note->setAlignment(Align(4)); 1355 Note->setDSOLocal(true); 1356 1357 // The pointers in the note need to be relative so that the note ends up being 1358 // placed in rodata, which is the standard location for notes. 1359 auto CreateRelPtr = [&](Constant *Ptr) { 1360 return ConstantExpr::getTrunc( 1361 ConstantExpr::getSub(ConstantExpr::getPtrToInt(Ptr, Int64Ty), 1362 ConstantExpr::getPtrToInt(Note, Int64Ty)), 1363 Int32Ty); 1364 }; 1365 Note->setInitializer(ConstantStruct::getAnon( 1366 {ConstantInt::get(Int32Ty, 8), // n_namesz 1367 ConstantInt::get(Int32Ty, 8), // n_descsz 1368 ConstantInt::get(Int32Ty, ELF::NT_LLVM_HWASAN_GLOBALS), // n_type 1369 Name, CreateRelPtr(Start), CreateRelPtr(Stop)})); 1370 appendToCompilerUsed(M, Note); 1371 1372 // Create a zero-length global in hwasan_globals so that the linker will 1373 // always create start and stop symbols. 1374 auto Dummy = new GlobalVariable( 1375 M, Int8Arr0Ty, /*isConstantGlobal*/ true, GlobalVariable::PrivateLinkage, 1376 Constant::getNullValue(Int8Arr0Ty), "hwasan.dummy.global"); 1377 Dummy->setSection("hwasan_globals"); 1378 Dummy->setComdat(NoteComdat); 1379 Dummy->setMetadata(LLVMContext::MD_associated, 1380 MDNode::get(*C, ValueAsMetadata::get(Note))); 1381 appendToCompilerUsed(M, Dummy); 1382 1383 std::vector<GlobalVariable *> Globals; 1384 for (GlobalVariable &GV : M.globals()) { 1385 if (GV.isDeclarationForLinker() || GV.getName().startswith("llvm.") || 1386 GV.isThreadLocal()) 1387 continue; 1388 1389 // Common symbols can't have aliases point to them, so they can't be tagged. 1390 if (GV.hasCommonLinkage()) 1391 continue; 1392 1393 // Globals with custom sections may be used in __start_/__stop_ enumeration, 1394 // which would be broken both by adding tags and potentially by the extra 1395 // padding/alignment that we insert. 1396 if (GV.hasSection()) 1397 continue; 1398 1399 Globals.push_back(&GV); 1400 } 1401 1402 MD5 Hasher; 1403 Hasher.update(M.getSourceFileName()); 1404 MD5::MD5Result Hash; 1405 Hasher.final(Hash); 1406 uint8_t Tag = Hash[0]; 1407 1408 for (GlobalVariable *GV : Globals) { 1409 // Skip tag 0 in order to avoid collisions with untagged memory. 1410 if (Tag == 0) 1411 Tag = 1; 1412 instrumentGlobal(GV, Tag++); 1413 } 1414 } 1415 1416 void HWAddressSanitizer::instrumentPersonalityFunctions() { 1417 // We need to untag stack frames as we unwind past them. That is the job of 1418 // the personality function wrapper, which either wraps an existing 1419 // personality function or acts as a personality function on its own. Each 1420 // function that has a personality function or that can be unwound past has 1421 // its personality function changed to a thunk that calls the personality 1422 // function wrapper in the runtime. 1423 MapVector<Constant *, std::vector<Function *>> PersonalityFns; 1424 for (Function &F : M) { 1425 if (F.isDeclaration() || !F.hasFnAttribute(Attribute::SanitizeHWAddress)) 1426 continue; 1427 1428 if (F.hasPersonalityFn()) { 1429 PersonalityFns[F.getPersonalityFn()->stripPointerCasts()].push_back(&F); 1430 } else if (!F.hasFnAttribute(Attribute::NoUnwind)) { 1431 PersonalityFns[nullptr].push_back(&F); 1432 } 1433 } 1434 1435 if (PersonalityFns.empty()) 1436 return; 1437 1438 FunctionCallee HwasanPersonalityWrapper = M.getOrInsertFunction( 1439 "__hwasan_personality_wrapper", Int32Ty, Int32Ty, Int32Ty, Int64Ty, 1440 Int8PtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy); 1441 FunctionCallee UnwindGetGR = M.getOrInsertFunction("_Unwind_GetGR", VoidTy); 1442 FunctionCallee UnwindGetCFA = M.getOrInsertFunction("_Unwind_GetCFA", VoidTy); 1443 1444 for (auto &P : PersonalityFns) { 1445 std::string ThunkName = kHwasanPersonalityThunkName; 1446 if (P.first) 1447 ThunkName += ("." + P.first->getName()).str(); 1448 FunctionType *ThunkFnTy = FunctionType::get( 1449 Int32Ty, {Int32Ty, Int32Ty, Int64Ty, Int8PtrTy, Int8PtrTy}, false); 1450 bool IsLocal = P.first && (!isa<GlobalValue>(P.first) || 1451 cast<GlobalValue>(P.first)->hasLocalLinkage()); 1452 auto *ThunkFn = Function::Create(ThunkFnTy, 1453 IsLocal ? GlobalValue::InternalLinkage 1454 : GlobalValue::LinkOnceODRLinkage, 1455 ThunkName, &M); 1456 if (!IsLocal) { 1457 ThunkFn->setVisibility(GlobalValue::HiddenVisibility); 1458 ThunkFn->setComdat(M.getOrInsertComdat(ThunkName)); 1459 } 1460 1461 auto *BB = BasicBlock::Create(*C, "entry", ThunkFn); 1462 IRBuilder<> IRB(BB); 1463 CallInst *WrapperCall = IRB.CreateCall( 1464 HwasanPersonalityWrapper, 1465 {ThunkFn->getArg(0), ThunkFn->getArg(1), ThunkFn->getArg(2), 1466 ThunkFn->getArg(3), ThunkFn->getArg(4), 1467 P.first ? IRB.CreateBitCast(P.first, Int8PtrTy) 1468 : Constant::getNullValue(Int8PtrTy), 1469 IRB.CreateBitCast(UnwindGetGR.getCallee(), Int8PtrTy), 1470 IRB.CreateBitCast(UnwindGetCFA.getCallee(), Int8PtrTy)}); 1471 WrapperCall->setTailCall(); 1472 IRB.CreateRet(WrapperCall); 1473 1474 for (Function *F : P.second) 1475 F->setPersonalityFn(ThunkFn); 1476 } 1477 } 1478 1479 void HWAddressSanitizer::ShadowMapping::init(Triple &TargetTriple) { 1480 Scale = kDefaultShadowScale; 1481 if (ClMappingOffset.getNumOccurrences() > 0) { 1482 InGlobal = false; 1483 InTls = false; 1484 Offset = ClMappingOffset; 1485 } else if (ClEnableKhwasan || ClInstrumentWithCalls) { 1486 InGlobal = false; 1487 InTls = false; 1488 Offset = 0; 1489 } else if (ClWithIfunc) { 1490 InGlobal = true; 1491 InTls = false; 1492 Offset = kDynamicShadowSentinel; 1493 } else if (ClWithTls) { 1494 InGlobal = false; 1495 InTls = true; 1496 Offset = kDynamicShadowSentinel; 1497 } else { 1498 InGlobal = false; 1499 InTls = false; 1500 Offset = kDynamicShadowSentinel; 1501 } 1502 } 1503