1 //===- MemProfInstrumentation.cpp - memory alloc and access instrumentation ==//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file is a part of MemProf. Memory accesses are instrumented
10 // to increment the access count held in a shadow memory location, or
11 // alternatively to call into the runtime. Memory intrinsic calls (memmove,
12 // memcpy, memset) are changed to call the memory profiling runtime version
13 // instead.
14 //
15 //===----------------------------------------------------------------------===//
16
17 #include "llvm/Transforms/Instrumentation/MemProfInstrumentation.h"
18 #include "llvm/ADT/SmallVector.h"
19 #include "llvm/ADT/Statistic.h"
20 #include "llvm/ADT/StringRef.h"
21 #include "llvm/Analysis/MemoryBuiltins.h"
22 #include "llvm/Analysis/TargetLibraryInfo.h"
23 #include "llvm/Analysis/ValueTracking.h"
24 #include "llvm/IR/Constant.h"
25 #include "llvm/IR/DataLayout.h"
26 #include "llvm/IR/DiagnosticInfo.h"
27 #include "llvm/IR/Function.h"
28 #include "llvm/IR/GlobalValue.h"
29 #include "llvm/IR/IRBuilder.h"
30 #include "llvm/IR/Instruction.h"
31 #include "llvm/IR/IntrinsicInst.h"
32 #include "llvm/IR/Module.h"
33 #include "llvm/IR/Type.h"
34 #include "llvm/IR/Value.h"
35 #include "llvm/ProfileData/InstrProf.h"
36 #include "llvm/ProfileData/MemProf.h"
37 #include "llvm/Support/CommandLine.h"
38 #include "llvm/Support/Debug.h"
39 #include "llvm/TargetParser/Triple.h"
40 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
41 #include "llvm/Transforms/Utils/ModuleUtils.h"
42
43 using namespace llvm;
44 using namespace llvm::memprof;
45
46 #define DEBUG_TYPE "memprof"
47
48 constexpr int LLVM_MEM_PROFILER_VERSION = 1;
49
50 // Size of memory mapped to a single shadow location.
51 constexpr uint64_t DefaultMemGranularity = 64;
52
53 // Size of memory mapped to a single histogram bucket.
54 constexpr uint64_t HistogramGranularity = 8;
55
56 // Scale from granularity down to shadow size.
57 constexpr uint64_t DefaultShadowScale = 3;
58
59 constexpr char MemProfModuleCtorName[] = "memprof.module_ctor";
60 constexpr uint64_t MemProfCtorAndDtorPriority = 1;
61 // On Emscripten, the system needs more than one priorities for constructors.
62 constexpr uint64_t MemProfEmscriptenCtorAndDtorPriority = 50;
63 constexpr char MemProfInitName[] = "__memprof_init";
64 constexpr char MemProfVersionCheckNamePrefix[] =
65 "__memprof_version_mismatch_check_v";
66
67 constexpr char MemProfShadowMemoryDynamicAddress[] =
68 "__memprof_shadow_memory_dynamic_address";
69
70 constexpr char MemProfFilenameVar[] = "__memprof_profile_filename";
71
72 constexpr char MemProfHistogramFlagVar[] = "__memprof_histogram";
73
74 // Command-line flags.
75
76 static cl::opt<bool> ClInsertVersionCheck(
77 "memprof-guard-against-version-mismatch",
78 cl::desc("Guard against compiler/runtime version mismatch."), cl::Hidden,
79 cl::init(true));
80
81 // This flag may need to be replaced with -f[no-]memprof-reads.
82 static cl::opt<bool> ClInstrumentReads("memprof-instrument-reads",
83 cl::desc("instrument read instructions"),
84 cl::Hidden, cl::init(true));
85
86 static cl::opt<bool>
87 ClInstrumentWrites("memprof-instrument-writes",
88 cl::desc("instrument write instructions"), cl::Hidden,
89 cl::init(true));
90
91 static cl::opt<bool> ClInstrumentAtomics(
92 "memprof-instrument-atomics",
93 cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden,
94 cl::init(true));
95
96 static cl::opt<bool> ClUseCalls(
97 "memprof-use-callbacks",
98 cl::desc("Use callbacks instead of inline instrumentation sequences."),
99 cl::Hidden, cl::init(false));
100
101 static cl::opt<std::string>
102 ClMemoryAccessCallbackPrefix("memprof-memory-access-callback-prefix",
103 cl::desc("Prefix for memory access callbacks"),
104 cl::Hidden, cl::init("__memprof_"));
105
106 // These flags allow to change the shadow mapping.
107 // The shadow mapping looks like
108 // Shadow = ((Mem & mask) >> scale) + offset
109
110 static cl::opt<int> ClMappingScale("memprof-mapping-scale",
111 cl::desc("scale of memprof shadow mapping"),
112 cl::Hidden, cl::init(DefaultShadowScale));
113
114 static cl::opt<int>
115 ClMappingGranularity("memprof-mapping-granularity",
116 cl::desc("granularity of memprof shadow mapping"),
117 cl::Hidden, cl::init(DefaultMemGranularity));
118
119 static cl::opt<bool> ClStack("memprof-instrument-stack",
120 cl::desc("Instrument scalar stack variables"),
121 cl::Hidden, cl::init(false));
122
123 // Debug flags.
124
125 static cl::opt<int> ClDebug("memprof-debug", cl::desc("debug"), cl::Hidden,
126 cl::init(0));
127
128 static cl::opt<std::string> ClDebugFunc("memprof-debug-func", cl::Hidden,
129 cl::desc("Debug func"));
130
131 static cl::opt<int> ClDebugMin("memprof-debug-min", cl::desc("Debug min inst"),
132 cl::Hidden, cl::init(-1));
133
134 static cl::opt<int> ClDebugMax("memprof-debug-max", cl::desc("Debug max inst"),
135 cl::Hidden, cl::init(-1));
136
137 static cl::opt<bool> ClHistogram("memprof-histogram",
138 cl::desc("Collect access count histograms"),
139 cl::Hidden, cl::init(false));
140
141 static cl::opt<std::string>
142 MemprofRuntimeDefaultOptions("memprof-runtime-default-options",
143 cl::desc("The default memprof options"),
144 cl::Hidden, cl::init(""));
145
146 // Instrumentation statistics
147 STATISTIC(NumInstrumentedReads, "Number of instrumented reads");
148 STATISTIC(NumInstrumentedWrites, "Number of instrumented writes");
149 STATISTIC(NumSkippedStackReads, "Number of non-instrumented stack reads");
150 STATISTIC(NumSkippedStackWrites, "Number of non-instrumented stack writes");
151
152 namespace {
153
154 /// This struct defines the shadow mapping using the rule:
155 /// shadow = ((mem & mask) >> Scale) ADD DynamicShadowOffset.
156 struct ShadowMapping {
ShadowMapping__anonef5aa81b0111::ShadowMapping157 ShadowMapping() {
158 Scale = ClMappingScale;
159 Granularity = ClHistogram ? HistogramGranularity : ClMappingGranularity;
160 Mask = ~(Granularity - 1);
161 }
162
163 int Scale;
164 int Granularity;
165 uint64_t Mask; // Computed as ~(Granularity-1)
166 };
167
getCtorAndDtorPriority(Triple & TargetTriple)168 static uint64_t getCtorAndDtorPriority(Triple &TargetTriple) {
169 return TargetTriple.isOSEmscripten() ? MemProfEmscriptenCtorAndDtorPriority
170 : MemProfCtorAndDtorPriority;
171 }
172
173 struct InterestingMemoryAccess {
174 Value *Addr = nullptr;
175 bool IsWrite;
176 Type *AccessTy;
177 Value *MaybeMask = nullptr;
178 };
179
180 /// Instrument the code in module to profile memory accesses.
181 class MemProfiler {
182 public:
MemProfiler(Module & M)183 MemProfiler(Module &M) {
184 C = &(M.getContext());
185 LongSize = M.getDataLayout().getPointerSizeInBits();
186 IntptrTy = Type::getIntNTy(*C, LongSize);
187 PtrTy = PointerType::getUnqual(*C);
188 }
189
190 /// If it is an interesting memory access, populate information
191 /// about the access and return a InterestingMemoryAccess struct.
192 /// Otherwise return std::nullopt.
193 std::optional<InterestingMemoryAccess>
194 isInterestingMemoryAccess(Instruction *I) const;
195
196 void instrumentMop(Instruction *I, const DataLayout &DL,
197 InterestingMemoryAccess &Access);
198 void instrumentAddress(Instruction *OrigIns, Instruction *InsertBefore,
199 Value *Addr, bool IsWrite);
200 void instrumentMaskedLoadOrStore(const DataLayout &DL, Value *Mask,
201 Instruction *I, Value *Addr, Type *AccessTy,
202 bool IsWrite);
203 void instrumentMemIntrinsic(MemIntrinsic *MI);
204 Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
205 bool instrumentFunction(Function &F);
206 bool maybeInsertMemProfInitAtFunctionEntry(Function &F);
207 bool insertDynamicShadowAtFunctionEntry(Function &F);
208
209 private:
210 void initializeCallbacks(Module &M);
211
212 LLVMContext *C;
213 int LongSize;
214 Type *IntptrTy;
215 PointerType *PtrTy;
216 ShadowMapping Mapping;
217
218 // These arrays is indexed by AccessIsWrite
219 FunctionCallee MemProfMemoryAccessCallback[2];
220
221 FunctionCallee MemProfMemmove, MemProfMemcpy, MemProfMemset;
222 Value *DynamicShadowOffset = nullptr;
223 };
224
225 class ModuleMemProfiler {
226 public:
ModuleMemProfiler(Module & M)227 ModuleMemProfiler(Module &M) { TargetTriple = M.getTargetTriple(); }
228
229 bool instrumentModule(Module &);
230
231 private:
232 Triple TargetTriple;
233 ShadowMapping Mapping;
234 Function *MemProfCtorFunction = nullptr;
235 };
236
237 } // end anonymous namespace
238
239 MemProfilerPass::MemProfilerPass() = default;
240
run(Function & F,AnalysisManager<Function> & AM)241 PreservedAnalyses MemProfilerPass::run(Function &F,
242 AnalysisManager<Function> &AM) {
243 assert((!ClHistogram || ClMappingGranularity == DefaultMemGranularity) &&
244 "Memprof with histogram only supports default mapping granularity");
245 Module &M = *F.getParent();
246 MemProfiler Profiler(M);
247 if (Profiler.instrumentFunction(F))
248 return PreservedAnalyses::none();
249 return PreservedAnalyses::all();
250 }
251
252 ModuleMemProfilerPass::ModuleMemProfilerPass() = default;
253
run(Module & M,AnalysisManager<Module> & AM)254 PreservedAnalyses ModuleMemProfilerPass::run(Module &M,
255 AnalysisManager<Module> &AM) {
256
257 ModuleMemProfiler Profiler(M);
258 if (Profiler.instrumentModule(M))
259 return PreservedAnalyses::none();
260 return PreservedAnalyses::all();
261 }
262
memToShadow(Value * Shadow,IRBuilder<> & IRB)263 Value *MemProfiler::memToShadow(Value *Shadow, IRBuilder<> &IRB) {
264 // (Shadow & mask) >> scale
265 Shadow = IRB.CreateAnd(Shadow, Mapping.Mask);
266 Shadow = IRB.CreateLShr(Shadow, Mapping.Scale);
267 // (Shadow >> scale) | offset
268 assert(DynamicShadowOffset);
269 return IRB.CreateAdd(Shadow, DynamicShadowOffset);
270 }
271
272 // Instrument memset/memmove/memcpy
instrumentMemIntrinsic(MemIntrinsic * MI)273 void MemProfiler::instrumentMemIntrinsic(MemIntrinsic *MI) {
274 IRBuilder<> IRB(MI);
275 if (isa<MemTransferInst>(MI)) {
276 IRB.CreateCall(isa<MemMoveInst>(MI) ? MemProfMemmove : MemProfMemcpy,
277 {MI->getOperand(0), MI->getOperand(1),
278 IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
279 } else if (isa<MemSetInst>(MI)) {
280 IRB.CreateCall(
281 MemProfMemset,
282 {MI->getOperand(0),
283 IRB.CreateIntCast(MI->getOperand(1), IRB.getInt32Ty(), false),
284 IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
285 }
286 MI->eraseFromParent();
287 }
288
289 std::optional<InterestingMemoryAccess>
isInterestingMemoryAccess(Instruction * I) const290 MemProfiler::isInterestingMemoryAccess(Instruction *I) const {
291 // Do not instrument the load fetching the dynamic shadow address.
292 if (DynamicShadowOffset == I)
293 return std::nullopt;
294
295 InterestingMemoryAccess Access;
296
297 if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
298 if (!ClInstrumentReads)
299 return std::nullopt;
300 Access.IsWrite = false;
301 Access.AccessTy = LI->getType();
302 Access.Addr = LI->getPointerOperand();
303 } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
304 if (!ClInstrumentWrites)
305 return std::nullopt;
306 Access.IsWrite = true;
307 Access.AccessTy = SI->getValueOperand()->getType();
308 Access.Addr = SI->getPointerOperand();
309 } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I)) {
310 if (!ClInstrumentAtomics)
311 return std::nullopt;
312 Access.IsWrite = true;
313 Access.AccessTy = RMW->getValOperand()->getType();
314 Access.Addr = RMW->getPointerOperand();
315 } else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I)) {
316 if (!ClInstrumentAtomics)
317 return std::nullopt;
318 Access.IsWrite = true;
319 Access.AccessTy = XCHG->getCompareOperand()->getType();
320 Access.Addr = XCHG->getPointerOperand();
321 } else if (auto *CI = dyn_cast<CallInst>(I)) {
322 auto *F = CI->getCalledFunction();
323 if (F && (F->getIntrinsicID() == Intrinsic::masked_load ||
324 F->getIntrinsicID() == Intrinsic::masked_store)) {
325 unsigned OpOffset = 0;
326 if (F->getIntrinsicID() == Intrinsic::masked_store) {
327 if (!ClInstrumentWrites)
328 return std::nullopt;
329 // Masked store has an initial operand for the value.
330 OpOffset = 1;
331 Access.AccessTy = CI->getArgOperand(0)->getType();
332 Access.IsWrite = true;
333 } else {
334 if (!ClInstrumentReads)
335 return std::nullopt;
336 Access.AccessTy = CI->getType();
337 Access.IsWrite = false;
338 }
339
340 auto *BasePtr = CI->getOperand(0 + OpOffset);
341 Access.MaybeMask = CI->getOperand(2 + OpOffset);
342 Access.Addr = BasePtr;
343 }
344 }
345
346 if (!Access.Addr)
347 return std::nullopt;
348
349 // Do not instrument accesses from different address spaces; we cannot deal
350 // with them.
351 Type *PtrTy = cast<PointerType>(Access.Addr->getType()->getScalarType());
352 if (PtrTy->getPointerAddressSpace() != 0)
353 return std::nullopt;
354
355 // Ignore swifterror addresses.
356 // swifterror memory addresses are mem2reg promoted by instruction
357 // selection. As such they cannot have regular uses like an instrumentation
358 // function and it makes no sense to track them as memory.
359 if (Access.Addr->isSwiftError())
360 return std::nullopt;
361
362 // Peel off GEPs and BitCasts.
363 auto *Addr = Access.Addr->stripInBoundsOffsets();
364
365 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Addr)) {
366 // Do not instrument PGO counter updates.
367 if (GV->hasSection()) {
368 StringRef SectionName = GV->getSection();
369 // Check if the global is in the PGO counters section.
370 auto OF = I->getModule()->getTargetTriple().getObjectFormat();
371 if (SectionName.ends_with(
372 getInstrProfSectionName(IPSK_cnts, OF, /*AddSegmentInfo=*/false)))
373 return std::nullopt;
374 }
375
376 // Do not instrument accesses to LLVM internal variables.
377 if (GV->getName().starts_with("__llvm"))
378 return std::nullopt;
379 }
380
381 return Access;
382 }
383
instrumentMaskedLoadOrStore(const DataLayout & DL,Value * Mask,Instruction * I,Value * Addr,Type * AccessTy,bool IsWrite)384 void MemProfiler::instrumentMaskedLoadOrStore(const DataLayout &DL, Value *Mask,
385 Instruction *I, Value *Addr,
386 Type *AccessTy, bool IsWrite) {
387 auto *VTy = cast<FixedVectorType>(AccessTy);
388 unsigned Num = VTy->getNumElements();
389 auto *Zero = ConstantInt::get(IntptrTy, 0);
390 for (unsigned Idx = 0; Idx < Num; ++Idx) {
391 Value *InstrumentedAddress = nullptr;
392 Instruction *InsertBefore = I;
393 if (auto *Vector = dyn_cast<ConstantVector>(Mask)) {
394 // dyn_cast as we might get UndefValue
395 if (auto *Masked = dyn_cast<ConstantInt>(Vector->getOperand(Idx))) {
396 if (Masked->isZero())
397 // Mask is constant false, so no instrumentation needed.
398 continue;
399 // If we have a true or undef value, fall through to instrumentAddress.
400 // with InsertBefore == I
401 }
402 } else {
403 IRBuilder<> IRB(I);
404 Value *MaskElem = IRB.CreateExtractElement(Mask, Idx);
405 Instruction *ThenTerm = SplitBlockAndInsertIfThen(MaskElem, I, false);
406 InsertBefore = ThenTerm;
407 }
408
409 IRBuilder<> IRB(InsertBefore);
410 InstrumentedAddress =
411 IRB.CreateGEP(VTy, Addr, {Zero, ConstantInt::get(IntptrTy, Idx)});
412 instrumentAddress(I, InsertBefore, InstrumentedAddress, IsWrite);
413 }
414 }
415
instrumentMop(Instruction * I,const DataLayout & DL,InterestingMemoryAccess & Access)416 void MemProfiler::instrumentMop(Instruction *I, const DataLayout &DL,
417 InterestingMemoryAccess &Access) {
418 // Skip instrumentation of stack accesses unless requested.
419 if (!ClStack && isa<AllocaInst>(getUnderlyingObject(Access.Addr))) {
420 if (Access.IsWrite)
421 ++NumSkippedStackWrites;
422 else
423 ++NumSkippedStackReads;
424 return;
425 }
426
427 if (Access.IsWrite)
428 NumInstrumentedWrites++;
429 else
430 NumInstrumentedReads++;
431
432 if (Access.MaybeMask) {
433 instrumentMaskedLoadOrStore(DL, Access.MaybeMask, I, Access.Addr,
434 Access.AccessTy, Access.IsWrite);
435 } else {
436 // Since the access counts will be accumulated across the entire allocation,
437 // we only update the shadow access count for the first location and thus
438 // don't need to worry about alignment and type size.
439 instrumentAddress(I, I, Access.Addr, Access.IsWrite);
440 }
441 }
442
instrumentAddress(Instruction * OrigIns,Instruction * InsertBefore,Value * Addr,bool IsWrite)443 void MemProfiler::instrumentAddress(Instruction *OrigIns,
444 Instruction *InsertBefore, Value *Addr,
445 bool IsWrite) {
446 IRBuilder<> IRB(InsertBefore);
447 Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy);
448
449 if (ClUseCalls) {
450 IRB.CreateCall(MemProfMemoryAccessCallback[IsWrite], AddrLong);
451 return;
452 }
453
454 Type *ShadowTy = ClHistogram ? Type::getInt8Ty(*C) : Type::getInt64Ty(*C);
455 Type *ShadowPtrTy = PointerType::get(*C, 0);
456
457 Value *ShadowPtr = memToShadow(AddrLong, IRB);
458 Value *ShadowAddr = IRB.CreateIntToPtr(ShadowPtr, ShadowPtrTy);
459 Value *ShadowValue = IRB.CreateLoad(ShadowTy, ShadowAddr);
460 // If we are profiling with histograms, add overflow protection at 255.
461 if (ClHistogram) {
462 Value *MaxCount = ConstantInt::get(Type::getInt8Ty(*C), 255);
463 Value *Cmp = IRB.CreateICmpULT(ShadowValue, MaxCount);
464 Instruction *IncBlock =
465 SplitBlockAndInsertIfThen(Cmp, InsertBefore, /*Unreachable=*/false);
466 IRB.SetInsertPoint(IncBlock);
467 }
468 Value *Inc = ConstantInt::get(ShadowTy, 1);
469 ShadowValue = IRB.CreateAdd(ShadowValue, Inc);
470 IRB.CreateStore(ShadowValue, ShadowAddr);
471 }
472
473 // Create the variable for the profile file name.
createProfileFileNameVar(Module & M)474 void createProfileFileNameVar(Module &M) {
475 const MDString *MemProfFilename =
476 dyn_cast_or_null<MDString>(M.getModuleFlag("MemProfProfileFilename"));
477 if (!MemProfFilename)
478 return;
479 assert(!MemProfFilename->getString().empty() &&
480 "Unexpected MemProfProfileFilename metadata with empty string");
481 Constant *ProfileNameConst = ConstantDataArray::getString(
482 M.getContext(), MemProfFilename->getString(), true);
483 GlobalVariable *ProfileNameVar = new GlobalVariable(
484 M, ProfileNameConst->getType(), /*isConstant=*/true,
485 GlobalValue::WeakAnyLinkage, ProfileNameConst, MemProfFilenameVar);
486 const Triple &TT = M.getTargetTriple();
487 if (TT.supportsCOMDAT()) {
488 ProfileNameVar->setLinkage(GlobalValue::ExternalLinkage);
489 ProfileNameVar->setComdat(M.getOrInsertComdat(MemProfFilenameVar));
490 }
491 }
492
493 // Set MemprofHistogramFlag as a Global veriable in IR. This makes it accessible
494 // to the runtime, changing shadow count behavior.
createMemprofHistogramFlagVar(Module & M)495 void createMemprofHistogramFlagVar(Module &M) {
496 const StringRef VarName(MemProfHistogramFlagVar);
497 Type *IntTy1 = Type::getInt1Ty(M.getContext());
498 auto MemprofHistogramFlag = new GlobalVariable(
499 M, IntTy1, true, GlobalValue::WeakAnyLinkage,
500 Constant::getIntegerValue(IntTy1, APInt(1, ClHistogram)), VarName);
501 const Triple &TT = M.getTargetTriple();
502 if (TT.supportsCOMDAT()) {
503 MemprofHistogramFlag->setLinkage(GlobalValue::ExternalLinkage);
504 MemprofHistogramFlag->setComdat(M.getOrInsertComdat(VarName));
505 }
506 appendToCompilerUsed(M, MemprofHistogramFlag);
507 }
508
createMemprofDefaultOptionsVar(Module & M)509 void createMemprofDefaultOptionsVar(Module &M) {
510 Constant *OptionsConst = ConstantDataArray::getString(
511 M.getContext(), MemprofRuntimeDefaultOptions, /*AddNull=*/true);
512 GlobalVariable *OptionsVar =
513 new GlobalVariable(M, OptionsConst->getType(), /*isConstant=*/true,
514 GlobalValue::WeakAnyLinkage, OptionsConst,
515 memprof::getMemprofOptionsSymbolName());
516 const Triple &TT = M.getTargetTriple();
517 if (TT.supportsCOMDAT()) {
518 OptionsVar->setLinkage(GlobalValue::ExternalLinkage);
519 OptionsVar->setComdat(M.getOrInsertComdat(OptionsVar->getName()));
520 }
521 }
522
instrumentModule(Module & M)523 bool ModuleMemProfiler::instrumentModule(Module &M) {
524
525 // Create a module constructor.
526 std::string MemProfVersion = std::to_string(LLVM_MEM_PROFILER_VERSION);
527 std::string VersionCheckName =
528 ClInsertVersionCheck ? (MemProfVersionCheckNamePrefix + MemProfVersion)
529 : "";
530 std::tie(MemProfCtorFunction, std::ignore) =
531 createSanitizerCtorAndInitFunctions(M, MemProfModuleCtorName,
532 MemProfInitName, /*InitArgTypes=*/{},
533 /*InitArgs=*/{}, VersionCheckName);
534
535 const uint64_t Priority = getCtorAndDtorPriority(TargetTriple);
536 appendToGlobalCtors(M, MemProfCtorFunction, Priority);
537
538 createProfileFileNameVar(M);
539
540 createMemprofHistogramFlagVar(M);
541
542 createMemprofDefaultOptionsVar(M);
543
544 return true;
545 }
546
initializeCallbacks(Module & M)547 void MemProfiler::initializeCallbacks(Module &M) {
548 IRBuilder<> IRB(*C);
549
550 for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
551 const std::string TypeStr = AccessIsWrite ? "store" : "load";
552 const std::string HistPrefix = ClHistogram ? "hist_" : "";
553
554 SmallVector<Type *, 2> Args1{1, IntptrTy};
555 MemProfMemoryAccessCallback[AccessIsWrite] = M.getOrInsertFunction(
556 ClMemoryAccessCallbackPrefix + HistPrefix + TypeStr,
557 FunctionType::get(IRB.getVoidTy(), Args1, false));
558 }
559 MemProfMemmove = M.getOrInsertFunction(
560 ClMemoryAccessCallbackPrefix + "memmove", PtrTy, PtrTy, PtrTy, IntptrTy);
561 MemProfMemcpy = M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + "memcpy",
562 PtrTy, PtrTy, PtrTy, IntptrTy);
563 MemProfMemset =
564 M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + "memset", PtrTy,
565 PtrTy, IRB.getInt32Ty(), IntptrTy);
566 }
567
maybeInsertMemProfInitAtFunctionEntry(Function & F)568 bool MemProfiler::maybeInsertMemProfInitAtFunctionEntry(Function &F) {
569 // For each NSObject descendant having a +load method, this method is invoked
570 // by the ObjC runtime before any of the static constructors is called.
571 // Therefore we need to instrument such methods with a call to __memprof_init
572 // at the beginning in order to initialize our runtime before any access to
573 // the shadow memory.
574 // We cannot just ignore these methods, because they may call other
575 // instrumented functions.
576 if (F.getName().contains(" load]")) {
577 FunctionCallee MemProfInitFunction =
578 declareSanitizerInitFunction(*F.getParent(), MemProfInitName, {});
579 IRBuilder<> IRB(&F.front(), F.front().begin());
580 IRB.CreateCall(MemProfInitFunction, {});
581 return true;
582 }
583 return false;
584 }
585
insertDynamicShadowAtFunctionEntry(Function & F)586 bool MemProfiler::insertDynamicShadowAtFunctionEntry(Function &F) {
587 IRBuilder<> IRB(&F.front().front());
588 Value *GlobalDynamicAddress = F.getParent()->getOrInsertGlobal(
589 MemProfShadowMemoryDynamicAddress, IntptrTy);
590 if (F.getParent()->getPICLevel() == PICLevel::NotPIC)
591 cast<GlobalVariable>(GlobalDynamicAddress)->setDSOLocal(true);
592 DynamicShadowOffset = IRB.CreateLoad(IntptrTy, GlobalDynamicAddress);
593 return true;
594 }
595
instrumentFunction(Function & F)596 bool MemProfiler::instrumentFunction(Function &F) {
597 if (F.getLinkage() == GlobalValue::AvailableExternallyLinkage)
598 return false;
599 if (ClDebugFunc == F.getName())
600 return false;
601 if (F.getName().starts_with("__memprof_"))
602 return false;
603
604 bool FunctionModified = false;
605
606 // If needed, insert __memprof_init.
607 // This function needs to be called even if the function body is not
608 // instrumented.
609 if (maybeInsertMemProfInitAtFunctionEntry(F))
610 FunctionModified = true;
611
612 LLVM_DEBUG(dbgs() << "MEMPROF instrumenting:\n" << F << "\n");
613
614 initializeCallbacks(*F.getParent());
615
616 SmallVector<Instruction *, 16> ToInstrument;
617
618 // Fill the set of memory operations to instrument.
619 for (auto &BB : F) {
620 for (auto &Inst : BB) {
621 if (isInterestingMemoryAccess(&Inst) || isa<MemIntrinsic>(Inst))
622 ToInstrument.push_back(&Inst);
623 }
624 }
625
626 if (ToInstrument.empty()) {
627 LLVM_DEBUG(dbgs() << "MEMPROF done instrumenting: " << FunctionModified
628 << " " << F << "\n");
629
630 return FunctionModified;
631 }
632
633 FunctionModified |= insertDynamicShadowAtFunctionEntry(F);
634
635 int NumInstrumented = 0;
636 for (auto *Inst : ToInstrument) {
637 if (ClDebugMin < 0 || ClDebugMax < 0 ||
638 (NumInstrumented >= ClDebugMin && NumInstrumented <= ClDebugMax)) {
639 std::optional<InterestingMemoryAccess> Access =
640 isInterestingMemoryAccess(Inst);
641 if (Access)
642 instrumentMop(Inst, F.getDataLayout(), *Access);
643 else
644 instrumentMemIntrinsic(cast<MemIntrinsic>(Inst));
645 }
646 NumInstrumented++;
647 }
648
649 if (NumInstrumented > 0)
650 FunctionModified = true;
651
652 LLVM_DEBUG(dbgs() << "MEMPROF done instrumenting: " << FunctionModified << " "
653 << F << "\n");
654
655 return FunctionModified;
656 }
657