xref: /freebsd/contrib/llvm-project/llvm/lib/Transforms/Instrumentation/MemProfInstrumentation.cpp (revision 700637cbb5e582861067a11aaca4d053546871d2) 
1 //===- MemProfInstrumentation.cpp - memory alloc and access instrumentation ==//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file is a part of MemProf. Memory accesses are instrumented
10 // to increment the access count held in a shadow memory location, or
11 // alternatively to call into the runtime. Memory intrinsic calls (memmove,
12 // memcpy, memset) are changed to call the memory profiling runtime version
13 // instead.
14 //
15 //===----------------------------------------------------------------------===//
16 
17 #include "llvm/Transforms/Instrumentation/MemProfInstrumentation.h"
18 #include "llvm/ADT/SmallVector.h"
19 #include "llvm/ADT/Statistic.h"
20 #include "llvm/ADT/StringRef.h"
21 #include "llvm/Analysis/MemoryBuiltins.h"
22 #include "llvm/Analysis/TargetLibraryInfo.h"
23 #include "llvm/Analysis/ValueTracking.h"
24 #include "llvm/IR/Constant.h"
25 #include "llvm/IR/DataLayout.h"
26 #include "llvm/IR/DiagnosticInfo.h"
27 #include "llvm/IR/Function.h"
28 #include "llvm/IR/GlobalValue.h"
29 #include "llvm/IR/IRBuilder.h"
30 #include "llvm/IR/Instruction.h"
31 #include "llvm/IR/IntrinsicInst.h"
32 #include "llvm/IR/Module.h"
33 #include "llvm/IR/Type.h"
34 #include "llvm/IR/Value.h"
35 #include "llvm/ProfileData/InstrProf.h"
36 #include "llvm/ProfileData/MemProf.h"
37 #include "llvm/Support/CommandLine.h"
38 #include "llvm/Support/Debug.h"
39 #include "llvm/TargetParser/Triple.h"
40 #include "llvm/Transforms/Utils/BasicBlockUtils.h"
41 #include "llvm/Transforms/Utils/ModuleUtils.h"
42 
43 using namespace llvm;
44 using namespace llvm::memprof;
45 
46 #define DEBUG_TYPE "memprof"
47 
48 constexpr int LLVM_MEM_PROFILER_VERSION = 1;
49 
50 // Size of memory mapped to a single shadow location.
51 constexpr uint64_t DefaultMemGranularity = 64;
52 
53 // Size of memory mapped to a single histogram bucket.
54 constexpr uint64_t HistogramGranularity = 8;
55 
56 // Scale from granularity down to shadow size.
57 constexpr uint64_t DefaultShadowScale = 3;
58 
59 constexpr char MemProfModuleCtorName[] = "memprof.module_ctor";
60 constexpr uint64_t MemProfCtorAndDtorPriority = 1;
61 // On Emscripten, the system needs more than one priorities for constructors.
62 constexpr uint64_t MemProfEmscriptenCtorAndDtorPriority = 50;
63 constexpr char MemProfInitName[] = "__memprof_init";
64 constexpr char MemProfVersionCheckNamePrefix[] =
65     "__memprof_version_mismatch_check_v";
66 
67 constexpr char MemProfShadowMemoryDynamicAddress[] =
68     "__memprof_shadow_memory_dynamic_address";
69 
70 constexpr char MemProfFilenameVar[] = "__memprof_profile_filename";
71 
72 constexpr char MemProfHistogramFlagVar[] = "__memprof_histogram";
73 
74 // Command-line flags.
75 
76 static cl::opt<bool> ClInsertVersionCheck(
77     "memprof-guard-against-version-mismatch",
78     cl::desc("Guard against compiler/runtime version mismatch."), cl::Hidden,
79     cl::init(true));
80 
81 // This flag may need to be replaced with -f[no-]memprof-reads.
82 static cl::opt<bool> ClInstrumentReads("memprof-instrument-reads",
83                                        cl::desc("instrument read instructions"),
84                                        cl::Hidden, cl::init(true));
85 
86 static cl::opt<bool>
87     ClInstrumentWrites("memprof-instrument-writes",
88                        cl::desc("instrument write instructions"), cl::Hidden,
89                        cl::init(true));
90 
91 static cl::opt<bool> ClInstrumentAtomics(
92     "memprof-instrument-atomics",
93     cl::desc("instrument atomic instructions (rmw, cmpxchg)"), cl::Hidden,
94     cl::init(true));
95 
96 static cl::opt<bool> ClUseCalls(
97     "memprof-use-callbacks",
98     cl::desc("Use callbacks instead of inline instrumentation sequences."),
99     cl::Hidden, cl::init(false));
100 
101 static cl::opt<std::string>
102     ClMemoryAccessCallbackPrefix("memprof-memory-access-callback-prefix",
103                                  cl::desc("Prefix for memory access callbacks"),
104                                  cl::Hidden, cl::init("__memprof_"));
105 
106 // These flags allow to change the shadow mapping.
107 // The shadow mapping looks like
108 //    Shadow = ((Mem & mask) >> scale) + offset
109 
110 static cl::opt<int> ClMappingScale("memprof-mapping-scale",
111                                    cl::desc("scale of memprof shadow mapping"),
112                                    cl::Hidden, cl::init(DefaultShadowScale));
113 
114 static cl::opt<int>
115     ClMappingGranularity("memprof-mapping-granularity",
116                          cl::desc("granularity of memprof shadow mapping"),
117                          cl::Hidden, cl::init(DefaultMemGranularity));
118 
119 static cl::opt<bool> ClStack("memprof-instrument-stack",
120                              cl::desc("Instrument scalar stack variables"),
121                              cl::Hidden, cl::init(false));
122 
123 // Debug flags.
124 
125 static cl::opt<int> ClDebug("memprof-debug", cl::desc("debug"), cl::Hidden,
126                             cl::init(0));
127 
128 static cl::opt<std::string> ClDebugFunc("memprof-debug-func", cl::Hidden,
129                                         cl::desc("Debug func"));
130 
131 static cl::opt<int> ClDebugMin("memprof-debug-min", cl::desc("Debug min inst"),
132                                cl::Hidden, cl::init(-1));
133 
134 static cl::opt<int> ClDebugMax("memprof-debug-max", cl::desc("Debug max inst"),
135                                cl::Hidden, cl::init(-1));
136 
137 static cl::opt<bool> ClHistogram("memprof-histogram",
138                                  cl::desc("Collect access count histograms"),
139                                  cl::Hidden, cl::init(false));
140 
141 static cl::opt<std::string>
142     MemprofRuntimeDefaultOptions("memprof-runtime-default-options",
143                                  cl::desc("The default memprof options"),
144                                  cl::Hidden, cl::init(""));
145 
146 // Instrumentation statistics
147 STATISTIC(NumInstrumentedReads, "Number of instrumented reads");
148 STATISTIC(NumInstrumentedWrites, "Number of instrumented writes");
149 STATISTIC(NumSkippedStackReads, "Number of non-instrumented stack reads");
150 STATISTIC(NumSkippedStackWrites, "Number of non-instrumented stack writes");
151 
152 namespace {
153 
154 /// This struct defines the shadow mapping using the rule:
155 ///   shadow = ((mem & mask) >> Scale) ADD DynamicShadowOffset.
156 struct ShadowMapping {
ShadowMapping__anonef5aa81b0111::ShadowMapping157   ShadowMapping() {
158     Scale = ClMappingScale;
159     Granularity = ClHistogram ? HistogramGranularity : ClMappingGranularity;
160     Mask = ~(Granularity - 1);
161   }
162 
163   int Scale;
164   int Granularity;
165   uint64_t Mask; // Computed as ~(Granularity-1)
166 };
167 
getCtorAndDtorPriority(Triple & TargetTriple)168 static uint64_t getCtorAndDtorPriority(Triple &TargetTriple) {
169   return TargetTriple.isOSEmscripten() ? MemProfEmscriptenCtorAndDtorPriority
170                                        : MemProfCtorAndDtorPriority;
171 }
172 
173 struct InterestingMemoryAccess {
174   Value *Addr = nullptr;
175   bool IsWrite;
176   Type *AccessTy;
177   Value *MaybeMask = nullptr;
178 };
179 
180 /// Instrument the code in module to profile memory accesses.
181 class MemProfiler {
182 public:
MemProfiler(Module & M)183   MemProfiler(Module &M) {
184     C = &(M.getContext());
185     LongSize = M.getDataLayout().getPointerSizeInBits();
186     IntptrTy = Type::getIntNTy(*C, LongSize);
187     PtrTy = PointerType::getUnqual(*C);
188   }
189 
190   /// If it is an interesting memory access, populate information
191   /// about the access and return a InterestingMemoryAccess struct.
192   /// Otherwise return std::nullopt.
193   std::optional<InterestingMemoryAccess>
194   isInterestingMemoryAccess(Instruction *I) const;
195 
196   void instrumentMop(Instruction *I, const DataLayout &DL,
197                      InterestingMemoryAccess &Access);
198   void instrumentAddress(Instruction *OrigIns, Instruction *InsertBefore,
199                          Value *Addr, bool IsWrite);
200   void instrumentMaskedLoadOrStore(const DataLayout &DL, Value *Mask,
201                                    Instruction *I, Value *Addr, Type *AccessTy,
202                                    bool IsWrite);
203   void instrumentMemIntrinsic(MemIntrinsic *MI);
204   Value *memToShadow(Value *Shadow, IRBuilder<> &IRB);
205   bool instrumentFunction(Function &F);
206   bool maybeInsertMemProfInitAtFunctionEntry(Function &F);
207   bool insertDynamicShadowAtFunctionEntry(Function &F);
208 
209 private:
210   void initializeCallbacks(Module &M);
211 
212   LLVMContext *C;
213   int LongSize;
214   Type *IntptrTy;
215   PointerType *PtrTy;
216   ShadowMapping Mapping;
217 
218   // These arrays is indexed by AccessIsWrite
219   FunctionCallee MemProfMemoryAccessCallback[2];
220 
221   FunctionCallee MemProfMemmove, MemProfMemcpy, MemProfMemset;
222   Value *DynamicShadowOffset = nullptr;
223 };
224 
225 class ModuleMemProfiler {
226 public:
ModuleMemProfiler(Module & M)227   ModuleMemProfiler(Module &M) { TargetTriple = M.getTargetTriple(); }
228 
229   bool instrumentModule(Module &);
230 
231 private:
232   Triple TargetTriple;
233   ShadowMapping Mapping;
234   Function *MemProfCtorFunction = nullptr;
235 };
236 
237 } // end anonymous namespace
238 
239 MemProfilerPass::MemProfilerPass() = default;
240 
run(Function & F,AnalysisManager<Function> & AM)241 PreservedAnalyses MemProfilerPass::run(Function &F,
242                                        AnalysisManager<Function> &AM) {
243   assert((!ClHistogram || ClMappingGranularity == DefaultMemGranularity) &&
244          "Memprof with histogram only supports default mapping granularity");
245   Module &M = *F.getParent();
246   MemProfiler Profiler(M);
247   if (Profiler.instrumentFunction(F))
248     return PreservedAnalyses::none();
249   return PreservedAnalyses::all();
250 }
251 
252 ModuleMemProfilerPass::ModuleMemProfilerPass() = default;
253 
run(Module & M,AnalysisManager<Module> & AM)254 PreservedAnalyses ModuleMemProfilerPass::run(Module &M,
255                                              AnalysisManager<Module> &AM) {
256 
257   ModuleMemProfiler Profiler(M);
258   if (Profiler.instrumentModule(M))
259     return PreservedAnalyses::none();
260   return PreservedAnalyses::all();
261 }
262 
memToShadow(Value * Shadow,IRBuilder<> & IRB)263 Value *MemProfiler::memToShadow(Value *Shadow, IRBuilder<> &IRB) {
264   // (Shadow & mask) >> scale
265   Shadow = IRB.CreateAnd(Shadow, Mapping.Mask);
266   Shadow = IRB.CreateLShr(Shadow, Mapping.Scale);
267   // (Shadow >> scale) | offset
268   assert(DynamicShadowOffset);
269   return IRB.CreateAdd(Shadow, DynamicShadowOffset);
270 }
271 
272 // Instrument memset/memmove/memcpy
instrumentMemIntrinsic(MemIntrinsic * MI)273 void MemProfiler::instrumentMemIntrinsic(MemIntrinsic *MI) {
274   IRBuilder<> IRB(MI);
275   if (isa<MemTransferInst>(MI)) {
276     IRB.CreateCall(isa<MemMoveInst>(MI) ? MemProfMemmove : MemProfMemcpy,
277                    {MI->getOperand(0), MI->getOperand(1),
278                     IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
279   } else if (isa<MemSetInst>(MI)) {
280     IRB.CreateCall(
281         MemProfMemset,
282         {MI->getOperand(0),
283          IRB.CreateIntCast(MI->getOperand(1), IRB.getInt32Ty(), false),
284          IRB.CreateIntCast(MI->getOperand(2), IntptrTy, false)});
285   }
286   MI->eraseFromParent();
287 }
288 
289 std::optional<InterestingMemoryAccess>
isInterestingMemoryAccess(Instruction * I) const290 MemProfiler::isInterestingMemoryAccess(Instruction *I) const {
291   // Do not instrument the load fetching the dynamic shadow address.
292   if (DynamicShadowOffset == I)
293     return std::nullopt;
294 
295   InterestingMemoryAccess Access;
296 
297   if (LoadInst *LI = dyn_cast<LoadInst>(I)) {
298     if (!ClInstrumentReads)
299       return std::nullopt;
300     Access.IsWrite = false;
301     Access.AccessTy = LI->getType();
302     Access.Addr = LI->getPointerOperand();
303   } else if (StoreInst *SI = dyn_cast<StoreInst>(I)) {
304     if (!ClInstrumentWrites)
305       return std::nullopt;
306     Access.IsWrite = true;
307     Access.AccessTy = SI->getValueOperand()->getType();
308     Access.Addr = SI->getPointerOperand();
309   } else if (AtomicRMWInst *RMW = dyn_cast<AtomicRMWInst>(I)) {
310     if (!ClInstrumentAtomics)
311       return std::nullopt;
312     Access.IsWrite = true;
313     Access.AccessTy = RMW->getValOperand()->getType();
314     Access.Addr = RMW->getPointerOperand();
315   } else if (AtomicCmpXchgInst *XCHG = dyn_cast<AtomicCmpXchgInst>(I)) {
316     if (!ClInstrumentAtomics)
317       return std::nullopt;
318     Access.IsWrite = true;
319     Access.AccessTy = XCHG->getCompareOperand()->getType();
320     Access.Addr = XCHG->getPointerOperand();
321   } else if (auto *CI = dyn_cast<CallInst>(I)) {
322     auto *F = CI->getCalledFunction();
323     if (F && (F->getIntrinsicID() == Intrinsic::masked_load ||
324               F->getIntrinsicID() == Intrinsic::masked_store)) {
325       unsigned OpOffset = 0;
326       if (F->getIntrinsicID() == Intrinsic::masked_store) {
327         if (!ClInstrumentWrites)
328           return std::nullopt;
329         // Masked store has an initial operand for the value.
330         OpOffset = 1;
331         Access.AccessTy = CI->getArgOperand(0)->getType();
332         Access.IsWrite = true;
333       } else {
334         if (!ClInstrumentReads)
335           return std::nullopt;
336         Access.AccessTy = CI->getType();
337         Access.IsWrite = false;
338       }
339 
340       auto *BasePtr = CI->getOperand(0 + OpOffset);
341       Access.MaybeMask = CI->getOperand(2 + OpOffset);
342       Access.Addr = BasePtr;
343     }
344   }
345 
346   if (!Access.Addr)
347     return std::nullopt;
348 
349   // Do not instrument accesses from different address spaces; we cannot deal
350   // with them.
351   Type *PtrTy = cast<PointerType>(Access.Addr->getType()->getScalarType());
352   if (PtrTy->getPointerAddressSpace() != 0)
353     return std::nullopt;
354 
355   // Ignore swifterror addresses.
356   // swifterror memory addresses are mem2reg promoted by instruction
357   // selection. As such they cannot have regular uses like an instrumentation
358   // function and it makes no sense to track them as memory.
359   if (Access.Addr->isSwiftError())
360     return std::nullopt;
361 
362   // Peel off GEPs and BitCasts.
363   auto *Addr = Access.Addr->stripInBoundsOffsets();
364 
365   if (GlobalVariable *GV = dyn_cast<GlobalVariable>(Addr)) {
366     // Do not instrument PGO counter updates.
367     if (GV->hasSection()) {
368       StringRef SectionName = GV->getSection();
369       // Check if the global is in the PGO counters section.
370       auto OF = I->getModule()->getTargetTriple().getObjectFormat();
371       if (SectionName.ends_with(
372               getInstrProfSectionName(IPSK_cnts, OF, /*AddSegmentInfo=*/false)))
373         return std::nullopt;
374     }
375 
376     // Do not instrument accesses to LLVM internal variables.
377     if (GV->getName().starts_with("__llvm"))
378       return std::nullopt;
379   }
380 
381   return Access;
382 }
383 
instrumentMaskedLoadOrStore(const DataLayout & DL,Value * Mask,Instruction * I,Value * Addr,Type * AccessTy,bool IsWrite)384 void MemProfiler::instrumentMaskedLoadOrStore(const DataLayout &DL, Value *Mask,
385                                               Instruction *I, Value *Addr,
386                                               Type *AccessTy, bool IsWrite) {
387   auto *VTy = cast<FixedVectorType>(AccessTy);
388   unsigned Num = VTy->getNumElements();
389   auto *Zero = ConstantInt::get(IntptrTy, 0);
390   for (unsigned Idx = 0; Idx < Num; ++Idx) {
391     Value *InstrumentedAddress = nullptr;
392     Instruction *InsertBefore = I;
393     if (auto *Vector = dyn_cast<ConstantVector>(Mask)) {
394       // dyn_cast as we might get UndefValue
395       if (auto *Masked = dyn_cast<ConstantInt>(Vector->getOperand(Idx))) {
396         if (Masked->isZero())
397           // Mask is constant false, so no instrumentation needed.
398           continue;
399         // If we have a true or undef value, fall through to instrumentAddress.
400         // with InsertBefore == I
401       }
402     } else {
403       IRBuilder<> IRB(I);
404       Value *MaskElem = IRB.CreateExtractElement(Mask, Idx);
405       Instruction *ThenTerm = SplitBlockAndInsertIfThen(MaskElem, I, false);
406       InsertBefore = ThenTerm;
407     }
408 
409     IRBuilder<> IRB(InsertBefore);
410     InstrumentedAddress =
411         IRB.CreateGEP(VTy, Addr, {Zero, ConstantInt::get(IntptrTy, Idx)});
412     instrumentAddress(I, InsertBefore, InstrumentedAddress, IsWrite);
413   }
414 }
415 
instrumentMop(Instruction * I,const DataLayout & DL,InterestingMemoryAccess & Access)416 void MemProfiler::instrumentMop(Instruction *I, const DataLayout &DL,
417                                 InterestingMemoryAccess &Access) {
418   // Skip instrumentation of stack accesses unless requested.
419   if (!ClStack && isa<AllocaInst>(getUnderlyingObject(Access.Addr))) {
420     if (Access.IsWrite)
421       ++NumSkippedStackWrites;
422     else
423       ++NumSkippedStackReads;
424     return;
425   }
426 
427   if (Access.IsWrite)
428     NumInstrumentedWrites++;
429   else
430     NumInstrumentedReads++;
431 
432   if (Access.MaybeMask) {
433     instrumentMaskedLoadOrStore(DL, Access.MaybeMask, I, Access.Addr,
434                                 Access.AccessTy, Access.IsWrite);
435   } else {
436     // Since the access counts will be accumulated across the entire allocation,
437     // we only update the shadow access count for the first location and thus
438     // don't need to worry about alignment and type size.
439     instrumentAddress(I, I, Access.Addr, Access.IsWrite);
440   }
441 }
442 
instrumentAddress(Instruction * OrigIns,Instruction * InsertBefore,Value * Addr,bool IsWrite)443 void MemProfiler::instrumentAddress(Instruction *OrigIns,
444                                     Instruction *InsertBefore, Value *Addr,
445                                     bool IsWrite) {
446   IRBuilder<> IRB(InsertBefore);
447   Value *AddrLong = IRB.CreatePointerCast(Addr, IntptrTy);
448 
449   if (ClUseCalls) {
450     IRB.CreateCall(MemProfMemoryAccessCallback[IsWrite], AddrLong);
451     return;
452   }
453 
454   Type *ShadowTy = ClHistogram ? Type::getInt8Ty(*C) : Type::getInt64Ty(*C);
455   Type *ShadowPtrTy = PointerType::get(*C, 0);
456 
457   Value *ShadowPtr = memToShadow(AddrLong, IRB);
458   Value *ShadowAddr = IRB.CreateIntToPtr(ShadowPtr, ShadowPtrTy);
459   Value *ShadowValue = IRB.CreateLoad(ShadowTy, ShadowAddr);
460   // If we are profiling with histograms, add overflow protection at 255.
461   if (ClHistogram) {
462     Value *MaxCount = ConstantInt::get(Type::getInt8Ty(*C), 255);
463     Value *Cmp = IRB.CreateICmpULT(ShadowValue, MaxCount);
464     Instruction *IncBlock =
465         SplitBlockAndInsertIfThen(Cmp, InsertBefore, /*Unreachable=*/false);
466     IRB.SetInsertPoint(IncBlock);
467   }
468   Value *Inc = ConstantInt::get(ShadowTy, 1);
469   ShadowValue = IRB.CreateAdd(ShadowValue, Inc);
470   IRB.CreateStore(ShadowValue, ShadowAddr);
471 }
472 
473 // Create the variable for the profile file name.
createProfileFileNameVar(Module & M)474 void createProfileFileNameVar(Module &M) {
475   const MDString *MemProfFilename =
476       dyn_cast_or_null<MDString>(M.getModuleFlag("MemProfProfileFilename"));
477   if (!MemProfFilename)
478     return;
479   assert(!MemProfFilename->getString().empty() &&
480          "Unexpected MemProfProfileFilename metadata with empty string");
481   Constant *ProfileNameConst = ConstantDataArray::getString(
482       M.getContext(), MemProfFilename->getString(), true);
483   GlobalVariable *ProfileNameVar = new GlobalVariable(
484       M, ProfileNameConst->getType(), /*isConstant=*/true,
485       GlobalValue::WeakAnyLinkage, ProfileNameConst, MemProfFilenameVar);
486   const Triple &TT = M.getTargetTriple();
487   if (TT.supportsCOMDAT()) {
488     ProfileNameVar->setLinkage(GlobalValue::ExternalLinkage);
489     ProfileNameVar->setComdat(M.getOrInsertComdat(MemProfFilenameVar));
490   }
491 }
492 
493 // Set MemprofHistogramFlag as a Global veriable in IR. This makes it accessible
494 // to the runtime, changing shadow count behavior.
createMemprofHistogramFlagVar(Module & M)495 void createMemprofHistogramFlagVar(Module &M) {
496   const StringRef VarName(MemProfHistogramFlagVar);
497   Type *IntTy1 = Type::getInt1Ty(M.getContext());
498   auto MemprofHistogramFlag = new GlobalVariable(
499       M, IntTy1, true, GlobalValue::WeakAnyLinkage,
500       Constant::getIntegerValue(IntTy1, APInt(1, ClHistogram)), VarName);
501   const Triple &TT = M.getTargetTriple();
502   if (TT.supportsCOMDAT()) {
503     MemprofHistogramFlag->setLinkage(GlobalValue::ExternalLinkage);
504     MemprofHistogramFlag->setComdat(M.getOrInsertComdat(VarName));
505   }
506   appendToCompilerUsed(M, MemprofHistogramFlag);
507 }
508 
createMemprofDefaultOptionsVar(Module & M)509 void createMemprofDefaultOptionsVar(Module &M) {
510   Constant *OptionsConst = ConstantDataArray::getString(
511       M.getContext(), MemprofRuntimeDefaultOptions, /*AddNull=*/true);
512   GlobalVariable *OptionsVar =
513       new GlobalVariable(M, OptionsConst->getType(), /*isConstant=*/true,
514                          GlobalValue::WeakAnyLinkage, OptionsConst,
515                          memprof::getMemprofOptionsSymbolName());
516   const Triple &TT = M.getTargetTriple();
517   if (TT.supportsCOMDAT()) {
518     OptionsVar->setLinkage(GlobalValue::ExternalLinkage);
519     OptionsVar->setComdat(M.getOrInsertComdat(OptionsVar->getName()));
520   }
521 }
522 
instrumentModule(Module & M)523 bool ModuleMemProfiler::instrumentModule(Module &M) {
524 
525   // Create a module constructor.
526   std::string MemProfVersion = std::to_string(LLVM_MEM_PROFILER_VERSION);
527   std::string VersionCheckName =
528       ClInsertVersionCheck ? (MemProfVersionCheckNamePrefix + MemProfVersion)
529                            : "";
530   std::tie(MemProfCtorFunction, std::ignore) =
531       createSanitizerCtorAndInitFunctions(M, MemProfModuleCtorName,
532                                           MemProfInitName, /*InitArgTypes=*/{},
533                                           /*InitArgs=*/{}, VersionCheckName);
534 
535   const uint64_t Priority = getCtorAndDtorPriority(TargetTriple);
536   appendToGlobalCtors(M, MemProfCtorFunction, Priority);
537 
538   createProfileFileNameVar(M);
539 
540   createMemprofHistogramFlagVar(M);
541 
542   createMemprofDefaultOptionsVar(M);
543 
544   return true;
545 }
546 
initializeCallbacks(Module & M)547 void MemProfiler::initializeCallbacks(Module &M) {
548   IRBuilder<> IRB(*C);
549 
550   for (size_t AccessIsWrite = 0; AccessIsWrite <= 1; AccessIsWrite++) {
551     const std::string TypeStr = AccessIsWrite ? "store" : "load";
552     const std::string HistPrefix = ClHistogram ? "hist_" : "";
553 
554     SmallVector<Type *, 2> Args1{1, IntptrTy};
555     MemProfMemoryAccessCallback[AccessIsWrite] = M.getOrInsertFunction(
556         ClMemoryAccessCallbackPrefix + HistPrefix + TypeStr,
557         FunctionType::get(IRB.getVoidTy(), Args1, false));
558   }
559   MemProfMemmove = M.getOrInsertFunction(
560       ClMemoryAccessCallbackPrefix + "memmove", PtrTy, PtrTy, PtrTy, IntptrTy);
561   MemProfMemcpy = M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + "memcpy",
562                                         PtrTy, PtrTy, PtrTy, IntptrTy);
563   MemProfMemset =
564       M.getOrInsertFunction(ClMemoryAccessCallbackPrefix + "memset", PtrTy,
565                             PtrTy, IRB.getInt32Ty(), IntptrTy);
566 }
567 
maybeInsertMemProfInitAtFunctionEntry(Function & F)568 bool MemProfiler::maybeInsertMemProfInitAtFunctionEntry(Function &F) {
569   // For each NSObject descendant having a +load method, this method is invoked
570   // by the ObjC runtime before any of the static constructors is called.
571   // Therefore we need to instrument such methods with a call to __memprof_init
572   // at the beginning in order to initialize our runtime before any access to
573   // the shadow memory.
574   // We cannot just ignore these methods, because they may call other
575   // instrumented functions.
576   if (F.getName().contains(" load]")) {
577     FunctionCallee MemProfInitFunction =
578         declareSanitizerInitFunction(*F.getParent(), MemProfInitName, {});
579     IRBuilder<> IRB(&F.front(), F.front().begin());
580     IRB.CreateCall(MemProfInitFunction, {});
581     return true;
582   }
583   return false;
584 }
585 
insertDynamicShadowAtFunctionEntry(Function & F)586 bool MemProfiler::insertDynamicShadowAtFunctionEntry(Function &F) {
587   IRBuilder<> IRB(&F.front().front());
588   Value *GlobalDynamicAddress = F.getParent()->getOrInsertGlobal(
589       MemProfShadowMemoryDynamicAddress, IntptrTy);
590   if (F.getParent()->getPICLevel() == PICLevel::NotPIC)
591     cast<GlobalVariable>(GlobalDynamicAddress)->setDSOLocal(true);
592   DynamicShadowOffset = IRB.CreateLoad(IntptrTy, GlobalDynamicAddress);
593   return true;
594 }
595 
instrumentFunction(Function & F)596 bool MemProfiler::instrumentFunction(Function &F) {
597   if (F.getLinkage() == GlobalValue::AvailableExternallyLinkage)
598     return false;
599   if (ClDebugFunc == F.getName())
600     return false;
601   if (F.getName().starts_with("__memprof_"))
602     return false;
603 
604   bool FunctionModified = false;
605 
606   // If needed, insert __memprof_init.
607   // This function needs to be called even if the function body is not
608   // instrumented.
609   if (maybeInsertMemProfInitAtFunctionEntry(F))
610     FunctionModified = true;
611 
612   LLVM_DEBUG(dbgs() << "MEMPROF instrumenting:\n" << F << "\n");
613 
614   initializeCallbacks(*F.getParent());
615 
616   SmallVector<Instruction *, 16> ToInstrument;
617 
618   // Fill the set of memory operations to instrument.
619   for (auto &BB : F) {
620     for (auto &Inst : BB) {
621       if (isInterestingMemoryAccess(&Inst) || isa<MemIntrinsic>(Inst))
622         ToInstrument.push_back(&Inst);
623     }
624   }
625 
626   if (ToInstrument.empty()) {
627     LLVM_DEBUG(dbgs() << "MEMPROF done instrumenting: " << FunctionModified
628                       << " " << F << "\n");
629 
630     return FunctionModified;
631   }
632 
633   FunctionModified |= insertDynamicShadowAtFunctionEntry(F);
634 
635   int NumInstrumented = 0;
636   for (auto *Inst : ToInstrument) {
637     if (ClDebugMin < 0 || ClDebugMax < 0 ||
638         (NumInstrumented >= ClDebugMin && NumInstrumented <= ClDebugMax)) {
639       std::optional<InterestingMemoryAccess> Access =
640           isInterestingMemoryAccess(Inst);
641       if (Access)
642         instrumentMop(Inst, F.getDataLayout(), *Access);
643       else
644         instrumentMemIntrinsic(cast<MemIntrinsic>(Inst));
645     }
646     NumInstrumented++;
647   }
648 
649   if (NumInstrumented > 0)
650     FunctionModified = true;
651 
652   LLVM_DEBUG(dbgs() << "MEMPROF done instrumenting: " << FunctionModified << " "
653                     << F << "\n");
654 
655   return FunctionModified;
656 }
657