1 #include <stdint.h> 2 #include <stdlib.h> 3 #include <string.h> 4 5 #include "memprof_rawprofile.h" 6 #include "profile/MemProfData.inc" 7 #include "sanitizer_common/sanitizer_allocator_internal.h" 8 #include "sanitizer_common/sanitizer_array_ref.h" 9 #include "sanitizer_common/sanitizer_common.h" 10 #include "sanitizer_common/sanitizer_linux.h" 11 #include "sanitizer_common/sanitizer_procmaps.h" 12 #include "sanitizer_common/sanitizer_stackdepot.h" 13 #include "sanitizer_common/sanitizer_stackdepotbase.h" 14 #include "sanitizer_common/sanitizer_stacktrace.h" 15 #include "sanitizer_common/sanitizer_vector.h" 16 17 namespace __memprof { 18 using ::__sanitizer::Vector; 19 using ::llvm::memprof::MemInfoBlock; 20 using SegmentEntry = ::llvm::memprof::SegmentEntry; 21 using Header = ::llvm::memprof::Header; 22 23 namespace { 24 template <class T> char *WriteBytes(const T &Pod, char *Buffer) { 25 *(T *)Buffer = Pod; 26 return Buffer + sizeof(T); 27 } 28 29 void RecordStackId(const uptr Key, UNUSED LockedMemInfoBlock *const &MIB, 30 void *Arg) { 31 // No need to touch the MIB value here since we are only recording the key. 32 auto *StackIds = reinterpret_cast<Vector<u64> *>(Arg); 33 StackIds->PushBack(Key); 34 } 35 } // namespace 36 37 u64 SegmentSizeBytes(ArrayRef<LoadedModule> Modules) { 38 u64 NumSegmentsToRecord = 0; 39 for (const auto &Module : Modules) { 40 for (const auto &Segment : Module.ranges()) { 41 if (Segment.executable) 42 NumSegmentsToRecord++; 43 } 44 } 45 46 return sizeof(u64) // A header which stores the number of records. 47 + sizeof(SegmentEntry) * NumSegmentsToRecord; 48 } 49 50 // The segment section uses the following format: 51 // ---------- Segment Info 52 // Num Entries 53 // ---------- Segment Entry 54 // Start 55 // End 56 // Offset 57 // UuidSize 58 // Uuid 32B 59 // ---------- 60 // ... 61 void SerializeSegmentsToBuffer(ArrayRef<LoadedModule> Modules, 62 const u64 ExpectedNumBytes, char *&Buffer) { 63 char *Ptr = Buffer; 64 // Reserve space for the final count. 65 Ptr += sizeof(u64); 66 67 u64 NumSegmentsRecorded = 0; 68 69 for (const auto &Module : Modules) { 70 for (const auto &Segment : Module.ranges()) { 71 if (Segment.executable) { 72 SegmentEntry Entry(Segment.beg, Segment.end, Module.base_address()); 73 CHECK(Module.uuid_size() <= MEMPROF_BUILDID_MAX_SIZE); 74 Entry.BuildIdSize = Module.uuid_size(); 75 memcpy(Entry.BuildId, Module.uuid(), Module.uuid_size()); 76 memcpy(Ptr, &Entry, sizeof(SegmentEntry)); 77 Ptr += sizeof(SegmentEntry); 78 NumSegmentsRecorded++; 79 } 80 } 81 } 82 // Store the number of segments we recorded in the space we reserved. 83 *((u64 *)Buffer) = NumSegmentsRecorded; 84 CHECK(ExpectedNumBytes >= static_cast<u64>(Ptr - Buffer) && 85 "Expected num bytes != actual bytes written"); 86 } 87 88 u64 StackSizeBytes(const Vector<u64> &StackIds) { 89 u64 NumBytesToWrite = sizeof(u64); 90 91 const u64 NumIds = StackIds.Size(); 92 for (unsigned k = 0; k < NumIds; ++k) { 93 const u64 Id = StackIds[k]; 94 // One entry for the id and then one more for the number of stack pcs. 95 NumBytesToWrite += 2 * sizeof(u64); 96 const StackTrace St = StackDepotGet(Id); 97 98 CHECK(St.trace != nullptr && St.size > 0 && "Empty stack trace"); 99 for (uptr i = 0; i < St.size && St.trace[i] != 0; i++) { 100 NumBytesToWrite += sizeof(u64); 101 } 102 } 103 return NumBytesToWrite; 104 } 105 106 // The stack info section uses the following format: 107 // 108 // ---------- Stack Info 109 // Num Entries 110 // ---------- Stack Entry 111 // Num Stacks 112 // PC1 113 // PC2 114 // ... 115 // ---------- 116 void SerializeStackToBuffer(const Vector<u64> &StackIds, 117 const u64 ExpectedNumBytes, char *&Buffer) { 118 const u64 NumIds = StackIds.Size(); 119 char *Ptr = Buffer; 120 Ptr = WriteBytes(static_cast<u64>(NumIds), Ptr); 121 122 for (unsigned k = 0; k < NumIds; ++k) { 123 const u64 Id = StackIds[k]; 124 Ptr = WriteBytes(Id, Ptr); 125 Ptr += sizeof(u64); // Bump it by u64, we will fill this in later. 126 u64 Count = 0; 127 const StackTrace St = StackDepotGet(Id); 128 for (uptr i = 0; i < St.size && St.trace[i] != 0; i++) { 129 // PCs in stack traces are actually the return addresses, that is, 130 // addresses of the next instructions after the call. 131 uptr pc = StackTrace::GetPreviousInstructionPc(St.trace[i]); 132 Ptr = WriteBytes(static_cast<u64>(pc), Ptr); 133 ++Count; 134 } 135 // Store the count in the space we reserved earlier. 136 *(u64 *)(Ptr - (Count + 1) * sizeof(u64)) = Count; 137 } 138 139 CHECK(ExpectedNumBytes >= static_cast<u64>(Ptr - Buffer) && 140 "Expected num bytes != actual bytes written"); 141 } 142 143 // The MIB section has the following format: 144 // ---------- MIB Info 145 // Num Entries 146 // ---------- MIB Entry 0 147 // Alloc Count 148 // ... 149 // ---- AccessHistogram Entry 0 150 // ... 151 // ---- AccessHistogram Entry AccessHistogramSize - 1 152 // ---------- MIB Entry 1 153 // Alloc Count 154 // ... 155 // ---- AccessHistogram Entry 0 156 // ... 157 // ---- AccessHistogram Entry AccessHistogramSize - 1 158 // ---------- 159 void SerializeMIBInfoToBuffer(MIBMapTy &MIBMap, const Vector<u64> &StackIds, 160 const u64 ExpectedNumBytes, char *&Buffer) { 161 char *Ptr = Buffer; 162 const u64 NumEntries = StackIds.Size(); 163 Ptr = WriteBytes(NumEntries, Ptr); 164 for (u64 i = 0; i < NumEntries; i++) { 165 const u64 Key = StackIds[i]; 166 MIBMapTy::Handle h(&MIBMap, Key, /*remove=*/true, /*create=*/false); 167 CHECK(h.exists()); 168 Ptr = WriteBytes(Key, Ptr); 169 // FIXME: We unnecessarily serialize the AccessHistogram pointer. Adding a 170 // serialization schema will fix this issue. See also FIXME in 171 // deserialization. 172 Ptr = WriteBytes((*h)->mib, Ptr); 173 for (u64 j = 0; j < (*h)->mib.AccessHistogramSize; ++j) { 174 u64 HistogramEntry = ((u64 *)((*h)->mib.AccessHistogram))[j]; 175 Ptr = WriteBytes(HistogramEntry, Ptr); 176 } 177 if ((*h)->mib.AccessHistogramSize > 0) { 178 InternalFree((void *)((*h)->mib.AccessHistogram)); 179 } 180 } 181 CHECK(ExpectedNumBytes >= static_cast<u64>(Ptr - Buffer) && 182 "Expected num bytes != actual bytes written"); 183 } 184 185 // Format 186 // ---------- Header 187 // Magic 188 // Version 189 // Total Size 190 // Segment Offset 191 // MIB Info Offset 192 // Stack Offset 193 // ---------- Segment Info 194 // Num Entries 195 // ---------- Segment Entry 196 // Start 197 // End 198 // Offset 199 // BuildID 32B 200 // ---------- 201 // ... 202 // ---------- 203 // Optional Padding Bytes 204 // ---------- MIB Info 205 // Num Entries 206 // ---------- MIB Entry 207 // Alloc Count 208 // ... 209 // ---- AccessHistogram Entry 0 210 // ... 211 // ---- AccessHistogram Entry AccessHistogramSize - 1 212 // ---------- MIB Entry 1 213 // Alloc Count 214 // ... 215 // ---- AccessHistogram Entry 0 216 // ... 217 // ---- AccessHistogram Entry AccessHistogramSize - 1 218 // Optional Padding Bytes 219 // ---------- Stack Info 220 // Num Entries 221 // ---------- Stack Entry 222 // Num Stacks 223 // PC1 224 // PC2 225 // ... 226 // ---------- 227 // Optional Padding Bytes 228 // ... 229 u64 SerializeToRawProfile(MIBMapTy &MIBMap, ArrayRef<LoadedModule> Modules, 230 char *&Buffer) { 231 // Each section size is rounded up to 8b since the first entry in each section 232 // is a u64 which holds the number of entries in the section by convention. 233 const u64 NumSegmentBytes = RoundUpTo(SegmentSizeBytes(Modules), 8); 234 235 Vector<u64> StackIds; 236 MIBMap.ForEach(RecordStackId, reinterpret_cast<void *>(&StackIds)); 237 // The first 8b are for the total number of MIB records. Each MIB record is 238 // preceded by a 8b stack id which is associated with stack frames in the next 239 // section. 240 const u64 NumMIBInfoBytes = RoundUpTo( 241 sizeof(u64) + StackIds.Size() * (sizeof(u64) + sizeof(MemInfoBlock)), 8); 242 243 // Get Number of AccessHistogram entries in total 244 u64 TotalAccessHistogramEntries = 0; 245 MIBMap.ForEach( 246 [](const uptr Key, UNUSED LockedMemInfoBlock *const &MIB, void *Arg) { 247 u64 *TotalAccessHistogramEntries = (u64 *)Arg; 248 *TotalAccessHistogramEntries += MIB->mib.AccessHistogramSize; 249 }, 250 reinterpret_cast<void *>(&TotalAccessHistogramEntries)); 251 const u64 NumHistogramBytes = 252 RoundUpTo(TotalAccessHistogramEntries * sizeof(uint64_t), 8); 253 254 const u64 NumStackBytes = RoundUpTo(StackSizeBytes(StackIds), 8); 255 256 // Ensure that the profile is 8b aligned. We allow for some optional padding 257 // at the end so that any subsequent profile serialized to the same file does 258 // not incur unaligned accesses. 259 const u64 TotalSizeBytes = 260 RoundUpTo(sizeof(Header) + NumSegmentBytes + NumStackBytes + 261 NumMIBInfoBytes + NumHistogramBytes, 262 8); 263 264 // Allocate the memory for the entire buffer incl. info blocks. 265 Buffer = (char *)InternalAlloc(TotalSizeBytes); 266 char *Ptr = Buffer; 267 268 Header header{MEMPROF_RAW_MAGIC_64, 269 MEMPROF_RAW_VERSION, 270 static_cast<u64>(TotalSizeBytes), 271 sizeof(Header), 272 sizeof(Header) + NumSegmentBytes, 273 sizeof(Header) + NumSegmentBytes + NumMIBInfoBytes + 274 NumHistogramBytes}; 275 Ptr = WriteBytes(header, Ptr); 276 277 SerializeSegmentsToBuffer(Modules, NumSegmentBytes, Ptr); 278 Ptr += NumSegmentBytes; 279 280 SerializeMIBInfoToBuffer(MIBMap, StackIds, 281 NumMIBInfoBytes + NumHistogramBytes, Ptr); 282 Ptr += NumMIBInfoBytes + NumHistogramBytes; 283 284 SerializeStackToBuffer(StackIds, NumStackBytes, Ptr); 285 286 return TotalSizeBytes; 287 } 288 289 } // namespace __memprof 290