1 //===-- tsan_platform.h -----------------------------------------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file is a part of ThreadSanitizer (TSan), a race detector. 10 // 11 // Platform-specific code. 12 //===----------------------------------------------------------------------===// 13 14 #ifndef TSAN_PLATFORM_H 15 #define TSAN_PLATFORM_H 16 17 #if !defined(__LP64__) && !defined(_WIN64) 18 # error "Only 64-bit is supported" 19 #endif 20 21 #include "sanitizer_common/sanitizer_common.h" 22 #include "tsan_defs.h" 23 24 namespace __tsan { 25 26 enum { 27 // App memory is not mapped onto shadow memory range. 28 kBrokenMapping = 1 << 0, 29 // Mapping app memory and back does not produce the same address, 30 // this can lead to wrong addresses in reports and potentially 31 // other bad consequences. 32 kBrokenReverseMapping = 1 << 1, 33 // Mapping is non-linear for linear user range. 34 // This is bad and can lead to unpredictable memory corruptions, etc 35 // because range access functions assume linearity. 36 kBrokenLinearity = 1 << 2, 37 // Meta for an app region overlaps with the meta of another app region. 38 // This is determined by recomputing the individual meta regions for 39 // each app region. 40 // 41 // N.B. There is no "kBrokenReverseMetaMapping" constant because there 42 // is no MetaToMem function. However, note that (!kBrokenLinearity 43 // && !kBrokenAliasedMetas) implies that MemToMeta is invertible. 44 kBrokenAliasedMetas = 1 << 3, 45 }; 46 47 /* 48 C/C++ on linux/x86_64 and freebsd/x86_64 49 0000 0000 1000 - 0200 0000 0000: main binary and/or MAP_32BIT mappings (2TB) 50 0200 0000 0000 - 1000 0000 0000: - 51 1000 0000 0000 - 3000 0000 0000: shadow (32TB) 52 3000 0000 0000 - 3800 0000 0000: metainfo (memory blocks and sync objects; 8TB) 53 3800 0000 0000 - 5500 0000 0000: - 54 5500 0000 0000 - 5a00 0000 0000: pie binaries without ASLR or on 4.1+ kernels 55 5a00 0000 0000 - 7200 0000 0000: - 56 7200 0000 0000 - 7300 0000 0000: heap (1TB) 57 7300 0000 0000 - 7a00 0000 0000: - 58 7a00 0000 0000 - 8000 0000 0000: modules and main thread stack (6TB) 59 60 C/C++ on netbsd/amd64 can reuse the same mapping: 61 * The address space starts from 0x1000 (option with 0x0) and ends with 62 0x7f7ffffff000. 63 * LoAppMem-kHeapMemEnd can be reused as it is. 64 * No VDSO support. 65 * No MidAppMem region. 66 * No additional HeapMem region. 67 * HiAppMem contains the stack, loader, shared libraries and heap. 68 * Stack on NetBSD/amd64 has prereserved 128MB. 69 * Heap grows downwards (top-down). 70 * ASLR must be disabled per-process or globally. 71 */ 72 struct Mapping48AddressSpace { 73 static const uptr kMetaShadowBeg = 0x300000000000ull; 74 static const uptr kMetaShadowEnd = 0x380000000000ull; 75 static const uptr kShadowBeg = 0x100000000000ull; 76 static const uptr kShadowEnd = 0x300000000000ull; 77 static const uptr kHeapMemBeg = 0x720000000000ull; 78 static const uptr kHeapMemEnd = 0x730000000000ull; 79 static const uptr kLoAppMemBeg = 0x000000001000ull; 80 static const uptr kLoAppMemEnd = 0x020000000000ull; 81 static const uptr kMidAppMemBeg = 0x550000000000ull; 82 static const uptr kMidAppMemEnd = 0x5a0000000000ull; 83 static const uptr kHiAppMemBeg = 0x7a0000000000ull; 84 static const uptr kHiAppMemEnd = 0x800000000000ull; 85 static const uptr kShadowMsk = 0x700000000000ull; 86 static const uptr kShadowXor = 0x000000000000ull; 87 static const uptr kShadowAdd = 0x100000000000ull; 88 static const uptr kVdsoBeg = 0xf000000000000000ull; 89 }; 90 91 /* 92 C/C++ on linux/mips64 (40-bit VMA) 93 0000 0000 00 - 0100 0000 00: - (4 GB) 94 0100 0000 00 - 0200 0000 00: main binary (4 GB) 95 0200 0000 00 - 1200 0000 00: - (64 GB) 96 1200 0000 00 - 2200 0000 00: shadow (64 GB) 97 2200 0000 00 - 4000 0000 00: - (120 GB) 98 4000 0000 00 - 5000 0000 00: metainfo (memory blocks and sync objects) (64 GB) 99 5000 0000 00 - aa00 0000 00: - (360 GB) 100 aa00 0000 00 - ab00 0000 00: main binary (PIE) (4 GB) 101 ab00 0000 00 - fe00 0000 00: - (332 GB) 102 fe00 0000 00 - ff00 0000 00: heap (4 GB) 103 ff00 0000 00 - ff80 0000 00: - (2 GB) 104 ff80 0000 00 - ffff ffff ff: modules and main thread stack (<2 GB) 105 */ 106 struct MappingMips64_40 { 107 static const uptr kMetaShadowBeg = 0x4000000000ull; 108 static const uptr kMetaShadowEnd = 0x5000000000ull; 109 static const uptr kShadowBeg = 0x1200000000ull; 110 static const uptr kShadowEnd = 0x2200000000ull; 111 static const uptr kHeapMemBeg = 0xfe00000000ull; 112 static const uptr kHeapMemEnd = 0xff00000000ull; 113 static const uptr kLoAppMemBeg = 0x0100000000ull; 114 static const uptr kLoAppMemEnd = 0x0200000000ull; 115 static const uptr kMidAppMemBeg = 0xaa00000000ull; 116 static const uptr kMidAppMemEnd = 0xab00000000ull; 117 static const uptr kHiAppMemBeg = 0xff80000000ull; 118 static const uptr kHiAppMemEnd = 0xffffffffffull; 119 static const uptr kShadowMsk = 0xf800000000ull; 120 static const uptr kShadowXor = 0x0800000000ull; 121 static const uptr kShadowAdd = 0x0000000000ull; 122 static const uptr kVdsoBeg = 0xfffff00000ull; 123 }; 124 125 /* 126 C/C++ on Darwin/iOS/ARM64 (36-bit VMA, 64 GB VM) 127 0000 0000 00 - 0100 0000 00: - (4 GB) 128 0100 0000 00 - 0200 0000 00: main binary, modules, thread stacks (4 GB) 129 0200 0000 00 - 0300 0000 00: heap (4 GB) 130 0300 0000 00 - 0400 0000 00: - (4 GB) 131 0400 0000 00 - 0800 0000 00: shadow memory (16 GB) 132 0800 0000 00 - 0d00 0000 00: - (20 GB) 133 0d00 0000 00 - 0e00 0000 00: metainfo (4 GB) 134 0e00 0000 00 - 1000 0000 00: - 135 */ 136 struct MappingAppleAarch64 { 137 static const uptr kLoAppMemBeg = 0x0100000000ull; 138 static const uptr kLoAppMemEnd = 0x0200000000ull; 139 static const uptr kHeapMemBeg = 0x0200000000ull; 140 static const uptr kHeapMemEnd = 0x0300000000ull; 141 static const uptr kShadowBeg = 0x0400000000ull; 142 static const uptr kShadowEnd = 0x0800000000ull; 143 static const uptr kMetaShadowBeg = 0x0d00000000ull; 144 static const uptr kMetaShadowEnd = 0x0e00000000ull; 145 static const uptr kHiAppMemBeg = 0x0fc0000000ull; 146 static const uptr kHiAppMemEnd = 0x0fc0000000ull; 147 static const uptr kShadowMsk = 0x0ull; 148 static const uptr kShadowXor = 0x0ull; 149 static const uptr kShadowAdd = 0x0200000000ull; 150 static const uptr kVdsoBeg = 0x7000000000000000ull; 151 static const uptr kMidAppMemBeg = 0; 152 static const uptr kMidAppMemEnd = 0; 153 }; 154 155 /* 156 C/C++ on linux/aarch64 (39-bit VMA) 157 0000 0010 00 - 0500 0000 00: main binary (20 GB) 158 0100 0000 00 - 2000 0000 00: - 159 2000 0000 00 - 4000 0000 00: shadow memory (128 GB) 160 4000 0000 00 - 4800 0000 00: metainfo (32 GB) 161 4800 0000 00 - 5500 0000 00: - 162 5500 0000 00 - 5a00 0000 00: main binary (PIE) (20 GB) 163 5600 0000 00 - 7c00 0000 00: - 164 7a00 0000 00 - 7d00 0000 00: heap (12 GB) 165 7d00 0000 00 - 7fff ffff ff: modules and main thread stack (12 GB) 166 */ 167 struct MappingAarch64_39 { 168 static const uptr kLoAppMemBeg = 0x0000001000ull; 169 static const uptr kLoAppMemEnd = 0x0500000000ull; 170 static const uptr kShadowBeg = 0x2000000000ull; 171 static const uptr kShadowEnd = 0x4000000000ull; 172 static const uptr kMetaShadowBeg = 0x4000000000ull; 173 static const uptr kMetaShadowEnd = 0x4800000000ull; 174 static const uptr kMidAppMemBeg = 0x5500000000ull; 175 static const uptr kMidAppMemEnd = 0x5a00000000ull; 176 static const uptr kHeapMemBeg = 0x7a00000000ull; 177 static const uptr kHeapMemEnd = 0x7d00000000ull; 178 static const uptr kHiAppMemBeg = 0x7d00000000ull; 179 static const uptr kHiAppMemEnd = 0x7fffffffffull; 180 static const uptr kShadowMsk = 0x7000000000ull; 181 static const uptr kShadowXor = 0x1000000000ull; 182 static const uptr kShadowAdd = 0x0000000000ull; 183 static const uptr kVdsoBeg = 0x7f00000000ull; 184 }; 185 186 /* 187 C/C++ on linux/aarch64 (42-bit VMA) 188 00000 0010 00 - 02000 0000 00: main binary (128 GB) 189 02000 0000 00 - 08000 0000 00: - 190 10000 0000 00 - 20000 0000 00: shadow memory (1024 GB) 191 20000 0000 00 - 24000 0000 00: metainfo (256 GB) 192 24000 0000 00 - 2aa00 0000 00: - 193 2aa00 0000 00 - 2c000 0000 00: main binary (PIE) (88 GB) 194 2c000 0000 00 - 3c000 0000 00: - 195 3c000 0000 00 - 3f000 0000 00: heap (192 GB) 196 3f000 0000 00 - 3ffff ffff ff: modules and main thread stack (64 GB) 197 */ 198 struct MappingAarch64_42 { 199 static const uptr kLoAppMemBeg = 0x00000001000ull; 200 static const uptr kLoAppMemEnd = 0x02000000000ull; 201 static const uptr kShadowBeg = 0x10000000000ull; 202 static const uptr kShadowEnd = 0x20000000000ull; 203 static const uptr kMetaShadowBeg = 0x20000000000ull; 204 static const uptr kMetaShadowEnd = 0x24000000000ull; 205 static const uptr kMidAppMemBeg = 0x2aa00000000ull; 206 static const uptr kMidAppMemEnd = 0x2c000000000ull; 207 static const uptr kHeapMemBeg = 0x3c000000000ull; 208 static const uptr kHeapMemEnd = 0x3f000000000ull; 209 static const uptr kHiAppMemBeg = 0x3f000000000ull; 210 static const uptr kHiAppMemEnd = 0x3ffffffffffull; 211 static const uptr kShadowMsk = 0x38000000000ull; 212 static const uptr kShadowXor = 0x08000000000ull; 213 static const uptr kShadowAdd = 0x00000000000ull; 214 static const uptr kVdsoBeg = 0x37f00000000ull; 215 }; 216 217 /* 218 C/C++ on linux/aarch64 (48-bit VMA) 219 0000 0000 1000 - 0a00 0000 0000: main binary (10240 GB) 220 0a00 0000 1000 - 1554 0000 0000: - 221 1554 0000 1000 - 5400 0000 0000: shadow memory (64176 GB) 222 5400 0000 1000 - 8000 0000 0000: - 223 8000 0000 1000 - 0a00 0000 0000: metainfo (32768 GB) 224 a000 0000 1000 - aaaa 0000 0000: - 225 aaaa 0000 1000 - ac00 0000 0000: main binary (PIE) (1368 GB) 226 ac00 0000 1000 - fc00 0000 0000: - 227 fc00 0000 1000 - ffff ffff ffff: modules and main thread stack (4096 GB) 228 229 N.B. the shadow memory region has a strange start address, because it 230 contains the shadows for the mid, high and low app regions (in this 231 unusual order). 232 */ 233 struct MappingAarch64_48 { 234 static const uptr kLoAppMemBeg = 0x0000000001000ull; 235 static const uptr kLoAppMemEnd = 0x00a0000000000ull; 236 static const uptr kShadowBeg = 0x0155400000000ull; 237 static const uptr kShadowEnd = 0x0540000000000ull; 238 static const uptr kMetaShadowBeg = 0x0800000000000ull; 239 static const uptr kMetaShadowEnd = 0x0a00000000000ull; 240 static const uptr kMidAppMemBeg = 0x0aaaa00000000ull; 241 static const uptr kMidAppMemEnd = 0x0ac0000000000ull; 242 static const uptr kHiAppMemBeg = 0x0fc0000000000ull; 243 static const uptr kHiAppMemEnd = 0x1000000000000ull; 244 static const uptr kHeapMemBeg = 0x0fc0000000000ull; 245 static const uptr kHeapMemEnd = 0x0fc0000000000ull; 246 static const uptr kShadowMsk = 0x0c00000000000ull; 247 static const uptr kShadowXor = 0x0200000000000ull; 248 static const uptr kShadowAdd = 0x0000000000000ull; 249 static const uptr kVdsoBeg = 0xffff000000000ull; 250 }; 251 252 /* C/C++ on linux/loongarch64 (47-bit VMA) 253 0000 0000 4000 - 0080 0000 0000: main binary 254 0080 0000 0000 - 0100 0000 0000: - 255 0100 0000 0000 - 1000 0000 0000: shadow memory 256 1000 0000 0000 - 3000 0000 0000: - 257 3000 0000 0000 - 3400 0000 0000: metainfo 258 3400 0000 0000 - 5555 0000 0000: - 259 5555 0000 0000 - 5556 0000 0000: main binary (PIE) 260 5556 0000 0000 - 7ffe 0000 0000: - 261 7ffe 0000 0000 - 7fff 0000 0000: heap 262 7fff 0000 0000 - 7fff 8000 0000: - 263 7fff 8000 0000 - 8000 0000 0000: modules and main thread stack 264 */ 265 struct MappingLoongArch64_47 { 266 static const uptr kMetaShadowBeg = 0x300000000000ull; 267 static const uptr kMetaShadowEnd = 0x340000000000ull; 268 static const uptr kShadowBeg = 0x010000000000ull; 269 static const uptr kShadowEnd = 0x100000000000ull; 270 static const uptr kHeapMemBeg = 0x7ffe00000000ull; 271 static const uptr kHeapMemEnd = 0x7fff00000000ull; 272 static const uptr kLoAppMemBeg = 0x000000004000ull; 273 static const uptr kLoAppMemEnd = 0x008000000000ull; 274 static const uptr kMidAppMemBeg = 0x555500000000ull; 275 static const uptr kMidAppMemEnd = 0x555600000000ull; 276 static const uptr kHiAppMemBeg = 0x7fff80000000ull; 277 static const uptr kHiAppMemEnd = 0x800000000000ull; 278 static const uptr kShadowMsk = 0x780000000000ull; 279 static const uptr kShadowXor = 0x040000000000ull; 280 static const uptr kShadowAdd = 0x000000000000ull; 281 static const uptr kVdsoBeg = 0x7fffffffc000ull; 282 }; 283 284 /* 285 C/C++ on linux/powerpc64 (44-bit VMA) 286 0000 0000 0100 - 0001 0000 0000: main binary 287 0001 0000 0000 - 0001 0000 0000: - 288 0001 0000 0000 - 0b00 0000 0000: shadow 289 0b00 0000 0000 - 0b00 0000 0000: - 290 0b00 0000 0000 - 0d00 0000 0000: metainfo (memory blocks and sync objects) 291 0d00 0000 0000 - 0f00 0000 0000: - 292 0f00 0000 0000 - 0f50 0000 0000: heap 293 0f50 0000 0000 - 0f60 0000 0000: - 294 0f60 0000 0000 - 1000 0000 0000: modules and main thread stack 295 */ 296 struct MappingPPC64_44 { 297 static const uptr kBroken = kBrokenMapping | kBrokenReverseMapping | 298 kBrokenLinearity | kBrokenAliasedMetas; 299 static const uptr kMetaShadowBeg = 0x0b0000000000ull; 300 static const uptr kMetaShadowEnd = 0x0d0000000000ull; 301 static const uptr kShadowBeg = 0x000100000000ull; 302 static const uptr kShadowEnd = 0x0b0000000000ull; 303 static const uptr kLoAppMemBeg = 0x000000000100ull; 304 static const uptr kLoAppMemEnd = 0x000100000000ull; 305 static const uptr kHeapMemBeg = 0x0f0000000000ull; 306 static const uptr kHeapMemEnd = 0x0f5000000000ull; 307 static const uptr kHiAppMemBeg = 0x0f6000000000ull; 308 static const uptr kHiAppMemEnd = 0x100000000000ull; // 44 bits 309 static const uptr kShadowMsk = 0x0f0000000000ull; 310 static const uptr kShadowXor = 0x002100000000ull; 311 static const uptr kShadowAdd = 0x000000000000ull; 312 static const uptr kVdsoBeg = 0x3c0000000000000ull; 313 static const uptr kMidAppMemBeg = 0; 314 static const uptr kMidAppMemEnd = 0; 315 }; 316 317 /* 318 C/C++ on linux/powerpc64 (46-bit VMA) 319 0000 0000 1000 - 0100 0000 0000: main binary 320 0100 0000 0000 - 0200 0000 0000: - 321 0100 0000 0000 - 0800 0000 0000: shadow 322 0800 0000 0000 - 1000 0000 0000: - 323 1000 0000 0000 - 1200 0000 0000: metainfo (memory blocks and sync objects) 324 1200 0000 0000 - 3d00 0000 0000: - 325 3d00 0000 0000 - 3e00 0000 0000: heap 326 3e00 0000 0000 - 3e80 0000 0000: - 327 3e80 0000 0000 - 4000 0000 0000: modules and main thread stack 328 */ 329 struct MappingPPC64_46 { 330 static const uptr kMetaShadowBeg = 0x100000000000ull; 331 static const uptr kMetaShadowEnd = 0x120000000000ull; 332 static const uptr kShadowBeg = 0x010000000000ull; 333 static const uptr kShadowEnd = 0x080000000000ull; 334 static const uptr kHeapMemBeg = 0x3d0000000000ull; 335 static const uptr kHeapMemEnd = 0x3e0000000000ull; 336 static const uptr kLoAppMemBeg = 0x000000001000ull; 337 static const uptr kLoAppMemEnd = 0x010000000000ull; 338 static const uptr kHiAppMemBeg = 0x3e8000000000ull; 339 static const uptr kHiAppMemEnd = 0x400000000000ull; // 46 bits 340 static const uptr kShadowMsk = 0x3c0000000000ull; 341 static const uptr kShadowXor = 0x020000000000ull; 342 static const uptr kShadowAdd = 0x000000000000ull; 343 static const uptr kVdsoBeg = 0x7800000000000000ull; 344 static const uptr kMidAppMemBeg = 0; 345 static const uptr kMidAppMemEnd = 0; 346 }; 347 348 /* 349 C/C++ on linux/powerpc64 (47-bit VMA) 350 0000 0000 1000 - 0100 0000 0000: main binary 351 0100 0000 0000 - 0200 0000 0000: - 352 0100 0000 0000 - 0800 0000 0000: shadow 353 0800 0000 0000 - 1000 0000 0000: - 354 1000 0000 0000 - 1200 0000 0000: metainfo (memory blocks and sync objects) 355 1200 0000 0000 - 7d00 0000 0000: - 356 7d00 0000 0000 - 7e00 0000 0000: heap 357 7e00 0000 0000 - 7e80 0000 0000: - 358 7e80 0000 0000 - 8000 0000 0000: modules and main thread stack 359 */ 360 struct MappingPPC64_47 { 361 static const uptr kMetaShadowBeg = 0x100000000000ull; 362 static const uptr kMetaShadowEnd = 0x120000000000ull; 363 static const uptr kShadowBeg = 0x010000000000ull; 364 static const uptr kShadowEnd = 0x080000000000ull; 365 static const uptr kHeapMemBeg = 0x7d0000000000ull; 366 static const uptr kHeapMemEnd = 0x7e0000000000ull; 367 static const uptr kLoAppMemBeg = 0x000000001000ull; 368 static const uptr kLoAppMemEnd = 0x010000000000ull; 369 static const uptr kHiAppMemBeg = 0x7e8000000000ull; 370 static const uptr kHiAppMemEnd = 0x800000000000ull; // 47 bits 371 static const uptr kShadowMsk = 0x7c0000000000ull; 372 static const uptr kShadowXor = 0x020000000000ull; 373 static const uptr kShadowAdd = 0x000000000000ull; 374 static const uptr kVdsoBeg = 0x7800000000000000ull; 375 static const uptr kMidAppMemBeg = 0; 376 static const uptr kMidAppMemEnd = 0; 377 }; 378 379 /* 380 C/C++ on linux/riscv64 (39-bit VMA) 381 0000 0010 00 - 0200 0000 00: main binary ( 8 GB) 382 0200 0000 00 - 1000 0000 00: - 383 1000 0000 00 - 4000 0000 00: shadow memory (64 GB) 384 4000 0000 00 - 4800 0000 00: metainfo (16 GB) 385 4800 0000 00 - 5500 0000 00: - 386 5500 0000 00 - 5a00 0000 00: main binary (PIE) (~8 GB) 387 5600 0000 00 - 7c00 0000 00: - 388 7d00 0000 00 - 7fff ffff ff: libraries and main thread stack ( 8 GB) 389 390 mmap by default allocates from top downwards 391 VDSO sits below loader and above dynamic libraries, within HiApp region. 392 Heap starts after program region whose position depends on pie or non-pie. 393 Disable tracking them since their locations are not fixed. 394 */ 395 struct MappingRiscv64_39 { 396 static const uptr kLoAppMemBeg = 0x0000001000ull; 397 static const uptr kLoAppMemEnd = 0x0200000000ull; 398 static const uptr kShadowBeg = 0x1000000000ull; 399 static const uptr kShadowEnd = 0x2000000000ull; 400 static const uptr kMetaShadowBeg = 0x2000000000ull; 401 static const uptr kMetaShadowEnd = 0x2400000000ull; 402 static const uptr kMidAppMemBeg = 0x2aaaaaa000ull; 403 static const uptr kMidAppMemEnd = 0x2c00000000ull; 404 static const uptr kHeapMemBeg = 0x2c00000000ull; 405 static const uptr kHeapMemEnd = 0x2c00000000ull; 406 static const uptr kHiAppMemBeg = 0x3c00000000ull; 407 static const uptr kHiAppMemEnd = 0x3fffffffffull; 408 static const uptr kShadowMsk = 0x3800000000ull; 409 static const uptr kShadowXor = 0x0800000000ull; 410 static const uptr kShadowAdd = 0x0000000000ull; 411 static const uptr kVdsoBeg = 0x4000000000ull; 412 }; 413 414 /* 415 C/C++ on linux/riscv64 (48-bit VMA) 416 0000 0000 1000 - 0400 0000 0000: main binary ( 4 TB) 417 0500 0000 0000 - 2000 0000 0000: - 418 2000 0000 0000 - 4000 0000 0000: shadow memory (32 TB) 419 4000 0000 0000 - 4800 0000 0000: metainfo ( 8 TB) 420 4800 0000 0000 - 5555 5555 5000: - 421 5555 5555 5000 - 5a00 0000 0000: main binary (PIE) (~5 TB) 422 5a00 0000 0000 - 7a00 0000 0000: - 423 7a00 0000 0000 - 7fff ffff ffff: libraries and main thread stack ( 6 TB) 424 */ 425 struct MappingRiscv64_48 { 426 static const uptr kLoAppMemBeg = 0x000000001000ull; 427 static const uptr kLoAppMemEnd = 0x040000000000ull; 428 static const uptr kShadowBeg = 0x200000000000ull; 429 static const uptr kShadowEnd = 0x400000000000ull; 430 static const uptr kMetaShadowBeg = 0x400000000000ull; 431 static const uptr kMetaShadowEnd = 0x480000000000ull; 432 static const uptr kMidAppMemBeg = 0x555555555000ull; 433 static const uptr kMidAppMemEnd = 0x5a0000000000ull; 434 static const uptr kHeapMemBeg = 0x5a0000000000ull; 435 static const uptr kHeapMemEnd = 0x5a0000000000ull; 436 static const uptr kHiAppMemBeg = 0x7a0000000000ull; 437 static const uptr kHiAppMemEnd = 0x7fffffffffffull; 438 static const uptr kShadowMsk = 0x700000000000ull; 439 static const uptr kShadowXor = 0x100000000000ull; 440 static const uptr kShadowAdd = 0x000000000000ull; 441 static const uptr kVdsoBeg = 0x800000000000ull; 442 }; 443 444 /* 445 C/C++ on linux/s390x 446 While the kernel provides a 64-bit address space, we have to restrict ourselves 447 to 48 bits due to how e.g. SyncVar::GetId() works. 448 0000 0000 1000 - 0e00 0000 0000: binary, modules, stacks - 14 TiB 449 0e00 0000 0000 - 2000 0000 0000: - 450 2000 0000 0000 - 4000 0000 0000: shadow - 32TiB (2 * app) 451 4000 0000 0000 - 9000 0000 0000: - 452 9000 0000 0000 - 9800 0000 0000: metainfo - 8TiB (0.5 * app) 453 9800 0000 0000 - be00 0000 0000: - 454 be00 0000 0000 - c000 0000 0000: heap - 2TiB (max supported by the allocator) 455 */ 456 struct MappingS390x { 457 static const uptr kMetaShadowBeg = 0x900000000000ull; 458 static const uptr kMetaShadowEnd = 0x980000000000ull; 459 static const uptr kShadowBeg = 0x200000000000ull; 460 static const uptr kShadowEnd = 0x400000000000ull; 461 static const uptr kHeapMemBeg = 0xbe0000000000ull; 462 static const uptr kHeapMemEnd = 0xc00000000000ull; 463 static const uptr kLoAppMemBeg = 0x000000001000ull; 464 static const uptr kLoAppMemEnd = 0x0e0000000000ull; 465 static const uptr kHiAppMemBeg = 0xc00000004000ull; 466 static const uptr kHiAppMemEnd = 0xc00000004000ull; 467 static const uptr kShadowMsk = 0xb00000000000ull; 468 static const uptr kShadowXor = 0x100000000000ull; 469 static const uptr kShadowAdd = 0x000000000000ull; 470 static const uptr kVdsoBeg = 0xfffffffff000ull; 471 static const uptr kMidAppMemBeg = 0; 472 static const uptr kMidAppMemEnd = 0; 473 }; 474 475 /* Go on linux, darwin and freebsd on x86_64 476 0000 0000 1000 - 0000 1000 0000: executable 477 0000 1000 0000 - 00c0 0000 0000: - 478 00c0 0000 0000 - 00e0 0000 0000: heap 479 00e0 0000 0000 - 2000 0000 0000: - 480 2000 0000 0000 - 21c0 0000 0000: shadow 481 21c0 0000 0000 - 3000 0000 0000: - 482 3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects) 483 4000 0000 0000 - 8000 0000 0000: - 484 */ 485 486 struct MappingGo48 { 487 static const uptr kMetaShadowBeg = 0x300000000000ull; 488 static const uptr kMetaShadowEnd = 0x400000000000ull; 489 static const uptr kShadowBeg = 0x200000000000ull; 490 static const uptr kShadowEnd = 0x21c000000000ull; 491 static const uptr kLoAppMemBeg = 0x000000001000ull; 492 static const uptr kLoAppMemEnd = 0x00e000000000ull; 493 static const uptr kMidAppMemBeg = 0; 494 static const uptr kMidAppMemEnd = 0; 495 static const uptr kHiAppMemBeg = 0; 496 static const uptr kHiAppMemEnd = 0; 497 static const uptr kHeapMemBeg = 0; 498 static const uptr kHeapMemEnd = 0; 499 static const uptr kVdsoBeg = 0; 500 static const uptr kShadowMsk = 0; 501 static const uptr kShadowXor = 0; 502 static const uptr kShadowAdd = 0x200000000000ull; 503 }; 504 505 /* Go on windows 506 0000 0000 1000 - 0000 1000 0000: executable 507 0000 1000 0000 - 00f8 0000 0000: - 508 00c0 0000 0000 - 00e0 0000 0000: heap 509 00e0 0000 0000 - 0100 0000 0000: - 510 0100 0000 0000 - 0300 0000 0000: shadow 511 0300 0000 0000 - 0700 0000 0000: - 512 0700 0000 0000 - 0770 0000 0000: metainfo (memory blocks and sync objects) 513 07d0 0000 0000 - 8000 0000 0000: - 514 PIE binaries currently not supported, but it should be theoretically possible. 515 */ 516 517 struct MappingGoWindows { 518 static const uptr kMetaShadowBeg = 0x070000000000ull; 519 static const uptr kMetaShadowEnd = 0x077000000000ull; 520 static const uptr kShadowBeg = 0x010000000000ull; 521 static const uptr kShadowEnd = 0x030000000000ull; 522 static const uptr kLoAppMemBeg = 0x000000001000ull; 523 static const uptr kLoAppMemEnd = 0x00e000000000ull; 524 static const uptr kMidAppMemBeg = 0; 525 static const uptr kMidAppMemEnd = 0; 526 static const uptr kHiAppMemBeg = 0; 527 static const uptr kHiAppMemEnd = 0; 528 static const uptr kHeapMemBeg = 0; 529 static const uptr kHeapMemEnd = 0; 530 static const uptr kVdsoBeg = 0; 531 static const uptr kShadowMsk = 0; 532 static const uptr kShadowXor = 0; 533 static const uptr kShadowAdd = 0x010000000000ull; 534 }; 535 536 /* Go on linux/powerpc64 (46-bit VMA) 537 0000 0000 1000 - 0000 1000 0000: executable 538 0000 1000 0000 - 00c0 0000 0000: - 539 00c0 0000 0000 - 00e0 0000 0000: heap 540 00e0 0000 0000 - 2000 0000 0000: - 541 2000 0000 0000 - 21c0 0000 0000: shadow 542 21c0 0000 0000 - 2400 0000 0000: - 543 2400 0000 0000 - 2470 0000 0000: metainfo (memory blocks and sync objects) 544 2470 0000 0000 - 4000 0000 0000: - 545 */ 546 547 struct MappingGoPPC64_46 { 548 static const uptr kMetaShadowBeg = 0x240000000000ull; 549 static const uptr kMetaShadowEnd = 0x247000000000ull; 550 static const uptr kShadowBeg = 0x200000000000ull; 551 static const uptr kShadowEnd = 0x21c000000000ull; 552 static const uptr kLoAppMemBeg = 0x000000001000ull; 553 static const uptr kLoAppMemEnd = 0x00e000000000ull; 554 static const uptr kMidAppMemBeg = 0; 555 static const uptr kMidAppMemEnd = 0; 556 static const uptr kHiAppMemBeg = 0; 557 static const uptr kHiAppMemEnd = 0; 558 static const uptr kHeapMemBeg = 0; 559 static const uptr kHeapMemEnd = 0; 560 static const uptr kVdsoBeg = 0; 561 static const uptr kShadowMsk = 0; 562 static const uptr kShadowXor = 0; 563 static const uptr kShadowAdd = 0x200000000000ull; 564 }; 565 566 /* Go on linux/powerpc64 (47-bit VMA) 567 0000 0000 1000 - 0000 1000 0000: executable 568 0000 1000 0000 - 00c0 0000 0000: - 569 00c0 0000 0000 - 00e0 0000 0000: heap 570 00e0 0000 0000 - 2000 0000 0000: - 571 2000 0000 0000 - 2800 0000 0000: shadow 572 2800 0000 0000 - 3000 0000 0000: - 573 3000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects) 574 3200 0000 0000 - 8000 0000 0000: - 575 */ 576 577 struct MappingGoPPC64_47 { 578 static const uptr kMetaShadowBeg = 0x300000000000ull; 579 static const uptr kMetaShadowEnd = 0x320000000000ull; 580 static const uptr kShadowBeg = 0x200000000000ull; 581 static const uptr kShadowEnd = 0x280000000000ull; 582 static const uptr kLoAppMemBeg = 0x000000001000ull; 583 static const uptr kLoAppMemEnd = 0x00e000000000ull; 584 static const uptr kMidAppMemBeg = 0; 585 static const uptr kMidAppMemEnd = 0; 586 static const uptr kHiAppMemBeg = 0; 587 static const uptr kHiAppMemEnd = 0; 588 static const uptr kHeapMemBeg = 0; 589 static const uptr kHeapMemEnd = 0; 590 static const uptr kVdsoBeg = 0; 591 static const uptr kShadowMsk = 0; 592 static const uptr kShadowXor = 0; 593 static const uptr kShadowAdd = 0x200000000000ull; 594 }; 595 596 /* Go on linux/aarch64 (48-bit VMA) and darwin/aarch64 (47-bit VMA) 597 0000 0000 1000 - 0000 1000 0000: executable 598 0000 1000 0000 - 00c0 0000 0000: - 599 00c0 0000 0000 - 00e0 0000 0000: heap 600 00e0 0000 0000 - 2000 0000 0000: - 601 2000 0000 0000 - 2800 0000 0000: shadow 602 2800 0000 0000 - 3000 0000 0000: - 603 3000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects) 604 3200 0000 0000 - 8000 0000 0000: - 605 */ 606 struct MappingGoAarch64 { 607 static const uptr kMetaShadowBeg = 0x300000000000ull; 608 static const uptr kMetaShadowEnd = 0x320000000000ull; 609 static const uptr kShadowBeg = 0x200000000000ull; 610 static const uptr kShadowEnd = 0x280000000000ull; 611 static const uptr kLoAppMemBeg = 0x000000001000ull; 612 static const uptr kLoAppMemEnd = 0x00e000000000ull; 613 static const uptr kMidAppMemBeg = 0; 614 static const uptr kMidAppMemEnd = 0; 615 static const uptr kHiAppMemBeg = 0; 616 static const uptr kHiAppMemEnd = 0; 617 static const uptr kHeapMemBeg = 0; 618 static const uptr kHeapMemEnd = 0; 619 static const uptr kVdsoBeg = 0; 620 static const uptr kShadowMsk = 0; 621 static const uptr kShadowXor = 0; 622 static const uptr kShadowAdd = 0x200000000000ull; 623 }; 624 625 /* Go on linux/loongarch64 (47-bit VMA) 626 0000 0000 1000 - 0000 1000 0000: executable 627 0000 1000 0000 - 00c0 0000 0000: - 628 00c0 0000 0000 - 00e0 0000 0000: heap 629 00e0 0000 0000 - 2000 0000 0000: - 630 2000 0000 0000 - 2800 0000 0000: shadow 631 2800 0000 0000 - 3000 0000 0000: - 632 3000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects) 633 3200 0000 0000 - 8000 0000 0000: - 634 */ 635 struct MappingGoLoongArch64_47 { 636 static const uptr kMetaShadowBeg = 0x300000000000ull; 637 static const uptr kMetaShadowEnd = 0x320000000000ull; 638 static const uptr kShadowBeg = 0x200000000000ull; 639 static const uptr kShadowEnd = 0x280000000000ull; 640 static const uptr kLoAppMemBeg = 0x000000001000ull; 641 static const uptr kLoAppMemEnd = 0x00e000000000ull; 642 static const uptr kMidAppMemBeg = 0; 643 static const uptr kMidAppMemEnd = 0; 644 static const uptr kHiAppMemBeg = 0; 645 static const uptr kHiAppMemEnd = 0; 646 static const uptr kHeapMemBeg = 0; 647 static const uptr kHeapMemEnd = 0; 648 static const uptr kVdsoBeg = 0; 649 static const uptr kShadowMsk = 0; 650 static const uptr kShadowXor = 0; 651 static const uptr kShadowAdd = 0x200000000000ull; 652 }; 653 654 /* 655 Go on linux/mips64 (47-bit VMA) 656 0000 0000 1000 - 0000 1000 0000: executable 657 0000 1000 0000 - 00c0 0000 0000: - 658 00c0 0000 0000 - 00e0 0000 0000: heap 659 00e0 0000 0000 - 2000 0000 0000: - 660 2000 0000 0000 - 2800 0000 0000: shadow 661 2800 0000 0000 - 3000 0000 0000: - 662 3000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects) 663 3200 0000 0000 - 8000 0000 0000: - 664 */ 665 struct MappingGoMips64_47 { 666 static const uptr kMetaShadowBeg = 0x300000000000ull; 667 static const uptr kMetaShadowEnd = 0x320000000000ull; 668 static const uptr kShadowBeg = 0x200000000000ull; 669 static const uptr kShadowEnd = 0x280000000000ull; 670 static const uptr kLoAppMemBeg = 0x000000001000ull; 671 static const uptr kLoAppMemEnd = 0x00e000000000ull; 672 static const uptr kMidAppMemBeg = 0; 673 static const uptr kMidAppMemEnd = 0; 674 static const uptr kHiAppMemBeg = 0; 675 static const uptr kHiAppMemEnd = 0; 676 static const uptr kHeapMemBeg = 0; 677 static const uptr kHeapMemEnd = 0; 678 static const uptr kVdsoBeg = 0; 679 static const uptr kShadowMsk = 0; 680 static const uptr kShadowXor = 0; 681 static const uptr kShadowAdd = 0x200000000000ull; 682 }; 683 684 /* 685 Go on linux/s390x 686 0000 0000 1000 - 1000 0000 0000: executable and heap - 16 TiB 687 1000 0000 0000 - 4000 0000 0000: - 688 4000 0000 0000 - 6000 0000 0000: shadow - 64TiB (4 * app) 689 6000 0000 0000 - 9000 0000 0000: - 690 9000 0000 0000 - 9800 0000 0000: metainfo - 8TiB (0.5 * app) 691 */ 692 struct MappingGoS390x { 693 static const uptr kMetaShadowBeg = 0x900000000000ull; 694 static const uptr kMetaShadowEnd = 0x980000000000ull; 695 static const uptr kShadowBeg = 0x400000000000ull; 696 static const uptr kShadowEnd = 0x600000000000ull; 697 static const uptr kLoAppMemBeg = 0x000000001000ull; 698 static const uptr kLoAppMemEnd = 0x100000000000ull; 699 static const uptr kMidAppMemBeg = 0; 700 static const uptr kMidAppMemEnd = 0; 701 static const uptr kHiAppMemBeg = 0; 702 static const uptr kHiAppMemEnd = 0; 703 static const uptr kHeapMemBeg = 0; 704 static const uptr kHeapMemEnd = 0; 705 static const uptr kVdsoBeg = 0; 706 static const uptr kShadowMsk = 0; 707 static const uptr kShadowXor = 0; 708 static const uptr kShadowAdd = 0x400000000000ull; 709 }; 710 711 extern uptr vmaSize; 712 713 template <typename Func, typename Arg> 714 ALWAYS_INLINE auto SelectMapping(Arg arg) { 715 #if SANITIZER_GO 716 # if defined(__powerpc64__) 717 switch (vmaSize) { 718 case 46: 719 return Func::template Apply<MappingGoPPC64_46>(arg); 720 case 47: 721 return Func::template Apply<MappingGoPPC64_47>(arg); 722 } 723 # elif defined(__mips64) 724 return Func::template Apply<MappingGoMips64_47>(arg); 725 # elif defined(__s390x__) 726 return Func::template Apply<MappingGoS390x>(arg); 727 # elif defined(__aarch64__) 728 return Func::template Apply<MappingGoAarch64>(arg); 729 # elif defined(__loongarch_lp64) 730 return Func::template Apply<MappingGoLoongArch64_47>(arg); 731 # elif SANITIZER_WINDOWS 732 return Func::template Apply<MappingGoWindows>(arg); 733 # else 734 return Func::template Apply<MappingGo48>(arg); 735 # endif 736 #else // SANITIZER_GO 737 # if SANITIZER_IOS && !SANITIZER_IOSSIM 738 return Func::template Apply<MappingAppleAarch64>(arg); 739 # elif defined(__x86_64__) || SANITIZER_APPLE 740 return Func::template Apply<Mapping48AddressSpace>(arg); 741 # elif defined(__aarch64__) 742 switch (vmaSize) { 743 case 39: 744 return Func::template Apply<MappingAarch64_39>(arg); 745 case 42: 746 return Func::template Apply<MappingAarch64_42>(arg); 747 case 48: 748 return Func::template Apply<MappingAarch64_48>(arg); 749 } 750 # elif SANITIZER_LOONGARCH64 751 return Func::template Apply<MappingLoongArch64_47>(arg); 752 # elif defined(__powerpc64__) 753 switch (vmaSize) { 754 case 44: 755 return Func::template Apply<MappingPPC64_44>(arg); 756 case 46: 757 return Func::template Apply<MappingPPC64_46>(arg); 758 case 47: 759 return Func::template Apply<MappingPPC64_47>(arg); 760 } 761 # elif defined(__mips64) 762 return Func::template Apply<MappingMips64_40>(arg); 763 # elif SANITIZER_RISCV64 764 switch (vmaSize) { 765 case 39: 766 return Func::template Apply<MappingRiscv64_39>(arg); 767 case 48: 768 return Func::template Apply<MappingRiscv64_48>(arg); 769 } 770 # elif defined(__s390x__) 771 return Func::template Apply<MappingS390x>(arg); 772 # else 773 # error "unsupported platform" 774 # endif 775 #endif 776 Die(); 777 } 778 779 template <typename Func> 780 void ForEachMapping() { 781 Func::template Apply<Mapping48AddressSpace>(); 782 Func::template Apply<MappingMips64_40>(); 783 Func::template Apply<MappingAppleAarch64>(); 784 Func::template Apply<MappingAarch64_39>(); 785 Func::template Apply<MappingAarch64_42>(); 786 Func::template Apply<MappingAarch64_48>(); 787 Func::template Apply<MappingLoongArch64_47>(); 788 Func::template Apply<MappingPPC64_44>(); 789 Func::template Apply<MappingPPC64_46>(); 790 Func::template Apply<MappingPPC64_47>(); 791 Func::template Apply<MappingRiscv64_39>(); 792 Func::template Apply<MappingRiscv64_48>(); 793 Func::template Apply<MappingS390x>(); 794 Func::template Apply<MappingGo48>(); 795 Func::template Apply<MappingGoWindows>(); 796 Func::template Apply<MappingGoPPC64_46>(); 797 Func::template Apply<MappingGoPPC64_47>(); 798 Func::template Apply<MappingGoAarch64>(); 799 Func::template Apply<MappingGoLoongArch64_47>(); 800 Func::template Apply<MappingGoMips64_47>(); 801 Func::template Apply<MappingGoS390x>(); 802 } 803 804 enum MappingType { 805 kLoAppMemBeg, 806 kLoAppMemEnd, 807 kHiAppMemBeg, 808 kHiAppMemEnd, 809 kMidAppMemBeg, 810 kMidAppMemEnd, 811 kHeapMemBeg, 812 kHeapMemEnd, 813 kShadowBeg, 814 kShadowEnd, 815 kMetaShadowBeg, 816 kMetaShadowEnd, 817 kVdsoBeg, 818 }; 819 820 struct MappingField { 821 template <typename Mapping> 822 static uptr Apply(MappingType type) { 823 switch (type) { 824 case kLoAppMemBeg: 825 return Mapping::kLoAppMemBeg; 826 case kLoAppMemEnd: 827 return Mapping::kLoAppMemEnd; 828 case kMidAppMemBeg: 829 return Mapping::kMidAppMemBeg; 830 case kMidAppMemEnd: 831 return Mapping::kMidAppMemEnd; 832 case kHiAppMemBeg: 833 return Mapping::kHiAppMemBeg; 834 case kHiAppMemEnd: 835 return Mapping::kHiAppMemEnd; 836 case kHeapMemBeg: 837 return Mapping::kHeapMemBeg; 838 case kHeapMemEnd: 839 return Mapping::kHeapMemEnd; 840 case kVdsoBeg: 841 return Mapping::kVdsoBeg; 842 case kShadowBeg: 843 return Mapping::kShadowBeg; 844 case kShadowEnd: 845 return Mapping::kShadowEnd; 846 case kMetaShadowBeg: 847 return Mapping::kMetaShadowBeg; 848 case kMetaShadowEnd: 849 return Mapping::kMetaShadowEnd; 850 } 851 Die(); 852 } 853 }; 854 855 ALWAYS_INLINE 856 uptr LoAppMemBeg(void) { return SelectMapping<MappingField>(kLoAppMemBeg); } 857 ALWAYS_INLINE 858 uptr LoAppMemEnd(void) { return SelectMapping<MappingField>(kLoAppMemEnd); } 859 860 ALWAYS_INLINE 861 uptr MidAppMemBeg(void) { return SelectMapping<MappingField>(kMidAppMemBeg); } 862 ALWAYS_INLINE 863 uptr MidAppMemEnd(void) { return SelectMapping<MappingField>(kMidAppMemEnd); } 864 865 ALWAYS_INLINE 866 uptr HeapMemBeg(void) { return SelectMapping<MappingField>(kHeapMemBeg); } 867 ALWAYS_INLINE 868 uptr HeapMemEnd(void) { return SelectMapping<MappingField>(kHeapMemEnd); } 869 870 ALWAYS_INLINE 871 uptr HiAppMemBeg(void) { return SelectMapping<MappingField>(kHiAppMemBeg); } 872 ALWAYS_INLINE 873 uptr HiAppMemEnd(void) { return SelectMapping<MappingField>(kHiAppMemEnd); } 874 875 ALWAYS_INLINE 876 uptr VdsoBeg(void) { return SelectMapping<MappingField>(kVdsoBeg); } 877 878 ALWAYS_INLINE 879 uptr ShadowBeg(void) { return SelectMapping<MappingField>(kShadowBeg); } 880 ALWAYS_INLINE 881 uptr ShadowEnd(void) { return SelectMapping<MappingField>(kShadowEnd); } 882 883 ALWAYS_INLINE 884 uptr MetaShadowBeg(void) { return SelectMapping<MappingField>(kMetaShadowBeg); } 885 ALWAYS_INLINE 886 uptr MetaShadowEnd(void) { return SelectMapping<MappingField>(kMetaShadowEnd); } 887 888 struct IsAppMemImpl { 889 template <typename Mapping> 890 static bool Apply(uptr mem) { 891 return (mem >= Mapping::kHeapMemBeg && mem < Mapping::kHeapMemEnd) || 892 (mem >= Mapping::kMidAppMemBeg && mem < Mapping::kMidAppMemEnd) || 893 (mem >= Mapping::kLoAppMemBeg && mem < Mapping::kLoAppMemEnd) || 894 (mem >= Mapping::kHiAppMemBeg && mem < Mapping::kHiAppMemEnd); 895 } 896 }; 897 898 ALWAYS_INLINE 899 bool IsAppMem(uptr mem) { return SelectMapping<IsAppMemImpl>(mem); } 900 901 struct IsShadowMemImpl { 902 template <typename Mapping> 903 static bool Apply(uptr mem) { 904 return mem >= Mapping::kShadowBeg && mem <= Mapping::kShadowEnd; 905 } 906 }; 907 908 ALWAYS_INLINE 909 bool IsShadowMem(RawShadow *p) { 910 return SelectMapping<IsShadowMemImpl>(reinterpret_cast<uptr>(p)); 911 } 912 913 struct IsMetaMemImpl { 914 template <typename Mapping> 915 static bool Apply(uptr mem) { 916 return mem >= Mapping::kMetaShadowBeg && mem <= Mapping::kMetaShadowEnd; 917 } 918 }; 919 920 ALWAYS_INLINE 921 bool IsMetaMem(const u32 *p) { 922 return SelectMapping<IsMetaMemImpl>(reinterpret_cast<uptr>(p)); 923 } 924 925 struct MemToShadowImpl { 926 template <typename Mapping> 927 static uptr Apply(uptr x) { 928 DCHECK(IsAppMemImpl::Apply<Mapping>(x)); 929 return (((x) & ~(Mapping::kShadowMsk | (kShadowCell - 1))) ^ 930 Mapping::kShadowXor) * 931 kShadowMultiplier + 932 Mapping::kShadowAdd; 933 } 934 }; 935 936 ALWAYS_INLINE 937 RawShadow *MemToShadow(uptr x) { 938 return reinterpret_cast<RawShadow *>(SelectMapping<MemToShadowImpl>(x)); 939 } 940 941 struct MemToMetaImpl { 942 template <typename Mapping> 943 static u32 *Apply(uptr x) { 944 DCHECK(IsAppMemImpl::Apply<Mapping>(x)); 945 return (u32 *)(((((x) & ~(Mapping::kShadowMsk | (kMetaShadowCell - 1)))) / 946 kMetaShadowCell * kMetaShadowSize) | 947 Mapping::kMetaShadowBeg); 948 } 949 }; 950 951 ALWAYS_INLINE 952 u32 *MemToMeta(uptr x) { return SelectMapping<MemToMetaImpl>(x); } 953 954 struct ShadowToMemImpl { 955 template <typename Mapping> 956 static uptr Apply(uptr sp) { 957 if (!IsShadowMemImpl::Apply<Mapping>(sp)) 958 return 0; 959 // The shadow mapping is non-linear and we've lost some bits, so we don't 960 // have an easy way to restore the original app address. But the mapping is 961 // a bijection, so we try to restore the address as belonging to 962 // low/mid/high range consecutively and see if shadow->app->shadow mapping 963 // gives us the same address. 964 uptr p = 965 ((sp - Mapping::kShadowAdd) / kShadowMultiplier) ^ Mapping::kShadowXor; 966 if (p >= Mapping::kLoAppMemBeg && p < Mapping::kLoAppMemEnd && 967 MemToShadowImpl::Apply<Mapping>(p) == sp) 968 return p; 969 if (Mapping::kMidAppMemBeg) { 970 uptr p_mid = p + (Mapping::kMidAppMemBeg & Mapping::kShadowMsk); 971 if (p_mid >= Mapping::kMidAppMemBeg && p_mid < Mapping::kMidAppMemEnd && 972 MemToShadowImpl::Apply<Mapping>(p_mid) == sp) 973 return p_mid; 974 } 975 return p | Mapping::kShadowMsk; 976 } 977 }; 978 979 ALWAYS_INLINE 980 uptr ShadowToMem(RawShadow *s) { 981 return SelectMapping<ShadowToMemImpl>(reinterpret_cast<uptr>(s)); 982 } 983 984 // Compresses addr to kCompressedAddrBits stored in least significant bits. 985 ALWAYS_INLINE uptr CompressAddr(uptr addr) { 986 return addr & ((1ull << kCompressedAddrBits) - 1); 987 } 988 989 struct RestoreAddrImpl { 990 typedef uptr Result; 991 template <typename Mapping> 992 static Result Apply(uptr addr) { 993 // To restore the address we go over all app memory ranges and check if top 994 // 3 bits of the compressed addr match that of the app range. If yes, we 995 // assume that the compressed address come from that range and restore the 996 // missing top bits to match the app range address. 997 const uptr ranges[] = { 998 Mapping::kLoAppMemBeg, Mapping::kLoAppMemEnd, Mapping::kMidAppMemBeg, 999 Mapping::kMidAppMemEnd, Mapping::kHiAppMemBeg, Mapping::kHiAppMemEnd, 1000 Mapping::kHeapMemBeg, Mapping::kHeapMemEnd, 1001 }; 1002 const uptr indicator = 0x0e0000000000ull; 1003 const uptr ind_lsb = 1ull << LeastSignificantSetBitIndex(indicator); 1004 for (uptr i = 0; i < ARRAY_SIZE(ranges); i += 2) { 1005 uptr beg = ranges[i]; 1006 uptr end = ranges[i + 1]; 1007 if (beg == end) 1008 continue; 1009 for (uptr p = beg; p < end; p = RoundDown(p + ind_lsb, ind_lsb)) { 1010 if ((addr & indicator) == (p & indicator)) 1011 return addr | (p & ~(ind_lsb - 1)); 1012 } 1013 } 1014 Printf("ThreadSanitizer: failed to restore address 0x%zx\n", addr); 1015 Die(); 1016 } 1017 }; 1018 1019 // Restores compressed addr from kCompressedAddrBits to full representation. 1020 // This is called only during reporting and is not performance-critical. 1021 inline uptr RestoreAddr(uptr addr) { 1022 return SelectMapping<RestoreAddrImpl>(addr); 1023 } 1024 1025 void InitializePlatform(); 1026 void InitializePlatformEarly(); 1027 bool CheckAndProtect(bool protect, bool ignore_heap, bool print_warnings); 1028 void InitializeShadowMemoryPlatform(); 1029 void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns); 1030 int ExtractResolvFDs(void *state, int *fds, int nfd); 1031 int ExtractRecvmsgFDs(void *msg, int *fds, int nfd); 1032 uptr ExtractLongJmpSp(uptr *env); 1033 void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size); 1034 1035 int call_pthread_cancel_with_cleanup(int (*fn)(void *arg), 1036 void (*cleanup)(void *arg), void *arg); 1037 1038 void DestroyThreadState(); 1039 void PlatformCleanUpThreadState(ThreadState *thr); 1040 1041 } // namespace __tsan 1042 1043 #endif // TSAN_PLATFORM_H 1044