1 //===-- tsan_platform.h -----------------------------------------*- C++ -*-===// 2 // 3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4 // See https://llvm.org/LICENSE.txt for license information. 5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6 // 7 //===----------------------------------------------------------------------===// 8 // 9 // This file is a part of ThreadSanitizer (TSan), a race detector. 10 // 11 // Platform-specific code. 12 //===----------------------------------------------------------------------===// 13 14 #ifndef TSAN_PLATFORM_H 15 #define TSAN_PLATFORM_H 16 17 #if !defined(__LP64__) && !defined(_WIN64) 18 # error "Only 64-bit is supported" 19 #endif 20 21 #include "sanitizer_common/sanitizer_common.h" 22 #include "tsan_defs.h" 23 24 namespace __tsan { 25 26 enum { 27 // App memory is not mapped onto shadow memory range. 28 kBrokenMapping = 1 << 0, 29 // Mapping app memory and back does not produce the same address, 30 // this can lead to wrong addresses in reports and potentially 31 // other bad consequences. 32 kBrokenReverseMapping = 1 << 1, 33 // Mapping is non-linear for linear user range. 34 // This is bad and can lead to unpredictable memory corruptions, etc 35 // because range access functions assume linearity. 36 kBrokenLinearity = 1 << 2, 37 }; 38 39 /* 40 C/C++ on linux/x86_64 and freebsd/x86_64 41 0000 0000 1000 - 0080 0000 0000: main binary and/or MAP_32BIT mappings (512GB) 42 0040 0000 0000 - 0100 0000 0000: - 43 0100 0000 0000 - 1000 0000 0000: shadow 44 1000 0000 0000 - 3000 0000 0000: - 45 3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects) 46 4000 0000 0000 - 5500 0000 0000: - 47 5500 0000 0000 - 5680 0000 0000: pie binaries without ASLR or on 4.1+ kernels 48 5680 0000 0000 - 7d00 0000 0000: - 49 7b00 0000 0000 - 7c00 0000 0000: heap 50 7c00 0000 0000 - 7e80 0000 0000: - 51 7e80 0000 0000 - 8000 0000 0000: modules and main thread stack 52 53 C/C++ on netbsd/amd64 can reuse the same mapping: 54 * The address space starts from 0x1000 (option with 0x0) and ends with 55 0x7f7ffffff000. 56 * LoAppMem-kHeapMemEnd can be reused as it is. 57 * No VDSO support. 58 * No MidAppMem region. 59 * No additional HeapMem region. 60 * HiAppMem contains the stack, loader, shared libraries and heap. 61 * Stack on NetBSD/amd64 has prereserved 128MB. 62 * Heap grows downwards (top-down). 63 * ASLR must be disabled per-process or globally. 64 */ 65 struct Mapping48AddressSpace { 66 static const uptr kMetaShadowBeg = 0x300000000000ull; 67 static const uptr kMetaShadowEnd = 0x340000000000ull; 68 static const uptr kShadowBeg = 0x010000000000ull; 69 static const uptr kShadowEnd = 0x100000000000ull; 70 static const uptr kHeapMemBeg = 0x7b0000000000ull; 71 static const uptr kHeapMemEnd = 0x7c0000000000ull; 72 static const uptr kLoAppMemBeg = 0x000000001000ull; 73 static const uptr kLoAppMemEnd = 0x008000000000ull; 74 static const uptr kMidAppMemBeg = 0x550000000000ull; 75 static const uptr kMidAppMemEnd = 0x568000000000ull; 76 static const uptr kHiAppMemBeg = 0x7e8000000000ull; 77 static const uptr kHiAppMemEnd = 0x800000000000ull; 78 static const uptr kShadowMsk = 0x780000000000ull; 79 static const uptr kShadowXor = 0x040000000000ull; 80 static const uptr kShadowAdd = 0x000000000000ull; 81 static const uptr kVdsoBeg = 0xf000000000000000ull; 82 }; 83 84 /* 85 C/C++ on linux/mips64 (40-bit VMA) 86 0000 0000 00 - 0100 0000 00: - (4 GB) 87 0100 0000 00 - 0200 0000 00: main binary (4 GB) 88 0200 0000 00 - 1200 0000 00: - (64 GB) 89 1200 0000 00 - 2200 0000 00: shadow (64 GB) 90 2200 0000 00 - 4000 0000 00: - (120 GB) 91 4000 0000 00 - 5000 0000 00: metainfo (memory blocks and sync objects) (64 GB) 92 5000 0000 00 - aa00 0000 00: - (360 GB) 93 aa00 0000 00 - ab00 0000 00: main binary (PIE) (4 GB) 94 ab00 0000 00 - fe00 0000 00: - (332 GB) 95 fe00 0000 00 - ff00 0000 00: heap (4 GB) 96 ff00 0000 00 - ff80 0000 00: - (2 GB) 97 ff80 0000 00 - ffff ffff ff: modules and main thread stack (<2 GB) 98 */ 99 struct MappingMips64_40 { 100 static const uptr kMetaShadowBeg = 0x4000000000ull; 101 static const uptr kMetaShadowEnd = 0x5000000000ull; 102 static const uptr kShadowBeg = 0x1200000000ull; 103 static const uptr kShadowEnd = 0x2200000000ull; 104 static const uptr kHeapMemBeg = 0xfe00000000ull; 105 static const uptr kHeapMemEnd = 0xff00000000ull; 106 static const uptr kLoAppMemBeg = 0x0100000000ull; 107 static const uptr kLoAppMemEnd = 0x0200000000ull; 108 static const uptr kMidAppMemBeg = 0xaa00000000ull; 109 static const uptr kMidAppMemEnd = 0xab00000000ull; 110 static const uptr kHiAppMemBeg = 0xff80000000ull; 111 static const uptr kHiAppMemEnd = 0xffffffffffull; 112 static const uptr kShadowMsk = 0xf800000000ull; 113 static const uptr kShadowXor = 0x0800000000ull; 114 static const uptr kShadowAdd = 0x0000000000ull; 115 static const uptr kVdsoBeg = 0xfffff00000ull; 116 }; 117 118 /* 119 C/C++ on Darwin/iOS/ARM64 (36-bit VMA, 64 GB VM) 120 0000 0000 00 - 0100 0000 00: - (4 GB) 121 0100 0000 00 - 0200 0000 00: main binary, modules, thread stacks (4 GB) 122 0200 0000 00 - 0300 0000 00: heap (4 GB) 123 0300 0000 00 - 0400 0000 00: - (4 GB) 124 0400 0000 00 - 0800 0000 00: shadow memory (16 GB) 125 0800 0000 00 - 0d00 0000 00: - (20 GB) 126 0d00 0000 00 - 0e00 0000 00: metainfo (4 GB) 127 0e00 0000 00 - 1000 0000 00: - 128 */ 129 struct MappingAppleAarch64 { 130 static const uptr kLoAppMemBeg = 0x0100000000ull; 131 static const uptr kLoAppMemEnd = 0x0200000000ull; 132 static const uptr kHeapMemBeg = 0x0200000000ull; 133 static const uptr kHeapMemEnd = 0x0300000000ull; 134 static const uptr kShadowBeg = 0x0400000000ull; 135 static const uptr kShadowEnd = 0x0800000000ull; 136 static const uptr kMetaShadowBeg = 0x0d00000000ull; 137 static const uptr kMetaShadowEnd = 0x0e00000000ull; 138 static const uptr kHiAppMemBeg = 0x0fc0000000ull; 139 static const uptr kHiAppMemEnd = 0x0fc0000000ull; 140 static const uptr kShadowMsk = 0x0ull; 141 static const uptr kShadowXor = 0x0ull; 142 static const uptr kShadowAdd = 0x0200000000ull; 143 static const uptr kVdsoBeg = 0x7000000000000000ull; 144 static const uptr kMidAppMemBeg = 0; 145 static const uptr kMidAppMemEnd = 0; 146 }; 147 148 /* 149 C/C++ on linux/aarch64 (39-bit VMA) 150 0000 0010 00 - 0100 0000 00: main binary 151 0100 0000 00 - 0400 0000 00: - 152 0400 0000 00 - 1000 0000 00: shadow memory 153 2000 0000 00 - 3100 0000 00: - 154 3100 0000 00 - 3400 0000 00: metainfo 155 3400 0000 00 - 5500 0000 00: - 156 5500 0000 00 - 5600 0000 00: main binary (PIE) 157 5600 0000 00 - 7c00 0000 00: - 158 7c00 0000 00 - 7d00 0000 00: heap 159 7d00 0000 00 - 7fff ffff ff: modules and main thread stack 160 */ 161 struct MappingAarch64_39 { 162 static const uptr kLoAppMemBeg = 0x0000001000ull; 163 static const uptr kLoAppMemEnd = 0x0100000000ull; 164 static const uptr kShadowBeg = 0x0400000000ull; 165 static const uptr kShadowEnd = 0x1000000000ull; 166 static const uptr kMetaShadowBeg = 0x3100000000ull; 167 static const uptr kMetaShadowEnd = 0x3400000000ull; 168 static const uptr kMidAppMemBeg = 0x5500000000ull; 169 static const uptr kMidAppMemEnd = 0x5600000000ull; 170 static const uptr kHeapMemBeg = 0x7c00000000ull; 171 static const uptr kHeapMemEnd = 0x7d00000000ull; 172 static const uptr kHiAppMemBeg = 0x7e00000000ull; 173 static const uptr kHiAppMemEnd = 0x7fffffffffull; 174 static const uptr kShadowMsk = 0x7800000000ull; 175 static const uptr kShadowXor = 0x0200000000ull; 176 static const uptr kShadowAdd = 0x0000000000ull; 177 static const uptr kVdsoBeg = 0x7f00000000ull; 178 }; 179 180 /* 181 C/C++ on linux/aarch64 (42-bit VMA) 182 00000 0010 00 - 01000 0000 00: main binary 183 01000 0000 00 - 08000 0000 00: - 184 08000 0000 00 - 10000 0000 00: shadow memory 185 10000 0000 00 - 26000 0000 00: - 186 26000 0000 00 - 28000 0000 00: metainfo 187 28000 0000 00 - 2aa00 0000 00: - 188 2aa00 0000 00 - 2ab00 0000 00: main binary (PIE) 189 2ab00 0000 00 - 3e000 0000 00: - 190 3e000 0000 00 - 3f000 0000 00: heap 191 3f000 0000 00 - 3ffff ffff ff: modules and main thread stack 192 */ 193 struct MappingAarch64_42 { 194 static const uptr kBroken = kBrokenReverseMapping; 195 static const uptr kLoAppMemBeg = 0x00000001000ull; 196 static const uptr kLoAppMemEnd = 0x01000000000ull; 197 static const uptr kShadowBeg = 0x08000000000ull; 198 static const uptr kShadowEnd = 0x10000000000ull; 199 static const uptr kMetaShadowBeg = 0x26000000000ull; 200 static const uptr kMetaShadowEnd = 0x28000000000ull; 201 static const uptr kMidAppMemBeg = 0x2aa00000000ull; 202 static const uptr kMidAppMemEnd = 0x2ab00000000ull; 203 static const uptr kHeapMemBeg = 0x3e000000000ull; 204 static const uptr kHeapMemEnd = 0x3f000000000ull; 205 static const uptr kHiAppMemBeg = 0x3f000000000ull; 206 static const uptr kHiAppMemEnd = 0x3ffffffffffull; 207 static const uptr kShadowMsk = 0x3c000000000ull; 208 static const uptr kShadowXor = 0x04000000000ull; 209 static const uptr kShadowAdd = 0x00000000000ull; 210 static const uptr kVdsoBeg = 0x37f00000000ull; 211 }; 212 213 struct MappingAarch64_48 { 214 static const uptr kLoAppMemBeg = 0x0000000001000ull; 215 static const uptr kLoAppMemEnd = 0x0000200000000ull; 216 static const uptr kShadowBeg = 0x0001000000000ull; 217 static const uptr kShadowEnd = 0x0002000000000ull; 218 static const uptr kMetaShadowBeg = 0x0005000000000ull; 219 static const uptr kMetaShadowEnd = 0x0006000000000ull; 220 static const uptr kMidAppMemBeg = 0x0aaaa00000000ull; 221 static const uptr kMidAppMemEnd = 0x0aaaf00000000ull; 222 static const uptr kHeapMemBeg = 0x0ffff00000000ull; 223 static const uptr kHeapMemEnd = 0x0ffff00000000ull; 224 static const uptr kHiAppMemBeg = 0x0ffff00000000ull; 225 static const uptr kHiAppMemEnd = 0x1000000000000ull; 226 static const uptr kShadowMsk = 0x0fff800000000ull; 227 static const uptr kShadowXor = 0x0000800000000ull; 228 static const uptr kShadowAdd = 0x0000000000000ull; 229 static const uptr kVdsoBeg = 0xffff000000000ull; 230 }; 231 232 /* 233 C/C++ on linux/powerpc64 (44-bit VMA) 234 0000 0000 0100 - 0001 0000 0000: main binary 235 0001 0000 0000 - 0001 0000 0000: - 236 0001 0000 0000 - 0b00 0000 0000: shadow 237 0b00 0000 0000 - 0b00 0000 0000: - 238 0b00 0000 0000 - 0d00 0000 0000: metainfo (memory blocks and sync objects) 239 0d00 0000 0000 - 0f00 0000 0000: - 240 0f00 0000 0000 - 0f50 0000 0000: heap 241 0f50 0000 0000 - 0f60 0000 0000: - 242 0f60 0000 0000 - 1000 0000 0000: modules and main thread stack 243 */ 244 struct MappingPPC64_44 { 245 static const uptr kBroken = 246 kBrokenMapping | kBrokenReverseMapping | kBrokenLinearity; 247 static const uptr kMetaShadowBeg = 0x0b0000000000ull; 248 static const uptr kMetaShadowEnd = 0x0d0000000000ull; 249 static const uptr kShadowBeg = 0x000100000000ull; 250 static const uptr kShadowEnd = 0x0b0000000000ull; 251 static const uptr kLoAppMemBeg = 0x000000000100ull; 252 static const uptr kLoAppMemEnd = 0x000100000000ull; 253 static const uptr kHeapMemBeg = 0x0f0000000000ull; 254 static const uptr kHeapMemEnd = 0x0f5000000000ull; 255 static const uptr kHiAppMemBeg = 0x0f6000000000ull; 256 static const uptr kHiAppMemEnd = 0x100000000000ull; // 44 bits 257 static const uptr kShadowMsk = 0x0f0000000000ull; 258 static const uptr kShadowXor = 0x002100000000ull; 259 static const uptr kShadowAdd = 0x000000000000ull; 260 static const uptr kVdsoBeg = 0x3c0000000000000ull; 261 static const uptr kMidAppMemBeg = 0; 262 static const uptr kMidAppMemEnd = 0; 263 }; 264 265 /* 266 C/C++ on linux/powerpc64 (46-bit VMA) 267 0000 0000 1000 - 0100 0000 0000: main binary 268 0100 0000 0000 - 0200 0000 0000: - 269 0100 0000 0000 - 0800 0000 0000: shadow 270 0800 0000 0000 - 1000 0000 0000: - 271 1000 0000 0000 - 1200 0000 0000: metainfo (memory blocks and sync objects) 272 1200 0000 0000 - 3d00 0000 0000: - 273 3d00 0000 0000 - 3e00 0000 0000: heap 274 3e00 0000 0000 - 3e80 0000 0000: - 275 3e80 0000 0000 - 4000 0000 0000: modules and main thread stack 276 */ 277 struct MappingPPC64_46 { 278 static const uptr kMetaShadowBeg = 0x100000000000ull; 279 static const uptr kMetaShadowEnd = 0x120000000000ull; 280 static const uptr kShadowBeg = 0x010000000000ull; 281 static const uptr kShadowEnd = 0x080000000000ull; 282 static const uptr kHeapMemBeg = 0x3d0000000000ull; 283 static const uptr kHeapMemEnd = 0x3e0000000000ull; 284 static const uptr kLoAppMemBeg = 0x000000001000ull; 285 static const uptr kLoAppMemEnd = 0x010000000000ull; 286 static const uptr kHiAppMemBeg = 0x3e8000000000ull; 287 static const uptr kHiAppMemEnd = 0x400000000000ull; // 46 bits 288 static const uptr kShadowMsk = 0x3c0000000000ull; 289 static const uptr kShadowXor = 0x020000000000ull; 290 static const uptr kShadowAdd = 0x000000000000ull; 291 static const uptr kVdsoBeg = 0x7800000000000000ull; 292 static const uptr kMidAppMemBeg = 0; 293 static const uptr kMidAppMemEnd = 0; 294 }; 295 296 /* 297 C/C++ on linux/powerpc64 (47-bit VMA) 298 0000 0000 1000 - 0100 0000 0000: main binary 299 0100 0000 0000 - 0200 0000 0000: - 300 0100 0000 0000 - 0800 0000 0000: shadow 301 0800 0000 0000 - 1000 0000 0000: - 302 1000 0000 0000 - 1200 0000 0000: metainfo (memory blocks and sync objects) 303 1200 0000 0000 - 7d00 0000 0000: - 304 7d00 0000 0000 - 7e00 0000 0000: heap 305 7e00 0000 0000 - 7e80 0000 0000: - 306 7e80 0000 0000 - 8000 0000 0000: modules and main thread stack 307 */ 308 struct MappingPPC64_47 { 309 static const uptr kMetaShadowBeg = 0x100000000000ull; 310 static const uptr kMetaShadowEnd = 0x120000000000ull; 311 static const uptr kShadowBeg = 0x010000000000ull; 312 static const uptr kShadowEnd = 0x080000000000ull; 313 static const uptr kHeapMemBeg = 0x7d0000000000ull; 314 static const uptr kHeapMemEnd = 0x7e0000000000ull; 315 static const uptr kLoAppMemBeg = 0x000000001000ull; 316 static const uptr kLoAppMemEnd = 0x010000000000ull; 317 static const uptr kHiAppMemBeg = 0x7e8000000000ull; 318 static const uptr kHiAppMemEnd = 0x800000000000ull; // 47 bits 319 static const uptr kShadowMsk = 0x7c0000000000ull; 320 static const uptr kShadowXor = 0x020000000000ull; 321 static const uptr kShadowAdd = 0x000000000000ull; 322 static const uptr kVdsoBeg = 0x7800000000000000ull; 323 static const uptr kMidAppMemBeg = 0; 324 static const uptr kMidAppMemEnd = 0; 325 }; 326 327 /* 328 C/C++ on linux/s390x 329 While the kernel provides a 64-bit address space, we have to restrict ourselves 330 to 48 bits due to how e.g. SyncVar::GetId() works. 331 0000 0000 1000 - 0e00 0000 0000: binary, modules, stacks - 14 TiB 332 0e00 0000 0000 - 2000 0000 0000: - 333 2000 0000 0000 - 4000 0000 0000: shadow - 32TiB (2 * app) 334 4000 0000 0000 - 9000 0000 0000: - 335 9000 0000 0000 - 9800 0000 0000: metainfo - 8TiB (0.5 * app) 336 9800 0000 0000 - be00 0000 0000: - 337 be00 0000 0000 - c000 0000 0000: heap - 2TiB (max supported by the allocator) 338 */ 339 struct MappingS390x { 340 static const uptr kMetaShadowBeg = 0x900000000000ull; 341 static const uptr kMetaShadowEnd = 0x980000000000ull; 342 static const uptr kShadowBeg = 0x200000000000ull; 343 static const uptr kShadowEnd = 0x400000000000ull; 344 static const uptr kHeapMemBeg = 0xbe0000000000ull; 345 static const uptr kHeapMemEnd = 0xc00000000000ull; 346 static const uptr kLoAppMemBeg = 0x000000001000ull; 347 static const uptr kLoAppMemEnd = 0x0e0000000000ull; 348 static const uptr kHiAppMemBeg = 0xc00000004000ull; 349 static const uptr kHiAppMemEnd = 0xc00000004000ull; 350 static const uptr kShadowMsk = 0xb00000000000ull; 351 static const uptr kShadowXor = 0x100000000000ull; 352 static const uptr kShadowAdd = 0x000000000000ull; 353 static const uptr kVdsoBeg = 0xfffffffff000ull; 354 static const uptr kMidAppMemBeg = 0; 355 static const uptr kMidAppMemEnd = 0; 356 }; 357 358 /* Go on linux, darwin and freebsd on x86_64 359 0000 0000 1000 - 0000 1000 0000: executable 360 0000 1000 0000 - 00c0 0000 0000: - 361 00c0 0000 0000 - 00e0 0000 0000: heap 362 00e0 0000 0000 - 2000 0000 0000: - 363 2000 0000 0000 - 21c0 0000 0000: shadow 364 21c0 0000 0000 - 3000 0000 0000: - 365 3000 0000 0000 - 4000 0000 0000: metainfo (memory blocks and sync objects) 366 4000 0000 0000 - 8000 0000 0000: - 367 */ 368 369 struct MappingGo48 { 370 static const uptr kMetaShadowBeg = 0x300000000000ull; 371 static const uptr kMetaShadowEnd = 0x400000000000ull; 372 static const uptr kShadowBeg = 0x200000000000ull; 373 static const uptr kShadowEnd = 0x21c000000000ull; 374 static const uptr kLoAppMemBeg = 0x000000001000ull; 375 static const uptr kLoAppMemEnd = 0x00e000000000ull; 376 static const uptr kMidAppMemBeg = 0; 377 static const uptr kMidAppMemEnd = 0; 378 static const uptr kHiAppMemBeg = 0; 379 static const uptr kHiAppMemEnd = 0; 380 static const uptr kHeapMemBeg = 0; 381 static const uptr kHeapMemEnd = 0; 382 static const uptr kVdsoBeg = 0; 383 static const uptr kShadowMsk = 0; 384 static const uptr kShadowXor = 0; 385 static const uptr kShadowAdd = 0x200000000000ull; 386 }; 387 388 /* Go on windows 389 0000 0000 1000 - 0000 1000 0000: executable 390 0000 1000 0000 - 00f8 0000 0000: - 391 00c0 0000 0000 - 00e0 0000 0000: heap 392 00e0 0000 0000 - 0100 0000 0000: - 393 0100 0000 0000 - 0300 0000 0000: shadow 394 0300 0000 0000 - 0700 0000 0000: - 395 0700 0000 0000 - 0770 0000 0000: metainfo (memory blocks and sync objects) 396 07d0 0000 0000 - 8000 0000 0000: - 397 */ 398 399 struct MappingGoWindows { 400 static const uptr kMetaShadowBeg = 0x070000000000ull; 401 static const uptr kMetaShadowEnd = 0x077000000000ull; 402 static const uptr kShadowBeg = 0x010000000000ull; 403 static const uptr kShadowEnd = 0x030000000000ull; 404 static const uptr kLoAppMemBeg = 0x000000001000ull; 405 static const uptr kLoAppMemEnd = 0x00e000000000ull; 406 static const uptr kMidAppMemBeg = 0; 407 static const uptr kMidAppMemEnd = 0; 408 static const uptr kHiAppMemBeg = 0; 409 static const uptr kHiAppMemEnd = 0; 410 static const uptr kHeapMemBeg = 0; 411 static const uptr kHeapMemEnd = 0; 412 static const uptr kVdsoBeg = 0; 413 static const uptr kShadowMsk = 0; 414 static const uptr kShadowXor = 0; 415 static const uptr kShadowAdd = 0x010000000000ull; 416 }; 417 418 /* Go on linux/powerpc64 (46-bit VMA) 419 0000 0000 1000 - 0000 1000 0000: executable 420 0000 1000 0000 - 00c0 0000 0000: - 421 00c0 0000 0000 - 00e0 0000 0000: heap 422 00e0 0000 0000 - 2000 0000 0000: - 423 2000 0000 0000 - 21c0 0000 0000: shadow 424 21c0 0000 0000 - 2400 0000 0000: - 425 2400 0000 0000 - 2470 0000 0000: metainfo (memory blocks and sync objects) 426 2470 0000 0000 - 4000 0000 0000: - 427 */ 428 429 struct MappingGoPPC64_46 { 430 static const uptr kMetaShadowBeg = 0x240000000000ull; 431 static const uptr kMetaShadowEnd = 0x247000000000ull; 432 static const uptr kShadowBeg = 0x200000000000ull; 433 static const uptr kShadowEnd = 0x21c000000000ull; 434 static const uptr kLoAppMemBeg = 0x000000001000ull; 435 static const uptr kLoAppMemEnd = 0x00e000000000ull; 436 static const uptr kMidAppMemBeg = 0; 437 static const uptr kMidAppMemEnd = 0; 438 static const uptr kHiAppMemBeg = 0; 439 static const uptr kHiAppMemEnd = 0; 440 static const uptr kHeapMemBeg = 0; 441 static const uptr kHeapMemEnd = 0; 442 static const uptr kVdsoBeg = 0; 443 static const uptr kShadowMsk = 0; 444 static const uptr kShadowXor = 0; 445 static const uptr kShadowAdd = 0x200000000000ull; 446 }; 447 448 /* Go on linux/powerpc64 (47-bit VMA) 449 0000 0000 1000 - 0000 1000 0000: executable 450 0000 1000 0000 - 00c0 0000 0000: - 451 00c0 0000 0000 - 00e0 0000 0000: heap 452 00e0 0000 0000 - 2000 0000 0000: - 453 2000 0000 0000 - 2800 0000 0000: shadow 454 2800 0000 0000 - 3000 0000 0000: - 455 3000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects) 456 3200 0000 0000 - 8000 0000 0000: - 457 */ 458 459 struct MappingGoPPC64_47 { 460 static const uptr kMetaShadowBeg = 0x300000000000ull; 461 static const uptr kMetaShadowEnd = 0x320000000000ull; 462 static const uptr kShadowBeg = 0x200000000000ull; 463 static const uptr kShadowEnd = 0x280000000000ull; 464 static const uptr kLoAppMemBeg = 0x000000001000ull; 465 static const uptr kLoAppMemEnd = 0x00e000000000ull; 466 static const uptr kMidAppMemBeg = 0; 467 static const uptr kMidAppMemEnd = 0; 468 static const uptr kHiAppMemBeg = 0; 469 static const uptr kHiAppMemEnd = 0; 470 static const uptr kHeapMemBeg = 0; 471 static const uptr kHeapMemEnd = 0; 472 static const uptr kVdsoBeg = 0; 473 static const uptr kShadowMsk = 0; 474 static const uptr kShadowXor = 0; 475 static const uptr kShadowAdd = 0x200000000000ull; 476 }; 477 478 /* Go on linux/aarch64 (48-bit VMA) and darwin/aarch64 (47-bit VMA) 479 0000 0000 1000 - 0000 1000 0000: executable 480 0000 1000 0000 - 00c0 0000 0000: - 481 00c0 0000 0000 - 00e0 0000 0000: heap 482 00e0 0000 0000 - 2000 0000 0000: - 483 2000 0000 0000 - 2800 0000 0000: shadow 484 2800 0000 0000 - 3000 0000 0000: - 485 3000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects) 486 3200 0000 0000 - 8000 0000 0000: - 487 */ 488 struct MappingGoAarch64 { 489 static const uptr kMetaShadowBeg = 0x300000000000ull; 490 static const uptr kMetaShadowEnd = 0x320000000000ull; 491 static const uptr kShadowBeg = 0x200000000000ull; 492 static const uptr kShadowEnd = 0x280000000000ull; 493 static const uptr kLoAppMemBeg = 0x000000001000ull; 494 static const uptr kLoAppMemEnd = 0x00e000000000ull; 495 static const uptr kMidAppMemBeg = 0; 496 static const uptr kMidAppMemEnd = 0; 497 static const uptr kHiAppMemBeg = 0; 498 static const uptr kHiAppMemEnd = 0; 499 static const uptr kHeapMemBeg = 0; 500 static const uptr kHeapMemEnd = 0; 501 static const uptr kVdsoBeg = 0; 502 static const uptr kShadowMsk = 0; 503 static const uptr kShadowXor = 0; 504 static const uptr kShadowAdd = 0x200000000000ull; 505 }; 506 507 /* 508 Go on linux/mips64 (47-bit VMA) 509 0000 0000 1000 - 0000 1000 0000: executable 510 0000 1000 0000 - 00c0 0000 0000: - 511 00c0 0000 0000 - 00e0 0000 0000: heap 512 00e0 0000 0000 - 2000 0000 0000: - 513 2000 0000 0000 - 2800 0000 0000: shadow 514 2800 0000 0000 - 3000 0000 0000: - 515 3000 0000 0000 - 3200 0000 0000: metainfo (memory blocks and sync objects) 516 3200 0000 0000 - 8000 0000 0000: - 517 */ 518 struct MappingGoMips64_47 { 519 static const uptr kMetaShadowBeg = 0x300000000000ull; 520 static const uptr kMetaShadowEnd = 0x320000000000ull; 521 static const uptr kShadowBeg = 0x200000000000ull; 522 static const uptr kShadowEnd = 0x280000000000ull; 523 static const uptr kLoAppMemBeg = 0x000000001000ull; 524 static const uptr kLoAppMemEnd = 0x00e000000000ull; 525 static const uptr kMidAppMemBeg = 0; 526 static const uptr kMidAppMemEnd = 0; 527 static const uptr kHiAppMemBeg = 0; 528 static const uptr kHiAppMemEnd = 0; 529 static const uptr kHeapMemBeg = 0; 530 static const uptr kHeapMemEnd = 0; 531 static const uptr kVdsoBeg = 0; 532 static const uptr kShadowMsk = 0; 533 static const uptr kShadowXor = 0; 534 static const uptr kShadowAdd = 0x200000000000ull; 535 }; 536 537 /* 538 Go on linux/s390x 539 0000 0000 1000 - 1000 0000 0000: executable and heap - 16 TiB 540 1000 0000 0000 - 4000 0000 0000: - 541 4000 0000 0000 - 6000 0000 0000: shadow - 64TiB (4 * app) 542 6000 0000 0000 - 9000 0000 0000: - 543 9000 0000 0000 - 9800 0000 0000: metainfo - 8TiB (0.5 * app) 544 */ 545 struct MappingGoS390x { 546 static const uptr kMetaShadowBeg = 0x900000000000ull; 547 static const uptr kMetaShadowEnd = 0x980000000000ull; 548 static const uptr kShadowBeg = 0x400000000000ull; 549 static const uptr kShadowEnd = 0x600000000000ull; 550 static const uptr kLoAppMemBeg = 0x000000001000ull; 551 static const uptr kLoAppMemEnd = 0x100000000000ull; 552 static const uptr kMidAppMemBeg = 0; 553 static const uptr kMidAppMemEnd = 0; 554 static const uptr kHiAppMemBeg = 0; 555 static const uptr kHiAppMemEnd = 0; 556 static const uptr kHeapMemBeg = 0; 557 static const uptr kHeapMemEnd = 0; 558 static const uptr kVdsoBeg = 0; 559 static const uptr kShadowMsk = 0; 560 static const uptr kShadowXor = 0; 561 static const uptr kShadowAdd = 0x400000000000ull; 562 }; 563 564 extern uptr vmaSize; 565 566 template <typename Func, typename Arg> 567 ALWAYS_INLINE auto SelectMapping(Arg arg) { 568 #if SANITIZER_GO 569 # if defined(__powerpc64__) 570 switch (vmaSize) { 571 case 46: 572 return Func::template Apply<MappingGoPPC64_46>(arg); 573 case 47: 574 return Func::template Apply<MappingGoPPC64_47>(arg); 575 } 576 # elif defined(__mips64) 577 return Func::template Apply<MappingGoMips64_47>(arg); 578 # elif defined(__s390x__) 579 return Func::template Apply<MappingGoS390x>(arg); 580 # elif defined(__aarch64__) 581 return Func::template Apply<MappingGoAarch64>(arg); 582 # elif SANITIZER_WINDOWS 583 return Func::template Apply<MappingGoWindows>(arg); 584 # else 585 return Func::template Apply<MappingGo48>(arg); 586 # endif 587 #else // SANITIZER_GO 588 # if SANITIZER_IOS && !SANITIZER_IOSSIM 589 return Func::template Apply<MappingAppleAarch64>(arg); 590 # elif defined(__x86_64__) || SANITIZER_MAC 591 return Func::template Apply<Mapping48AddressSpace>(arg); 592 # elif defined(__aarch64__) 593 switch (vmaSize) { 594 case 39: 595 return Func::template Apply<MappingAarch64_39>(arg); 596 case 42: 597 return Func::template Apply<MappingAarch64_42>(arg); 598 case 48: 599 return Func::template Apply<MappingAarch64_48>(arg); 600 } 601 # elif defined(__powerpc64__) 602 switch (vmaSize) { 603 case 44: 604 return Func::template Apply<MappingPPC64_44>(arg); 605 case 46: 606 return Func::template Apply<MappingPPC64_46>(arg); 607 case 47: 608 return Func::template Apply<MappingPPC64_47>(arg); 609 } 610 # elif defined(__mips64) 611 return Func::template Apply<MappingMips64_40>(arg); 612 # elif defined(__s390x__) 613 return Func::template Apply<MappingS390x>(arg); 614 # else 615 # error "unsupported platform" 616 # endif 617 #endif 618 Die(); 619 } 620 621 template <typename Func> 622 void ForEachMapping() { 623 Func::template Apply<Mapping48AddressSpace>(); 624 Func::template Apply<MappingMips64_40>(); 625 Func::template Apply<MappingAppleAarch64>(); 626 Func::template Apply<MappingAarch64_39>(); 627 Func::template Apply<MappingAarch64_42>(); 628 Func::template Apply<MappingAarch64_48>(); 629 Func::template Apply<MappingPPC64_44>(); 630 Func::template Apply<MappingPPC64_46>(); 631 Func::template Apply<MappingPPC64_47>(); 632 Func::template Apply<MappingS390x>(); 633 Func::template Apply<MappingGo48>(); 634 Func::template Apply<MappingGoWindows>(); 635 Func::template Apply<MappingGoPPC64_46>(); 636 Func::template Apply<MappingGoPPC64_47>(); 637 Func::template Apply<MappingGoAarch64>(); 638 Func::template Apply<MappingGoMips64_47>(); 639 Func::template Apply<MappingGoS390x>(); 640 } 641 642 enum MappingType { 643 kLoAppMemBeg, 644 kLoAppMemEnd, 645 kHiAppMemBeg, 646 kHiAppMemEnd, 647 kMidAppMemBeg, 648 kMidAppMemEnd, 649 kHeapMemBeg, 650 kHeapMemEnd, 651 kShadowBeg, 652 kShadowEnd, 653 kMetaShadowBeg, 654 kMetaShadowEnd, 655 kVdsoBeg, 656 }; 657 658 struct MappingField { 659 template <typename Mapping> 660 static uptr Apply(MappingType type) { 661 switch (type) { 662 case kLoAppMemBeg: 663 return Mapping::kLoAppMemBeg; 664 case kLoAppMemEnd: 665 return Mapping::kLoAppMemEnd; 666 case kMidAppMemBeg: 667 return Mapping::kMidAppMemBeg; 668 case kMidAppMemEnd: 669 return Mapping::kMidAppMemEnd; 670 case kHiAppMemBeg: 671 return Mapping::kHiAppMemBeg; 672 case kHiAppMemEnd: 673 return Mapping::kHiAppMemEnd; 674 case kHeapMemBeg: 675 return Mapping::kHeapMemBeg; 676 case kHeapMemEnd: 677 return Mapping::kHeapMemEnd; 678 case kVdsoBeg: 679 return Mapping::kVdsoBeg; 680 case kShadowBeg: 681 return Mapping::kShadowBeg; 682 case kShadowEnd: 683 return Mapping::kShadowEnd; 684 case kMetaShadowBeg: 685 return Mapping::kMetaShadowBeg; 686 case kMetaShadowEnd: 687 return Mapping::kMetaShadowEnd; 688 } 689 Die(); 690 } 691 }; 692 693 ALWAYS_INLINE 694 uptr LoAppMemBeg(void) { return SelectMapping<MappingField>(kLoAppMemBeg); } 695 ALWAYS_INLINE 696 uptr LoAppMemEnd(void) { return SelectMapping<MappingField>(kLoAppMemEnd); } 697 698 ALWAYS_INLINE 699 uptr MidAppMemBeg(void) { return SelectMapping<MappingField>(kMidAppMemBeg); } 700 ALWAYS_INLINE 701 uptr MidAppMemEnd(void) { return SelectMapping<MappingField>(kMidAppMemEnd); } 702 703 ALWAYS_INLINE 704 uptr HeapMemBeg(void) { return SelectMapping<MappingField>(kHeapMemBeg); } 705 ALWAYS_INLINE 706 uptr HeapMemEnd(void) { return SelectMapping<MappingField>(kHeapMemEnd); } 707 708 ALWAYS_INLINE 709 uptr HiAppMemBeg(void) { return SelectMapping<MappingField>(kHiAppMemBeg); } 710 ALWAYS_INLINE 711 uptr HiAppMemEnd(void) { return SelectMapping<MappingField>(kHiAppMemEnd); } 712 713 ALWAYS_INLINE 714 uptr VdsoBeg(void) { return SelectMapping<MappingField>(kVdsoBeg); } 715 716 ALWAYS_INLINE 717 uptr ShadowBeg(void) { return SelectMapping<MappingField>(kShadowBeg); } 718 ALWAYS_INLINE 719 uptr ShadowEnd(void) { return SelectMapping<MappingField>(kShadowEnd); } 720 721 ALWAYS_INLINE 722 uptr MetaShadowBeg(void) { return SelectMapping<MappingField>(kMetaShadowBeg); } 723 ALWAYS_INLINE 724 uptr MetaShadowEnd(void) { return SelectMapping<MappingField>(kMetaShadowEnd); } 725 726 struct IsAppMemImpl { 727 template <typename Mapping> 728 static bool Apply(uptr mem) { 729 return (mem >= Mapping::kHeapMemBeg && mem < Mapping::kHeapMemEnd) || 730 (mem >= Mapping::kMidAppMemBeg && mem < Mapping::kMidAppMemEnd) || 731 (mem >= Mapping::kLoAppMemBeg && mem < Mapping::kLoAppMemEnd) || 732 (mem >= Mapping::kHiAppMemBeg && mem < Mapping::kHiAppMemEnd); 733 } 734 }; 735 736 ALWAYS_INLINE 737 bool IsAppMem(uptr mem) { return SelectMapping<IsAppMemImpl>(mem); } 738 739 struct IsShadowMemImpl { 740 template <typename Mapping> 741 static bool Apply(uptr mem) { 742 return mem >= Mapping::kShadowBeg && mem <= Mapping::kShadowEnd; 743 } 744 }; 745 746 ALWAYS_INLINE 747 bool IsShadowMem(RawShadow *p) { 748 return SelectMapping<IsShadowMemImpl>(reinterpret_cast<uptr>(p)); 749 } 750 751 struct IsMetaMemImpl { 752 template <typename Mapping> 753 static bool Apply(uptr mem) { 754 return mem >= Mapping::kMetaShadowBeg && mem <= Mapping::kMetaShadowEnd; 755 } 756 }; 757 758 ALWAYS_INLINE 759 bool IsMetaMem(const u32 *p) { 760 return SelectMapping<IsMetaMemImpl>(reinterpret_cast<uptr>(p)); 761 } 762 763 struct MemToShadowImpl { 764 template <typename Mapping> 765 static uptr Apply(uptr x) { 766 DCHECK(IsAppMemImpl::Apply<Mapping>(x)); 767 return (((x) & ~(Mapping::kShadowMsk | (kShadowCell - 1))) ^ 768 Mapping::kShadowXor) * 769 kShadowMultiplier + 770 Mapping::kShadowAdd; 771 } 772 }; 773 774 ALWAYS_INLINE 775 RawShadow *MemToShadow(uptr x) { 776 return reinterpret_cast<RawShadow *>(SelectMapping<MemToShadowImpl>(x)); 777 } 778 779 struct MemToMetaImpl { 780 template <typename Mapping> 781 static u32 *Apply(uptr x) { 782 DCHECK(IsAppMemImpl::Apply<Mapping>(x)); 783 return (u32 *)(((((x) & ~(Mapping::kShadowMsk | (kMetaShadowCell - 1)))) / 784 kMetaShadowCell * kMetaShadowSize) | 785 Mapping::kMetaShadowBeg); 786 } 787 }; 788 789 ALWAYS_INLINE 790 u32 *MemToMeta(uptr x) { return SelectMapping<MemToMetaImpl>(x); } 791 792 struct ShadowToMemImpl { 793 template <typename Mapping> 794 static uptr Apply(uptr sp) { 795 if (!IsShadowMemImpl::Apply<Mapping>(sp)) 796 return 0; 797 // The shadow mapping is non-linear and we've lost some bits, so we don't 798 // have an easy way to restore the original app address. But the mapping is 799 // a bijection, so we try to restore the address as belonging to 800 // low/mid/high range consecutively and see if shadow->app->shadow mapping 801 // gives us the same address. 802 uptr p = 803 ((sp - Mapping::kShadowAdd) / kShadowMultiplier) ^ Mapping::kShadowXor; 804 if (p >= Mapping::kLoAppMemBeg && p < Mapping::kLoAppMemEnd && 805 MemToShadowImpl::Apply<Mapping>(p) == sp) 806 return p; 807 if (Mapping::kMidAppMemBeg) { 808 uptr p_mid = p + (Mapping::kMidAppMemBeg & Mapping::kShadowMsk); 809 if (p_mid >= Mapping::kMidAppMemBeg && p_mid < Mapping::kMidAppMemEnd && 810 MemToShadowImpl::Apply<Mapping>(p_mid) == sp) 811 return p_mid; 812 } 813 return p | Mapping::kShadowMsk; 814 } 815 }; 816 817 ALWAYS_INLINE 818 uptr ShadowToMem(RawShadow *s) { 819 return SelectMapping<ShadowToMemImpl>(reinterpret_cast<uptr>(s)); 820 } 821 822 // Compresses addr to kCompressedAddrBits stored in least significant bits. 823 ALWAYS_INLINE uptr CompressAddr(uptr addr) { 824 return addr & ((1ull << kCompressedAddrBits) - 1); 825 } 826 827 struct RestoreAddrImpl { 828 typedef uptr Result; 829 template <typename Mapping> 830 static Result Apply(uptr addr) { 831 // To restore the address we go over all app memory ranges and check if top 832 // 3 bits of the compressed addr match that of the app range. If yes, we 833 // assume that the compressed address come from that range and restore the 834 // missing top bits to match the app range address. 835 const uptr ranges[] = { 836 Mapping::kLoAppMemBeg, Mapping::kLoAppMemEnd, Mapping::kMidAppMemBeg, 837 Mapping::kMidAppMemEnd, Mapping::kHiAppMemBeg, Mapping::kHiAppMemEnd, 838 Mapping::kHeapMemBeg, Mapping::kHeapMemEnd, 839 }; 840 const uptr indicator = 0x0e0000000000ull; 841 const uptr ind_lsb = 1ull << LeastSignificantSetBitIndex(indicator); 842 for (uptr i = 0; i < ARRAY_SIZE(ranges); i += 2) { 843 uptr beg = ranges[i]; 844 uptr end = ranges[i + 1]; 845 if (beg == end) 846 continue; 847 for (uptr p = beg; p < end; p = RoundDown(p + ind_lsb, ind_lsb)) { 848 if ((addr & indicator) == (p & indicator)) 849 return addr | (p & ~(ind_lsb - 1)); 850 } 851 } 852 Printf("ThreadSanitizer: failed to restore address 0x%zx\n", addr); 853 Die(); 854 } 855 }; 856 857 // Restores compressed addr from kCompressedAddrBits to full representation. 858 // This is called only during reporting and is not performance-critical. 859 inline uptr RestoreAddr(uptr addr) { 860 return SelectMapping<RestoreAddrImpl>(addr); 861 } 862 863 void InitializePlatform(); 864 void InitializePlatformEarly(); 865 void CheckAndProtect(); 866 void InitializeShadowMemoryPlatform(); 867 void WriteMemoryProfile(char *buf, uptr buf_size, u64 uptime_ns); 868 int ExtractResolvFDs(void *state, int *fds, int nfd); 869 int ExtractRecvmsgFDs(void *msg, int *fds, int nfd); 870 uptr ExtractLongJmpSp(uptr *env); 871 void ImitateTlsWrite(ThreadState *thr, uptr tls_addr, uptr tls_size); 872 873 int call_pthread_cancel_with_cleanup(int (*fn)(void *arg), 874 void (*cleanup)(void *arg), void *arg); 875 876 void DestroyThreadState(); 877 void PlatformCleanUpThreadState(ThreadState *thr); 878 879 } // namespace __tsan 880 881 #endif // TSAN_PLATFORM_H 882