1 #include "jemalloc/internal/jemalloc_preamble.h" 2 #include "jemalloc/internal/jemalloc_internal_includes.h" 3 4 #include "jemalloc/internal/malloc_io.h" 5 #include "jemalloc/internal/util.h" 6 7 #ifdef assert 8 # undef assert 9 #endif 10 #ifdef not_reached 11 # undef not_reached 12 #endif 13 #ifdef not_implemented 14 # undef not_implemented 15 #endif 16 #ifdef assert_not_implemented 17 # undef assert_not_implemented 18 #endif 19 20 /* 21 * Define simple versions of assertion macros that won't recurse in case 22 * of assertion failures in malloc_*printf(). 23 */ 24 #define assert(e) do { \ 25 if (config_debug && !(e)) { \ 26 malloc_write("<jemalloc>: Failed assertion\n"); \ 27 abort(); \ 28 } \ 29 } while (0) 30 31 #define not_reached() do { \ 32 if (config_debug) { \ 33 malloc_write("<jemalloc>: Unreachable code reached\n"); \ 34 abort(); \ 35 } \ 36 unreachable(); \ 37 } while (0) 38 39 #define not_implemented() do { \ 40 if (config_debug) { \ 41 malloc_write("<jemalloc>: Not implemented\n"); \ 42 abort(); \ 43 } \ 44 } while (0) 45 46 #define assert_not_implemented(e) do { \ 47 if (unlikely(config_debug && !(e))) { \ 48 not_implemented(); \ 49 } \ 50 } while (0) 51 52 /******************************************************************************/ 53 /* Function prototypes for non-inline static functions. */ 54 55 #define U2S_BUFSIZE ((1U << (LG_SIZEOF_INTMAX_T + 3)) + 1) 56 static char *u2s(uintmax_t x, unsigned base, bool uppercase, char *s, 57 size_t *slen_p); 58 #define D2S_BUFSIZE (1 + U2S_BUFSIZE) 59 static char *d2s(intmax_t x, char sign, char *s, size_t *slen_p); 60 #define O2S_BUFSIZE (1 + U2S_BUFSIZE) 61 static char *o2s(uintmax_t x, bool alt_form, char *s, size_t *slen_p); 62 #define X2S_BUFSIZE (2 + U2S_BUFSIZE) 63 static char *x2s(uintmax_t x, bool alt_form, bool uppercase, char *s, 64 size_t *slen_p); 65 66 /******************************************************************************/ 67 68 /* malloc_message() setup. */ 69 void 70 wrtmessage(void *cbopaque, const char *s) { 71 malloc_write_fd(STDERR_FILENO, s, strlen(s)); 72 } 73 74 JEMALLOC_EXPORT void (*je_malloc_message)(void *, const char *s); 75 76 JEMALLOC_ATTR(visibility("hidden")) 77 void 78 wrtmessage_1_0(const char *s1, const char *s2, const char *s3, const char *s4) { 79 80 wrtmessage(NULL, s1); 81 wrtmessage(NULL, s2); 82 wrtmessage(NULL, s3); 83 wrtmessage(NULL, s4); 84 } 85 86 void (*__malloc_message_1_0)(const char *s1, const char *s2, const char *s3, 87 const char *s4) = wrtmessage_1_0; 88 __sym_compat(_malloc_message, __malloc_message_1_0, FBSD_1.0); 89 90 /* 91 * Wrapper around malloc_message() that avoids the need for 92 * je_malloc_message(...) throughout the code. 93 */ 94 void 95 malloc_write(const char *s) { 96 if (je_malloc_message != NULL) { 97 je_malloc_message(NULL, s); 98 } else { 99 wrtmessage(NULL, s); 100 } 101 } 102 103 /* 104 * glibc provides a non-standard strerror_r() when _GNU_SOURCE is defined, so 105 * provide a wrapper. 106 */ 107 int 108 buferror(int err, char *buf, size_t buflen) { 109 #ifdef _WIN32 110 FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM, NULL, err, 0, 111 (LPSTR)buf, (DWORD)buflen, NULL); 112 return 0; 113 #elif defined(JEMALLOC_STRERROR_R_RETURNS_CHAR_WITH_GNU_SOURCE) && defined(_GNU_SOURCE) 114 char *b = strerror_r(err, buf, buflen); 115 if (b != buf) { 116 strncpy(buf, b, buflen); 117 buf[buflen-1] = '\0'; 118 } 119 return 0; 120 #else 121 return strerror_r(err, buf, buflen); 122 #endif 123 } 124 125 uintmax_t 126 malloc_strtoumax(const char *restrict nptr, char **restrict endptr, int base) { 127 uintmax_t ret, digit; 128 unsigned b; 129 bool neg; 130 const char *p, *ns; 131 132 p = nptr; 133 if (base < 0 || base == 1 || base > 36) { 134 ns = p; 135 set_errno(EINVAL); 136 ret = UINTMAX_MAX; 137 goto label_return; 138 } 139 b = base; 140 141 /* Swallow leading whitespace and get sign, if any. */ 142 neg = false; 143 while (true) { 144 switch (*p) { 145 case '\t': case '\n': case '\v': case '\f': case '\r': case ' ': 146 p++; 147 break; 148 case '-': 149 neg = true; 150 JEMALLOC_FALLTHROUGH; 151 case '+': 152 p++; 153 JEMALLOC_FALLTHROUGH; 154 default: 155 goto label_prefix; 156 } 157 } 158 159 /* Get prefix, if any. */ 160 label_prefix: 161 /* 162 * Note where the first non-whitespace/sign character is so that it is 163 * possible to tell whether any digits are consumed (e.g., " 0" vs. 164 * " -x"). 165 */ 166 ns = p; 167 if (*p == '0') { 168 switch (p[1]) { 169 case '0': case '1': case '2': case '3': case '4': case '5': 170 case '6': case '7': 171 if (b == 0) { 172 b = 8; 173 } 174 if (b == 8) { 175 p++; 176 } 177 break; 178 case 'X': case 'x': 179 switch (p[2]) { 180 case '0': case '1': case '2': case '3': case '4': 181 case '5': case '6': case '7': case '8': case '9': 182 case 'A': case 'B': case 'C': case 'D': case 'E': 183 case 'F': 184 case 'a': case 'b': case 'c': case 'd': case 'e': 185 case 'f': 186 if (b == 0) { 187 b = 16; 188 } 189 if (b == 16) { 190 p += 2; 191 } 192 break; 193 default: 194 break; 195 } 196 break; 197 default: 198 p++; 199 ret = 0; 200 goto label_return; 201 } 202 } 203 if (b == 0) { 204 b = 10; 205 } 206 207 /* Convert. */ 208 ret = 0; 209 while ((*p >= '0' && *p <= '9' && (digit = *p - '0') < b) 210 || (*p >= 'A' && *p <= 'Z' && (digit = 10 + *p - 'A') < b) 211 || (*p >= 'a' && *p <= 'z' && (digit = 10 + *p - 'a') < b)) { 212 uintmax_t pret = ret; 213 ret *= b; 214 ret += digit; 215 if (ret < pret) { 216 /* Overflow. */ 217 set_errno(ERANGE); 218 ret = UINTMAX_MAX; 219 goto label_return; 220 } 221 p++; 222 } 223 if (neg) { 224 ret = (uintmax_t)(-((intmax_t)ret)); 225 } 226 227 if (p == ns) { 228 /* No conversion performed. */ 229 set_errno(EINVAL); 230 ret = UINTMAX_MAX; 231 goto label_return; 232 } 233 234 label_return: 235 if (endptr != NULL) { 236 if (p == ns) { 237 /* No characters were converted. */ 238 *endptr = (char *)nptr; 239 } else { 240 *endptr = (char *)p; 241 } 242 } 243 return ret; 244 } 245 246 static char * 247 u2s(uintmax_t x, unsigned base, bool uppercase, char *s, size_t *slen_p) { 248 unsigned i; 249 250 i = U2S_BUFSIZE - 1; 251 s[i] = '\0'; 252 switch (base) { 253 case 10: 254 do { 255 i--; 256 s[i] = "0123456789"[x % (uint64_t)10]; 257 x /= (uint64_t)10; 258 } while (x > 0); 259 break; 260 case 16: { 261 const char *digits = (uppercase) 262 ? "0123456789ABCDEF" 263 : "0123456789abcdef"; 264 265 do { 266 i--; 267 s[i] = digits[x & 0xf]; 268 x >>= 4; 269 } while (x > 0); 270 break; 271 } default: { 272 const char *digits = (uppercase) 273 ? "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" 274 : "0123456789abcdefghijklmnopqrstuvwxyz"; 275 276 assert(base >= 2 && base <= 36); 277 do { 278 i--; 279 s[i] = digits[x % (uint64_t)base]; 280 x /= (uint64_t)base; 281 } while (x > 0); 282 }} 283 284 *slen_p = U2S_BUFSIZE - 1 - i; 285 return &s[i]; 286 } 287 288 static char * 289 d2s(intmax_t x, char sign, char *s, size_t *slen_p) { 290 bool neg; 291 292 if ((neg = (x < 0))) { 293 x = -x; 294 } 295 s = u2s(x, 10, false, s, slen_p); 296 if (neg) { 297 sign = '-'; 298 } 299 switch (sign) { 300 case '-': 301 if (!neg) { 302 break; 303 } 304 JEMALLOC_FALLTHROUGH; 305 case ' ': 306 case '+': 307 s--; 308 (*slen_p)++; 309 *s = sign; 310 break; 311 default: not_reached(); 312 } 313 return s; 314 } 315 316 static char * 317 o2s(uintmax_t x, bool alt_form, char *s, size_t *slen_p) { 318 s = u2s(x, 8, false, s, slen_p); 319 if (alt_form && *s != '0') { 320 s--; 321 (*slen_p)++; 322 *s = '0'; 323 } 324 return s; 325 } 326 327 static char * 328 x2s(uintmax_t x, bool alt_form, bool uppercase, char *s, size_t *slen_p) { 329 s = u2s(x, 16, uppercase, s, slen_p); 330 if (alt_form) { 331 s -= 2; 332 (*slen_p) += 2; 333 memcpy(s, uppercase ? "0X" : "0x", 2); 334 } 335 return s; 336 } 337 338 JEMALLOC_COLD 339 size_t 340 malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) { 341 size_t i; 342 const char *f; 343 344 #define APPEND_C(c) do { \ 345 if (i < size) { \ 346 str[i] = (c); \ 347 } \ 348 i++; \ 349 } while (0) 350 #define APPEND_S(s, slen) do { \ 351 if (i < size) { \ 352 size_t cpylen = (slen <= size - i) ? slen : size - i; \ 353 memcpy(&str[i], s, cpylen); \ 354 } \ 355 i += slen; \ 356 } while (0) 357 #define APPEND_PADDED_S(s, slen, width, left_justify) do { \ 358 /* Left padding. */ \ 359 size_t pad_len = (width == -1) ? 0 : ((slen < (size_t)width) ? \ 360 (size_t)width - slen : 0); \ 361 if (!left_justify && pad_len != 0) { \ 362 size_t j; \ 363 for (j = 0; j < pad_len; j++) { \ 364 if (pad_zero) { \ 365 APPEND_C('0'); \ 366 } else { \ 367 APPEND_C(' '); \ 368 } \ 369 } \ 370 } \ 371 /* Value. */ \ 372 APPEND_S(s, slen); \ 373 /* Right padding. */ \ 374 if (left_justify && pad_len != 0) { \ 375 size_t j; \ 376 for (j = 0; j < pad_len; j++) { \ 377 APPEND_C(' '); \ 378 } \ 379 } \ 380 } while (0) 381 #define GET_ARG_NUMERIC(val, len) do { \ 382 switch ((unsigned char)len) { \ 383 case '?': \ 384 val = va_arg(ap, int); \ 385 break; \ 386 case '?' | 0x80: \ 387 val = va_arg(ap, unsigned int); \ 388 break; \ 389 case 'l': \ 390 val = va_arg(ap, long); \ 391 break; \ 392 case 'l' | 0x80: \ 393 val = va_arg(ap, unsigned long); \ 394 break; \ 395 case 'q': \ 396 val = va_arg(ap, long long); \ 397 break; \ 398 case 'q' | 0x80: \ 399 val = va_arg(ap, unsigned long long); \ 400 break; \ 401 case 'j': \ 402 val = va_arg(ap, intmax_t); \ 403 break; \ 404 case 'j' | 0x80: \ 405 val = va_arg(ap, uintmax_t); \ 406 break; \ 407 case 't': \ 408 val = va_arg(ap, ptrdiff_t); \ 409 break; \ 410 case 'z': \ 411 val = va_arg(ap, ssize_t); \ 412 break; \ 413 case 'z' | 0x80: \ 414 val = va_arg(ap, size_t); \ 415 break; \ 416 case 'p': /* Synthetic; used for %p. */ \ 417 val = va_arg(ap, uintptr_t); \ 418 break; \ 419 default: \ 420 not_reached(); \ 421 val = 0; \ 422 } \ 423 } while (0) 424 425 i = 0; 426 f = format; 427 while (true) { 428 switch (*f) { 429 case '\0': goto label_out; 430 case '%': { 431 bool alt_form = false; 432 bool left_justify = false; 433 bool plus_space = false; 434 bool plus_plus = false; 435 int prec = -1; 436 int width = -1; 437 unsigned char len = '?'; 438 char *s; 439 size_t slen; 440 bool first_width_digit = true; 441 bool pad_zero = false; 442 443 f++; 444 /* Flags. */ 445 while (true) { 446 switch (*f) { 447 case '#': 448 assert(!alt_form); 449 alt_form = true; 450 break; 451 case '-': 452 assert(!left_justify); 453 left_justify = true; 454 break; 455 case ' ': 456 assert(!plus_space); 457 plus_space = true; 458 break; 459 case '+': 460 assert(!plus_plus); 461 plus_plus = true; 462 break; 463 default: goto label_width; 464 } 465 f++; 466 } 467 /* Width. */ 468 label_width: 469 switch (*f) { 470 case '*': 471 width = va_arg(ap, int); 472 f++; 473 if (width < 0) { 474 left_justify = true; 475 width = -width; 476 } 477 break; 478 case '0': 479 if (first_width_digit) { 480 pad_zero = true; 481 } 482 JEMALLOC_FALLTHROUGH; 483 case '1': case '2': case '3': case '4': 484 case '5': case '6': case '7': case '8': case '9': { 485 uintmax_t uwidth; 486 set_errno(0); 487 uwidth = malloc_strtoumax(f, (char **)&f, 10); 488 assert(uwidth != UINTMAX_MAX || get_errno() != 489 ERANGE); 490 width = (int)uwidth; 491 first_width_digit = false; 492 break; 493 } default: 494 break; 495 } 496 /* Width/precision separator. */ 497 if (*f == '.') { 498 f++; 499 } else { 500 goto label_length; 501 } 502 /* Precision. */ 503 switch (*f) { 504 case '*': 505 prec = va_arg(ap, int); 506 f++; 507 break; 508 case '0': case '1': case '2': case '3': case '4': 509 case '5': case '6': case '7': case '8': case '9': { 510 uintmax_t uprec; 511 set_errno(0); 512 uprec = malloc_strtoumax(f, (char **)&f, 10); 513 assert(uprec != UINTMAX_MAX || get_errno() != 514 ERANGE); 515 prec = (int)uprec; 516 break; 517 } 518 default: break; 519 } 520 /* Length. */ 521 label_length: 522 switch (*f) { 523 case 'l': 524 f++; 525 if (*f == 'l') { 526 len = 'q'; 527 f++; 528 } else { 529 len = 'l'; 530 } 531 break; 532 case 'q': case 'j': case 't': case 'z': 533 len = *f; 534 f++; 535 break; 536 default: break; 537 } 538 /* Conversion specifier. */ 539 switch (*f) { 540 case '%': 541 /* %% */ 542 APPEND_C(*f); 543 f++; 544 break; 545 case 'd': case 'i': { 546 intmax_t val JEMALLOC_CC_SILENCE_INIT(0); 547 char buf[D2S_BUFSIZE]; 548 549 /* 550 * Outputting negative, zero-padded numbers 551 * would require a nontrivial rework of the 552 * interaction between the width and padding 553 * (since 0 padding goes between the '-' and the 554 * number, while ' ' padding goes either before 555 * the - or after the number. Since we 556 * currently don't ever need 0-padded negative 557 * numbers, just don't bother supporting it. 558 */ 559 assert(!pad_zero); 560 561 GET_ARG_NUMERIC(val, len); 562 s = d2s(val, (plus_plus ? '+' : (plus_space ? 563 ' ' : '-')), buf, &slen); 564 APPEND_PADDED_S(s, slen, width, left_justify); 565 f++; 566 break; 567 } case 'o': { 568 uintmax_t val JEMALLOC_CC_SILENCE_INIT(0); 569 char buf[O2S_BUFSIZE]; 570 571 GET_ARG_NUMERIC(val, len | 0x80); 572 s = o2s(val, alt_form, buf, &slen); 573 APPEND_PADDED_S(s, slen, width, left_justify); 574 f++; 575 break; 576 } case 'u': { 577 uintmax_t val JEMALLOC_CC_SILENCE_INIT(0); 578 char buf[U2S_BUFSIZE]; 579 580 GET_ARG_NUMERIC(val, len | 0x80); 581 s = u2s(val, 10, false, buf, &slen); 582 APPEND_PADDED_S(s, slen, width, left_justify); 583 f++; 584 break; 585 } case 'x': case 'X': { 586 uintmax_t val JEMALLOC_CC_SILENCE_INIT(0); 587 char buf[X2S_BUFSIZE]; 588 589 GET_ARG_NUMERIC(val, len | 0x80); 590 s = x2s(val, alt_form, *f == 'X', buf, &slen); 591 APPEND_PADDED_S(s, slen, width, left_justify); 592 f++; 593 break; 594 } case 'c': { 595 unsigned char val; 596 char buf[2]; 597 598 assert(len == '?' || len == 'l'); 599 assert_not_implemented(len != 'l'); 600 val = va_arg(ap, int); 601 buf[0] = val; 602 buf[1] = '\0'; 603 APPEND_PADDED_S(buf, 1, width, left_justify); 604 f++; 605 break; 606 } case 's': 607 assert(len == '?' || len == 'l'); 608 assert_not_implemented(len != 'l'); 609 s = va_arg(ap, char *); 610 slen = (prec < 0) ? strlen(s) : (size_t)prec; 611 APPEND_PADDED_S(s, slen, width, left_justify); 612 f++; 613 break; 614 case 'p': { 615 uintmax_t val; 616 char buf[X2S_BUFSIZE]; 617 618 GET_ARG_NUMERIC(val, 'p'); 619 s = x2s(val, true, false, buf, &slen); 620 APPEND_PADDED_S(s, slen, width, left_justify); 621 f++; 622 break; 623 } default: not_reached(); 624 } 625 break; 626 } default: { 627 APPEND_C(*f); 628 f++; 629 break; 630 }} 631 } 632 label_out: 633 if (i < size) { 634 str[i] = '\0'; 635 } else { 636 str[size - 1] = '\0'; 637 } 638 639 #undef APPEND_C 640 #undef APPEND_S 641 #undef APPEND_PADDED_S 642 #undef GET_ARG_NUMERIC 643 return i; 644 } 645 646 JEMALLOC_FORMAT_PRINTF(3, 4) 647 size_t 648 malloc_snprintf(char *str, size_t size, const char *format, ...) { 649 size_t ret; 650 va_list ap; 651 652 va_start(ap, format); 653 ret = malloc_vsnprintf(str, size, format, ap); 654 va_end(ap); 655 656 return ret; 657 } 658 659 void 660 malloc_vcprintf(write_cb_t *write_cb, void *cbopaque, const char *format, 661 va_list ap) { 662 char buf[MALLOC_PRINTF_BUFSIZE]; 663 664 if (write_cb == NULL) { 665 /* 666 * The caller did not provide an alternate write_cb callback 667 * function, so use the default one. malloc_write() is an 668 * inline function, so use malloc_message() directly here. 669 */ 670 write_cb = (je_malloc_message != NULL) ? je_malloc_message : 671 wrtmessage; 672 } 673 674 malloc_vsnprintf(buf, sizeof(buf), format, ap); 675 write_cb(cbopaque, buf); 676 } 677 678 /* 679 * Print to a callback function in such a way as to (hopefully) avoid memory 680 * allocation. 681 */ 682 JEMALLOC_FORMAT_PRINTF(3, 4) 683 void 684 malloc_cprintf(write_cb_t *write_cb, void *cbopaque, const char *format, ...) { 685 va_list ap; 686 687 va_start(ap, format); 688 malloc_vcprintf(write_cb, cbopaque, format, ap); 689 va_end(ap); 690 } 691 692 /* Print to stderr in such a way as to avoid memory allocation. */ 693 JEMALLOC_FORMAT_PRINTF(1, 2) 694 void 695 malloc_printf(const char *format, ...) { 696 va_list ap; 697 698 va_start(ap, format); 699 malloc_vcprintf(NULL, NULL, format, ap); 700 va_end(ap); 701 } 702 703 /* 704 * Restore normal assertion macros, in order to make it possible to compile all 705 * C files as a single concatenation. 706 */ 707 #undef assert 708 #undef not_reached 709 #undef not_implemented 710 #undef assert_not_implemented 711 #include "jemalloc/internal/assert.h" 712