1 #define JEMALLOC_MALLOC_IO_C_ 2 #include "jemalloc/internal/jemalloc_preamble.h" 3 #include "jemalloc/internal/jemalloc_internal_includes.h" 4 5 #include "jemalloc/internal/malloc_io.h" 6 #include "jemalloc/internal/util.h" 7 8 #ifdef assert 9 # undef assert 10 #endif 11 #ifdef not_reached 12 # undef not_reached 13 #endif 14 #ifdef not_implemented 15 # undef not_implemented 16 #endif 17 #ifdef assert_not_implemented 18 # undef assert_not_implemented 19 #endif 20 21 /* 22 * Define simple versions of assertion macros that won't recurse in case 23 * of assertion failures in malloc_*printf(). 24 */ 25 #define assert(e) do { \ 26 if (config_debug && !(e)) { \ 27 malloc_write("<jemalloc>: Failed assertion\n"); \ 28 abort(); \ 29 } \ 30 } while (0) 31 32 #define not_reached() do { \ 33 if (config_debug) { \ 34 malloc_write("<jemalloc>: Unreachable code reached\n"); \ 35 abort(); \ 36 } \ 37 unreachable(); \ 38 } while (0) 39 40 #define not_implemented() do { \ 41 if (config_debug) { \ 42 malloc_write("<jemalloc>: Not implemented\n"); \ 43 abort(); \ 44 } \ 45 } while (0) 46 47 #define assert_not_implemented(e) do { \ 48 if (unlikely(config_debug && !(e))) { \ 49 not_implemented(); \ 50 } \ 51 } while (0) 52 53 /******************************************************************************/ 54 /* Function prototypes for non-inline static functions. */ 55 56 static void wrtmessage(void *cbopaque, const char *s); 57 #define U2S_BUFSIZE ((1U << (LG_SIZEOF_INTMAX_T + 3)) + 1) 58 static char *u2s(uintmax_t x, unsigned base, bool uppercase, char *s, 59 size_t *slen_p); 60 #define D2S_BUFSIZE (1 + U2S_BUFSIZE) 61 static char *d2s(intmax_t x, char sign, char *s, size_t *slen_p); 62 #define O2S_BUFSIZE (1 + U2S_BUFSIZE) 63 static char *o2s(uintmax_t x, bool alt_form, char *s, size_t *slen_p); 64 #define X2S_BUFSIZE (2 + U2S_BUFSIZE) 65 static char *x2s(uintmax_t x, bool alt_form, bool uppercase, char *s, 66 size_t *slen_p); 67 68 /******************************************************************************/ 69 70 /* malloc_message() setup. */ 71 static void 72 wrtmessage(void *cbopaque, const char *s) { 73 #if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_write) 74 /* 75 * Use syscall(2) rather than write(2) when possible in order to avoid 76 * the possibility of memory allocation within libc. This is necessary 77 * on FreeBSD; most operating systems do not have this problem though. 78 * 79 * syscall() returns long or int, depending on platform, so capture the 80 * unused result in the widest plausible type to avoid compiler 81 * warnings. 82 */ 83 UNUSED long result = syscall(SYS_write, STDERR_FILENO, s, strlen(s)); 84 #else 85 UNUSED ssize_t result = write(STDERR_FILENO, s, strlen(s)); 86 #endif 87 } 88 89 JEMALLOC_EXPORT void (*je_malloc_message)(void *, const char *s); 90 91 JEMALLOC_ATTR(visibility("hidden")) 92 void 93 wrtmessage_1_0(const char *s1, const char *s2, const char *s3, const char *s4) { 94 95 wrtmessage(NULL, s1); 96 wrtmessage(NULL, s2); 97 wrtmessage(NULL, s3); 98 wrtmessage(NULL, s4); 99 } 100 101 void (*__malloc_message_1_0)(const char *s1, const char *s2, const char *s3, 102 const char *s4) = wrtmessage_1_0; 103 __sym_compat(_malloc_message, __malloc_message_1_0, FBSD_1.0); 104 105 /* 106 * Wrapper around malloc_message() that avoids the need for 107 * je_malloc_message(...) throughout the code. 108 */ 109 void 110 malloc_write(const char *s) { 111 if (je_malloc_message != NULL) { 112 je_malloc_message(NULL, s); 113 } else { 114 wrtmessage(NULL, s); 115 } 116 } 117 118 /* 119 * glibc provides a non-standard strerror_r() when _GNU_SOURCE is defined, so 120 * provide a wrapper. 121 */ 122 int 123 buferror(int err, char *buf, size_t buflen) { 124 #ifdef _WIN32 125 FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM, NULL, err, 0, 126 (LPSTR)buf, (DWORD)buflen, NULL); 127 return 0; 128 #elif defined(__GLIBC__) && defined(_GNU_SOURCE) 129 char *b = strerror_r(err, buf, buflen); 130 if (b != buf) { 131 strncpy(buf, b, buflen); 132 buf[buflen-1] = '\0'; 133 } 134 return 0; 135 #else 136 return strerror_r(err, buf, buflen); 137 #endif 138 } 139 140 uintmax_t 141 malloc_strtoumax(const char *restrict nptr, char **restrict endptr, int base) { 142 uintmax_t ret, digit; 143 unsigned b; 144 bool neg; 145 const char *p, *ns; 146 147 p = nptr; 148 if (base < 0 || base == 1 || base > 36) { 149 ns = p; 150 set_errno(EINVAL); 151 ret = UINTMAX_MAX; 152 goto label_return; 153 } 154 b = base; 155 156 /* Swallow leading whitespace and get sign, if any. */ 157 neg = false; 158 while (true) { 159 switch (*p) { 160 case '\t': case '\n': case '\v': case '\f': case '\r': case ' ': 161 p++; 162 break; 163 case '-': 164 neg = true; 165 /* Fall through. */ 166 case '+': 167 p++; 168 /* Fall through. */ 169 default: 170 goto label_prefix; 171 } 172 } 173 174 /* Get prefix, if any. */ 175 label_prefix: 176 /* 177 * Note where the first non-whitespace/sign character is so that it is 178 * possible to tell whether any digits are consumed (e.g., " 0" vs. 179 * " -x"). 180 */ 181 ns = p; 182 if (*p == '0') { 183 switch (p[1]) { 184 case '0': case '1': case '2': case '3': case '4': case '5': 185 case '6': case '7': 186 if (b == 0) { 187 b = 8; 188 } 189 if (b == 8) { 190 p++; 191 } 192 break; 193 case 'X': case 'x': 194 switch (p[2]) { 195 case '0': case '1': case '2': case '3': case '4': 196 case '5': case '6': case '7': case '8': case '9': 197 case 'A': case 'B': case 'C': case 'D': case 'E': 198 case 'F': 199 case 'a': case 'b': case 'c': case 'd': case 'e': 200 case 'f': 201 if (b == 0) { 202 b = 16; 203 } 204 if (b == 16) { 205 p += 2; 206 } 207 break; 208 default: 209 break; 210 } 211 break; 212 default: 213 p++; 214 ret = 0; 215 goto label_return; 216 } 217 } 218 if (b == 0) { 219 b = 10; 220 } 221 222 /* Convert. */ 223 ret = 0; 224 while ((*p >= '0' && *p <= '9' && (digit = *p - '0') < b) 225 || (*p >= 'A' && *p <= 'Z' && (digit = 10 + *p - 'A') < b) 226 || (*p >= 'a' && *p <= 'z' && (digit = 10 + *p - 'a') < b)) { 227 uintmax_t pret = ret; 228 ret *= b; 229 ret += digit; 230 if (ret < pret) { 231 /* Overflow. */ 232 set_errno(ERANGE); 233 ret = UINTMAX_MAX; 234 goto label_return; 235 } 236 p++; 237 } 238 if (neg) { 239 ret = (uintmax_t)(-((intmax_t)ret)); 240 } 241 242 if (p == ns) { 243 /* No conversion performed. */ 244 set_errno(EINVAL); 245 ret = UINTMAX_MAX; 246 goto label_return; 247 } 248 249 label_return: 250 if (endptr != NULL) { 251 if (p == ns) { 252 /* No characters were converted. */ 253 *endptr = (char *)nptr; 254 } else { 255 *endptr = (char *)p; 256 } 257 } 258 return ret; 259 } 260 261 static char * 262 u2s(uintmax_t x, unsigned base, bool uppercase, char *s, size_t *slen_p) { 263 unsigned i; 264 265 i = U2S_BUFSIZE - 1; 266 s[i] = '\0'; 267 switch (base) { 268 case 10: 269 do { 270 i--; 271 s[i] = "0123456789"[x % (uint64_t)10]; 272 x /= (uint64_t)10; 273 } while (x > 0); 274 break; 275 case 16: { 276 const char *digits = (uppercase) 277 ? "0123456789ABCDEF" 278 : "0123456789abcdef"; 279 280 do { 281 i--; 282 s[i] = digits[x & 0xf]; 283 x >>= 4; 284 } while (x > 0); 285 break; 286 } default: { 287 const char *digits = (uppercase) 288 ? "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" 289 : "0123456789abcdefghijklmnopqrstuvwxyz"; 290 291 assert(base >= 2 && base <= 36); 292 do { 293 i--; 294 s[i] = digits[x % (uint64_t)base]; 295 x /= (uint64_t)base; 296 } while (x > 0); 297 }} 298 299 *slen_p = U2S_BUFSIZE - 1 - i; 300 return &s[i]; 301 } 302 303 static char * 304 d2s(intmax_t x, char sign, char *s, size_t *slen_p) { 305 bool neg; 306 307 if ((neg = (x < 0))) { 308 x = -x; 309 } 310 s = u2s(x, 10, false, s, slen_p); 311 if (neg) { 312 sign = '-'; 313 } 314 switch (sign) { 315 case '-': 316 if (!neg) { 317 break; 318 } 319 /* Fall through. */ 320 case ' ': 321 case '+': 322 s--; 323 (*slen_p)++; 324 *s = sign; 325 break; 326 default: not_reached(); 327 } 328 return s; 329 } 330 331 static char * 332 o2s(uintmax_t x, bool alt_form, char *s, size_t *slen_p) { 333 s = u2s(x, 8, false, s, slen_p); 334 if (alt_form && *s != '0') { 335 s--; 336 (*slen_p)++; 337 *s = '0'; 338 } 339 return s; 340 } 341 342 static char * 343 x2s(uintmax_t x, bool alt_form, bool uppercase, char *s, size_t *slen_p) { 344 s = u2s(x, 16, uppercase, s, slen_p); 345 if (alt_form) { 346 s -= 2; 347 (*slen_p) += 2; 348 memcpy(s, uppercase ? "0X" : "0x", 2); 349 } 350 return s; 351 } 352 353 size_t 354 malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) { 355 size_t i; 356 const char *f; 357 358 #define APPEND_C(c) do { \ 359 if (i < size) { \ 360 str[i] = (c); \ 361 } \ 362 i++; \ 363 } while (0) 364 #define APPEND_S(s, slen) do { \ 365 if (i < size) { \ 366 size_t cpylen = (slen <= size - i) ? slen : size - i; \ 367 memcpy(&str[i], s, cpylen); \ 368 } \ 369 i += slen; \ 370 } while (0) 371 #define APPEND_PADDED_S(s, slen, width, left_justify) do { \ 372 /* Left padding. */ \ 373 size_t pad_len = (width == -1) ? 0 : ((slen < (size_t)width) ? \ 374 (size_t)width - slen : 0); \ 375 if (!left_justify && pad_len != 0) { \ 376 size_t j; \ 377 for (j = 0; j < pad_len; j++) { \ 378 APPEND_C(' '); \ 379 } \ 380 } \ 381 /* Value. */ \ 382 APPEND_S(s, slen); \ 383 /* Right padding. */ \ 384 if (left_justify && pad_len != 0) { \ 385 size_t j; \ 386 for (j = 0; j < pad_len; j++) { \ 387 APPEND_C(' '); \ 388 } \ 389 } \ 390 } while (0) 391 #define GET_ARG_NUMERIC(val, len) do { \ 392 switch (len) { \ 393 case '?': \ 394 val = va_arg(ap, int); \ 395 break; \ 396 case '?' | 0x80: \ 397 val = va_arg(ap, unsigned int); \ 398 break; \ 399 case 'l': \ 400 val = va_arg(ap, long); \ 401 break; \ 402 case 'l' | 0x80: \ 403 val = va_arg(ap, unsigned long); \ 404 break; \ 405 case 'q': \ 406 val = va_arg(ap, long long); \ 407 break; \ 408 case 'q' | 0x80: \ 409 val = va_arg(ap, unsigned long long); \ 410 break; \ 411 case 'j': \ 412 val = va_arg(ap, intmax_t); \ 413 break; \ 414 case 'j' | 0x80: \ 415 val = va_arg(ap, uintmax_t); \ 416 break; \ 417 case 't': \ 418 val = va_arg(ap, ptrdiff_t); \ 419 break; \ 420 case 'z': \ 421 val = va_arg(ap, ssize_t); \ 422 break; \ 423 case 'z' | 0x80: \ 424 val = va_arg(ap, size_t); \ 425 break; \ 426 case 'p': /* Synthetic; used for %p. */ \ 427 val = va_arg(ap, uintptr_t); \ 428 break; \ 429 default: \ 430 not_reached(); \ 431 val = 0; \ 432 } \ 433 } while (0) 434 435 i = 0; 436 f = format; 437 while (true) { 438 switch (*f) { 439 case '\0': goto label_out; 440 case '%': { 441 bool alt_form = false; 442 bool left_justify = false; 443 bool plus_space = false; 444 bool plus_plus = false; 445 int prec = -1; 446 int width = -1; 447 unsigned char len = '?'; 448 char *s; 449 size_t slen; 450 451 f++; 452 /* Flags. */ 453 while (true) { 454 switch (*f) { 455 case '#': 456 assert(!alt_form); 457 alt_form = true; 458 break; 459 case '-': 460 assert(!left_justify); 461 left_justify = true; 462 break; 463 case ' ': 464 assert(!plus_space); 465 plus_space = true; 466 break; 467 case '+': 468 assert(!plus_plus); 469 plus_plus = true; 470 break; 471 default: goto label_width; 472 } 473 f++; 474 } 475 /* Width. */ 476 label_width: 477 switch (*f) { 478 case '*': 479 width = va_arg(ap, int); 480 f++; 481 if (width < 0) { 482 left_justify = true; 483 width = -width; 484 } 485 break; 486 case '0': case '1': case '2': case '3': case '4': 487 case '5': case '6': case '7': case '8': case '9': { 488 uintmax_t uwidth; 489 set_errno(0); 490 uwidth = malloc_strtoumax(f, (char **)&f, 10); 491 assert(uwidth != UINTMAX_MAX || get_errno() != 492 ERANGE); 493 width = (int)uwidth; 494 break; 495 } default: 496 break; 497 } 498 /* Width/precision separator. */ 499 if (*f == '.') { 500 f++; 501 } else { 502 goto label_length; 503 } 504 /* Precision. */ 505 switch (*f) { 506 case '*': 507 prec = va_arg(ap, int); 508 f++; 509 break; 510 case '0': case '1': case '2': case '3': case '4': 511 case '5': case '6': case '7': case '8': case '9': { 512 uintmax_t uprec; 513 set_errno(0); 514 uprec = malloc_strtoumax(f, (char **)&f, 10); 515 assert(uprec != UINTMAX_MAX || get_errno() != 516 ERANGE); 517 prec = (int)uprec; 518 break; 519 } 520 default: break; 521 } 522 /* Length. */ 523 label_length: 524 switch (*f) { 525 case 'l': 526 f++; 527 if (*f == 'l') { 528 len = 'q'; 529 f++; 530 } else { 531 len = 'l'; 532 } 533 break; 534 case 'q': case 'j': case 't': case 'z': 535 len = *f; 536 f++; 537 break; 538 default: break; 539 } 540 /* Conversion specifier. */ 541 switch (*f) { 542 case '%': 543 /* %% */ 544 APPEND_C(*f); 545 f++; 546 break; 547 case 'd': case 'i': { 548 intmax_t val JEMALLOC_CC_SILENCE_INIT(0); 549 char buf[D2S_BUFSIZE]; 550 551 GET_ARG_NUMERIC(val, len); 552 s = d2s(val, (plus_plus ? '+' : (plus_space ? 553 ' ' : '-')), buf, &slen); 554 APPEND_PADDED_S(s, slen, width, left_justify); 555 f++; 556 break; 557 } case 'o': { 558 uintmax_t val JEMALLOC_CC_SILENCE_INIT(0); 559 char buf[O2S_BUFSIZE]; 560 561 GET_ARG_NUMERIC(val, len | 0x80); 562 s = o2s(val, alt_form, buf, &slen); 563 APPEND_PADDED_S(s, slen, width, left_justify); 564 f++; 565 break; 566 } case 'u': { 567 uintmax_t val JEMALLOC_CC_SILENCE_INIT(0); 568 char buf[U2S_BUFSIZE]; 569 570 GET_ARG_NUMERIC(val, len | 0x80); 571 s = u2s(val, 10, false, buf, &slen); 572 APPEND_PADDED_S(s, slen, width, left_justify); 573 f++; 574 break; 575 } case 'x': case 'X': { 576 uintmax_t val JEMALLOC_CC_SILENCE_INIT(0); 577 char buf[X2S_BUFSIZE]; 578 579 GET_ARG_NUMERIC(val, len | 0x80); 580 s = x2s(val, alt_form, *f == 'X', buf, &slen); 581 APPEND_PADDED_S(s, slen, width, left_justify); 582 f++; 583 break; 584 } case 'c': { 585 unsigned char val; 586 char buf[2]; 587 588 assert(len == '?' || len == 'l'); 589 assert_not_implemented(len != 'l'); 590 val = va_arg(ap, int); 591 buf[0] = val; 592 buf[1] = '\0'; 593 APPEND_PADDED_S(buf, 1, width, left_justify); 594 f++; 595 break; 596 } case 's': 597 assert(len == '?' || len == 'l'); 598 assert_not_implemented(len != 'l'); 599 s = va_arg(ap, char *); 600 slen = (prec < 0) ? strlen(s) : (size_t)prec; 601 APPEND_PADDED_S(s, slen, width, left_justify); 602 f++; 603 break; 604 case 'p': { 605 uintmax_t val; 606 char buf[X2S_BUFSIZE]; 607 608 GET_ARG_NUMERIC(val, 'p'); 609 s = x2s(val, true, false, buf, &slen); 610 APPEND_PADDED_S(s, slen, width, left_justify); 611 f++; 612 break; 613 } default: not_reached(); 614 } 615 break; 616 } default: { 617 APPEND_C(*f); 618 f++; 619 break; 620 }} 621 } 622 label_out: 623 if (i < size) { 624 str[i] = '\0'; 625 } else { 626 str[size - 1] = '\0'; 627 } 628 629 #undef APPEND_C 630 #undef APPEND_S 631 #undef APPEND_PADDED_S 632 #undef GET_ARG_NUMERIC 633 return i; 634 } 635 636 JEMALLOC_FORMAT_PRINTF(3, 4) 637 size_t 638 malloc_snprintf(char *str, size_t size, const char *format, ...) { 639 size_t ret; 640 va_list ap; 641 642 va_start(ap, format); 643 ret = malloc_vsnprintf(str, size, format, ap); 644 va_end(ap); 645 646 return ret; 647 } 648 649 void 650 malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque, 651 const char *format, va_list ap) { 652 char buf[MALLOC_PRINTF_BUFSIZE]; 653 654 if (write_cb == NULL) { 655 /* 656 * The caller did not provide an alternate write_cb callback 657 * function, so use the default one. malloc_write() is an 658 * inline function, so use malloc_message() directly here. 659 */ 660 write_cb = (je_malloc_message != NULL) ? je_malloc_message : 661 wrtmessage; 662 cbopaque = NULL; 663 } 664 665 malloc_vsnprintf(buf, sizeof(buf), format, ap); 666 write_cb(cbopaque, buf); 667 } 668 669 /* 670 * Print to a callback function in such a way as to (hopefully) avoid memory 671 * allocation. 672 */ 673 JEMALLOC_FORMAT_PRINTF(3, 4) 674 void 675 malloc_cprintf(void (*write_cb)(void *, const char *), void *cbopaque, 676 const char *format, ...) { 677 va_list ap; 678 679 va_start(ap, format); 680 malloc_vcprintf(write_cb, cbopaque, format, ap); 681 va_end(ap); 682 } 683 684 /* Print to stderr in such a way as to avoid memory allocation. */ 685 JEMALLOC_FORMAT_PRINTF(1, 2) 686 void 687 malloc_printf(const char *format, ...) { 688 va_list ap; 689 690 va_start(ap, format); 691 malloc_vcprintf(NULL, NULL, format, ap); 692 va_end(ap); 693 } 694 695 /* 696 * Restore normal assertion macros, in order to make it possible to compile all 697 * C files as a single concatenation. 698 */ 699 #undef assert 700 #undef not_reached 701 #undef not_implemented 702 #undef assert_not_implemented 703 #include "jemalloc/internal/assert.h" 704