1 #define JEMALLOC_MALLOC_IO_C_ 2 #include "jemalloc/internal/jemalloc_preamble.h" 3 #include "jemalloc/internal/jemalloc_internal_includes.h" 4 5 #include "jemalloc/internal/malloc_io.h" 6 #include "jemalloc/internal/util.h" 7 8 #ifdef assert 9 # undef assert 10 #endif 11 #ifdef not_reached 12 # undef not_reached 13 #endif 14 #ifdef not_implemented 15 # undef not_implemented 16 #endif 17 #ifdef assert_not_implemented 18 # undef assert_not_implemented 19 #endif 20 21 /* 22 * Define simple versions of assertion macros that won't recurse in case 23 * of assertion failures in malloc_*printf(). 24 */ 25 #define assert(e) do { \ 26 if (config_debug && !(e)) { \ 27 malloc_write("<jemalloc>: Failed assertion\n"); \ 28 abort(); \ 29 } \ 30 } while (0) 31 32 #define not_reached() do { \ 33 if (config_debug) { \ 34 malloc_write("<jemalloc>: Unreachable code reached\n"); \ 35 abort(); \ 36 } \ 37 unreachable(); \ 38 } while (0) 39 40 #define not_implemented() do { \ 41 if (config_debug) { \ 42 malloc_write("<jemalloc>: Not implemented\n"); \ 43 abort(); \ 44 } \ 45 } while (0) 46 47 #define assert_not_implemented(e) do { \ 48 if (unlikely(config_debug && !(e))) { \ 49 not_implemented(); \ 50 } \ 51 } while (0) 52 53 /******************************************************************************/ 54 /* Function prototypes for non-inline static functions. */ 55 56 static void wrtmessage(void *cbopaque, const char *s); 57 #define U2S_BUFSIZE ((1U << (LG_SIZEOF_INTMAX_T + 3)) + 1) 58 static char *u2s(uintmax_t x, unsigned base, bool uppercase, char *s, 59 size_t *slen_p); 60 #define D2S_BUFSIZE (1 + U2S_BUFSIZE) 61 static char *d2s(intmax_t x, char sign, char *s, size_t *slen_p); 62 #define O2S_BUFSIZE (1 + U2S_BUFSIZE) 63 static char *o2s(uintmax_t x, bool alt_form, char *s, size_t *slen_p); 64 #define X2S_BUFSIZE (2 + U2S_BUFSIZE) 65 static char *x2s(uintmax_t x, bool alt_form, bool uppercase, char *s, 66 size_t *slen_p); 67 68 /******************************************************************************/ 69 70 /* malloc_message() setup. */ 71 static void 72 wrtmessage(void *cbopaque, const char *s) { 73 malloc_write_fd(STDERR_FILENO, s, strlen(s)); 74 } 75 76 JEMALLOC_EXPORT void (*je_malloc_message)(void *, const char *s); 77 78 JEMALLOC_ATTR(visibility("hidden")) 79 void 80 wrtmessage_1_0(const char *s1, const char *s2, const char *s3, const char *s4) { 81 82 wrtmessage(NULL, s1); 83 wrtmessage(NULL, s2); 84 wrtmessage(NULL, s3); 85 wrtmessage(NULL, s4); 86 } 87 88 void (*__malloc_message_1_0)(const char *s1, const char *s2, const char *s3, 89 const char *s4) = wrtmessage_1_0; 90 __sym_compat(_malloc_message, __malloc_message_1_0, FBSD_1.0); 91 92 /* 93 * Wrapper around malloc_message() that avoids the need for 94 * je_malloc_message(...) throughout the code. 95 */ 96 void 97 malloc_write(const char *s) { 98 if (je_malloc_message != NULL) { 99 je_malloc_message(NULL, s); 100 } else { 101 wrtmessage(NULL, s); 102 } 103 } 104 105 /* 106 * glibc provides a non-standard strerror_r() when _GNU_SOURCE is defined, so 107 * provide a wrapper. 108 */ 109 int 110 buferror(int err, char *buf, size_t buflen) { 111 #ifdef _WIN32 112 FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM, NULL, err, 0, 113 (LPSTR)buf, (DWORD)buflen, NULL); 114 return 0; 115 #elif defined(JEMALLOC_STRERROR_R_RETURNS_CHAR_WITH_GNU_SOURCE) && defined(_GNU_SOURCE) 116 char *b = strerror_r(err, buf, buflen); 117 if (b != buf) { 118 strncpy(buf, b, buflen); 119 buf[buflen-1] = '\0'; 120 } 121 return 0; 122 #else 123 return strerror_r(err, buf, buflen); 124 #endif 125 } 126 127 uintmax_t 128 malloc_strtoumax(const char *restrict nptr, char **restrict endptr, int base) { 129 uintmax_t ret, digit; 130 unsigned b; 131 bool neg; 132 const char *p, *ns; 133 134 p = nptr; 135 if (base < 0 || base == 1 || base > 36) { 136 ns = p; 137 set_errno(EINVAL); 138 ret = UINTMAX_MAX; 139 goto label_return; 140 } 141 b = base; 142 143 /* Swallow leading whitespace and get sign, if any. */ 144 neg = false; 145 while (true) { 146 switch (*p) { 147 case '\t': case '\n': case '\v': case '\f': case '\r': case ' ': 148 p++; 149 break; 150 case '-': 151 neg = true; 152 /* Fall through. */ 153 case '+': 154 p++; 155 /* Fall through. */ 156 default: 157 goto label_prefix; 158 } 159 } 160 161 /* Get prefix, if any. */ 162 label_prefix: 163 /* 164 * Note where the first non-whitespace/sign character is so that it is 165 * possible to tell whether any digits are consumed (e.g., " 0" vs. 166 * " -x"). 167 */ 168 ns = p; 169 if (*p == '0') { 170 switch (p[1]) { 171 case '0': case '1': case '2': case '3': case '4': case '5': 172 case '6': case '7': 173 if (b == 0) { 174 b = 8; 175 } 176 if (b == 8) { 177 p++; 178 } 179 break; 180 case 'X': case 'x': 181 switch (p[2]) { 182 case '0': case '1': case '2': case '3': case '4': 183 case '5': case '6': case '7': case '8': case '9': 184 case 'A': case 'B': case 'C': case 'D': case 'E': 185 case 'F': 186 case 'a': case 'b': case 'c': case 'd': case 'e': 187 case 'f': 188 if (b == 0) { 189 b = 16; 190 } 191 if (b == 16) { 192 p += 2; 193 } 194 break; 195 default: 196 break; 197 } 198 break; 199 default: 200 p++; 201 ret = 0; 202 goto label_return; 203 } 204 } 205 if (b == 0) { 206 b = 10; 207 } 208 209 /* Convert. */ 210 ret = 0; 211 while ((*p >= '0' && *p <= '9' && (digit = *p - '0') < b) 212 || (*p >= 'A' && *p <= 'Z' && (digit = 10 + *p - 'A') < b) 213 || (*p >= 'a' && *p <= 'z' && (digit = 10 + *p - 'a') < b)) { 214 uintmax_t pret = ret; 215 ret *= b; 216 ret += digit; 217 if (ret < pret) { 218 /* Overflow. */ 219 set_errno(ERANGE); 220 ret = UINTMAX_MAX; 221 goto label_return; 222 } 223 p++; 224 } 225 if (neg) { 226 ret = (uintmax_t)(-((intmax_t)ret)); 227 } 228 229 if (p == ns) { 230 /* No conversion performed. */ 231 set_errno(EINVAL); 232 ret = UINTMAX_MAX; 233 goto label_return; 234 } 235 236 label_return: 237 if (endptr != NULL) { 238 if (p == ns) { 239 /* No characters were converted. */ 240 *endptr = (char *)nptr; 241 } else { 242 *endptr = (char *)p; 243 } 244 } 245 return ret; 246 } 247 248 static char * 249 u2s(uintmax_t x, unsigned base, bool uppercase, char *s, size_t *slen_p) { 250 unsigned i; 251 252 i = U2S_BUFSIZE - 1; 253 s[i] = '\0'; 254 switch (base) { 255 case 10: 256 do { 257 i--; 258 s[i] = "0123456789"[x % (uint64_t)10]; 259 x /= (uint64_t)10; 260 } while (x > 0); 261 break; 262 case 16: { 263 const char *digits = (uppercase) 264 ? "0123456789ABCDEF" 265 : "0123456789abcdef"; 266 267 do { 268 i--; 269 s[i] = digits[x & 0xf]; 270 x >>= 4; 271 } while (x > 0); 272 break; 273 } default: { 274 const char *digits = (uppercase) 275 ? "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" 276 : "0123456789abcdefghijklmnopqrstuvwxyz"; 277 278 assert(base >= 2 && base <= 36); 279 do { 280 i--; 281 s[i] = digits[x % (uint64_t)base]; 282 x /= (uint64_t)base; 283 } while (x > 0); 284 }} 285 286 *slen_p = U2S_BUFSIZE - 1 - i; 287 return &s[i]; 288 } 289 290 static char * 291 d2s(intmax_t x, char sign, char *s, size_t *slen_p) { 292 bool neg; 293 294 if ((neg = (x < 0))) { 295 x = -x; 296 } 297 s = u2s(x, 10, false, s, slen_p); 298 if (neg) { 299 sign = '-'; 300 } 301 switch (sign) { 302 case '-': 303 if (!neg) { 304 break; 305 } 306 /* Fall through. */ 307 case ' ': 308 case '+': 309 s--; 310 (*slen_p)++; 311 *s = sign; 312 break; 313 default: not_reached(); 314 } 315 return s; 316 } 317 318 static char * 319 o2s(uintmax_t x, bool alt_form, char *s, size_t *slen_p) { 320 s = u2s(x, 8, false, s, slen_p); 321 if (alt_form && *s != '0') { 322 s--; 323 (*slen_p)++; 324 *s = '0'; 325 } 326 return s; 327 } 328 329 static char * 330 x2s(uintmax_t x, bool alt_form, bool uppercase, char *s, size_t *slen_p) { 331 s = u2s(x, 16, uppercase, s, slen_p); 332 if (alt_form) { 333 s -= 2; 334 (*slen_p) += 2; 335 memcpy(s, uppercase ? "0X" : "0x", 2); 336 } 337 return s; 338 } 339 340 size_t 341 malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap) { 342 size_t i; 343 const char *f; 344 345 #define APPEND_C(c) do { \ 346 if (i < size) { \ 347 str[i] = (c); \ 348 } \ 349 i++; \ 350 } while (0) 351 #define APPEND_S(s, slen) do { \ 352 if (i < size) { \ 353 size_t cpylen = (slen <= size - i) ? slen : size - i; \ 354 memcpy(&str[i], s, cpylen); \ 355 } \ 356 i += slen; \ 357 } while (0) 358 #define APPEND_PADDED_S(s, slen, width, left_justify) do { \ 359 /* Left padding. */ \ 360 size_t pad_len = (width == -1) ? 0 : ((slen < (size_t)width) ? \ 361 (size_t)width - slen : 0); \ 362 if (!left_justify && pad_len != 0) { \ 363 size_t j; \ 364 for (j = 0; j < pad_len; j++) { \ 365 APPEND_C(' '); \ 366 } \ 367 } \ 368 /* Value. */ \ 369 APPEND_S(s, slen); \ 370 /* Right padding. */ \ 371 if (left_justify && pad_len != 0) { \ 372 size_t j; \ 373 for (j = 0; j < pad_len; j++) { \ 374 APPEND_C(' '); \ 375 } \ 376 } \ 377 } while (0) 378 #define GET_ARG_NUMERIC(val, len) do { \ 379 switch ((unsigned char)len) { \ 380 case '?': \ 381 val = va_arg(ap, int); \ 382 break; \ 383 case '?' | 0x80: \ 384 val = va_arg(ap, unsigned int); \ 385 break; \ 386 case 'l': \ 387 val = va_arg(ap, long); \ 388 break; \ 389 case 'l' | 0x80: \ 390 val = va_arg(ap, unsigned long); \ 391 break; \ 392 case 'q': \ 393 val = va_arg(ap, long long); \ 394 break; \ 395 case 'q' | 0x80: \ 396 val = va_arg(ap, unsigned long long); \ 397 break; \ 398 case 'j': \ 399 val = va_arg(ap, intmax_t); \ 400 break; \ 401 case 'j' | 0x80: \ 402 val = va_arg(ap, uintmax_t); \ 403 break; \ 404 case 't': \ 405 val = va_arg(ap, ptrdiff_t); \ 406 break; \ 407 case 'z': \ 408 val = va_arg(ap, ssize_t); \ 409 break; \ 410 case 'z' | 0x80: \ 411 val = va_arg(ap, size_t); \ 412 break; \ 413 case 'p': /* Synthetic; used for %p. */ \ 414 val = va_arg(ap, uintptr_t); \ 415 break; \ 416 default: \ 417 not_reached(); \ 418 val = 0; \ 419 } \ 420 } while (0) 421 422 i = 0; 423 f = format; 424 while (true) { 425 switch (*f) { 426 case '\0': goto label_out; 427 case '%': { 428 bool alt_form = false; 429 bool left_justify = false; 430 bool plus_space = false; 431 bool plus_plus = false; 432 int prec = -1; 433 int width = -1; 434 unsigned char len = '?'; 435 char *s; 436 size_t slen; 437 438 f++; 439 /* Flags. */ 440 while (true) { 441 switch (*f) { 442 case '#': 443 assert(!alt_form); 444 alt_form = true; 445 break; 446 case '-': 447 assert(!left_justify); 448 left_justify = true; 449 break; 450 case ' ': 451 assert(!plus_space); 452 plus_space = true; 453 break; 454 case '+': 455 assert(!plus_plus); 456 plus_plus = true; 457 break; 458 default: goto label_width; 459 } 460 f++; 461 } 462 /* Width. */ 463 label_width: 464 switch (*f) { 465 case '*': 466 width = va_arg(ap, int); 467 f++; 468 if (width < 0) { 469 left_justify = true; 470 width = -width; 471 } 472 break; 473 case '0': case '1': case '2': case '3': case '4': 474 case '5': case '6': case '7': case '8': case '9': { 475 uintmax_t uwidth; 476 set_errno(0); 477 uwidth = malloc_strtoumax(f, (char **)&f, 10); 478 assert(uwidth != UINTMAX_MAX || get_errno() != 479 ERANGE); 480 width = (int)uwidth; 481 break; 482 } default: 483 break; 484 } 485 /* Width/precision separator. */ 486 if (*f == '.') { 487 f++; 488 } else { 489 goto label_length; 490 } 491 /* Precision. */ 492 switch (*f) { 493 case '*': 494 prec = va_arg(ap, int); 495 f++; 496 break; 497 case '0': case '1': case '2': case '3': case '4': 498 case '5': case '6': case '7': case '8': case '9': { 499 uintmax_t uprec; 500 set_errno(0); 501 uprec = malloc_strtoumax(f, (char **)&f, 10); 502 assert(uprec != UINTMAX_MAX || get_errno() != 503 ERANGE); 504 prec = (int)uprec; 505 break; 506 } 507 default: break; 508 } 509 /* Length. */ 510 label_length: 511 switch (*f) { 512 case 'l': 513 f++; 514 if (*f == 'l') { 515 len = 'q'; 516 f++; 517 } else { 518 len = 'l'; 519 } 520 break; 521 case 'q': case 'j': case 't': case 'z': 522 len = *f; 523 f++; 524 break; 525 default: break; 526 } 527 /* Conversion specifier. */ 528 switch (*f) { 529 case '%': 530 /* %% */ 531 APPEND_C(*f); 532 f++; 533 break; 534 case 'd': case 'i': { 535 intmax_t val JEMALLOC_CC_SILENCE_INIT(0); 536 char buf[D2S_BUFSIZE]; 537 538 GET_ARG_NUMERIC(val, len); 539 s = d2s(val, (plus_plus ? '+' : (plus_space ? 540 ' ' : '-')), buf, &slen); 541 APPEND_PADDED_S(s, slen, width, left_justify); 542 f++; 543 break; 544 } case 'o': { 545 uintmax_t val JEMALLOC_CC_SILENCE_INIT(0); 546 char buf[O2S_BUFSIZE]; 547 548 GET_ARG_NUMERIC(val, len | 0x80); 549 s = o2s(val, alt_form, buf, &slen); 550 APPEND_PADDED_S(s, slen, width, left_justify); 551 f++; 552 break; 553 } case 'u': { 554 uintmax_t val JEMALLOC_CC_SILENCE_INIT(0); 555 char buf[U2S_BUFSIZE]; 556 557 GET_ARG_NUMERIC(val, len | 0x80); 558 s = u2s(val, 10, false, buf, &slen); 559 APPEND_PADDED_S(s, slen, width, left_justify); 560 f++; 561 break; 562 } case 'x': case 'X': { 563 uintmax_t val JEMALLOC_CC_SILENCE_INIT(0); 564 char buf[X2S_BUFSIZE]; 565 566 GET_ARG_NUMERIC(val, len | 0x80); 567 s = x2s(val, alt_form, *f == 'X', buf, &slen); 568 APPEND_PADDED_S(s, slen, width, left_justify); 569 f++; 570 break; 571 } case 'c': { 572 unsigned char val; 573 char buf[2]; 574 575 assert(len == '?' || len == 'l'); 576 assert_not_implemented(len != 'l'); 577 val = va_arg(ap, int); 578 buf[0] = val; 579 buf[1] = '\0'; 580 APPEND_PADDED_S(buf, 1, width, left_justify); 581 f++; 582 break; 583 } case 's': 584 assert(len == '?' || len == 'l'); 585 assert_not_implemented(len != 'l'); 586 s = va_arg(ap, char *); 587 slen = (prec < 0) ? strlen(s) : (size_t)prec; 588 APPEND_PADDED_S(s, slen, width, left_justify); 589 f++; 590 break; 591 case 'p': { 592 uintmax_t val; 593 char buf[X2S_BUFSIZE]; 594 595 GET_ARG_NUMERIC(val, 'p'); 596 s = x2s(val, true, false, buf, &slen); 597 APPEND_PADDED_S(s, slen, width, left_justify); 598 f++; 599 break; 600 } default: not_reached(); 601 } 602 break; 603 } default: { 604 APPEND_C(*f); 605 f++; 606 break; 607 }} 608 } 609 label_out: 610 if (i < size) { 611 str[i] = '\0'; 612 } else { 613 str[size - 1] = '\0'; 614 } 615 616 #undef APPEND_C 617 #undef APPEND_S 618 #undef APPEND_PADDED_S 619 #undef GET_ARG_NUMERIC 620 return i; 621 } 622 623 JEMALLOC_FORMAT_PRINTF(3, 4) 624 size_t 625 malloc_snprintf(char *str, size_t size, const char *format, ...) { 626 size_t ret; 627 va_list ap; 628 629 va_start(ap, format); 630 ret = malloc_vsnprintf(str, size, format, ap); 631 va_end(ap); 632 633 return ret; 634 } 635 636 void 637 malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque, 638 const char *format, va_list ap) { 639 char buf[MALLOC_PRINTF_BUFSIZE]; 640 641 if (write_cb == NULL) { 642 /* 643 * The caller did not provide an alternate write_cb callback 644 * function, so use the default one. malloc_write() is an 645 * inline function, so use malloc_message() directly here. 646 */ 647 write_cb = (je_malloc_message != NULL) ? je_malloc_message : 648 wrtmessage; 649 } 650 651 malloc_vsnprintf(buf, sizeof(buf), format, ap); 652 write_cb(cbopaque, buf); 653 } 654 655 /* 656 * Print to a callback function in such a way as to (hopefully) avoid memory 657 * allocation. 658 */ 659 JEMALLOC_FORMAT_PRINTF(3, 4) 660 void 661 malloc_cprintf(void (*write_cb)(void *, const char *), void *cbopaque, 662 const char *format, ...) { 663 va_list ap; 664 665 va_start(ap, format); 666 malloc_vcprintf(write_cb, cbopaque, format, ap); 667 va_end(ap); 668 } 669 670 /* Print to stderr in such a way as to avoid memory allocation. */ 671 JEMALLOC_FORMAT_PRINTF(1, 2) 672 void 673 malloc_printf(const char *format, ...) { 674 va_list ap; 675 676 va_start(ap, format); 677 malloc_vcprintf(NULL, NULL, format, ap); 678 va_end(ap); 679 } 680 681 /* 682 * Restore normal assertion macros, in order to make it possible to compile all 683 * C files as a single concatenation. 684 */ 685 #undef assert 686 #undef not_reached 687 #undef not_implemented 688 #undef assert_not_implemented 689 #include "jemalloc/internal/assert.h" 690