1 #include "builtin.h" 2 #include "perf.h" 3 4 #include "util/evlist.h" 5 #include "util/evsel.h" 6 #include "util/util.h" 7 #include "util/cache.h" 8 #include "util/symbol.h" 9 #include "util/thread.h" 10 #include "util/header.h" 11 #include "util/session.h" 12 #include "util/tool.h" 13 #include "util/callchain.h" 14 15 #include "util/parse-options.h" 16 #include "util/trace-event.h" 17 #include "util/data.h" 18 #include "util/cpumap.h" 19 20 #include "util/debug.h" 21 22 #include <linux/rbtree.h> 23 #include <linux/string.h> 24 #include <locale.h> 25 #include <regex.h> 26 27 static int kmem_slab; 28 static int kmem_page; 29 30 static long kmem_page_size; 31 static enum { 32 KMEM_SLAB, 33 KMEM_PAGE, 34 } kmem_default = KMEM_SLAB; /* for backward compatibility */ 35 36 struct alloc_stat; 37 typedef int (*sort_fn_t)(void *, void *); 38 39 static int alloc_flag; 40 static int caller_flag; 41 42 static int alloc_lines = -1; 43 static int caller_lines = -1; 44 45 static bool raw_ip; 46 47 struct alloc_stat { 48 u64 call_site; 49 u64 ptr; 50 u64 bytes_req; 51 u64 bytes_alloc; 52 u32 hit; 53 u32 pingpong; 54 55 short alloc_cpu; 56 57 struct rb_node node; 58 }; 59 60 static struct rb_root root_alloc_stat; 61 static struct rb_root root_alloc_sorted; 62 static struct rb_root root_caller_stat; 63 static struct rb_root root_caller_sorted; 64 65 static unsigned long total_requested, total_allocated; 66 static unsigned long nr_allocs, nr_cross_allocs; 67 68 static int insert_alloc_stat(unsigned long call_site, unsigned long ptr, 69 int bytes_req, int bytes_alloc, int cpu) 70 { 71 struct rb_node **node = &root_alloc_stat.rb_node; 72 struct rb_node *parent = NULL; 73 struct alloc_stat *data = NULL; 74 75 while (*node) { 76 parent = *node; 77 data = rb_entry(*node, struct alloc_stat, node); 78 79 if (ptr > data->ptr) 80 node = &(*node)->rb_right; 81 else if (ptr < data->ptr) 82 node = &(*node)->rb_left; 83 else 84 break; 85 } 86 87 if (data && data->ptr == ptr) { 88 data->hit++; 89 data->bytes_req += bytes_req; 90 data->bytes_alloc += bytes_alloc; 91 } else { 92 data = malloc(sizeof(*data)); 93 if (!data) { 94 pr_err("%s: malloc failed\n", __func__); 95 return -1; 96 } 97 data->ptr = ptr; 98 data->pingpong = 0; 99 data->hit = 1; 100 data->bytes_req = bytes_req; 101 data->bytes_alloc = bytes_alloc; 102 103 rb_link_node(&data->node, parent, node); 104 rb_insert_color(&data->node, &root_alloc_stat); 105 } 106 data->call_site = call_site; 107 data->alloc_cpu = cpu; 108 return 0; 109 } 110 111 static int insert_caller_stat(unsigned long call_site, 112 int bytes_req, int bytes_alloc) 113 { 114 struct rb_node **node = &root_caller_stat.rb_node; 115 struct rb_node *parent = NULL; 116 struct alloc_stat *data = NULL; 117 118 while (*node) { 119 parent = *node; 120 data = rb_entry(*node, struct alloc_stat, node); 121 122 if (call_site > data->call_site) 123 node = &(*node)->rb_right; 124 else if (call_site < data->call_site) 125 node = &(*node)->rb_left; 126 else 127 break; 128 } 129 130 if (data && data->call_site == call_site) { 131 data->hit++; 132 data->bytes_req += bytes_req; 133 data->bytes_alloc += bytes_alloc; 134 } else { 135 data = malloc(sizeof(*data)); 136 if (!data) { 137 pr_err("%s: malloc failed\n", __func__); 138 return -1; 139 } 140 data->call_site = call_site; 141 data->pingpong = 0; 142 data->hit = 1; 143 data->bytes_req = bytes_req; 144 data->bytes_alloc = bytes_alloc; 145 146 rb_link_node(&data->node, parent, node); 147 rb_insert_color(&data->node, &root_caller_stat); 148 } 149 150 return 0; 151 } 152 153 static int perf_evsel__process_alloc_event(struct perf_evsel *evsel, 154 struct perf_sample *sample) 155 { 156 unsigned long ptr = perf_evsel__intval(evsel, sample, "ptr"), 157 call_site = perf_evsel__intval(evsel, sample, "call_site"); 158 int bytes_req = perf_evsel__intval(evsel, sample, "bytes_req"), 159 bytes_alloc = perf_evsel__intval(evsel, sample, "bytes_alloc"); 160 161 if (insert_alloc_stat(call_site, ptr, bytes_req, bytes_alloc, sample->cpu) || 162 insert_caller_stat(call_site, bytes_req, bytes_alloc)) 163 return -1; 164 165 total_requested += bytes_req; 166 total_allocated += bytes_alloc; 167 168 nr_allocs++; 169 return 0; 170 } 171 172 static int perf_evsel__process_alloc_node_event(struct perf_evsel *evsel, 173 struct perf_sample *sample) 174 { 175 int ret = perf_evsel__process_alloc_event(evsel, sample); 176 177 if (!ret) { 178 int node1 = cpu__get_node(sample->cpu), 179 node2 = perf_evsel__intval(evsel, sample, "node"); 180 181 if (node1 != node2) 182 nr_cross_allocs++; 183 } 184 185 return ret; 186 } 187 188 static int ptr_cmp(void *, void *); 189 static int slab_callsite_cmp(void *, void *); 190 191 static struct alloc_stat *search_alloc_stat(unsigned long ptr, 192 unsigned long call_site, 193 struct rb_root *root, 194 sort_fn_t sort_fn) 195 { 196 struct rb_node *node = root->rb_node; 197 struct alloc_stat key = { .ptr = ptr, .call_site = call_site }; 198 199 while (node) { 200 struct alloc_stat *data; 201 int cmp; 202 203 data = rb_entry(node, struct alloc_stat, node); 204 205 cmp = sort_fn(&key, data); 206 if (cmp < 0) 207 node = node->rb_left; 208 else if (cmp > 0) 209 node = node->rb_right; 210 else 211 return data; 212 } 213 return NULL; 214 } 215 216 static int perf_evsel__process_free_event(struct perf_evsel *evsel, 217 struct perf_sample *sample) 218 { 219 unsigned long ptr = perf_evsel__intval(evsel, sample, "ptr"); 220 struct alloc_stat *s_alloc, *s_caller; 221 222 s_alloc = search_alloc_stat(ptr, 0, &root_alloc_stat, ptr_cmp); 223 if (!s_alloc) 224 return 0; 225 226 if ((short)sample->cpu != s_alloc->alloc_cpu) { 227 s_alloc->pingpong++; 228 229 s_caller = search_alloc_stat(0, s_alloc->call_site, 230 &root_caller_stat, 231 slab_callsite_cmp); 232 if (!s_caller) 233 return -1; 234 s_caller->pingpong++; 235 } 236 s_alloc->alloc_cpu = -1; 237 238 return 0; 239 } 240 241 static u64 total_page_alloc_bytes; 242 static u64 total_page_free_bytes; 243 static u64 total_page_nomatch_bytes; 244 static u64 total_page_fail_bytes; 245 static unsigned long nr_page_allocs; 246 static unsigned long nr_page_frees; 247 static unsigned long nr_page_fails; 248 static unsigned long nr_page_nomatch; 249 250 static bool use_pfn; 251 static bool live_page; 252 static struct perf_session *kmem_session; 253 254 #define MAX_MIGRATE_TYPES 6 255 #define MAX_PAGE_ORDER 11 256 257 static int order_stats[MAX_PAGE_ORDER][MAX_MIGRATE_TYPES]; 258 259 struct page_stat { 260 struct rb_node node; 261 u64 page; 262 u64 callsite; 263 int order; 264 unsigned gfp_flags; 265 unsigned migrate_type; 266 u64 alloc_bytes; 267 u64 free_bytes; 268 int nr_alloc; 269 int nr_free; 270 }; 271 272 static struct rb_root page_live_tree; 273 static struct rb_root page_alloc_tree; 274 static struct rb_root page_alloc_sorted; 275 static struct rb_root page_caller_tree; 276 static struct rb_root page_caller_sorted; 277 278 struct alloc_func { 279 u64 start; 280 u64 end; 281 char *name; 282 }; 283 284 static int nr_alloc_funcs; 285 static struct alloc_func *alloc_func_list; 286 287 static int funcmp(const void *a, const void *b) 288 { 289 const struct alloc_func *fa = a; 290 const struct alloc_func *fb = b; 291 292 if (fa->start > fb->start) 293 return 1; 294 else 295 return -1; 296 } 297 298 static int callcmp(const void *a, const void *b) 299 { 300 const struct alloc_func *fa = a; 301 const struct alloc_func *fb = b; 302 303 if (fb->start <= fa->start && fa->end < fb->end) 304 return 0; 305 306 if (fa->start > fb->start) 307 return 1; 308 else 309 return -1; 310 } 311 312 static int build_alloc_func_list(void) 313 { 314 int ret; 315 struct map *kernel_map; 316 struct symbol *sym; 317 struct rb_node *node; 318 struct alloc_func *func; 319 struct machine *machine = &kmem_session->machines.host; 320 regex_t alloc_func_regex; 321 const char pattern[] = "^_?_?(alloc|get_free|get_zeroed)_pages?"; 322 323 ret = regcomp(&alloc_func_regex, pattern, REG_EXTENDED); 324 if (ret) { 325 char err[BUFSIZ]; 326 327 regerror(ret, &alloc_func_regex, err, sizeof(err)); 328 pr_err("Invalid regex: %s\n%s", pattern, err); 329 return -EINVAL; 330 } 331 332 kernel_map = machine->vmlinux_maps[MAP__FUNCTION]; 333 if (map__load(kernel_map, NULL) < 0) { 334 pr_err("cannot load kernel map\n"); 335 return -ENOENT; 336 } 337 338 map__for_each_symbol(kernel_map, sym, node) { 339 if (regexec(&alloc_func_regex, sym->name, 0, NULL, 0)) 340 continue; 341 342 func = realloc(alloc_func_list, 343 (nr_alloc_funcs + 1) * sizeof(*func)); 344 if (func == NULL) 345 return -ENOMEM; 346 347 pr_debug("alloc func: %s\n", sym->name); 348 func[nr_alloc_funcs].start = sym->start; 349 func[nr_alloc_funcs].end = sym->end; 350 func[nr_alloc_funcs].name = sym->name; 351 352 alloc_func_list = func; 353 nr_alloc_funcs++; 354 } 355 356 qsort(alloc_func_list, nr_alloc_funcs, sizeof(*func), funcmp); 357 358 regfree(&alloc_func_regex); 359 return 0; 360 } 361 362 /* 363 * Find first non-memory allocation function from callchain. 364 * The allocation functions are in the 'alloc_func_list'. 365 */ 366 static u64 find_callsite(struct perf_evsel *evsel, struct perf_sample *sample) 367 { 368 struct addr_location al; 369 struct machine *machine = &kmem_session->machines.host; 370 struct callchain_cursor_node *node; 371 372 if (alloc_func_list == NULL) { 373 if (build_alloc_func_list() < 0) 374 goto out; 375 } 376 377 al.thread = machine__findnew_thread(machine, sample->pid, sample->tid); 378 sample__resolve_callchain(sample, NULL, evsel, &al, 16); 379 380 callchain_cursor_commit(&callchain_cursor); 381 while (true) { 382 struct alloc_func key, *caller; 383 u64 addr; 384 385 node = callchain_cursor_current(&callchain_cursor); 386 if (node == NULL) 387 break; 388 389 key.start = key.end = node->ip; 390 caller = bsearch(&key, alloc_func_list, nr_alloc_funcs, 391 sizeof(key), callcmp); 392 if (!caller) { 393 /* found */ 394 if (node->map) 395 addr = map__unmap_ip(node->map, node->ip); 396 else 397 addr = node->ip; 398 399 return addr; 400 } else 401 pr_debug3("skipping alloc function: %s\n", caller->name); 402 403 callchain_cursor_advance(&callchain_cursor); 404 } 405 406 out: 407 pr_debug2("unknown callsite: %"PRIx64 "\n", sample->ip); 408 return sample->ip; 409 } 410 411 struct sort_dimension { 412 const char name[20]; 413 sort_fn_t cmp; 414 struct list_head list; 415 }; 416 417 static LIST_HEAD(page_alloc_sort_input); 418 static LIST_HEAD(page_caller_sort_input); 419 420 static struct page_stat * 421 __page_stat__findnew_page(struct page_stat *pstat, bool create) 422 { 423 struct rb_node **node = &page_live_tree.rb_node; 424 struct rb_node *parent = NULL; 425 struct page_stat *data; 426 427 while (*node) { 428 s64 cmp; 429 430 parent = *node; 431 data = rb_entry(*node, struct page_stat, node); 432 433 cmp = data->page - pstat->page; 434 if (cmp < 0) 435 node = &parent->rb_left; 436 else if (cmp > 0) 437 node = &parent->rb_right; 438 else 439 return data; 440 } 441 442 if (!create) 443 return NULL; 444 445 data = zalloc(sizeof(*data)); 446 if (data != NULL) { 447 data->page = pstat->page; 448 data->order = pstat->order; 449 data->gfp_flags = pstat->gfp_flags; 450 data->migrate_type = pstat->migrate_type; 451 452 rb_link_node(&data->node, parent, node); 453 rb_insert_color(&data->node, &page_live_tree); 454 } 455 456 return data; 457 } 458 459 static struct page_stat *page_stat__find_page(struct page_stat *pstat) 460 { 461 return __page_stat__findnew_page(pstat, false); 462 } 463 464 static struct page_stat *page_stat__findnew_page(struct page_stat *pstat) 465 { 466 return __page_stat__findnew_page(pstat, true); 467 } 468 469 static struct page_stat * 470 __page_stat__findnew_alloc(struct page_stat *pstat, bool create) 471 { 472 struct rb_node **node = &page_alloc_tree.rb_node; 473 struct rb_node *parent = NULL; 474 struct page_stat *data; 475 struct sort_dimension *sort; 476 477 while (*node) { 478 int cmp = 0; 479 480 parent = *node; 481 data = rb_entry(*node, struct page_stat, node); 482 483 list_for_each_entry(sort, &page_alloc_sort_input, list) { 484 cmp = sort->cmp(pstat, data); 485 if (cmp) 486 break; 487 } 488 489 if (cmp < 0) 490 node = &parent->rb_left; 491 else if (cmp > 0) 492 node = &parent->rb_right; 493 else 494 return data; 495 } 496 497 if (!create) 498 return NULL; 499 500 data = zalloc(sizeof(*data)); 501 if (data != NULL) { 502 data->page = pstat->page; 503 data->order = pstat->order; 504 data->gfp_flags = pstat->gfp_flags; 505 data->migrate_type = pstat->migrate_type; 506 507 rb_link_node(&data->node, parent, node); 508 rb_insert_color(&data->node, &page_alloc_tree); 509 } 510 511 return data; 512 } 513 514 static struct page_stat *page_stat__find_alloc(struct page_stat *pstat) 515 { 516 return __page_stat__findnew_alloc(pstat, false); 517 } 518 519 static struct page_stat *page_stat__findnew_alloc(struct page_stat *pstat) 520 { 521 return __page_stat__findnew_alloc(pstat, true); 522 } 523 524 static struct page_stat * 525 __page_stat__findnew_caller(struct page_stat *pstat, bool create) 526 { 527 struct rb_node **node = &page_caller_tree.rb_node; 528 struct rb_node *parent = NULL; 529 struct page_stat *data; 530 struct sort_dimension *sort; 531 532 while (*node) { 533 int cmp = 0; 534 535 parent = *node; 536 data = rb_entry(*node, struct page_stat, node); 537 538 list_for_each_entry(sort, &page_caller_sort_input, list) { 539 cmp = sort->cmp(pstat, data); 540 if (cmp) 541 break; 542 } 543 544 if (cmp < 0) 545 node = &parent->rb_left; 546 else if (cmp > 0) 547 node = &parent->rb_right; 548 else 549 return data; 550 } 551 552 if (!create) 553 return NULL; 554 555 data = zalloc(sizeof(*data)); 556 if (data != NULL) { 557 data->callsite = pstat->callsite; 558 data->order = pstat->order; 559 data->gfp_flags = pstat->gfp_flags; 560 data->migrate_type = pstat->migrate_type; 561 562 rb_link_node(&data->node, parent, node); 563 rb_insert_color(&data->node, &page_caller_tree); 564 } 565 566 return data; 567 } 568 569 static struct page_stat *page_stat__find_caller(struct page_stat *pstat) 570 { 571 return __page_stat__findnew_caller(pstat, false); 572 } 573 574 static struct page_stat *page_stat__findnew_caller(struct page_stat *pstat) 575 { 576 return __page_stat__findnew_caller(pstat, true); 577 } 578 579 static bool valid_page(u64 pfn_or_page) 580 { 581 if (use_pfn && pfn_or_page == -1UL) 582 return false; 583 if (!use_pfn && pfn_or_page == 0) 584 return false; 585 return true; 586 } 587 588 struct gfp_flag { 589 unsigned int flags; 590 char *compact_str; 591 char *human_readable; 592 }; 593 594 static struct gfp_flag *gfps; 595 static int nr_gfps; 596 597 static int gfpcmp(const void *a, const void *b) 598 { 599 const struct gfp_flag *fa = a; 600 const struct gfp_flag *fb = b; 601 602 return fa->flags - fb->flags; 603 } 604 605 /* see include/trace/events/gfpflags.h */ 606 static const struct { 607 const char *original; 608 const char *compact; 609 } gfp_compact_table[] = { 610 { "GFP_TRANSHUGE", "THP" }, 611 { "GFP_HIGHUSER_MOVABLE", "HUM" }, 612 { "GFP_HIGHUSER", "HU" }, 613 { "GFP_USER", "U" }, 614 { "GFP_TEMPORARY", "TMP" }, 615 { "GFP_KERNEL", "K" }, 616 { "GFP_NOFS", "NF" }, 617 { "GFP_ATOMIC", "A" }, 618 { "GFP_NOIO", "NI" }, 619 { "GFP_HIGH", "H" }, 620 { "GFP_WAIT", "W" }, 621 { "GFP_IO", "I" }, 622 { "GFP_COLD", "CO" }, 623 { "GFP_NOWARN", "NWR" }, 624 { "GFP_REPEAT", "R" }, 625 { "GFP_NOFAIL", "NF" }, 626 { "GFP_NORETRY", "NR" }, 627 { "GFP_COMP", "C" }, 628 { "GFP_ZERO", "Z" }, 629 { "GFP_NOMEMALLOC", "NMA" }, 630 { "GFP_MEMALLOC", "MA" }, 631 { "GFP_HARDWALL", "HW" }, 632 { "GFP_THISNODE", "TN" }, 633 { "GFP_RECLAIMABLE", "RC" }, 634 { "GFP_MOVABLE", "M" }, 635 { "GFP_NOTRACK", "NT" }, 636 { "GFP_NO_KSWAPD", "NK" }, 637 { "GFP_OTHER_NODE", "ON" }, 638 { "GFP_NOWAIT", "NW" }, 639 }; 640 641 static size_t max_gfp_len; 642 643 static char *compact_gfp_flags(char *gfp_flags) 644 { 645 char *orig_flags = strdup(gfp_flags); 646 char *new_flags = NULL; 647 char *str, *pos; 648 size_t len = 0; 649 650 if (orig_flags == NULL) 651 return NULL; 652 653 str = strtok_r(orig_flags, "|", &pos); 654 while (str) { 655 size_t i; 656 char *new; 657 const char *cpt; 658 659 for (i = 0; i < ARRAY_SIZE(gfp_compact_table); i++) { 660 if (strcmp(gfp_compact_table[i].original, str)) 661 continue; 662 663 cpt = gfp_compact_table[i].compact; 664 new = realloc(new_flags, len + strlen(cpt) + 2); 665 if (new == NULL) { 666 free(new_flags); 667 return NULL; 668 } 669 670 new_flags = new; 671 672 if (!len) { 673 strcpy(new_flags, cpt); 674 } else { 675 strcat(new_flags, "|"); 676 strcat(new_flags, cpt); 677 len++; 678 } 679 680 len += strlen(cpt); 681 } 682 683 str = strtok_r(NULL, "|", &pos); 684 } 685 686 if (max_gfp_len < len) 687 max_gfp_len = len; 688 689 free(orig_flags); 690 return new_flags; 691 } 692 693 static char *compact_gfp_string(unsigned long gfp_flags) 694 { 695 struct gfp_flag key = { 696 .flags = gfp_flags, 697 }; 698 struct gfp_flag *gfp; 699 700 gfp = bsearch(&key, gfps, nr_gfps, sizeof(*gfps), gfpcmp); 701 if (gfp) 702 return gfp->compact_str; 703 704 return NULL; 705 } 706 707 static int parse_gfp_flags(struct perf_evsel *evsel, struct perf_sample *sample, 708 unsigned int gfp_flags) 709 { 710 struct pevent_record record = { 711 .cpu = sample->cpu, 712 .data = sample->raw_data, 713 .size = sample->raw_size, 714 }; 715 struct trace_seq seq; 716 char *str, *pos; 717 718 if (nr_gfps) { 719 struct gfp_flag key = { 720 .flags = gfp_flags, 721 }; 722 723 if (bsearch(&key, gfps, nr_gfps, sizeof(*gfps), gfpcmp)) 724 return 0; 725 } 726 727 trace_seq_init(&seq); 728 pevent_event_info(&seq, evsel->tp_format, &record); 729 730 str = strtok_r(seq.buffer, " ", &pos); 731 while (str) { 732 if (!strncmp(str, "gfp_flags=", 10)) { 733 struct gfp_flag *new; 734 735 new = realloc(gfps, (nr_gfps + 1) * sizeof(*gfps)); 736 if (new == NULL) 737 return -ENOMEM; 738 739 gfps = new; 740 new += nr_gfps++; 741 742 new->flags = gfp_flags; 743 new->human_readable = strdup(str + 10); 744 new->compact_str = compact_gfp_flags(str + 10); 745 if (!new->human_readable || !new->compact_str) 746 return -ENOMEM; 747 748 qsort(gfps, nr_gfps, sizeof(*gfps), gfpcmp); 749 } 750 751 str = strtok_r(NULL, " ", &pos); 752 } 753 754 trace_seq_destroy(&seq); 755 return 0; 756 } 757 758 static int perf_evsel__process_page_alloc_event(struct perf_evsel *evsel, 759 struct perf_sample *sample) 760 { 761 u64 page; 762 unsigned int order = perf_evsel__intval(evsel, sample, "order"); 763 unsigned int gfp_flags = perf_evsel__intval(evsel, sample, "gfp_flags"); 764 unsigned int migrate_type = perf_evsel__intval(evsel, sample, 765 "migratetype"); 766 u64 bytes = kmem_page_size << order; 767 u64 callsite; 768 struct page_stat *pstat; 769 struct page_stat this = { 770 .order = order, 771 .gfp_flags = gfp_flags, 772 .migrate_type = migrate_type, 773 }; 774 775 if (use_pfn) 776 page = perf_evsel__intval(evsel, sample, "pfn"); 777 else 778 page = perf_evsel__intval(evsel, sample, "page"); 779 780 nr_page_allocs++; 781 total_page_alloc_bytes += bytes; 782 783 if (!valid_page(page)) { 784 nr_page_fails++; 785 total_page_fail_bytes += bytes; 786 787 return 0; 788 } 789 790 if (parse_gfp_flags(evsel, sample, gfp_flags) < 0) 791 return -1; 792 793 callsite = find_callsite(evsel, sample); 794 795 /* 796 * This is to find the current page (with correct gfp flags and 797 * migrate type) at free event. 798 */ 799 this.page = page; 800 pstat = page_stat__findnew_page(&this); 801 if (pstat == NULL) 802 return -ENOMEM; 803 804 pstat->nr_alloc++; 805 pstat->alloc_bytes += bytes; 806 pstat->callsite = callsite; 807 808 if (!live_page) { 809 pstat = page_stat__findnew_alloc(&this); 810 if (pstat == NULL) 811 return -ENOMEM; 812 813 pstat->nr_alloc++; 814 pstat->alloc_bytes += bytes; 815 pstat->callsite = callsite; 816 } 817 818 this.callsite = callsite; 819 pstat = page_stat__findnew_caller(&this); 820 if (pstat == NULL) 821 return -ENOMEM; 822 823 pstat->nr_alloc++; 824 pstat->alloc_bytes += bytes; 825 826 order_stats[order][migrate_type]++; 827 828 return 0; 829 } 830 831 static int perf_evsel__process_page_free_event(struct perf_evsel *evsel, 832 struct perf_sample *sample) 833 { 834 u64 page; 835 unsigned int order = perf_evsel__intval(evsel, sample, "order"); 836 u64 bytes = kmem_page_size << order; 837 struct page_stat *pstat; 838 struct page_stat this = { 839 .order = order, 840 }; 841 842 if (use_pfn) 843 page = perf_evsel__intval(evsel, sample, "pfn"); 844 else 845 page = perf_evsel__intval(evsel, sample, "page"); 846 847 nr_page_frees++; 848 total_page_free_bytes += bytes; 849 850 this.page = page; 851 pstat = page_stat__find_page(&this); 852 if (pstat == NULL) { 853 pr_debug2("missing free at page %"PRIx64" (order: %d)\n", 854 page, order); 855 856 nr_page_nomatch++; 857 total_page_nomatch_bytes += bytes; 858 859 return 0; 860 } 861 862 this.gfp_flags = pstat->gfp_flags; 863 this.migrate_type = pstat->migrate_type; 864 this.callsite = pstat->callsite; 865 866 rb_erase(&pstat->node, &page_live_tree); 867 free(pstat); 868 869 if (live_page) { 870 order_stats[this.order][this.migrate_type]--; 871 } else { 872 pstat = page_stat__find_alloc(&this); 873 if (pstat == NULL) 874 return -ENOMEM; 875 876 pstat->nr_free++; 877 pstat->free_bytes += bytes; 878 } 879 880 pstat = page_stat__find_caller(&this); 881 if (pstat == NULL) 882 return -ENOENT; 883 884 pstat->nr_free++; 885 pstat->free_bytes += bytes; 886 887 if (live_page) { 888 pstat->nr_alloc--; 889 pstat->alloc_bytes -= bytes; 890 891 if (pstat->nr_alloc == 0) { 892 rb_erase(&pstat->node, &page_caller_tree); 893 free(pstat); 894 } 895 } 896 897 return 0; 898 } 899 900 typedef int (*tracepoint_handler)(struct perf_evsel *evsel, 901 struct perf_sample *sample); 902 903 static int process_sample_event(struct perf_tool *tool __maybe_unused, 904 union perf_event *event, 905 struct perf_sample *sample, 906 struct perf_evsel *evsel, 907 struct machine *machine) 908 { 909 struct thread *thread = machine__findnew_thread(machine, sample->pid, 910 sample->tid); 911 912 if (thread == NULL) { 913 pr_debug("problem processing %d event, skipping it.\n", 914 event->header.type); 915 return -1; 916 } 917 918 dump_printf(" ... thread: %s:%d\n", thread__comm_str(thread), thread->tid); 919 920 if (evsel->handler != NULL) { 921 tracepoint_handler f = evsel->handler; 922 return f(evsel, sample); 923 } 924 925 return 0; 926 } 927 928 static struct perf_tool perf_kmem = { 929 .sample = process_sample_event, 930 .comm = perf_event__process_comm, 931 .mmap = perf_event__process_mmap, 932 .mmap2 = perf_event__process_mmap2, 933 .ordered_events = true, 934 }; 935 936 static double fragmentation(unsigned long n_req, unsigned long n_alloc) 937 { 938 if (n_alloc == 0) 939 return 0.0; 940 else 941 return 100.0 - (100.0 * n_req / n_alloc); 942 } 943 944 static void __print_slab_result(struct rb_root *root, 945 struct perf_session *session, 946 int n_lines, int is_caller) 947 { 948 struct rb_node *next; 949 struct machine *machine = &session->machines.host; 950 951 printf("%.105s\n", graph_dotted_line); 952 printf(" %-34s |", is_caller ? "Callsite": "Alloc Ptr"); 953 printf(" Total_alloc/Per | Total_req/Per | Hit | Ping-pong | Frag\n"); 954 printf("%.105s\n", graph_dotted_line); 955 956 next = rb_first(root); 957 958 while (next && n_lines--) { 959 struct alloc_stat *data = rb_entry(next, struct alloc_stat, 960 node); 961 struct symbol *sym = NULL; 962 struct map *map; 963 char buf[BUFSIZ]; 964 u64 addr; 965 966 if (is_caller) { 967 addr = data->call_site; 968 if (!raw_ip) 969 sym = machine__find_kernel_function(machine, addr, &map, NULL); 970 } else 971 addr = data->ptr; 972 973 if (sym != NULL) 974 snprintf(buf, sizeof(buf), "%s+%" PRIx64 "", sym->name, 975 addr - map->unmap_ip(map, sym->start)); 976 else 977 snprintf(buf, sizeof(buf), "%#" PRIx64 "", addr); 978 printf(" %-34s |", buf); 979 980 printf(" %9llu/%-5lu | %9llu/%-5lu | %8lu | %9lu | %6.3f%%\n", 981 (unsigned long long)data->bytes_alloc, 982 (unsigned long)data->bytes_alloc / data->hit, 983 (unsigned long long)data->bytes_req, 984 (unsigned long)data->bytes_req / data->hit, 985 (unsigned long)data->hit, 986 (unsigned long)data->pingpong, 987 fragmentation(data->bytes_req, data->bytes_alloc)); 988 989 next = rb_next(next); 990 } 991 992 if (n_lines == -1) 993 printf(" ... | ... | ... | ... | ... | ... \n"); 994 995 printf("%.105s\n", graph_dotted_line); 996 } 997 998 static const char * const migrate_type_str[] = { 999 "UNMOVABL", 1000 "RECLAIM", 1001 "MOVABLE", 1002 "RESERVED", 1003 "CMA/ISLT", 1004 "UNKNOWN", 1005 }; 1006 1007 static void __print_page_alloc_result(struct perf_session *session, int n_lines) 1008 { 1009 struct rb_node *next = rb_first(&page_alloc_sorted); 1010 struct machine *machine = &session->machines.host; 1011 const char *format; 1012 int gfp_len = max(strlen("GFP flags"), max_gfp_len); 1013 1014 printf("\n%.105s\n", graph_dotted_line); 1015 printf(" %-16s | %5s alloc (KB) | Hits | Order | Mig.type | %-*s | Callsite\n", 1016 use_pfn ? "PFN" : "Page", live_page ? "Live" : "Total", 1017 gfp_len, "GFP flags"); 1018 printf("%.105s\n", graph_dotted_line); 1019 1020 if (use_pfn) 1021 format = " %16llu | %'16llu | %'9d | %5d | %8s | %-*s | %s\n"; 1022 else 1023 format = " %016llx | %'16llu | %'9d | %5d | %8s | %-*s | %s\n"; 1024 1025 while (next && n_lines--) { 1026 struct page_stat *data; 1027 struct symbol *sym; 1028 struct map *map; 1029 char buf[32]; 1030 char *caller = buf; 1031 1032 data = rb_entry(next, struct page_stat, node); 1033 sym = machine__find_kernel_function(machine, data->callsite, 1034 &map, NULL); 1035 if (sym && sym->name) 1036 caller = sym->name; 1037 else 1038 scnprintf(buf, sizeof(buf), "%"PRIx64, data->callsite); 1039 1040 printf(format, (unsigned long long)data->page, 1041 (unsigned long long)data->alloc_bytes / 1024, 1042 data->nr_alloc, data->order, 1043 migrate_type_str[data->migrate_type], 1044 gfp_len, compact_gfp_string(data->gfp_flags), caller); 1045 1046 next = rb_next(next); 1047 } 1048 1049 if (n_lines == -1) { 1050 printf(" ... | ... | ... | ... | ... | %-*s | ...\n", 1051 gfp_len, "..."); 1052 } 1053 1054 printf("%.105s\n", graph_dotted_line); 1055 } 1056 1057 static void __print_page_caller_result(struct perf_session *session, int n_lines) 1058 { 1059 struct rb_node *next = rb_first(&page_caller_sorted); 1060 struct machine *machine = &session->machines.host; 1061 int gfp_len = max(strlen("GFP flags"), max_gfp_len); 1062 1063 printf("\n%.105s\n", graph_dotted_line); 1064 printf(" %5s alloc (KB) | Hits | Order | Mig.type | %-*s | Callsite\n", 1065 live_page ? "Live" : "Total", gfp_len, "GFP flags"); 1066 printf("%.105s\n", graph_dotted_line); 1067 1068 while (next && n_lines--) { 1069 struct page_stat *data; 1070 struct symbol *sym; 1071 struct map *map; 1072 char buf[32]; 1073 char *caller = buf; 1074 1075 data = rb_entry(next, struct page_stat, node); 1076 sym = machine__find_kernel_function(machine, data->callsite, 1077 &map, NULL); 1078 if (sym && sym->name) 1079 caller = sym->name; 1080 else 1081 scnprintf(buf, sizeof(buf), "%"PRIx64, data->callsite); 1082 1083 printf(" %'16llu | %'9d | %5d | %8s | %-*s | %s\n", 1084 (unsigned long long)data->alloc_bytes / 1024, 1085 data->nr_alloc, data->order, 1086 migrate_type_str[data->migrate_type], 1087 gfp_len, compact_gfp_string(data->gfp_flags), caller); 1088 1089 next = rb_next(next); 1090 } 1091 1092 if (n_lines == -1) { 1093 printf(" ... | ... | ... | ... | %-*s | ...\n", 1094 gfp_len, "..."); 1095 } 1096 1097 printf("%.105s\n", graph_dotted_line); 1098 } 1099 1100 static void print_gfp_flags(void) 1101 { 1102 int i; 1103 1104 printf("#\n"); 1105 printf("# GFP flags\n"); 1106 printf("# ---------\n"); 1107 for (i = 0; i < nr_gfps; i++) { 1108 printf("# %08x: %*s: %s\n", gfps[i].flags, 1109 (int) max_gfp_len, gfps[i].compact_str, 1110 gfps[i].human_readable); 1111 } 1112 } 1113 1114 static void print_slab_summary(void) 1115 { 1116 printf("\nSUMMARY (SLAB allocator)"); 1117 printf("\n========================\n"); 1118 printf("Total bytes requested: %'lu\n", total_requested); 1119 printf("Total bytes allocated: %'lu\n", total_allocated); 1120 printf("Total bytes wasted on internal fragmentation: %'lu\n", 1121 total_allocated - total_requested); 1122 printf("Internal fragmentation: %f%%\n", 1123 fragmentation(total_requested, total_allocated)); 1124 printf("Cross CPU allocations: %'lu/%'lu\n", nr_cross_allocs, nr_allocs); 1125 } 1126 1127 static void print_page_summary(void) 1128 { 1129 int o, m; 1130 u64 nr_alloc_freed = nr_page_frees - nr_page_nomatch; 1131 u64 total_alloc_freed_bytes = total_page_free_bytes - total_page_nomatch_bytes; 1132 1133 printf("\nSUMMARY (page allocator)"); 1134 printf("\n========================\n"); 1135 printf("%-30s: %'16lu [ %'16"PRIu64" KB ]\n", "Total allocation requests", 1136 nr_page_allocs, total_page_alloc_bytes / 1024); 1137 printf("%-30s: %'16lu [ %'16"PRIu64" KB ]\n", "Total free requests", 1138 nr_page_frees, total_page_free_bytes / 1024); 1139 printf("\n"); 1140 1141 printf("%-30s: %'16lu [ %'16"PRIu64" KB ]\n", "Total alloc+freed requests", 1142 nr_alloc_freed, (total_alloc_freed_bytes) / 1024); 1143 printf("%-30s: %'16lu [ %'16"PRIu64" KB ]\n", "Total alloc-only requests", 1144 nr_page_allocs - nr_alloc_freed, 1145 (total_page_alloc_bytes - total_alloc_freed_bytes) / 1024); 1146 printf("%-30s: %'16lu [ %'16"PRIu64" KB ]\n", "Total free-only requests", 1147 nr_page_nomatch, total_page_nomatch_bytes / 1024); 1148 printf("\n"); 1149 1150 printf("%-30s: %'16lu [ %'16"PRIu64" KB ]\n", "Total allocation failures", 1151 nr_page_fails, total_page_fail_bytes / 1024); 1152 printf("\n"); 1153 1154 printf("%5s %12s %12s %12s %12s %12s\n", "Order", "Unmovable", 1155 "Reclaimable", "Movable", "Reserved", "CMA/Isolated"); 1156 printf("%.5s %.12s %.12s %.12s %.12s %.12s\n", graph_dotted_line, 1157 graph_dotted_line, graph_dotted_line, graph_dotted_line, 1158 graph_dotted_line, graph_dotted_line); 1159 1160 for (o = 0; o < MAX_PAGE_ORDER; o++) { 1161 printf("%5d", o); 1162 for (m = 0; m < MAX_MIGRATE_TYPES - 1; m++) { 1163 if (order_stats[o][m]) 1164 printf(" %'12d", order_stats[o][m]); 1165 else 1166 printf(" %12c", '.'); 1167 } 1168 printf("\n"); 1169 } 1170 } 1171 1172 static void print_slab_result(struct perf_session *session) 1173 { 1174 if (caller_flag) 1175 __print_slab_result(&root_caller_sorted, session, caller_lines, 1); 1176 if (alloc_flag) 1177 __print_slab_result(&root_alloc_sorted, session, alloc_lines, 0); 1178 print_slab_summary(); 1179 } 1180 1181 static void print_page_result(struct perf_session *session) 1182 { 1183 if (caller_flag || alloc_flag) 1184 print_gfp_flags(); 1185 if (caller_flag) 1186 __print_page_caller_result(session, caller_lines); 1187 if (alloc_flag) 1188 __print_page_alloc_result(session, alloc_lines); 1189 print_page_summary(); 1190 } 1191 1192 static void print_result(struct perf_session *session) 1193 { 1194 if (kmem_slab) 1195 print_slab_result(session); 1196 if (kmem_page) 1197 print_page_result(session); 1198 } 1199 1200 static LIST_HEAD(slab_caller_sort); 1201 static LIST_HEAD(slab_alloc_sort); 1202 static LIST_HEAD(page_caller_sort); 1203 static LIST_HEAD(page_alloc_sort); 1204 1205 static void sort_slab_insert(struct rb_root *root, struct alloc_stat *data, 1206 struct list_head *sort_list) 1207 { 1208 struct rb_node **new = &(root->rb_node); 1209 struct rb_node *parent = NULL; 1210 struct sort_dimension *sort; 1211 1212 while (*new) { 1213 struct alloc_stat *this; 1214 int cmp = 0; 1215 1216 this = rb_entry(*new, struct alloc_stat, node); 1217 parent = *new; 1218 1219 list_for_each_entry(sort, sort_list, list) { 1220 cmp = sort->cmp(data, this); 1221 if (cmp) 1222 break; 1223 } 1224 1225 if (cmp > 0) 1226 new = &((*new)->rb_left); 1227 else 1228 new = &((*new)->rb_right); 1229 } 1230 1231 rb_link_node(&data->node, parent, new); 1232 rb_insert_color(&data->node, root); 1233 } 1234 1235 static void __sort_slab_result(struct rb_root *root, struct rb_root *root_sorted, 1236 struct list_head *sort_list) 1237 { 1238 struct rb_node *node; 1239 struct alloc_stat *data; 1240 1241 for (;;) { 1242 node = rb_first(root); 1243 if (!node) 1244 break; 1245 1246 rb_erase(node, root); 1247 data = rb_entry(node, struct alloc_stat, node); 1248 sort_slab_insert(root_sorted, data, sort_list); 1249 } 1250 } 1251 1252 static void sort_page_insert(struct rb_root *root, struct page_stat *data, 1253 struct list_head *sort_list) 1254 { 1255 struct rb_node **new = &root->rb_node; 1256 struct rb_node *parent = NULL; 1257 struct sort_dimension *sort; 1258 1259 while (*new) { 1260 struct page_stat *this; 1261 int cmp = 0; 1262 1263 this = rb_entry(*new, struct page_stat, node); 1264 parent = *new; 1265 1266 list_for_each_entry(sort, sort_list, list) { 1267 cmp = sort->cmp(data, this); 1268 if (cmp) 1269 break; 1270 } 1271 1272 if (cmp > 0) 1273 new = &parent->rb_left; 1274 else 1275 new = &parent->rb_right; 1276 } 1277 1278 rb_link_node(&data->node, parent, new); 1279 rb_insert_color(&data->node, root); 1280 } 1281 1282 static void __sort_page_result(struct rb_root *root, struct rb_root *root_sorted, 1283 struct list_head *sort_list) 1284 { 1285 struct rb_node *node; 1286 struct page_stat *data; 1287 1288 for (;;) { 1289 node = rb_first(root); 1290 if (!node) 1291 break; 1292 1293 rb_erase(node, root); 1294 data = rb_entry(node, struct page_stat, node); 1295 sort_page_insert(root_sorted, data, sort_list); 1296 } 1297 } 1298 1299 static void sort_result(void) 1300 { 1301 if (kmem_slab) { 1302 __sort_slab_result(&root_alloc_stat, &root_alloc_sorted, 1303 &slab_alloc_sort); 1304 __sort_slab_result(&root_caller_stat, &root_caller_sorted, 1305 &slab_caller_sort); 1306 } 1307 if (kmem_page) { 1308 if (live_page) 1309 __sort_page_result(&page_live_tree, &page_alloc_sorted, 1310 &page_alloc_sort); 1311 else 1312 __sort_page_result(&page_alloc_tree, &page_alloc_sorted, 1313 &page_alloc_sort); 1314 1315 __sort_page_result(&page_caller_tree, &page_caller_sorted, 1316 &page_caller_sort); 1317 } 1318 } 1319 1320 static int __cmd_kmem(struct perf_session *session) 1321 { 1322 int err = -EINVAL; 1323 struct perf_evsel *evsel; 1324 const struct perf_evsel_str_handler kmem_tracepoints[] = { 1325 /* slab allocator */ 1326 { "kmem:kmalloc", perf_evsel__process_alloc_event, }, 1327 { "kmem:kmem_cache_alloc", perf_evsel__process_alloc_event, }, 1328 { "kmem:kmalloc_node", perf_evsel__process_alloc_node_event, }, 1329 { "kmem:kmem_cache_alloc_node", perf_evsel__process_alloc_node_event, }, 1330 { "kmem:kfree", perf_evsel__process_free_event, }, 1331 { "kmem:kmem_cache_free", perf_evsel__process_free_event, }, 1332 /* page allocator */ 1333 { "kmem:mm_page_alloc", perf_evsel__process_page_alloc_event, }, 1334 { "kmem:mm_page_free", perf_evsel__process_page_free_event, }, 1335 }; 1336 1337 if (!perf_session__has_traces(session, "kmem record")) 1338 goto out; 1339 1340 if (perf_session__set_tracepoints_handlers(session, kmem_tracepoints)) { 1341 pr_err("Initializing perf session tracepoint handlers failed\n"); 1342 goto out; 1343 } 1344 1345 evlist__for_each(session->evlist, evsel) { 1346 if (!strcmp(perf_evsel__name(evsel), "kmem:mm_page_alloc") && 1347 perf_evsel__field(evsel, "pfn")) { 1348 use_pfn = true; 1349 break; 1350 } 1351 } 1352 1353 setup_pager(); 1354 err = perf_session__process_events(session); 1355 if (err != 0) { 1356 pr_err("error during process events: %d\n", err); 1357 goto out; 1358 } 1359 sort_result(); 1360 print_result(session); 1361 out: 1362 return err; 1363 } 1364 1365 /* slab sort keys */ 1366 static int ptr_cmp(void *a, void *b) 1367 { 1368 struct alloc_stat *l = a; 1369 struct alloc_stat *r = b; 1370 1371 if (l->ptr < r->ptr) 1372 return -1; 1373 else if (l->ptr > r->ptr) 1374 return 1; 1375 return 0; 1376 } 1377 1378 static struct sort_dimension ptr_sort_dimension = { 1379 .name = "ptr", 1380 .cmp = ptr_cmp, 1381 }; 1382 1383 static int slab_callsite_cmp(void *a, void *b) 1384 { 1385 struct alloc_stat *l = a; 1386 struct alloc_stat *r = b; 1387 1388 if (l->call_site < r->call_site) 1389 return -1; 1390 else if (l->call_site > r->call_site) 1391 return 1; 1392 return 0; 1393 } 1394 1395 static struct sort_dimension callsite_sort_dimension = { 1396 .name = "callsite", 1397 .cmp = slab_callsite_cmp, 1398 }; 1399 1400 static int hit_cmp(void *a, void *b) 1401 { 1402 struct alloc_stat *l = a; 1403 struct alloc_stat *r = b; 1404 1405 if (l->hit < r->hit) 1406 return -1; 1407 else if (l->hit > r->hit) 1408 return 1; 1409 return 0; 1410 } 1411 1412 static struct sort_dimension hit_sort_dimension = { 1413 .name = "hit", 1414 .cmp = hit_cmp, 1415 }; 1416 1417 static int bytes_cmp(void *a, void *b) 1418 { 1419 struct alloc_stat *l = a; 1420 struct alloc_stat *r = b; 1421 1422 if (l->bytes_alloc < r->bytes_alloc) 1423 return -1; 1424 else if (l->bytes_alloc > r->bytes_alloc) 1425 return 1; 1426 return 0; 1427 } 1428 1429 static struct sort_dimension bytes_sort_dimension = { 1430 .name = "bytes", 1431 .cmp = bytes_cmp, 1432 }; 1433 1434 static int frag_cmp(void *a, void *b) 1435 { 1436 double x, y; 1437 struct alloc_stat *l = a; 1438 struct alloc_stat *r = b; 1439 1440 x = fragmentation(l->bytes_req, l->bytes_alloc); 1441 y = fragmentation(r->bytes_req, r->bytes_alloc); 1442 1443 if (x < y) 1444 return -1; 1445 else if (x > y) 1446 return 1; 1447 return 0; 1448 } 1449 1450 static struct sort_dimension frag_sort_dimension = { 1451 .name = "frag", 1452 .cmp = frag_cmp, 1453 }; 1454 1455 static int pingpong_cmp(void *a, void *b) 1456 { 1457 struct alloc_stat *l = a; 1458 struct alloc_stat *r = b; 1459 1460 if (l->pingpong < r->pingpong) 1461 return -1; 1462 else if (l->pingpong > r->pingpong) 1463 return 1; 1464 return 0; 1465 } 1466 1467 static struct sort_dimension pingpong_sort_dimension = { 1468 .name = "pingpong", 1469 .cmp = pingpong_cmp, 1470 }; 1471 1472 /* page sort keys */ 1473 static int page_cmp(void *a, void *b) 1474 { 1475 struct page_stat *l = a; 1476 struct page_stat *r = b; 1477 1478 if (l->page < r->page) 1479 return -1; 1480 else if (l->page > r->page) 1481 return 1; 1482 return 0; 1483 } 1484 1485 static struct sort_dimension page_sort_dimension = { 1486 .name = "page", 1487 .cmp = page_cmp, 1488 }; 1489 1490 static int page_callsite_cmp(void *a, void *b) 1491 { 1492 struct page_stat *l = a; 1493 struct page_stat *r = b; 1494 1495 if (l->callsite < r->callsite) 1496 return -1; 1497 else if (l->callsite > r->callsite) 1498 return 1; 1499 return 0; 1500 } 1501 1502 static struct sort_dimension page_callsite_sort_dimension = { 1503 .name = "callsite", 1504 .cmp = page_callsite_cmp, 1505 }; 1506 1507 static int page_hit_cmp(void *a, void *b) 1508 { 1509 struct page_stat *l = a; 1510 struct page_stat *r = b; 1511 1512 if (l->nr_alloc < r->nr_alloc) 1513 return -1; 1514 else if (l->nr_alloc > r->nr_alloc) 1515 return 1; 1516 return 0; 1517 } 1518 1519 static struct sort_dimension page_hit_sort_dimension = { 1520 .name = "hit", 1521 .cmp = page_hit_cmp, 1522 }; 1523 1524 static int page_bytes_cmp(void *a, void *b) 1525 { 1526 struct page_stat *l = a; 1527 struct page_stat *r = b; 1528 1529 if (l->alloc_bytes < r->alloc_bytes) 1530 return -1; 1531 else if (l->alloc_bytes > r->alloc_bytes) 1532 return 1; 1533 return 0; 1534 } 1535 1536 static struct sort_dimension page_bytes_sort_dimension = { 1537 .name = "bytes", 1538 .cmp = page_bytes_cmp, 1539 }; 1540 1541 static int page_order_cmp(void *a, void *b) 1542 { 1543 struct page_stat *l = a; 1544 struct page_stat *r = b; 1545 1546 if (l->order < r->order) 1547 return -1; 1548 else if (l->order > r->order) 1549 return 1; 1550 return 0; 1551 } 1552 1553 static struct sort_dimension page_order_sort_dimension = { 1554 .name = "order", 1555 .cmp = page_order_cmp, 1556 }; 1557 1558 static int migrate_type_cmp(void *a, void *b) 1559 { 1560 struct page_stat *l = a; 1561 struct page_stat *r = b; 1562 1563 /* for internal use to find free'd page */ 1564 if (l->migrate_type == -1U) 1565 return 0; 1566 1567 if (l->migrate_type < r->migrate_type) 1568 return -1; 1569 else if (l->migrate_type > r->migrate_type) 1570 return 1; 1571 return 0; 1572 } 1573 1574 static struct sort_dimension migrate_type_sort_dimension = { 1575 .name = "migtype", 1576 .cmp = migrate_type_cmp, 1577 }; 1578 1579 static int gfp_flags_cmp(void *a, void *b) 1580 { 1581 struct page_stat *l = a; 1582 struct page_stat *r = b; 1583 1584 /* for internal use to find free'd page */ 1585 if (l->gfp_flags == -1U) 1586 return 0; 1587 1588 if (l->gfp_flags < r->gfp_flags) 1589 return -1; 1590 else if (l->gfp_flags > r->gfp_flags) 1591 return 1; 1592 return 0; 1593 } 1594 1595 static struct sort_dimension gfp_flags_sort_dimension = { 1596 .name = "gfp", 1597 .cmp = gfp_flags_cmp, 1598 }; 1599 1600 static struct sort_dimension *slab_sorts[] = { 1601 &ptr_sort_dimension, 1602 &callsite_sort_dimension, 1603 &hit_sort_dimension, 1604 &bytes_sort_dimension, 1605 &frag_sort_dimension, 1606 &pingpong_sort_dimension, 1607 }; 1608 1609 static struct sort_dimension *page_sorts[] = { 1610 &page_sort_dimension, 1611 &page_callsite_sort_dimension, 1612 &page_hit_sort_dimension, 1613 &page_bytes_sort_dimension, 1614 &page_order_sort_dimension, 1615 &migrate_type_sort_dimension, 1616 &gfp_flags_sort_dimension, 1617 }; 1618 1619 static int slab_sort_dimension__add(const char *tok, struct list_head *list) 1620 { 1621 struct sort_dimension *sort; 1622 int i; 1623 1624 for (i = 0; i < (int)ARRAY_SIZE(slab_sorts); i++) { 1625 if (!strcmp(slab_sorts[i]->name, tok)) { 1626 sort = memdup(slab_sorts[i], sizeof(*slab_sorts[i])); 1627 if (!sort) { 1628 pr_err("%s: memdup failed\n", __func__); 1629 return -1; 1630 } 1631 list_add_tail(&sort->list, list); 1632 return 0; 1633 } 1634 } 1635 1636 return -1; 1637 } 1638 1639 static int page_sort_dimension__add(const char *tok, struct list_head *list) 1640 { 1641 struct sort_dimension *sort; 1642 int i; 1643 1644 for (i = 0; i < (int)ARRAY_SIZE(page_sorts); i++) { 1645 if (!strcmp(page_sorts[i]->name, tok)) { 1646 sort = memdup(page_sorts[i], sizeof(*page_sorts[i])); 1647 if (!sort) { 1648 pr_err("%s: memdup failed\n", __func__); 1649 return -1; 1650 } 1651 list_add_tail(&sort->list, list); 1652 return 0; 1653 } 1654 } 1655 1656 return -1; 1657 } 1658 1659 static int setup_slab_sorting(struct list_head *sort_list, const char *arg) 1660 { 1661 char *tok; 1662 char *str = strdup(arg); 1663 char *pos = str; 1664 1665 if (!str) { 1666 pr_err("%s: strdup failed\n", __func__); 1667 return -1; 1668 } 1669 1670 while (true) { 1671 tok = strsep(&pos, ","); 1672 if (!tok) 1673 break; 1674 if (slab_sort_dimension__add(tok, sort_list) < 0) { 1675 error("Unknown slab --sort key: '%s'", tok); 1676 free(str); 1677 return -1; 1678 } 1679 } 1680 1681 free(str); 1682 return 0; 1683 } 1684 1685 static int setup_page_sorting(struct list_head *sort_list, const char *arg) 1686 { 1687 char *tok; 1688 char *str = strdup(arg); 1689 char *pos = str; 1690 1691 if (!str) { 1692 pr_err("%s: strdup failed\n", __func__); 1693 return -1; 1694 } 1695 1696 while (true) { 1697 tok = strsep(&pos, ","); 1698 if (!tok) 1699 break; 1700 if (page_sort_dimension__add(tok, sort_list) < 0) { 1701 error("Unknown page --sort key: '%s'", tok); 1702 free(str); 1703 return -1; 1704 } 1705 } 1706 1707 free(str); 1708 return 0; 1709 } 1710 1711 static int parse_sort_opt(const struct option *opt __maybe_unused, 1712 const char *arg, int unset __maybe_unused) 1713 { 1714 if (!arg) 1715 return -1; 1716 1717 if (kmem_page > kmem_slab || 1718 (kmem_page == 0 && kmem_slab == 0 && kmem_default == KMEM_PAGE)) { 1719 if (caller_flag > alloc_flag) 1720 return setup_page_sorting(&page_caller_sort, arg); 1721 else 1722 return setup_page_sorting(&page_alloc_sort, arg); 1723 } else { 1724 if (caller_flag > alloc_flag) 1725 return setup_slab_sorting(&slab_caller_sort, arg); 1726 else 1727 return setup_slab_sorting(&slab_alloc_sort, arg); 1728 } 1729 1730 return 0; 1731 } 1732 1733 static int parse_caller_opt(const struct option *opt __maybe_unused, 1734 const char *arg __maybe_unused, 1735 int unset __maybe_unused) 1736 { 1737 caller_flag = (alloc_flag + 1); 1738 return 0; 1739 } 1740 1741 static int parse_alloc_opt(const struct option *opt __maybe_unused, 1742 const char *arg __maybe_unused, 1743 int unset __maybe_unused) 1744 { 1745 alloc_flag = (caller_flag + 1); 1746 return 0; 1747 } 1748 1749 static int parse_slab_opt(const struct option *opt __maybe_unused, 1750 const char *arg __maybe_unused, 1751 int unset __maybe_unused) 1752 { 1753 kmem_slab = (kmem_page + 1); 1754 return 0; 1755 } 1756 1757 static int parse_page_opt(const struct option *opt __maybe_unused, 1758 const char *arg __maybe_unused, 1759 int unset __maybe_unused) 1760 { 1761 kmem_page = (kmem_slab + 1); 1762 return 0; 1763 } 1764 1765 static int parse_line_opt(const struct option *opt __maybe_unused, 1766 const char *arg, int unset __maybe_unused) 1767 { 1768 int lines; 1769 1770 if (!arg) 1771 return -1; 1772 1773 lines = strtoul(arg, NULL, 10); 1774 1775 if (caller_flag > alloc_flag) 1776 caller_lines = lines; 1777 else 1778 alloc_lines = lines; 1779 1780 return 0; 1781 } 1782 1783 static int __cmd_record(int argc, const char **argv) 1784 { 1785 const char * const record_args[] = { 1786 "record", "-a", "-R", "-c", "1", 1787 }; 1788 const char * const slab_events[] = { 1789 "-e", "kmem:kmalloc", 1790 "-e", "kmem:kmalloc_node", 1791 "-e", "kmem:kfree", 1792 "-e", "kmem:kmem_cache_alloc", 1793 "-e", "kmem:kmem_cache_alloc_node", 1794 "-e", "kmem:kmem_cache_free", 1795 }; 1796 const char * const page_events[] = { 1797 "-e", "kmem:mm_page_alloc", 1798 "-e", "kmem:mm_page_free", 1799 }; 1800 unsigned int rec_argc, i, j; 1801 const char **rec_argv; 1802 1803 rec_argc = ARRAY_SIZE(record_args) + argc - 1; 1804 if (kmem_slab) 1805 rec_argc += ARRAY_SIZE(slab_events); 1806 if (kmem_page) 1807 rec_argc += ARRAY_SIZE(page_events) + 1; /* for -g */ 1808 1809 rec_argv = calloc(rec_argc + 1, sizeof(char *)); 1810 1811 if (rec_argv == NULL) 1812 return -ENOMEM; 1813 1814 for (i = 0; i < ARRAY_SIZE(record_args); i++) 1815 rec_argv[i] = strdup(record_args[i]); 1816 1817 if (kmem_slab) { 1818 for (j = 0; j < ARRAY_SIZE(slab_events); j++, i++) 1819 rec_argv[i] = strdup(slab_events[j]); 1820 } 1821 if (kmem_page) { 1822 rec_argv[i++] = strdup("-g"); 1823 1824 for (j = 0; j < ARRAY_SIZE(page_events); j++, i++) 1825 rec_argv[i] = strdup(page_events[j]); 1826 } 1827 1828 for (j = 1; j < (unsigned int)argc; j++, i++) 1829 rec_argv[i] = argv[j]; 1830 1831 return cmd_record(i, rec_argv, NULL); 1832 } 1833 1834 static int kmem_config(const char *var, const char *value, void *cb) 1835 { 1836 if (!strcmp(var, "kmem.default")) { 1837 if (!strcmp(value, "slab")) 1838 kmem_default = KMEM_SLAB; 1839 else if (!strcmp(value, "page")) 1840 kmem_default = KMEM_PAGE; 1841 else 1842 pr_err("invalid default value ('slab' or 'page' required): %s\n", 1843 value); 1844 return 0; 1845 } 1846 1847 return perf_default_config(var, value, cb); 1848 } 1849 1850 int cmd_kmem(int argc, const char **argv, const char *prefix __maybe_unused) 1851 { 1852 const char * const default_slab_sort = "frag,hit,bytes"; 1853 const char * const default_page_sort = "bytes,hit"; 1854 struct perf_data_file file = { 1855 .mode = PERF_DATA_MODE_READ, 1856 }; 1857 const struct option kmem_options[] = { 1858 OPT_STRING('i', "input", &input_name, "file", "input file name"), 1859 OPT_INCR('v', "verbose", &verbose, 1860 "be more verbose (show symbol address, etc)"), 1861 OPT_CALLBACK_NOOPT(0, "caller", NULL, NULL, 1862 "show per-callsite statistics", parse_caller_opt), 1863 OPT_CALLBACK_NOOPT(0, "alloc", NULL, NULL, 1864 "show per-allocation statistics", parse_alloc_opt), 1865 OPT_CALLBACK('s', "sort", NULL, "key[,key2...]", 1866 "sort by keys: ptr, callsite, bytes, hit, pingpong, frag, " 1867 "page, order, migtype, gfp", parse_sort_opt), 1868 OPT_CALLBACK('l', "line", NULL, "num", "show n lines", parse_line_opt), 1869 OPT_BOOLEAN(0, "raw-ip", &raw_ip, "show raw ip instead of symbol"), 1870 OPT_BOOLEAN('f', "force", &file.force, "don't complain, do it"), 1871 OPT_CALLBACK_NOOPT(0, "slab", NULL, NULL, "Analyze slab allocator", 1872 parse_slab_opt), 1873 OPT_CALLBACK_NOOPT(0, "page", NULL, NULL, "Analyze page allocator", 1874 parse_page_opt), 1875 OPT_BOOLEAN(0, "live", &live_page, "Show live page stat"), 1876 OPT_END() 1877 }; 1878 const char *const kmem_subcommands[] = { "record", "stat", NULL }; 1879 const char *kmem_usage[] = { 1880 NULL, 1881 NULL 1882 }; 1883 struct perf_session *session; 1884 int ret = -1; 1885 const char errmsg[] = "No %s allocation events found. Have you run 'perf kmem record --%s'?\n"; 1886 1887 perf_config(kmem_config, NULL); 1888 argc = parse_options_subcommand(argc, argv, kmem_options, 1889 kmem_subcommands, kmem_usage, 0); 1890 1891 if (!argc) 1892 usage_with_options(kmem_usage, kmem_options); 1893 1894 if (kmem_slab == 0 && kmem_page == 0) { 1895 if (kmem_default == KMEM_SLAB) 1896 kmem_slab = 1; 1897 else 1898 kmem_page = 1; 1899 } 1900 1901 if (!strncmp(argv[0], "rec", 3)) { 1902 symbol__init(NULL); 1903 return __cmd_record(argc, argv); 1904 } 1905 1906 file.path = input_name; 1907 1908 kmem_session = session = perf_session__new(&file, false, &perf_kmem); 1909 if (session == NULL) 1910 return -1; 1911 1912 if (kmem_slab) { 1913 if (!perf_evlist__find_tracepoint_by_name(session->evlist, 1914 "kmem:kmalloc")) { 1915 pr_err(errmsg, "slab", "slab"); 1916 return -1; 1917 } 1918 } 1919 1920 if (kmem_page) { 1921 struct perf_evsel *evsel; 1922 1923 evsel = perf_evlist__find_tracepoint_by_name(session->evlist, 1924 "kmem:mm_page_alloc"); 1925 if (evsel == NULL) { 1926 pr_err(errmsg, "page", "page"); 1927 return -1; 1928 } 1929 1930 kmem_page_size = pevent_get_page_size(evsel->tp_format->pevent); 1931 symbol_conf.use_callchain = true; 1932 } 1933 1934 symbol__init(&session->header.env); 1935 1936 if (!strcmp(argv[0], "stat")) { 1937 setlocale(LC_ALL, ""); 1938 1939 if (cpu__setup_cpunode_map()) 1940 goto out_delete; 1941 1942 if (list_empty(&slab_caller_sort)) 1943 setup_slab_sorting(&slab_caller_sort, default_slab_sort); 1944 if (list_empty(&slab_alloc_sort)) 1945 setup_slab_sorting(&slab_alloc_sort, default_slab_sort); 1946 if (list_empty(&page_caller_sort)) 1947 setup_page_sorting(&page_caller_sort, default_page_sort); 1948 if (list_empty(&page_alloc_sort)) 1949 setup_page_sorting(&page_alloc_sort, default_page_sort); 1950 1951 if (kmem_page) { 1952 setup_page_sorting(&page_alloc_sort_input, 1953 "page,order,migtype,gfp"); 1954 setup_page_sorting(&page_caller_sort_input, 1955 "callsite,order,migtype,gfp"); 1956 } 1957 ret = __cmd_kmem(session); 1958 } else 1959 usage_with_options(kmem_usage, kmem_options); 1960 1961 out_delete: 1962 perf_session__delete(session); 1963 1964 return ret; 1965 } 1966 1967