1 // SPDX-License-Identifier: GPL-2.0 2 #include <api/fs/fs.h> 3 #include "cpumap.h" 4 #include "debug.h" 5 #include "event.h" 6 #include <assert.h> 7 #include <dirent.h> 8 #include <stdio.h> 9 #include <stdlib.h> 10 #include <linux/bitmap.h> 11 #include "asm/bug.h" 12 13 #include <linux/ctype.h> 14 #include <linux/zalloc.h> 15 #include <internal/cpumap.h> 16 17 static struct perf_cpu max_cpu_num; 18 static struct perf_cpu max_present_cpu_num; 19 static int max_node_num; 20 /** 21 * The numa node X as read from /sys/devices/system/node/nodeX indexed by the 22 * CPU number. 23 */ 24 static int *cpunode_map; 25 26 bool perf_record_cpu_map_data__test_bit(int i, 27 const struct perf_record_cpu_map_data *data) 28 { 29 int bit_word32 = i / 32; 30 __u32 bit_mask32 = 1U << (i & 31); 31 int bit_word64 = i / 64; 32 __u64 bit_mask64 = ((__u64)1) << (i & 63); 33 34 return (data->mask32_data.long_size == 4) 35 ? (bit_word32 < data->mask32_data.nr) && 36 (data->mask32_data.mask[bit_word32] & bit_mask32) != 0 37 : (bit_word64 < data->mask64_data.nr) && 38 (data->mask64_data.mask[bit_word64] & bit_mask64) != 0; 39 } 40 41 /* Read ith mask value from data into the given 64-bit sized bitmap */ 42 static void perf_record_cpu_map_data__read_one_mask(const struct perf_record_cpu_map_data *data, 43 int i, unsigned long *bitmap) 44 { 45 #if __SIZEOF_LONG__ == 8 46 if (data->mask32_data.long_size == 4) 47 bitmap[0] = data->mask32_data.mask[i]; 48 else 49 bitmap[0] = data->mask64_data.mask[i]; 50 #else 51 if (data->mask32_data.long_size == 4) { 52 bitmap[0] = data->mask32_data.mask[i]; 53 bitmap[1] = 0; 54 } else { 55 #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ 56 bitmap[0] = (unsigned long)(data->mask64_data.mask[i] >> 32); 57 bitmap[1] = (unsigned long)data->mask64_data.mask[i]; 58 #else 59 bitmap[0] = (unsigned long)data->mask64_data.mask[i]; 60 bitmap[1] = (unsigned long)(data->mask64_data.mask[i] >> 32); 61 #endif 62 } 63 #endif 64 } 65 static struct perf_cpu_map *cpu_map__from_entries(const struct perf_record_cpu_map_data *data) 66 { 67 struct perf_cpu_map *map; 68 69 map = perf_cpu_map__empty_new(data->cpus_data.nr); 70 if (map) { 71 unsigned i; 72 73 for (i = 0; i < data->cpus_data.nr; i++) { 74 /* 75 * Special treatment for -1, which is not real cpu number, 76 * and we need to use (int) -1 to initialize map[i], 77 * otherwise it would become 65535. 78 */ 79 if (data->cpus_data.cpu[i] == (u16) -1) 80 RC_CHK_ACCESS(map)->map[i].cpu = -1; 81 else 82 RC_CHK_ACCESS(map)->map[i].cpu = (int) data->cpus_data.cpu[i]; 83 } 84 } 85 86 return map; 87 } 88 89 static struct perf_cpu_map *cpu_map__from_mask(const struct perf_record_cpu_map_data *data) 90 { 91 DECLARE_BITMAP(local_copy, 64); 92 int weight = 0, mask_nr = data->mask32_data.nr; 93 struct perf_cpu_map *map; 94 95 for (int i = 0; i < mask_nr; i++) { 96 perf_record_cpu_map_data__read_one_mask(data, i, local_copy); 97 weight += bitmap_weight(local_copy, 64); 98 } 99 100 map = perf_cpu_map__empty_new(weight); 101 if (!map) 102 return NULL; 103 104 for (int i = 0, j = 0; i < mask_nr; i++) { 105 int cpus_per_i = (i * data->mask32_data.long_size * BITS_PER_BYTE); 106 int cpu; 107 108 perf_record_cpu_map_data__read_one_mask(data, i, local_copy); 109 for_each_set_bit(cpu, local_copy, 64) 110 RC_CHK_ACCESS(map)->map[j++].cpu = cpu + cpus_per_i; 111 } 112 return map; 113 114 } 115 116 static struct perf_cpu_map *cpu_map__from_range(const struct perf_record_cpu_map_data *data) 117 { 118 struct perf_cpu_map *map; 119 unsigned int i = 0; 120 121 map = perf_cpu_map__empty_new(data->range_cpu_data.end_cpu - 122 data->range_cpu_data.start_cpu + 1 + data->range_cpu_data.any_cpu); 123 if (!map) 124 return NULL; 125 126 if (data->range_cpu_data.any_cpu) 127 RC_CHK_ACCESS(map)->map[i++].cpu = -1; 128 129 for (int cpu = data->range_cpu_data.start_cpu; cpu <= data->range_cpu_data.end_cpu; 130 i++, cpu++) 131 RC_CHK_ACCESS(map)->map[i].cpu = cpu; 132 133 return map; 134 } 135 136 struct perf_cpu_map *cpu_map__new_data(const struct perf_record_cpu_map_data *data) 137 { 138 switch (data->type) { 139 case PERF_CPU_MAP__CPUS: 140 return cpu_map__from_entries(data); 141 case PERF_CPU_MAP__MASK: 142 return cpu_map__from_mask(data); 143 case PERF_CPU_MAP__RANGE_CPUS: 144 return cpu_map__from_range(data); 145 default: 146 pr_err("cpu_map__new_data unknown type %d\n", data->type); 147 return NULL; 148 } 149 } 150 151 size_t cpu_map__fprintf(struct perf_cpu_map *map, FILE *fp) 152 { 153 #define BUFSIZE 1024 154 char buf[BUFSIZE]; 155 156 cpu_map__snprint(map, buf, sizeof(buf)); 157 return fprintf(fp, "%s\n", buf); 158 #undef BUFSIZE 159 } 160 161 struct perf_cpu_map *perf_cpu_map__empty_new(int nr) 162 { 163 struct perf_cpu_map *cpus = perf_cpu_map__alloc(nr); 164 165 if (cpus != NULL) { 166 for (int i = 0; i < nr; i++) 167 RC_CHK_ACCESS(cpus)->map[i].cpu = -1; 168 } 169 170 return cpus; 171 } 172 173 struct cpu_aggr_map *cpu_aggr_map__empty_new(int nr) 174 { 175 struct cpu_aggr_map *cpus = malloc(sizeof(*cpus) + sizeof(struct aggr_cpu_id) * nr); 176 177 if (cpus != NULL) { 178 int i; 179 180 cpus->nr = nr; 181 for (i = 0; i < nr; i++) 182 cpus->map[i] = aggr_cpu_id__empty(); 183 184 refcount_set(&cpus->refcnt, 1); 185 } 186 187 return cpus; 188 } 189 190 static int cpu__get_topology_int(int cpu, const char *name, int *value) 191 { 192 char path[PATH_MAX]; 193 194 snprintf(path, PATH_MAX, 195 "devices/system/cpu/cpu%d/topology/%s", cpu, name); 196 197 return sysfs__read_int(path, value); 198 } 199 200 int cpu__get_socket_id(struct perf_cpu cpu) 201 { 202 int value, ret = cpu__get_topology_int(cpu.cpu, "physical_package_id", &value); 203 return ret ?: value; 204 } 205 206 struct aggr_cpu_id aggr_cpu_id__socket(struct perf_cpu cpu, void *data __maybe_unused) 207 { 208 struct aggr_cpu_id id = aggr_cpu_id__empty(); 209 210 id.socket = cpu__get_socket_id(cpu); 211 return id; 212 } 213 214 static int aggr_cpu_id__cmp(const void *a_pointer, const void *b_pointer) 215 { 216 struct aggr_cpu_id *a = (struct aggr_cpu_id *)a_pointer; 217 struct aggr_cpu_id *b = (struct aggr_cpu_id *)b_pointer; 218 219 if (a->node != b->node) 220 return a->node - b->node; 221 else if (a->socket != b->socket) 222 return a->socket - b->socket; 223 else if (a->die != b->die) 224 return a->die - b->die; 225 else if (a->cache_lvl != b->cache_lvl) 226 return a->cache_lvl - b->cache_lvl; 227 else if (a->cache != b->cache) 228 return a->cache - b->cache; 229 else if (a->core != b->core) 230 return a->core - b->core; 231 else 232 return a->thread_idx - b->thread_idx; 233 } 234 235 struct cpu_aggr_map *cpu_aggr_map__new(const struct perf_cpu_map *cpus, 236 aggr_cpu_id_get_t get_id, 237 void *data, bool needs_sort) 238 { 239 int idx; 240 struct perf_cpu cpu; 241 struct cpu_aggr_map *c = cpu_aggr_map__empty_new(perf_cpu_map__nr(cpus)); 242 243 if (!c) 244 return NULL; 245 246 /* Reset size as it may only be partially filled */ 247 c->nr = 0; 248 249 perf_cpu_map__for_each_cpu(cpu, idx, cpus) { 250 bool duplicate = false; 251 struct aggr_cpu_id cpu_id = get_id(cpu, data); 252 253 for (int j = 0; j < c->nr; j++) { 254 if (aggr_cpu_id__equal(&cpu_id, &c->map[j])) { 255 duplicate = true; 256 break; 257 } 258 } 259 if (!duplicate) { 260 c->map[c->nr] = cpu_id; 261 c->nr++; 262 } 263 } 264 /* Trim. */ 265 if (c->nr != perf_cpu_map__nr(cpus)) { 266 struct cpu_aggr_map *trimmed_c = 267 realloc(c, 268 sizeof(struct cpu_aggr_map) + sizeof(struct aggr_cpu_id) * c->nr); 269 270 if (trimmed_c) 271 c = trimmed_c; 272 } 273 274 /* ensure we process id in increasing order */ 275 if (needs_sort) 276 qsort(c->map, c->nr, sizeof(struct aggr_cpu_id), aggr_cpu_id__cmp); 277 278 return c; 279 280 } 281 282 int cpu__get_die_id(struct perf_cpu cpu) 283 { 284 int value, ret = cpu__get_topology_int(cpu.cpu, "die_id", &value); 285 286 return ret ?: value; 287 } 288 289 struct aggr_cpu_id aggr_cpu_id__die(struct perf_cpu cpu, void *data) 290 { 291 struct aggr_cpu_id id; 292 int die; 293 294 die = cpu__get_die_id(cpu); 295 /* There is no die_id on legacy system. */ 296 if (die == -1) 297 die = 0; 298 299 /* 300 * die_id is relative to socket, so start 301 * with the socket ID and then add die to 302 * make a unique ID. 303 */ 304 id = aggr_cpu_id__socket(cpu, data); 305 if (aggr_cpu_id__is_empty(&id)) 306 return id; 307 308 id.die = die; 309 return id; 310 } 311 312 int cpu__get_core_id(struct perf_cpu cpu) 313 { 314 int value, ret = cpu__get_topology_int(cpu.cpu, "core_id", &value); 315 return ret ?: value; 316 } 317 318 struct aggr_cpu_id aggr_cpu_id__core(struct perf_cpu cpu, void *data) 319 { 320 struct aggr_cpu_id id; 321 int core = cpu__get_core_id(cpu); 322 323 /* aggr_cpu_id__die returns a struct with socket and die set. */ 324 id = aggr_cpu_id__die(cpu, data); 325 if (aggr_cpu_id__is_empty(&id)) 326 return id; 327 328 /* 329 * core_id is relative to socket and die, we need a global id. 330 * So we combine the result from cpu_map__get_die with the core id 331 */ 332 id.core = core; 333 return id; 334 335 } 336 337 struct aggr_cpu_id aggr_cpu_id__cpu(struct perf_cpu cpu, void *data) 338 { 339 struct aggr_cpu_id id; 340 341 /* aggr_cpu_id__core returns a struct with socket, die and core set. */ 342 id = aggr_cpu_id__core(cpu, data); 343 if (aggr_cpu_id__is_empty(&id)) 344 return id; 345 346 id.cpu = cpu; 347 return id; 348 349 } 350 351 struct aggr_cpu_id aggr_cpu_id__node(struct perf_cpu cpu, void *data __maybe_unused) 352 { 353 struct aggr_cpu_id id = aggr_cpu_id__empty(); 354 355 id.node = cpu__get_node(cpu); 356 return id; 357 } 358 359 struct aggr_cpu_id aggr_cpu_id__global(struct perf_cpu cpu, void *data __maybe_unused) 360 { 361 struct aggr_cpu_id id = aggr_cpu_id__empty(); 362 363 /* it always aggregates to the cpu 0 */ 364 cpu.cpu = 0; 365 id.cpu = cpu; 366 return id; 367 } 368 369 /* setup simple routines to easily access node numbers given a cpu number */ 370 static int get_max_num(char *path, int *max) 371 { 372 size_t num; 373 char *buf; 374 int err = 0; 375 376 if (filename__read_str(path, &buf, &num)) 377 return -1; 378 379 buf[num] = '\0'; 380 381 /* start on the right, to find highest node num */ 382 while (--num) { 383 if ((buf[num] == ',') || (buf[num] == '-')) { 384 num++; 385 break; 386 } 387 } 388 if (sscanf(&buf[num], "%d", max) < 1) { 389 err = -1; 390 goto out; 391 } 392 393 /* convert from 0-based to 1-based */ 394 (*max)++; 395 396 out: 397 free(buf); 398 return err; 399 } 400 401 /* Determine highest possible cpu in the system for sparse allocation */ 402 static void set_max_cpu_num(void) 403 { 404 const char *mnt; 405 char path[PATH_MAX]; 406 int ret = -1; 407 408 /* set up default */ 409 max_cpu_num.cpu = 4096; 410 max_present_cpu_num.cpu = 4096; 411 412 mnt = sysfs__mountpoint(); 413 if (!mnt) 414 goto out; 415 416 /* get the highest possible cpu number for a sparse allocation */ 417 ret = snprintf(path, PATH_MAX, "%s/devices/system/cpu/possible", mnt); 418 if (ret >= PATH_MAX) { 419 pr_err("sysfs path crossed PATH_MAX(%d) size\n", PATH_MAX); 420 goto out; 421 } 422 423 ret = get_max_num(path, &max_cpu_num.cpu); 424 if (ret) 425 goto out; 426 427 /* get the highest present cpu number for a sparse allocation */ 428 ret = snprintf(path, PATH_MAX, "%s/devices/system/cpu/present", mnt); 429 if (ret >= PATH_MAX) { 430 pr_err("sysfs path crossed PATH_MAX(%d) size\n", PATH_MAX); 431 goto out; 432 } 433 434 ret = get_max_num(path, &max_present_cpu_num.cpu); 435 436 out: 437 if (ret) 438 pr_err("Failed to read max cpus, using default of %d\n", max_cpu_num.cpu); 439 } 440 441 /* Determine highest possible node in the system for sparse allocation */ 442 static void set_max_node_num(void) 443 { 444 const char *mnt; 445 char path[PATH_MAX]; 446 int ret = -1; 447 448 /* set up default */ 449 max_node_num = 8; 450 451 mnt = sysfs__mountpoint(); 452 if (!mnt) 453 goto out; 454 455 /* get the highest possible cpu number for a sparse allocation */ 456 ret = snprintf(path, PATH_MAX, "%s/devices/system/node/possible", mnt); 457 if (ret >= PATH_MAX) { 458 pr_err("sysfs path crossed PATH_MAX(%d) size\n", PATH_MAX); 459 goto out; 460 } 461 462 ret = get_max_num(path, &max_node_num); 463 464 out: 465 if (ret) 466 pr_err("Failed to read max nodes, using default of %d\n", max_node_num); 467 } 468 469 int cpu__max_node(void) 470 { 471 if (unlikely(!max_node_num)) 472 set_max_node_num(); 473 474 return max_node_num; 475 } 476 477 struct perf_cpu cpu__max_cpu(void) 478 { 479 if (unlikely(!max_cpu_num.cpu)) 480 set_max_cpu_num(); 481 482 return max_cpu_num; 483 } 484 485 struct perf_cpu cpu__max_present_cpu(void) 486 { 487 if (unlikely(!max_present_cpu_num.cpu)) 488 set_max_cpu_num(); 489 490 return max_present_cpu_num; 491 } 492 493 494 int cpu__get_node(struct perf_cpu cpu) 495 { 496 if (unlikely(cpunode_map == NULL)) { 497 pr_debug("cpu_map not initialized\n"); 498 return -1; 499 } 500 501 return cpunode_map[cpu.cpu]; 502 } 503 504 static int init_cpunode_map(void) 505 { 506 int i; 507 508 set_max_cpu_num(); 509 set_max_node_num(); 510 511 cpunode_map = calloc(max_cpu_num.cpu, sizeof(int)); 512 if (!cpunode_map) { 513 pr_err("%s: calloc failed\n", __func__); 514 return -1; 515 } 516 517 for (i = 0; i < max_cpu_num.cpu; i++) 518 cpunode_map[i] = -1; 519 520 return 0; 521 } 522 523 int cpu__setup_cpunode_map(void) 524 { 525 struct dirent *dent1, *dent2; 526 DIR *dir1, *dir2; 527 unsigned int cpu, mem; 528 char buf[PATH_MAX]; 529 char path[PATH_MAX]; 530 const char *mnt; 531 int n; 532 533 /* initialize globals */ 534 if (init_cpunode_map()) 535 return -1; 536 537 mnt = sysfs__mountpoint(); 538 if (!mnt) 539 return 0; 540 541 n = snprintf(path, PATH_MAX, "%s/devices/system/node", mnt); 542 if (n >= PATH_MAX) { 543 pr_err("sysfs path crossed PATH_MAX(%d) size\n", PATH_MAX); 544 return -1; 545 } 546 547 dir1 = opendir(path); 548 if (!dir1) 549 return 0; 550 551 /* walk tree and setup map */ 552 while ((dent1 = readdir(dir1)) != NULL) { 553 if (dent1->d_type != DT_DIR || sscanf(dent1->d_name, "node%u", &mem) < 1) 554 continue; 555 556 n = snprintf(buf, PATH_MAX, "%s/%s", path, dent1->d_name); 557 if (n >= PATH_MAX) { 558 pr_err("sysfs path crossed PATH_MAX(%d) size\n", PATH_MAX); 559 continue; 560 } 561 562 dir2 = opendir(buf); 563 if (!dir2) 564 continue; 565 while ((dent2 = readdir(dir2)) != NULL) { 566 if (dent2->d_type != DT_LNK || sscanf(dent2->d_name, "cpu%u", &cpu) < 1) 567 continue; 568 cpunode_map[cpu] = mem; 569 } 570 closedir(dir2); 571 } 572 closedir(dir1); 573 return 0; 574 } 575 576 size_t cpu_map__snprint(struct perf_cpu_map *map, char *buf, size_t size) 577 { 578 int i, start = -1; 579 bool first = true; 580 size_t ret = 0; 581 582 #define COMMA first ? "" : "," 583 584 for (i = 0; i < perf_cpu_map__nr(map) + 1; i++) { 585 struct perf_cpu cpu = { .cpu = INT_MAX }; 586 bool last = i == perf_cpu_map__nr(map); 587 588 if (!last) 589 cpu = perf_cpu_map__cpu(map, i); 590 591 if (start == -1) { 592 start = i; 593 if (last) { 594 ret += snprintf(buf + ret, size - ret, 595 "%s%d", COMMA, 596 perf_cpu_map__cpu(map, i).cpu); 597 } 598 } else if (((i - start) != (cpu.cpu - perf_cpu_map__cpu(map, start).cpu)) || last) { 599 int end = i - 1; 600 601 if (start == end) { 602 ret += snprintf(buf + ret, size - ret, 603 "%s%d", COMMA, 604 perf_cpu_map__cpu(map, start).cpu); 605 } else { 606 ret += snprintf(buf + ret, size - ret, 607 "%s%d-%d", COMMA, 608 perf_cpu_map__cpu(map, start).cpu, perf_cpu_map__cpu(map, end).cpu); 609 } 610 first = false; 611 start = i; 612 } 613 } 614 615 #undef COMMA 616 617 pr_debug2("cpumask list: %s\n", buf); 618 return ret; 619 } 620 621 static char hex_char(unsigned char val) 622 { 623 if (val < 10) 624 return val + '0'; 625 if (val < 16) 626 return val - 10 + 'a'; 627 return '?'; 628 } 629 630 size_t cpu_map__snprint_mask(struct perf_cpu_map *map, char *buf, size_t size) 631 { 632 int i, cpu; 633 char *ptr = buf; 634 unsigned char *bitmap; 635 struct perf_cpu last_cpu = perf_cpu_map__cpu(map, perf_cpu_map__nr(map) - 1); 636 637 if (buf == NULL) 638 return 0; 639 640 bitmap = zalloc(last_cpu.cpu / 8 + 1); 641 if (bitmap == NULL) { 642 buf[0] = '\0'; 643 return 0; 644 } 645 646 for (i = 0; i < perf_cpu_map__nr(map); i++) { 647 cpu = perf_cpu_map__cpu(map, i).cpu; 648 bitmap[cpu / 8] |= 1 << (cpu % 8); 649 } 650 651 for (cpu = last_cpu.cpu / 4 * 4; cpu >= 0; cpu -= 4) { 652 unsigned char bits = bitmap[cpu / 8]; 653 654 if (cpu % 8) 655 bits >>= 4; 656 else 657 bits &= 0xf; 658 659 *ptr++ = hex_char(bits); 660 if ((cpu % 32) == 0 && cpu > 0) 661 *ptr++ = ','; 662 } 663 *ptr = '\0'; 664 free(bitmap); 665 666 buf[size - 1] = '\0'; 667 return ptr - buf; 668 } 669 670 struct perf_cpu_map *cpu_map__online(void) /* thread unsafe */ 671 { 672 static struct perf_cpu_map *online; 673 674 if (!online) 675 online = perf_cpu_map__new(NULL); /* from /sys/devices/system/cpu/online */ 676 677 return online; 678 } 679 680 bool aggr_cpu_id__equal(const struct aggr_cpu_id *a, const struct aggr_cpu_id *b) 681 { 682 return a->thread_idx == b->thread_idx && 683 a->node == b->node && 684 a->socket == b->socket && 685 a->die == b->die && 686 a->cache_lvl == b->cache_lvl && 687 a->cache == b->cache && 688 a->core == b->core && 689 a->cpu.cpu == b->cpu.cpu; 690 } 691 692 bool aggr_cpu_id__is_empty(const struct aggr_cpu_id *a) 693 { 694 return a->thread_idx == -1 && 695 a->node == -1 && 696 a->socket == -1 && 697 a->die == -1 && 698 a->cache_lvl == -1 && 699 a->cache == -1 && 700 a->core == -1 && 701 a->cpu.cpu == -1; 702 } 703 704 struct aggr_cpu_id aggr_cpu_id__empty(void) 705 { 706 struct aggr_cpu_id ret = { 707 .thread_idx = -1, 708 .node = -1, 709 .socket = -1, 710 .die = -1, 711 .cache_lvl = -1, 712 .cache = -1, 713 .core = -1, 714 .cpu = (struct perf_cpu){ .cpu = -1 }, 715 }; 716 return ret; 717 } 718