1 // SPDX-License-Identifier: GPL-2.0 2 #include <api/fs/fs.h> 3 #include "cpumap.h" 4 #include "debug.h" 5 #include "event.h" 6 #include <assert.h> 7 #include <dirent.h> 8 #include <stdio.h> 9 #include <stdlib.h> 10 #include <linux/bitmap.h> 11 #include "asm/bug.h" 12 13 #include <linux/ctype.h> 14 #include <linux/zalloc.h> 15 #include <internal/cpumap.h> 16 17 static struct perf_cpu max_cpu_num; 18 static struct perf_cpu max_present_cpu_num; 19 static int max_node_num; 20 /** 21 * The numa node X as read from /sys/devices/system/node/nodeX indexed by the 22 * CPU number. 23 */ 24 static int *cpunode_map; 25 26 bool perf_record_cpu_map_data__test_bit(int i, 27 const struct perf_record_cpu_map_data *data) 28 { 29 int bit_word32 = i / 32; 30 __u32 bit_mask32 = 1U << (i & 31); 31 int bit_word64 = i / 64; 32 __u64 bit_mask64 = ((__u64)1) << (i & 63); 33 34 return (data->mask32_data.long_size == 4) 35 ? (bit_word32 < data->mask32_data.nr) && 36 (data->mask32_data.mask[bit_word32] & bit_mask32) != 0 37 : (bit_word64 < data->mask64_data.nr) && 38 (data->mask64_data.mask[bit_word64] & bit_mask64) != 0; 39 } 40 41 /* Read ith mask value from data into the given 64-bit sized bitmap */ 42 static void perf_record_cpu_map_data__read_one_mask(const struct perf_record_cpu_map_data *data, 43 int i, unsigned long *bitmap) 44 { 45 #if __SIZEOF_LONG__ == 8 46 if (data->mask32_data.long_size == 4) 47 bitmap[0] = data->mask32_data.mask[i]; 48 else 49 bitmap[0] = data->mask64_data.mask[i]; 50 #else 51 if (data->mask32_data.long_size == 4) { 52 bitmap[0] = data->mask32_data.mask[i]; 53 bitmap[1] = 0; 54 } else { 55 #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ 56 bitmap[0] = (unsigned long)(data->mask64_data.mask[i] >> 32); 57 bitmap[1] = (unsigned long)data->mask64_data.mask[i]; 58 #else 59 bitmap[0] = (unsigned long)data->mask64_data.mask[i]; 60 bitmap[1] = (unsigned long)(data->mask64_data.mask[i] >> 32); 61 #endif 62 } 63 #endif 64 } 65 static struct perf_cpu_map *cpu_map__from_entries(const struct perf_record_cpu_map_data *data) 66 { 67 struct perf_cpu_map *map; 68 69 map = perf_cpu_map__empty_new(data->cpus_data.nr); 70 if (map) { 71 unsigned i; 72 73 for (i = 0; i < data->cpus_data.nr; i++) { 74 /* 75 * Special treatment for -1, which is not real cpu number, 76 * and we need to use (int) -1 to initialize map[i], 77 * otherwise it would become 65535. 78 */ 79 if (data->cpus_data.cpu[i] == (u16) -1) 80 RC_CHK_ACCESS(map)->map[i].cpu = -1; 81 else 82 RC_CHK_ACCESS(map)->map[i].cpu = (int) data->cpus_data.cpu[i]; 83 } 84 } 85 86 return map; 87 } 88 89 static struct perf_cpu_map *cpu_map__from_mask(const struct perf_record_cpu_map_data *data) 90 { 91 DECLARE_BITMAP(local_copy, 64); 92 int weight = 0, mask_nr = data->mask32_data.nr; 93 struct perf_cpu_map *map; 94 95 for (int i = 0; i < mask_nr; i++) { 96 perf_record_cpu_map_data__read_one_mask(data, i, local_copy); 97 weight += bitmap_weight(local_copy, 64); 98 } 99 100 map = perf_cpu_map__empty_new(weight); 101 if (!map) 102 return NULL; 103 104 for (int i = 0, j = 0; i < mask_nr; i++) { 105 int cpus_per_i = (i * data->mask32_data.long_size * BITS_PER_BYTE); 106 int cpu; 107 108 perf_record_cpu_map_data__read_one_mask(data, i, local_copy); 109 for_each_set_bit(cpu, local_copy, 64) 110 RC_CHK_ACCESS(map)->map[j++].cpu = cpu + cpus_per_i; 111 } 112 return map; 113 114 } 115 116 static struct perf_cpu_map *cpu_map__from_range(const struct perf_record_cpu_map_data *data) 117 { 118 struct perf_cpu_map *map; 119 unsigned int i = 0; 120 121 map = perf_cpu_map__empty_new(data->range_cpu_data.end_cpu - 122 data->range_cpu_data.start_cpu + 1 + data->range_cpu_data.any_cpu); 123 if (!map) 124 return NULL; 125 126 if (data->range_cpu_data.any_cpu) 127 RC_CHK_ACCESS(map)->map[i++].cpu = -1; 128 129 for (int cpu = data->range_cpu_data.start_cpu; cpu <= data->range_cpu_data.end_cpu; 130 i++, cpu++) 131 RC_CHK_ACCESS(map)->map[i].cpu = cpu; 132 133 return map; 134 } 135 136 struct perf_cpu_map *cpu_map__new_data(const struct perf_record_cpu_map_data *data) 137 { 138 switch (data->type) { 139 case PERF_CPU_MAP__CPUS: 140 return cpu_map__from_entries(data); 141 case PERF_CPU_MAP__MASK: 142 return cpu_map__from_mask(data); 143 case PERF_CPU_MAP__RANGE_CPUS: 144 return cpu_map__from_range(data); 145 default: 146 pr_err("cpu_map__new_data unknown type %d\n", data->type); 147 return NULL; 148 } 149 } 150 151 size_t cpu_map__fprintf(struct perf_cpu_map *map, FILE *fp) 152 { 153 #define BUFSIZE 1024 154 char buf[BUFSIZE]; 155 156 cpu_map__snprint(map, buf, sizeof(buf)); 157 return fprintf(fp, "%s\n", buf); 158 #undef BUFSIZE 159 } 160 161 struct perf_cpu_map *perf_cpu_map__empty_new(int nr) 162 { 163 struct perf_cpu_map *cpus = perf_cpu_map__alloc(nr); 164 165 if (cpus != NULL) { 166 for (int i = 0; i < nr; i++) 167 RC_CHK_ACCESS(cpus)->map[i].cpu = -1; 168 } 169 170 return cpus; 171 } 172 173 struct cpu_aggr_map *cpu_aggr_map__empty_new(int nr) 174 { 175 struct cpu_aggr_map *cpus = malloc(sizeof(*cpus) + sizeof(struct aggr_cpu_id) * nr); 176 177 if (cpus != NULL) { 178 int i; 179 180 cpus->nr = nr; 181 for (i = 0; i < nr; i++) 182 cpus->map[i] = aggr_cpu_id__empty(); 183 184 refcount_set(&cpus->refcnt, 1); 185 } 186 187 return cpus; 188 } 189 190 static int cpu__get_topology_int(int cpu, const char *name, int *value) 191 { 192 char path[PATH_MAX]; 193 194 snprintf(path, PATH_MAX, 195 "devices/system/cpu/cpu%d/topology/%s", cpu, name); 196 197 return sysfs__read_int(path, value); 198 } 199 200 int cpu__get_socket_id(struct perf_cpu cpu) 201 { 202 int value, ret = cpu__get_topology_int(cpu.cpu, "physical_package_id", &value); 203 return ret ?: value; 204 } 205 206 struct aggr_cpu_id aggr_cpu_id__socket(struct perf_cpu cpu, void *data __maybe_unused) 207 { 208 struct aggr_cpu_id id = aggr_cpu_id__empty(); 209 210 id.socket = cpu__get_socket_id(cpu); 211 return id; 212 } 213 214 static int aggr_cpu_id__cmp(const void *a_pointer, const void *b_pointer) 215 { 216 struct aggr_cpu_id *a = (struct aggr_cpu_id *)a_pointer; 217 struct aggr_cpu_id *b = (struct aggr_cpu_id *)b_pointer; 218 219 if (a->node != b->node) 220 return a->node - b->node; 221 else if (a->socket != b->socket) 222 return a->socket - b->socket; 223 else if (a->die != b->die) 224 return a->die - b->die; 225 else if (a->core != b->core) 226 return a->core - b->core; 227 else 228 return a->thread_idx - b->thread_idx; 229 } 230 231 struct cpu_aggr_map *cpu_aggr_map__new(const struct perf_cpu_map *cpus, 232 aggr_cpu_id_get_t get_id, 233 void *data, bool needs_sort) 234 { 235 int idx; 236 struct perf_cpu cpu; 237 struct cpu_aggr_map *c = cpu_aggr_map__empty_new(perf_cpu_map__nr(cpus)); 238 239 if (!c) 240 return NULL; 241 242 /* Reset size as it may only be partially filled */ 243 c->nr = 0; 244 245 perf_cpu_map__for_each_cpu(cpu, idx, cpus) { 246 bool duplicate = false; 247 struct aggr_cpu_id cpu_id = get_id(cpu, data); 248 249 for (int j = 0; j < c->nr; j++) { 250 if (aggr_cpu_id__equal(&cpu_id, &c->map[j])) { 251 duplicate = true; 252 break; 253 } 254 } 255 if (!duplicate) { 256 c->map[c->nr] = cpu_id; 257 c->nr++; 258 } 259 } 260 /* Trim. */ 261 if (c->nr != perf_cpu_map__nr(cpus)) { 262 struct cpu_aggr_map *trimmed_c = 263 realloc(c, 264 sizeof(struct cpu_aggr_map) + sizeof(struct aggr_cpu_id) * c->nr); 265 266 if (trimmed_c) 267 c = trimmed_c; 268 } 269 270 /* ensure we process id in increasing order */ 271 if (needs_sort) 272 qsort(c->map, c->nr, sizeof(struct aggr_cpu_id), aggr_cpu_id__cmp); 273 274 return c; 275 276 } 277 278 int cpu__get_die_id(struct perf_cpu cpu) 279 { 280 int value, ret = cpu__get_topology_int(cpu.cpu, "die_id", &value); 281 282 return ret ?: value; 283 } 284 285 struct aggr_cpu_id aggr_cpu_id__die(struct perf_cpu cpu, void *data) 286 { 287 struct aggr_cpu_id id; 288 int die; 289 290 die = cpu__get_die_id(cpu); 291 /* There is no die_id on legacy system. */ 292 if (die == -1) 293 die = 0; 294 295 /* 296 * die_id is relative to socket, so start 297 * with the socket ID and then add die to 298 * make a unique ID. 299 */ 300 id = aggr_cpu_id__socket(cpu, data); 301 if (aggr_cpu_id__is_empty(&id)) 302 return id; 303 304 id.die = die; 305 return id; 306 } 307 308 int cpu__get_core_id(struct perf_cpu cpu) 309 { 310 int value, ret = cpu__get_topology_int(cpu.cpu, "core_id", &value); 311 return ret ?: value; 312 } 313 314 struct aggr_cpu_id aggr_cpu_id__core(struct perf_cpu cpu, void *data) 315 { 316 struct aggr_cpu_id id; 317 int core = cpu__get_core_id(cpu); 318 319 /* aggr_cpu_id__die returns a struct with socket and die set. */ 320 id = aggr_cpu_id__die(cpu, data); 321 if (aggr_cpu_id__is_empty(&id)) 322 return id; 323 324 /* 325 * core_id is relative to socket and die, we need a global id. 326 * So we combine the result from cpu_map__get_die with the core id 327 */ 328 id.core = core; 329 return id; 330 331 } 332 333 struct aggr_cpu_id aggr_cpu_id__cpu(struct perf_cpu cpu, void *data) 334 { 335 struct aggr_cpu_id id; 336 337 /* aggr_cpu_id__core returns a struct with socket, die and core set. */ 338 id = aggr_cpu_id__core(cpu, data); 339 if (aggr_cpu_id__is_empty(&id)) 340 return id; 341 342 id.cpu = cpu; 343 return id; 344 345 } 346 347 struct aggr_cpu_id aggr_cpu_id__node(struct perf_cpu cpu, void *data __maybe_unused) 348 { 349 struct aggr_cpu_id id = aggr_cpu_id__empty(); 350 351 id.node = cpu__get_node(cpu); 352 return id; 353 } 354 355 struct aggr_cpu_id aggr_cpu_id__global(struct perf_cpu cpu, void *data __maybe_unused) 356 { 357 struct aggr_cpu_id id = aggr_cpu_id__empty(); 358 359 /* it always aggregates to the cpu 0 */ 360 cpu.cpu = 0; 361 id.cpu = cpu; 362 return id; 363 } 364 365 /* setup simple routines to easily access node numbers given a cpu number */ 366 static int get_max_num(char *path, int *max) 367 { 368 size_t num; 369 char *buf; 370 int err = 0; 371 372 if (filename__read_str(path, &buf, &num)) 373 return -1; 374 375 buf[num] = '\0'; 376 377 /* start on the right, to find highest node num */ 378 while (--num) { 379 if ((buf[num] == ',') || (buf[num] == '-')) { 380 num++; 381 break; 382 } 383 } 384 if (sscanf(&buf[num], "%d", max) < 1) { 385 err = -1; 386 goto out; 387 } 388 389 /* convert from 0-based to 1-based */ 390 (*max)++; 391 392 out: 393 free(buf); 394 return err; 395 } 396 397 /* Determine highest possible cpu in the system for sparse allocation */ 398 static void set_max_cpu_num(void) 399 { 400 const char *mnt; 401 char path[PATH_MAX]; 402 int ret = -1; 403 404 /* set up default */ 405 max_cpu_num.cpu = 4096; 406 max_present_cpu_num.cpu = 4096; 407 408 mnt = sysfs__mountpoint(); 409 if (!mnt) 410 goto out; 411 412 /* get the highest possible cpu number for a sparse allocation */ 413 ret = snprintf(path, PATH_MAX, "%s/devices/system/cpu/possible", mnt); 414 if (ret >= PATH_MAX) { 415 pr_err("sysfs path crossed PATH_MAX(%d) size\n", PATH_MAX); 416 goto out; 417 } 418 419 ret = get_max_num(path, &max_cpu_num.cpu); 420 if (ret) 421 goto out; 422 423 /* get the highest present cpu number for a sparse allocation */ 424 ret = snprintf(path, PATH_MAX, "%s/devices/system/cpu/present", mnt); 425 if (ret >= PATH_MAX) { 426 pr_err("sysfs path crossed PATH_MAX(%d) size\n", PATH_MAX); 427 goto out; 428 } 429 430 ret = get_max_num(path, &max_present_cpu_num.cpu); 431 432 out: 433 if (ret) 434 pr_err("Failed to read max cpus, using default of %d\n", max_cpu_num.cpu); 435 } 436 437 /* Determine highest possible node in the system for sparse allocation */ 438 static void set_max_node_num(void) 439 { 440 const char *mnt; 441 char path[PATH_MAX]; 442 int ret = -1; 443 444 /* set up default */ 445 max_node_num = 8; 446 447 mnt = sysfs__mountpoint(); 448 if (!mnt) 449 goto out; 450 451 /* get the highest possible cpu number for a sparse allocation */ 452 ret = snprintf(path, PATH_MAX, "%s/devices/system/node/possible", mnt); 453 if (ret >= PATH_MAX) { 454 pr_err("sysfs path crossed PATH_MAX(%d) size\n", PATH_MAX); 455 goto out; 456 } 457 458 ret = get_max_num(path, &max_node_num); 459 460 out: 461 if (ret) 462 pr_err("Failed to read max nodes, using default of %d\n", max_node_num); 463 } 464 465 int cpu__max_node(void) 466 { 467 if (unlikely(!max_node_num)) 468 set_max_node_num(); 469 470 return max_node_num; 471 } 472 473 struct perf_cpu cpu__max_cpu(void) 474 { 475 if (unlikely(!max_cpu_num.cpu)) 476 set_max_cpu_num(); 477 478 return max_cpu_num; 479 } 480 481 struct perf_cpu cpu__max_present_cpu(void) 482 { 483 if (unlikely(!max_present_cpu_num.cpu)) 484 set_max_cpu_num(); 485 486 return max_present_cpu_num; 487 } 488 489 490 int cpu__get_node(struct perf_cpu cpu) 491 { 492 if (unlikely(cpunode_map == NULL)) { 493 pr_debug("cpu_map not initialized\n"); 494 return -1; 495 } 496 497 return cpunode_map[cpu.cpu]; 498 } 499 500 static int init_cpunode_map(void) 501 { 502 int i; 503 504 set_max_cpu_num(); 505 set_max_node_num(); 506 507 cpunode_map = calloc(max_cpu_num.cpu, sizeof(int)); 508 if (!cpunode_map) { 509 pr_err("%s: calloc failed\n", __func__); 510 return -1; 511 } 512 513 for (i = 0; i < max_cpu_num.cpu; i++) 514 cpunode_map[i] = -1; 515 516 return 0; 517 } 518 519 int cpu__setup_cpunode_map(void) 520 { 521 struct dirent *dent1, *dent2; 522 DIR *dir1, *dir2; 523 unsigned int cpu, mem; 524 char buf[PATH_MAX]; 525 char path[PATH_MAX]; 526 const char *mnt; 527 int n; 528 529 /* initialize globals */ 530 if (init_cpunode_map()) 531 return -1; 532 533 mnt = sysfs__mountpoint(); 534 if (!mnt) 535 return 0; 536 537 n = snprintf(path, PATH_MAX, "%s/devices/system/node", mnt); 538 if (n >= PATH_MAX) { 539 pr_err("sysfs path crossed PATH_MAX(%d) size\n", PATH_MAX); 540 return -1; 541 } 542 543 dir1 = opendir(path); 544 if (!dir1) 545 return 0; 546 547 /* walk tree and setup map */ 548 while ((dent1 = readdir(dir1)) != NULL) { 549 if (dent1->d_type != DT_DIR || sscanf(dent1->d_name, "node%u", &mem) < 1) 550 continue; 551 552 n = snprintf(buf, PATH_MAX, "%s/%s", path, dent1->d_name); 553 if (n >= PATH_MAX) { 554 pr_err("sysfs path crossed PATH_MAX(%d) size\n", PATH_MAX); 555 continue; 556 } 557 558 dir2 = opendir(buf); 559 if (!dir2) 560 continue; 561 while ((dent2 = readdir(dir2)) != NULL) { 562 if (dent2->d_type != DT_LNK || sscanf(dent2->d_name, "cpu%u", &cpu) < 1) 563 continue; 564 cpunode_map[cpu] = mem; 565 } 566 closedir(dir2); 567 } 568 closedir(dir1); 569 return 0; 570 } 571 572 size_t cpu_map__snprint(struct perf_cpu_map *map, char *buf, size_t size) 573 { 574 int i, start = -1; 575 bool first = true; 576 size_t ret = 0; 577 578 #define COMMA first ? "" : "," 579 580 for (i = 0; i < perf_cpu_map__nr(map) + 1; i++) { 581 struct perf_cpu cpu = { .cpu = INT_MAX }; 582 bool last = i == perf_cpu_map__nr(map); 583 584 if (!last) 585 cpu = perf_cpu_map__cpu(map, i); 586 587 if (start == -1) { 588 start = i; 589 if (last) { 590 ret += snprintf(buf + ret, size - ret, 591 "%s%d", COMMA, 592 perf_cpu_map__cpu(map, i).cpu); 593 } 594 } else if (((i - start) != (cpu.cpu - perf_cpu_map__cpu(map, start).cpu)) || last) { 595 int end = i - 1; 596 597 if (start == end) { 598 ret += snprintf(buf + ret, size - ret, 599 "%s%d", COMMA, 600 perf_cpu_map__cpu(map, start).cpu); 601 } else { 602 ret += snprintf(buf + ret, size - ret, 603 "%s%d-%d", COMMA, 604 perf_cpu_map__cpu(map, start).cpu, perf_cpu_map__cpu(map, end).cpu); 605 } 606 first = false; 607 start = i; 608 } 609 } 610 611 #undef COMMA 612 613 pr_debug2("cpumask list: %s\n", buf); 614 return ret; 615 } 616 617 static char hex_char(unsigned char val) 618 { 619 if (val < 10) 620 return val + '0'; 621 if (val < 16) 622 return val - 10 + 'a'; 623 return '?'; 624 } 625 626 size_t cpu_map__snprint_mask(struct perf_cpu_map *map, char *buf, size_t size) 627 { 628 int i, cpu; 629 char *ptr = buf; 630 unsigned char *bitmap; 631 struct perf_cpu last_cpu = perf_cpu_map__cpu(map, perf_cpu_map__nr(map) - 1); 632 633 if (buf == NULL) 634 return 0; 635 636 bitmap = zalloc(last_cpu.cpu / 8 + 1); 637 if (bitmap == NULL) { 638 buf[0] = '\0'; 639 return 0; 640 } 641 642 for (i = 0; i < perf_cpu_map__nr(map); i++) { 643 cpu = perf_cpu_map__cpu(map, i).cpu; 644 bitmap[cpu / 8] |= 1 << (cpu % 8); 645 } 646 647 for (cpu = last_cpu.cpu / 4 * 4; cpu >= 0; cpu -= 4) { 648 unsigned char bits = bitmap[cpu / 8]; 649 650 if (cpu % 8) 651 bits >>= 4; 652 else 653 bits &= 0xf; 654 655 *ptr++ = hex_char(bits); 656 if ((cpu % 32) == 0 && cpu > 0) 657 *ptr++ = ','; 658 } 659 *ptr = '\0'; 660 free(bitmap); 661 662 buf[size - 1] = '\0'; 663 return ptr - buf; 664 } 665 666 const struct perf_cpu_map *cpu_map__online(void) /* thread unsafe */ 667 { 668 static const struct perf_cpu_map *online = NULL; 669 670 if (!online) 671 online = perf_cpu_map__new(NULL); /* from /sys/devices/system/cpu/online */ 672 673 return online; 674 } 675 676 bool aggr_cpu_id__equal(const struct aggr_cpu_id *a, const struct aggr_cpu_id *b) 677 { 678 return a->thread_idx == b->thread_idx && 679 a->node == b->node && 680 a->socket == b->socket && 681 a->die == b->die && 682 a->core == b->core && 683 a->cpu.cpu == b->cpu.cpu; 684 } 685 686 bool aggr_cpu_id__is_empty(const struct aggr_cpu_id *a) 687 { 688 return a->thread_idx == -1 && 689 a->node == -1 && 690 a->socket == -1 && 691 a->die == -1 && 692 a->core == -1 && 693 a->cpu.cpu == -1; 694 } 695 696 struct aggr_cpu_id aggr_cpu_id__empty(void) 697 { 698 struct aggr_cpu_id ret = { 699 .thread_idx = -1, 700 .node = -1, 701 .socket = -1, 702 .die = -1, 703 .core = -1, 704 .cpu = (struct perf_cpu){ .cpu = -1 }, 705 }; 706 return ret; 707 } 708