1 // SPDX-License-Identifier: GPL-2.0 2 /* Manage affinity to optimize IPIs inside the kernel perf API. */ 3 #define _GNU_SOURCE 1 4 #include <sched.h> 5 #include <stdlib.h> 6 #include <linux/bitmap.h> 7 #include <linux/zalloc.h> 8 #include <perf/cpumap.h> 9 #include "perf.h" 10 #include "cpumap.h" 11 #include "affinity.h" 12 13 static int get_cpu_set_size(void) 14 { 15 int sz = cpu__max_cpu().cpu + 8 - 1; 16 /* 17 * sched_getaffinity doesn't like masks smaller than the kernel. 18 * Hopefully that's big enough. 19 */ 20 if (sz < 4096) 21 sz = 4096; 22 return sz / 8; 23 } 24 25 int affinity__setup(struct affinity *a) 26 { 27 int cpu_set_size = get_cpu_set_size(); 28 29 a->orig_cpus = bitmap_zalloc(cpu_set_size * 8); 30 if (!a->orig_cpus) 31 return -1; 32 sched_getaffinity(0, cpu_set_size, (cpu_set_t *)a->orig_cpus); 33 a->sched_cpus = bitmap_zalloc(cpu_set_size * 8); 34 if (!a->sched_cpus) { 35 zfree(&a->orig_cpus); 36 return -1; 37 } 38 bitmap_zero((unsigned long *)a->sched_cpus, cpu_set_size); 39 a->changed = false; 40 return 0; 41 } 42 43 /* 44 * perf_event_open does an IPI internally to the target CPU. 45 * It is more efficient to change perf's affinity to the target 46 * CPU and then set up all events on that CPU, so we amortize 47 * CPU communication. 48 */ 49 void affinity__set(struct affinity *a, int cpu) 50 { 51 int cpu_set_size = get_cpu_set_size(); 52 53 /* 54 * Return: 55 * - if cpu is -1 56 * - restrict out of bound access to sched_cpus 57 */ 58 if (cpu == -1 || ((cpu >= (cpu_set_size * 8)))) 59 return; 60 61 a->changed = true; 62 __set_bit(cpu, a->sched_cpus); 63 /* 64 * We ignore errors because affinity is just an optimization. 65 * This could happen for example with isolated CPUs or cpusets. 66 * In this case the IPIs inside the kernel's perf API still work. 67 */ 68 sched_setaffinity(0, cpu_set_size, (cpu_set_t *)a->sched_cpus); 69 __clear_bit(cpu, a->sched_cpus); 70 } 71 72 static void __affinity__cleanup(struct affinity *a) 73 { 74 int cpu_set_size = get_cpu_set_size(); 75 76 if (a->changed) 77 sched_setaffinity(0, cpu_set_size, (cpu_set_t *)a->orig_cpus); 78 zfree(&a->sched_cpus); 79 zfree(&a->orig_cpus); 80 } 81 82 void affinity__cleanup(struct affinity *a) 83 { 84 if (a != NULL) 85 __affinity__cleanup(a); 86 } 87 88 void cpu_map__set_affinity(const struct perf_cpu_map *cpumap) 89 { 90 int cpu_set_size = get_cpu_set_size(); 91 unsigned long *cpuset = bitmap_zalloc(cpu_set_size * 8); 92 struct perf_cpu cpu; 93 int idx; 94 95 if (!cpuset) 96 return; 97 98 perf_cpu_map__for_each_cpu_skip_any(cpu, idx, cpumap) 99 __set_bit(cpu.cpu, cpuset); 100 101 sched_setaffinity(0, cpu_set_size, (cpu_set_t *)cpuset); 102 zfree(&cpuset); 103 } 104