1 // SPDX-License-Identifier: GPL-2.0 2 3 /* Copyright (c) 2021 Facebook */ 4 /* Copyright (c) 2021 Google */ 5 6 #include <assert.h> 7 #include <errno.h> 8 #include <limits.h> 9 #include <unistd.h> 10 #include <sys/file.h> 11 #include <sys/time.h> 12 #include <sys/resource.h> 13 #include <linux/err.h> 14 #include <linux/perf_event.h> 15 #include <api/fs/fs.h> 16 #include <bpf/bpf.h> 17 #include <perf/bpf_perf.h> 18 19 #include "affinity.h" 20 #include "bpf_counter.h" 21 #include "cgroup.h" 22 #include "counts.h" 23 #include "debug.h" 24 #include "evsel.h" 25 #include "evlist.h" 26 #include "target.h" 27 #include "cpumap.h" 28 #include "thread_map.h" 29 30 #include "bpf_skel/bperf_cgroup.h" 31 #include "bpf_skel/bperf_cgroup.skel.h" 32 33 static struct perf_event_attr cgrp_switch_attr = { 34 .type = PERF_TYPE_SOFTWARE, 35 .config = PERF_COUNT_SW_CGROUP_SWITCHES, 36 .size = sizeof(cgrp_switch_attr), 37 .sample_period = 1, 38 .disabled = 1, 39 }; 40 41 static struct evsel *cgrp_switch; 42 static struct bperf_cgroup_bpf *skel; 43 44 #define FD(evt, cpu) (*(int *)xyarray__entry(evt->core.fd, cpu, 0)) 45 46 static void setup_rodata(struct bperf_cgroup_bpf *sk, int evlist_size) 47 { 48 int map_size, total_cpus = cpu__max_cpu().cpu; 49 50 sk->rodata->num_cpus = total_cpus; 51 sk->rodata->num_events = evlist_size / nr_cgroups; 52 53 if (cgroup_is_v2("perf_event") > 0) 54 sk->rodata->use_cgroup_v2 = 1; 55 56 BUG_ON(evlist_size % nr_cgroups != 0); 57 58 /* we need one copy of events per cpu for reading */ 59 map_size = total_cpus * evlist_size / nr_cgroups; 60 bpf_map__set_max_entries(sk->maps.events, map_size); 61 bpf_map__set_max_entries(sk->maps.cgrp_idx, nr_cgroups); 62 /* previous result is saved in a per-cpu array */ 63 map_size = evlist_size / nr_cgroups; 64 bpf_map__set_max_entries(sk->maps.prev_readings, map_size); 65 /* cgroup result needs all events (per-cpu) */ 66 map_size = evlist_size; 67 bpf_map__set_max_entries(sk->maps.cgrp_readings, map_size); 68 } 69 70 static void test_max_events_program_load(void) 71 { 72 #ifndef NDEBUG 73 /* 74 * Test that the program verifies with the maximum number of events. If 75 * this test fails unfortunately perf needs recompiling with a lower 76 * BPERF_CGROUP__MAX_EVENTS to avoid BPF verifier issues. 77 */ 78 int err, max_events = BPERF_CGROUP__MAX_EVENTS * nr_cgroups; 79 struct bperf_cgroup_bpf *test_skel = bperf_cgroup_bpf__open(); 80 81 if (!test_skel) { 82 pr_err("Failed to open cgroup skeleton\n"); 83 return; 84 } 85 setup_rodata(test_skel, max_events); 86 err = bperf_cgroup_bpf__load(test_skel); 87 if (err) { 88 pr_err("Failed to load cgroup skeleton with max events %d.\n", 89 BPERF_CGROUP__MAX_EVENTS); 90 } 91 bperf_cgroup_bpf__destroy(test_skel); 92 #endif 93 } 94 95 static int bperf_load_program(struct evlist *evlist) 96 { 97 struct bpf_link *link; 98 struct evsel *evsel; 99 struct cgroup *cgrp, *leader_cgrp; 100 unsigned int i; 101 struct perf_cpu cpu; 102 int total_cpus = cpu__max_cpu().cpu; 103 int map_fd, prog_fd, err; 104 105 set_max_rlimit(); 106 107 if (nr_cgroups == 0 || evlist__nr_entries(evlist) % nr_cgroups != 0) { 108 pr_err("Invalid cgroup or event count\n"); 109 return -EINVAL; 110 } 111 112 test_max_events_program_load(); 113 114 skel = bperf_cgroup_bpf__open(); 115 if (!skel) { 116 pr_err("Failed to open cgroup skeleton\n"); 117 return -1; 118 } 119 setup_rodata(skel, evlist__nr_entries(evlist)); 120 121 err = bperf_cgroup_bpf__load(skel); 122 if (err) { 123 pr_err("Failed to load cgroup skeleton\n"); 124 goto out; 125 } 126 127 err = -1; 128 129 cgrp_switch = evsel__new(&cgrp_switch_attr); 130 if (evsel__open_per_cpu(cgrp_switch, evlist__core(evlist)->all_cpus, -1) < 0) { 131 pr_err("Failed to open cgroup switches event\n"); 132 goto out; 133 } 134 135 perf_cpu_map__for_each_cpu(cpu, i, evlist__core(evlist)->all_cpus) { 136 link = bpf_program__attach_perf_event(skel->progs.on_cgrp_switch, 137 FD(cgrp_switch, i)); 138 if (IS_ERR(link)) { 139 pr_err("Failed to attach cgroup program\n"); 140 err = PTR_ERR(link); 141 goto out; 142 } 143 } 144 145 /* 146 * Update cgrp_idx map from cgroup-id to event index. 147 */ 148 cgrp = NULL; 149 i = 0; 150 151 evlist__for_each_entry(evlist, evsel) { 152 if (cgrp == NULL || evsel->cgrp == leader_cgrp) { 153 unsigned int j; 154 155 leader_cgrp = evsel->cgrp; 156 evsel->cgrp = NULL; 157 158 /* open single copy of the events w/o cgroup */ 159 err = evsel__open_per_cpu(evsel, evsel->core.cpus, -1); 160 if (err == 0) 161 evsel->supported = true; 162 163 map_fd = bpf_map__fd(skel->maps.events); 164 perf_cpu_map__for_each_cpu(cpu, j, evsel->core.cpus) { 165 int fd = FD(evsel, j); 166 __u32 idx = evsel->core.idx * total_cpus + cpu.cpu; 167 168 bpf_map_update_elem(map_fd, &idx, &fd, BPF_ANY); 169 } 170 171 evsel->cgrp = leader_cgrp; 172 } 173 174 if (evsel->cgrp == cgrp) 175 continue; 176 177 cgrp = evsel->cgrp; 178 179 if (read_cgroup_id(cgrp) < 0) { 180 pr_debug("Failed to get cgroup id for %s\n", cgrp->name); 181 cgrp->id = 0; 182 } 183 184 map_fd = bpf_map__fd(skel->maps.cgrp_idx); 185 err = bpf_map_update_elem(map_fd, &cgrp->id, &i, BPF_ANY); 186 if (err < 0) { 187 pr_err("Failed to update cgroup index map\n"); 188 goto out; 189 } 190 191 i++; 192 } 193 194 /* 195 * Propagate supported flag from leaders to followers. Follower events 196 * are not opened, so their supported flag remains false. 197 */ 198 { 199 struct evsel *leader; 200 int num_events = evlist__nr_entries(evlist) / nr_cgroups; 201 202 evlist__for_each_entry(evlist, evsel) { 203 leader = evlist__find_evsel(evlist, evsel->core.idx % num_events); 204 if (leader) 205 evsel->supported = leader->supported; 206 } 207 } 208 209 /* 210 * bperf uses BPF_PROG_TEST_RUN to get accurate reading. Check 211 * whether the kernel support it 212 */ 213 prog_fd = bpf_program__fd(skel->progs.trigger_read); 214 err = bperf_trigger_reading(prog_fd, 0); 215 if (err) { 216 pr_warning("The kernel does not support test_run for raw_tp BPF programs.\n" 217 "Therefore, --for-each-cgroup might show inaccurate readings\n"); 218 err = 0; 219 } 220 221 out: 222 return err; 223 } 224 225 static int bperf_cgrp__load(struct evsel *evsel, 226 struct target *target __maybe_unused) 227 { 228 static bool bperf_loaded = false; 229 230 evsel->bperf_leader_prog_fd = -1; 231 evsel->bperf_leader_link_fd = -1; 232 233 if (!bperf_loaded && bperf_load_program(evsel->evlist)) 234 return -1; 235 236 bperf_loaded = true; 237 /* just to bypass bpf_counter_skip() */ 238 evsel->follower_skel = (struct bperf_follower_bpf *)skel; 239 240 return 0; 241 } 242 243 static int bperf_cgrp__install_pe(struct evsel *evsel __maybe_unused, 244 int cpu_map_idx __maybe_unused, 245 int fd __maybe_unused) 246 { 247 /* nothing to do */ 248 return 0; 249 } 250 251 /* 252 * trigger the leader prog on each cpu, so the cgrp_reading map could get 253 * the latest results. 254 */ 255 static int bperf_cgrp__sync_counters(struct evlist *evlist) 256 { 257 struct perf_cpu cpu; 258 unsigned int idx; 259 int prog_fd = bpf_program__fd(skel->progs.trigger_read); 260 261 perf_cpu_map__for_each_cpu(cpu, idx, evlist__core(evlist)->all_cpus) 262 bperf_trigger_reading(prog_fd, cpu.cpu); 263 264 return 0; 265 } 266 267 static int bperf_cgrp__enable(struct evsel *evsel) 268 { 269 if (evsel->core.idx) 270 return 0; 271 272 bperf_cgrp__sync_counters(evsel->evlist); 273 274 skel->bss->enabled = 1; 275 return 0; 276 } 277 278 static int bperf_cgrp__disable(struct evsel *evsel) 279 { 280 if (evsel->core.idx) 281 return 0; 282 283 bperf_cgrp__sync_counters(evsel->evlist); 284 285 skel->bss->enabled = 0; 286 return 0; 287 } 288 289 static int bperf_cgrp__read(struct evsel *evsel) 290 { 291 struct evlist *evlist = evsel->evlist; 292 int total_cpus = cpu__max_cpu().cpu; 293 struct perf_counts_values *counts; 294 struct bpf_perf_event_value *values; 295 int reading_map_fd, err = 0; 296 297 if (evsel->core.idx) 298 return 0; 299 300 bperf_cgrp__sync_counters(evsel->evlist); 301 302 values = calloc(total_cpus, sizeof(*values)); 303 if (values == NULL) 304 return -ENOMEM; 305 306 reading_map_fd = bpf_map__fd(skel->maps.cgrp_readings); 307 308 evlist__for_each_entry(evlist, evsel) { 309 __u32 idx = evsel->core.idx; 310 unsigned int i; 311 struct perf_cpu cpu; 312 313 err = bpf_map_lookup_elem(reading_map_fd, &idx, values); 314 if (err) { 315 pr_err("bpf map lookup failed: idx=%u, event=%s, cgrp=%s\n", 316 idx, evsel__name(evsel), evsel->cgrp->name); 317 goto out; 318 } 319 320 perf_cpu_map__for_each_cpu(cpu, i, evsel->core.cpus) { 321 counts = perf_counts(evsel->counts, i, 0); 322 counts->val = values[cpu.cpu].counter; 323 counts->ena = values[cpu.cpu].enabled; 324 counts->run = values[cpu.cpu].running; 325 } 326 } 327 328 out: 329 free(values); 330 return err; 331 } 332 333 static int bperf_cgrp__destroy(struct evsel *evsel) 334 { 335 if (evsel->core.idx) 336 return 0; 337 338 bperf_cgroup_bpf__destroy(skel); 339 evsel__put(cgrp_switch); // it'll destroy on_switch progs too 340 341 return 0; 342 } 343 344 struct bpf_counter_ops bperf_cgrp_ops = { 345 .load = bperf_cgrp__load, 346 .enable = bperf_cgrp__enable, 347 .disable = bperf_cgrp__disable, 348 .read = bperf_cgrp__read, 349 .install_pe = bperf_cgrp__install_pe, 350 .destroy = bperf_cgrp__destroy, 351 }; 352