1407b36f6SNamhyung Kim // SPDX-License-Identifier: GPL-2.0
2d0c502e4SNamhyung Kim #include "util/cgroup.h"
3407b36f6SNamhyung Kim #include "util/debug.h"
46fda2405SNamhyung Kim #include "util/evlist.h"
5407b36f6SNamhyung Kim #include "util/machine.h"
6407b36f6SNamhyung Kim #include "util/map.h"
7407b36f6SNamhyung Kim #include "util/symbol.h"
86fda2405SNamhyung Kim #include "util/target.h"
9eca949b2SNamhyung Kim #include "util/thread.h"
106fda2405SNamhyung Kim #include "util/thread_map.h"
11407b36f6SNamhyung Kim #include "util/lock-contention.h"
12407b36f6SNamhyung Kim #include <linux/zalloc.h>
13a6eaf966SNamhyung Kim #include <linux/string.h>
14407b36f6SNamhyung Kim #include <bpf/bpf.h>
1529b8e94dSYang Jihong #include <inttypes.h>
16407b36f6SNamhyung Kim
17407b36f6SNamhyung Kim #include "bpf_skel/lock_contention.skel.h"
18fd507d3eSNamhyung Kim #include "bpf_skel/lock_data.h"
19407b36f6SNamhyung Kim
20407b36f6SNamhyung Kim static struct lock_contention_bpf *skel;
21407b36f6SNamhyung Kim
lock_contention_prepare(struct lock_contention * con)22447ec4e5SNamhyung Kim int lock_contention_prepare(struct lock_contention *con)
23407b36f6SNamhyung Kim {
246fda2405SNamhyung Kim int i, fd;
254fd06bd2SNamhyung Kim int ncpus = 1, ntasks = 1, ntypes = 1, naddrs = 1, ncgrps = 1;
26447ec4e5SNamhyung Kim struct evlist *evlist = con->evlist;
27447ec4e5SNamhyung Kim struct target *target = con->target;
286fda2405SNamhyung Kim
29407b36f6SNamhyung Kim skel = lock_contention_bpf__open();
30407b36f6SNamhyung Kim if (!skel) {
31407b36f6SNamhyung Kim pr_err("Failed to open lock-contention BPF skeleton\n");
32407b36f6SNamhyung Kim return -1;
33407b36f6SNamhyung Kim }
34407b36f6SNamhyung Kim
3596532a83SNamhyung Kim bpf_map__set_value_size(skel->maps.stacks, con->max_stack * sizeof(u64));
36ceb13bfcSNamhyung Kim bpf_map__set_max_entries(skel->maps.lock_stat, con->map_nr_entries);
37c66a36afSNamhyung Kim bpf_map__set_max_entries(skel->maps.tstamp, con->map_nr_entries);
38ceb13bfcSNamhyung Kim
39ebab2916SNamhyung Kim if (con->aggr_mode == LOCK_AGGR_TASK)
40eca949b2SNamhyung Kim bpf_map__set_max_entries(skel->maps.task_data, con->map_nr_entries);
41ebab2916SNamhyung Kim else
42eca949b2SNamhyung Kim bpf_map__set_max_entries(skel->maps.task_data, 1);
43ebab2916SNamhyung Kim
44ebab2916SNamhyung Kim if (con->save_callstack)
45eca949b2SNamhyung Kim bpf_map__set_max_entries(skel->maps.stacks, con->map_nr_entries);
46ebab2916SNamhyung Kim else
47ebab2916SNamhyung Kim bpf_map__set_max_entries(skel->maps.stacks, 1);
48eca949b2SNamhyung Kim
496fda2405SNamhyung Kim if (target__has_cpu(target)) {
506fda2405SNamhyung Kim skel->rodata->has_cpu = 1;
516fda2405SNamhyung Kim ncpus = perf_cpu_map__nr(evlist->core.user_requested_cpus);
526fda2405SNamhyung Kim }
53529772c4SNamhyung Kim if (target__has_task(target)) {
54529772c4SNamhyung Kim skel->rodata->has_task = 1;
554fd06bd2SNamhyung Kim ntasks = perf_thread_map__nr(evlist->core.threads);
564fd06bd2SNamhyung Kim }
576fda2405SNamhyung Kim if (con->filters->nr_types) {
585e3febe7SNamhyung Kim skel->rodata->has_type = 1;
595e3febe7SNamhyung Kim ntypes = con->filters->nr_types;
605e3febe7SNamhyung Kim }
615e3febe7SNamhyung Kim if (con->filters->nr_cgrps) {
625e3febe7SNamhyung Kim skel->rodata->has_cgroup = 1;
635e3febe7SNamhyung Kim ncgrps = con->filters->nr_cgrps;
645e3febe7SNamhyung Kim }
655e3febe7SNamhyung Kim
665e3febe7SNamhyung Kim /* resolve lock name filters to addr */
675e3febe7SNamhyung Kim if (con->filters->nr_syms) {
685e3febe7SNamhyung Kim struct symbol *sym;
695e3febe7SNamhyung Kim struct map *kmap;
705e3febe7SNamhyung Kim unsigned long *addrs;
715e3febe7SNamhyung Kim
725e3febe7SNamhyung Kim for (i = 0; i < con->filters->nr_syms; i++) {
735e3febe7SNamhyung Kim sym = machine__find_kernel_symbol_by_name(con->machine,
745e3febe7SNamhyung Kim con->filters->syms[i],
755e3febe7SNamhyung Kim &kmap);
765e3febe7SNamhyung Kim if (sym == NULL) {
775e3febe7SNamhyung Kim pr_warning("ignore unknown symbol: %s\n",
785e3febe7SNamhyung Kim con->filters->syms[i]);
795e3febe7SNamhyung Kim continue;
805e3febe7SNamhyung Kim }
8178a1f7cdSIan Rogers
825e3febe7SNamhyung Kim addrs = realloc(con->filters->addrs,
835e3febe7SNamhyung Kim (con->filters->nr_addrs + 1) * sizeof(*addrs));
845e3febe7SNamhyung Kim if (addrs == NULL) {
855e3febe7SNamhyung Kim pr_warning("memory allocation failure\n");
865e3febe7SNamhyung Kim continue;
876fda2405SNamhyung Kim }
886fda2405SNamhyung Kim
89529772c4SNamhyung Kim addrs[con->filters->nr_addrs++] = map__unmap_ip(kmap, sym->start);
905e3febe7SNamhyung Kim con->filters->addrs = addrs;
914fd06bd2SNamhyung Kim }
926fda2405SNamhyung Kim naddrs = con->filters->nr_addrs;
93407b36f6SNamhyung Kim skel->rodata->has_addr = 1;
94407b36f6SNamhyung Kim }
95407b36f6SNamhyung Kim
96407b36f6SNamhyung Kim bpf_map__set_max_entries(skel->maps.cpu_filter, ncpus);
97407b36f6SNamhyung Kim bpf_map__set_max_entries(skel->maps.task_filter, ntasks);
986fda2405SNamhyung Kim bpf_map__set_max_entries(skel->maps.type_filter, ntypes);
996fda2405SNamhyung Kim bpf_map__set_max_entries(skel->maps.addr_filter, naddrs);
1006fda2405SNamhyung Kim bpf_map__set_max_entries(skel->maps.cgroup_filter, ncgrps);
1016fda2405SNamhyung Kim
1026fda2405SNamhyung Kim skel->rodata->stack_skip = con->stack_skip;
1036fda2405SNamhyung Kim skel->rodata->aggr_mode = con->aggr_mode;
1046fda2405SNamhyung Kim skel->rodata->needs_callstack = con->save_callstack;
1056fda2405SNamhyung Kim skel->rodata->lock_owner = con->owner;
1066fda2405SNamhyung Kim
1076fda2405SNamhyung Kim if (con->aggr_mode == LOCK_AGGR_CGROUP || con->filters->nr_cgrps) {
1086fda2405SNamhyung Kim if (cgroup_is_v2("perf_event"))
1096fda2405SNamhyung Kim skel->rodata->use_cgroup_v2 = 1;
1106fda2405SNamhyung Kim }
1116fda2405SNamhyung Kim
1126fda2405SNamhyung Kim if (lock_contention_bpf__load(skel) < 0) {
1136fda2405SNamhyung Kim pr_err("Failed to load lock-contention BPF skeleton\n");
1146fda2405SNamhyung Kim return -1;
1156fda2405SNamhyung Kim }
1166fda2405SNamhyung Kim
1176fda2405SNamhyung Kim if (target__has_cpu(target)) {
1186fda2405SNamhyung Kim u32 cpu;
1196fda2405SNamhyung Kim u8 val = 1;
1206fda2405SNamhyung Kim
1216fda2405SNamhyung Kim fd = bpf_map__fd(skel->maps.cpu_filter);
1226fda2405SNamhyung Kim
1236fda2405SNamhyung Kim for (i = 0; i < ncpus; i++) {
1246fda2405SNamhyung Kim cpu = perf_cpu_map__cpu(evlist->core.user_requested_cpus, i).cpu;
1256fda2405SNamhyung Kim bpf_map_update_elem(fd, &cpu, &val, BPF_ANY);
1266fda2405SNamhyung Kim }
1276fda2405SNamhyung Kim }
1286fda2405SNamhyung Kim
1296fda2405SNamhyung Kim if (target__has_task(target)) {
1306fda2405SNamhyung Kim u32 pid;
1316fda2405SNamhyung Kim u8 val = 1;
1326fda2405SNamhyung Kim
133529772c4SNamhyung Kim fd = bpf_map__fd(skel->maps.task_filter);
134529772c4SNamhyung Kim
135529772c4SNamhyung Kim for (i = 0; i < ntasks; i++) {
136529772c4SNamhyung Kim pid = perf_thread_map__pid(evlist->core.threads, i);
137529772c4SNamhyung Kim bpf_map_update_elem(fd, &pid, &val, BPF_ANY);
138529772c4SNamhyung Kim }
139529772c4SNamhyung Kim }
140529772c4SNamhyung Kim
141529772c4SNamhyung Kim if (target__none(target) && evlist->workload.pid > 0) {
142529772c4SNamhyung Kim u32 pid = evlist->workload.pid;
1435e3febe7SNamhyung Kim u8 val = 1;
1445e3febe7SNamhyung Kim
1455e3febe7SNamhyung Kim fd = bpf_map__fd(skel->maps.task_filter);
1465e3febe7SNamhyung Kim bpf_map_update_elem(fd, &pid, &val, BPF_ANY);
1475e3febe7SNamhyung Kim }
1485e3febe7SNamhyung Kim
1495e3febe7SNamhyung Kim if (con->filters->nr_types) {
1505e3febe7SNamhyung Kim u8 val = 1;
1515e3febe7SNamhyung Kim
1525e3febe7SNamhyung Kim fd = bpf_map__fd(skel->maps.type_filter);
1534fd06bd2SNamhyung Kim
1544fd06bd2SNamhyung Kim for (i = 0; i < con->filters->nr_types; i++)
1554fd06bd2SNamhyung Kim bpf_map_update_elem(fd, &con->filters->types[i], &val, BPF_ANY);
1564fd06bd2SNamhyung Kim }
1574fd06bd2SNamhyung Kim
1584fd06bd2SNamhyung Kim if (con->filters->nr_addrs) {
1594fd06bd2SNamhyung Kim u8 val = 1;
1604fd06bd2SNamhyung Kim
1614fd06bd2SNamhyung Kim fd = bpf_map__fd(skel->maps.addr_filter);
1624fd06bd2SNamhyung Kim
163eca949b2SNamhyung Kim for (i = 0; i < con->filters->nr_addrs; i++)
164c1da8dd5SNamhyung Kim bpf_map_update_elem(fd, &con->filters->addrs[i], &val, BPF_ANY);
165eca949b2SNamhyung Kim }
166ebab2916SNamhyung Kim
1673477f079SNamhyung Kim if (con->filters->nr_cgrps) {
168c1da8dd5SNamhyung Kim u8 val = 1;
1694d1792d0SNamhyung Kim
1704d1792d0SNamhyung Kim fd = bpf_map__fd(skel->maps.cgroup_filter);
1714d1792d0SNamhyung Kim
1724d1792d0SNamhyung Kim for (i = 0; i < con->filters->nr_cgrps; i++)
173d0c502e4SNamhyung Kim bpf_map_update_elem(fd, &con->filters->cgrps[i], &val, BPF_ANY);
174d0c502e4SNamhyung Kim }
175d0c502e4SNamhyung Kim
176d24c0144SNamhyung Kim if (con->aggr_mode == LOCK_AGGR_CGROUP)
177d24c0144SNamhyung Kim read_all_cgroups(&con->cgroups);
178407b36f6SNamhyung Kim
179407b36f6SNamhyung Kim bpf_program__set_autoload(skel->progs.collect_lock_syms, false);
180407b36f6SNamhyung Kim
181407b36f6SNamhyung Kim lock_contention_bpf__attach(skel);
182b44d6653SNamhyung Kim return 0;
183b44d6653SNamhyung Kim }
184b44d6653SNamhyung Kim
185b44d6653SNamhyung Kim /*
186b44d6653SNamhyung Kim * Run the BPF program directly using BPF_PROG_TEST_RUN to update the end
187b44d6653SNamhyung Kim * timestamp in ktime so that it can calculate delta easily.
188b44d6653SNamhyung Kim */
mark_end_timestamp(void)189b44d6653SNamhyung Kim static void mark_end_timestamp(void)
190b44d6653SNamhyung Kim {
191b44d6653SNamhyung Kim DECLARE_LIBBPF_OPTS(bpf_test_run_opts, opts,
192b44d6653SNamhyung Kim .flags = BPF_F_TEST_RUN_ON_CPU,
193b44d6653SNamhyung Kim );
194b44d6653SNamhyung Kim int prog_fd = bpf_program__fd(skel->progs.end_timestamp);
195b44d6653SNamhyung Kim
196b44d6653SNamhyung Kim bpf_prog_test_run_opts(prog_fd, &opts);
197b44d6653SNamhyung Kim }
198b44d6653SNamhyung Kim
update_lock_stat(int map_fd,int pid,u64 end_ts,enum lock_aggr_mode aggr_mode,struct tstamp_data * ts_data)199b44d6653SNamhyung Kim static void update_lock_stat(int map_fd, int pid, u64 end_ts,
200b44d6653SNamhyung Kim enum lock_aggr_mode aggr_mode,
201b44d6653SNamhyung Kim struct tstamp_data *ts_data)
202b44d6653SNamhyung Kim {
203b44d6653SNamhyung Kim u64 delta;
204b44d6653SNamhyung Kim struct contention_key stat_key = {};
205b44d6653SNamhyung Kim struct contention_data stat_data;
206b44d6653SNamhyung Kim
207b44d6653SNamhyung Kim if (ts_data->timestamp >= end_ts)
208b44d6653SNamhyung Kim return;
209b44d6653SNamhyung Kim
210b44d6653SNamhyung Kim delta = end_ts - ts_data->timestamp;
211b44d6653SNamhyung Kim
212b44d6653SNamhyung Kim switch (aggr_mode) {
213b44d6653SNamhyung Kim case LOCK_AGGR_CALLER:
214b44d6653SNamhyung Kim stat_key.stack_id = ts_data->stack_id;
215b44d6653SNamhyung Kim break;
216b44d6653SNamhyung Kim case LOCK_AGGR_TASK:
217b44d6653SNamhyung Kim stat_key.pid = pid;
218b44d6653SNamhyung Kim break;
219b44d6653SNamhyung Kim case LOCK_AGGR_ADDR:
220b44d6653SNamhyung Kim stat_key.lock_addr_or_cgroup = ts_data->lock;
221b44d6653SNamhyung Kim break;
222b44d6653SNamhyung Kim case LOCK_AGGR_CGROUP:
223b44d6653SNamhyung Kim /* TODO */
224b44d6653SNamhyung Kim return;
225b44d6653SNamhyung Kim default:
226b44d6653SNamhyung Kim return;
227b44d6653SNamhyung Kim }
228b44d6653SNamhyung Kim
229b44d6653SNamhyung Kim if (bpf_map_lookup_elem(map_fd, &stat_key, &stat_data) < 0)
230b44d6653SNamhyung Kim return;
231b44d6653SNamhyung Kim
232b44d6653SNamhyung Kim stat_data.total_time += delta;
233b44d6653SNamhyung Kim stat_data.count++;
234b44d6653SNamhyung Kim
235b44d6653SNamhyung Kim if (delta > stat_data.max_time)
236b44d6653SNamhyung Kim stat_data.max_time = delta;
237b44d6653SNamhyung Kim if (delta < stat_data.min_time)
238b44d6653SNamhyung Kim stat_data.min_time = delta;
239b44d6653SNamhyung Kim
240b44d6653SNamhyung Kim bpf_map_update_elem(map_fd, &stat_key, &stat_data, BPF_EXIST);
241b44d6653SNamhyung Kim }
242b44d6653SNamhyung Kim
243b44d6653SNamhyung Kim /*
244b44d6653SNamhyung Kim * Account entries in the tstamp map (which didn't see the corresponding
245b44d6653SNamhyung Kim * lock:contention_end tracepoint) using end_ts.
246b44d6653SNamhyung Kim */
account_end_timestamp(struct lock_contention * con)247b44d6653SNamhyung Kim static void account_end_timestamp(struct lock_contention *con)
248b44d6653SNamhyung Kim {
249b44d6653SNamhyung Kim int ts_fd, stat_fd;
250b44d6653SNamhyung Kim int *prev_key, key;
251b44d6653SNamhyung Kim u64 end_ts = skel->bss->end_ts;
252b44d6653SNamhyung Kim int total_cpus;
253b44d6653SNamhyung Kim enum lock_aggr_mode aggr_mode = con->aggr_mode;
254b44d6653SNamhyung Kim struct tstamp_data ts_data, *cpu_data;
255b44d6653SNamhyung Kim
256b44d6653SNamhyung Kim /* Iterate per-task tstamp map (key = TID) */
257b44d6653SNamhyung Kim ts_fd = bpf_map__fd(skel->maps.tstamp);
258b44d6653SNamhyung Kim stat_fd = bpf_map__fd(skel->maps.lock_stat);
259b44d6653SNamhyung Kim
260b44d6653SNamhyung Kim prev_key = NULL;
261b44d6653SNamhyung Kim while (!bpf_map_get_next_key(ts_fd, prev_key, &key)) {
262b44d6653SNamhyung Kim if (bpf_map_lookup_elem(ts_fd, &key, &ts_data) == 0) {
263b44d6653SNamhyung Kim int pid = key;
264b44d6653SNamhyung Kim
265b44d6653SNamhyung Kim if (aggr_mode == LOCK_AGGR_TASK && con->owner)
266b44d6653SNamhyung Kim pid = ts_data.flags;
267b44d6653SNamhyung Kim
268b44d6653SNamhyung Kim update_lock_stat(stat_fd, pid, end_ts, aggr_mode,
269b44d6653SNamhyung Kim &ts_data);
270b44d6653SNamhyung Kim }
271b44d6653SNamhyung Kim
272b44d6653SNamhyung Kim prev_key = &key;
273b44d6653SNamhyung Kim }
274b44d6653SNamhyung Kim
275b44d6653SNamhyung Kim /* Now it'll check per-cpu tstamp map which doesn't have TID. */
276b44d6653SNamhyung Kim if (aggr_mode == LOCK_AGGR_TASK || aggr_mode == LOCK_AGGR_CGROUP)
277b44d6653SNamhyung Kim return;
278b44d6653SNamhyung Kim
279b44d6653SNamhyung Kim total_cpus = cpu__max_cpu().cpu;
280b44d6653SNamhyung Kim ts_fd = bpf_map__fd(skel->maps.tstamp_cpu);
281b44d6653SNamhyung Kim
282b44d6653SNamhyung Kim cpu_data = calloc(total_cpus, sizeof(*cpu_data));
283b44d6653SNamhyung Kim if (cpu_data == NULL)
284b44d6653SNamhyung Kim return;
285b44d6653SNamhyung Kim
286b44d6653SNamhyung Kim prev_key = NULL;
287b44d6653SNamhyung Kim while (!bpf_map_get_next_key(ts_fd, prev_key, &key)) {
288b44d6653SNamhyung Kim if (bpf_map_lookup_elem(ts_fd, &key, cpu_data) < 0)
289*287bd5cfSNamhyung Kim goto next;
290*287bd5cfSNamhyung Kim
291*287bd5cfSNamhyung Kim for (int i = 0; i < total_cpus; i++) {
292b44d6653SNamhyung Kim if (cpu_data[i].lock == 0)
293b44d6653SNamhyung Kim continue;
294b44d6653SNamhyung Kim
295b44d6653SNamhyung Kim update_lock_stat(stat_fd, -1, end_ts, aggr_mode,
296b44d6653SNamhyung Kim &cpu_data[i]);
297b44d6653SNamhyung Kim }
298b44d6653SNamhyung Kim
299b44d6653SNamhyung Kim next:
300b44d6653SNamhyung Kim prev_key = &key;
301b44d6653SNamhyung Kim }
302407b36f6SNamhyung Kim free(cpu_data);
303407b36f6SNamhyung Kim }
304407b36f6SNamhyung Kim
lock_contention_start(void)305407b36f6SNamhyung Kim int lock_contention_start(void)
306407b36f6SNamhyung Kim {
307407b36f6SNamhyung Kim skel->bss->enabled = 1;
308407b36f6SNamhyung Kim return 0;
309407b36f6SNamhyung Kim }
310407b36f6SNamhyung Kim
lock_contention_stop(void)311b44d6653SNamhyung Kim int lock_contention_stop(void)
312407b36f6SNamhyung Kim {
313407b36f6SNamhyung Kim skel->bss->enabled = 0;
314407b36f6SNamhyung Kim mark_end_timestamp();
315492fef21SNamhyung Kim return 0;
316492fef21SNamhyung Kim }
3171811e827SNamhyung Kim
lock_contention_get_name(struct lock_contention * con,struct contention_key * key,u64 * stack_trace,u32 flags)318492fef21SNamhyung Kim static const char *lock_contention_get_name(struct lock_contention *con,
319492fef21SNamhyung Kim struct contention_key *key,
320492fef21SNamhyung Kim u64 *stack_trace, u32 flags)
321492fef21SNamhyung Kim {
322492fef21SNamhyung Kim int idx = 0;
323492fef21SNamhyung Kim u64 addr;
324492fef21SNamhyung Kim const char *name = "";
325492fef21SNamhyung Kim static char name_buf[KSYM_NAME_LEN];
326492fef21SNamhyung Kim struct symbol *sym;
327492fef21SNamhyung Kim struct map *kmap;
328492fef21SNamhyung Kim struct machine *machine = con->machine;
329ebab2916SNamhyung Kim
330492fef21SNamhyung Kim if (con->aggr_mode == LOCK_AGGR_TASK) {
331492fef21SNamhyung Kim struct contention_task_data task;
332492fef21SNamhyung Kim int pid = key->pid;
333492fef21SNamhyung Kim int task_fd = bpf_map__fd(skel->maps.task_data);
334d436f90aSIan Rogers
335492fef21SNamhyung Kim /* do not update idle comm which contains CPU number */
336492fef21SNamhyung Kim if (pid) {
337492fef21SNamhyung Kim struct thread *t = machine__findnew_thread(machine, /*pid=*/-1, pid);
338492fef21SNamhyung Kim
339492fef21SNamhyung Kim if (t == NULL)
340492fef21SNamhyung Kim return name;
341492fef21SNamhyung Kim if (!bpf_map_lookup_elem(task_fd, &pid, &task) &&
342492fef21SNamhyung Kim thread__set_comm(t, task.comm, /*timestamp=*/0))
343492fef21SNamhyung Kim name = task.comm;
344492fef21SNamhyung Kim }
345492fef21SNamhyung Kim return name;
346d24c0144SNamhyung Kim }
347d24c0144SNamhyung Kim
348d24c0144SNamhyung Kim if (con->aggr_mode == LOCK_AGGR_ADDR) {
3491811e827SNamhyung Kim int lock_fd = bpf_map__fd(skel->maps.lock_syms);
3501811e827SNamhyung Kim
3511811e827SNamhyung Kim /* per-process locks set upper bits of the flags */
3521811e827SNamhyung Kim if (flags & LCD_F_MMAP_LOCK)
353d24c0144SNamhyung Kim return "mmap_lock";
354d24c0144SNamhyung Kim if (flags & LCD_F_SIGHAND_LOCK)
3554d1792d0SNamhyung Kim return "siglock";
356492fef21SNamhyung Kim
357d24c0144SNamhyung Kim /* global locks with symbols */
358d24c0144SNamhyung Kim sym = machine__find_kernel_symbol(machine, key->lock_addr_or_cgroup, &kmap);
359d24c0144SNamhyung Kim if (sym)
3604d1792d0SNamhyung Kim return sym->name;
361d24c0144SNamhyung Kim
362d24c0144SNamhyung Kim /* try semi-global locks collected separately */
363d24c0144SNamhyung Kim if (!bpf_map_lookup_elem(lock_fd, &key->lock_addr_or_cgroup, &flags)) {
364d24c0144SNamhyung Kim if (flags == LOCK_CLASS_RQLOCK)
365d24c0144SNamhyung Kim return "rq_lock";
366492fef21SNamhyung Kim }
367492fef21SNamhyung Kim
3684d1792d0SNamhyung Kim return "";
3694d1792d0SNamhyung Kim }
370d0c502e4SNamhyung Kim
371d0c502e4SNamhyung Kim if (con->aggr_mode == LOCK_AGGR_CGROUP) {
372d0c502e4SNamhyung Kim u64 cgrp_id = key->lock_addr_or_cgroup;
373d0c502e4SNamhyung Kim struct cgroup *cgrp = __cgroup__find(&con->cgroups, cgrp_id);
374d0c502e4SNamhyung Kim
37529b8e94dSYang Jihong if (cgrp)
376d0c502e4SNamhyung Kim return cgrp->name;
377d0c502e4SNamhyung Kim
378d0c502e4SNamhyung Kim snprintf(name_buf, sizeof(name_buf), "cgroup:%" PRIu64 "", cgrp_id);
379492fef21SNamhyung Kim return name_buf;
380492fef21SNamhyung Kim }
381492fef21SNamhyung Kim
382492fef21SNamhyung Kim /* LOCK_AGGR_CALLER: skip lock internal functions */
383492fef21SNamhyung Kim while (machine__is_lock_function(machine, stack_trace[idx]) &&
384492fef21SNamhyung Kim idx < con->max_stack - 1)
385492fef21SNamhyung Kim idx++;
386492fef21SNamhyung Kim
387492fef21SNamhyung Kim addr = stack_trace[idx];
388492fef21SNamhyung Kim sym = machine__find_kernel_symbol(machine, addr, &kmap);
389492fef21SNamhyung Kim
39078a1f7cdSIan Rogers if (sym) {
391492fef21SNamhyung Kim unsigned long offset;
392492fef21SNamhyung Kim
393492fef21SNamhyung Kim offset = map__map_ip(kmap, addr) - sym->start;
394492fef21SNamhyung Kim
395492fef21SNamhyung Kim if (offset == 0)
396492fef21SNamhyung Kim return sym->name;
397492fef21SNamhyung Kim
398492fef21SNamhyung Kim snprintf(name_buf, sizeof(name_buf), "%s+%#lx", sym->name, offset);
399492fef21SNamhyung Kim } else {
400492fef21SNamhyung Kim snprintf(name_buf, sizeof(name_buf), "%#lx", (unsigned long)addr);
401492fef21SNamhyung Kim }
402492fef21SNamhyung Kim
403447ec4e5SNamhyung Kim return name_buf;
404407b36f6SNamhyung Kim }
405492fef21SNamhyung Kim
lock_contention_read(struct lock_contention * con)4065d8c0f0eSNamhyung Kim int lock_contention_read(struct lock_contention *con)
407fd507d3eSNamhyung Kim {
4089e9c5f3cSNamhyung Kim int fd, stack, err = 0;
409447ec4e5SNamhyung Kim struct contention_key *prev_key, key = {};
4109e9c5f3cSNamhyung Kim struct contention_data data = {};
4119e9c5f3cSNamhyung Kim struct lock_stat *st = NULL;
412407b36f6SNamhyung Kim struct machine *machine = con->machine;
413407b36f6SNamhyung Kim u64 *stack_trace;
414407b36f6SNamhyung Kim size_t stack_size = con->max_stack * sizeof(*stack_trace);
415407b36f6SNamhyung Kim
41684c3a2bbSNamhyung Kim fd = bpf_map__fd(skel->maps.lock_stat);
41784c3a2bbSNamhyung Kim stack = bpf_map__fd(skel->maps.stacks);
41884c3a2bbSNamhyung Kim
419954cdac7SNamhyung Kim con->fails.task = skel->bss->task_fail;
4206d499a6bSNamhyung Kim con->fails.stack = skel->bss->stack_fail;
4219e9c5f3cSNamhyung Kim con->fails.time = skel->bss->time_fail;
4229e9c5f3cSNamhyung Kim con->fails.data = skel->bss->data_fail;
4239e9c5f3cSNamhyung Kim
4249e9c5f3cSNamhyung Kim stack_trace = zalloc(stack_size);
425b44d6653SNamhyung Kim if (stack_trace == NULL)
426b44d6653SNamhyung Kim return -1;
427eca949b2SNamhyung Kim
428d436f90aSIan Rogers account_end_timestamp(con);
429eca949b2SNamhyung Kim
430eca949b2SNamhyung Kim if (con->aggr_mode == LOCK_AGGR_TASK) {
431eca949b2SNamhyung Kim struct thread *idle = machine__findnew_thread(machine,
432eca949b2SNamhyung Kim /*pid=*/0,
433eca949b2SNamhyung Kim /*tid=*/0);
434d24c0144SNamhyung Kim thread__set_comm(idle, "swapper", /*timestamp=*/0);
435d24c0144SNamhyung Kim }
436d24c0144SNamhyung Kim
437d24c0144SNamhyung Kim if (con->aggr_mode == LOCK_AGGR_ADDR) {
438d24c0144SNamhyung Kim DECLARE_LIBBPF_OPTS(bpf_test_run_opts, opts,
439d24c0144SNamhyung Kim .flags = BPF_F_TEST_RUN_ON_CPU,
440d24c0144SNamhyung Kim );
441d24c0144SNamhyung Kim int prog_fd = bpf_program__fd(skel->progs.collect_lock_syms);
442d24c0144SNamhyung Kim
443688d2e8dSNamhyung Kim bpf_prog_test_run_opts(prog_fd, &opts);
444e77b0236SIan Rogers }
445688d2e8dSNamhyung Kim
446fd507d3eSNamhyung Kim /* make sure it loads the kernel map */
447fd507d3eSNamhyung Kim maps__load_first(machine->kmaps);
448ebab2916SNamhyung Kim
44916cad1d3SNamhyung Kim prev_key = NULL;
450407b36f6SNamhyung Kim while (!bpf_map_get_next_key(fd, prev_key, &key)) {
4519e9c5f3cSNamhyung Kim s64 ls_key;
4529e9c5f3cSNamhyung Kim const char *name;
4539e9c5f3cSNamhyung Kim
454407b36f6SNamhyung Kim /* to handle errors in the loop body */
45516cad1d3SNamhyung Kim err = -1;
456ebab2916SNamhyung Kim
457ebab2916SNamhyung Kim bpf_map_lookup_elem(fd, &key, &data);
458aae7e453SNamhyung Kim if (con->save_callstack) {
459aae7e453SNamhyung Kim bpf_map_lookup_elem(stack, &key.stack_id, stack_trace);
460ebab2916SNamhyung Kim
46116cad1d3SNamhyung Kim if (!match_callstack_filter(machine, stack_trace)) {
462aae7e453SNamhyung Kim con->nr_filtered += data.count;
46316cad1d3SNamhyung Kim goto next;
464ebab2916SNamhyung Kim }
465ebab2916SNamhyung Kim }
466ebab2916SNamhyung Kim
467ebab2916SNamhyung Kim switch (con->aggr_mode) {
468ebab2916SNamhyung Kim case LOCK_AGGR_CALLER:
469ebab2916SNamhyung Kim ls_key = key.stack_id;
470ebab2916SNamhyung Kim break;
471ebab2916SNamhyung Kim case LOCK_AGGR_TASK:
4724d1792d0SNamhyung Kim ls_key = key.pid;
4734d1792d0SNamhyung Kim break;
474ebab2916SNamhyung Kim case LOCK_AGGR_ADDR:
475ebab2916SNamhyung Kim case LOCK_AGGR_CGROUP:
476ebab2916SNamhyung Kim ls_key = key.lock_addr_or_cgroup;
477ebab2916SNamhyung Kim break;
478ebab2916SNamhyung Kim default:
479ebab2916SNamhyung Kim goto next;
48016cad1d3SNamhyung Kim }
48116cad1d3SNamhyung Kim
48216cad1d3SNamhyung Kim st = lock_stat_find(ls_key);
48316cad1d3SNamhyung Kim if (st != NULL) {
48416cad1d3SNamhyung Kim st->wait_time_total += data.total_time;
48516cad1d3SNamhyung Kim if (st->wait_time_max < data.max_time)
48616cad1d3SNamhyung Kim st->wait_time_max = data.max_time;
48716cad1d3SNamhyung Kim if (st->wait_time_min > data.min_time)
48816cad1d3SNamhyung Kim st->wait_time_min = data.min_time;
48916cad1d3SNamhyung Kim
49016cad1d3SNamhyung Kim st->nr_contended += data.count;
49116cad1d3SNamhyung Kim if (st->nr_contended)
49216cad1d3SNamhyung Kim st->avg_wait_time = st->wait_time_total / st->nr_contended;
4931811e827SNamhyung Kim goto next;
494ebab2916SNamhyung Kim }
495407b36f6SNamhyung Kim
4969e9c5f3cSNamhyung Kim name = lock_contention_get_name(con, &key, stack_trace, data.flags);
497407b36f6SNamhyung Kim st = lock_stat_findnew(ls_key, name, data.flags);
498407b36f6SNamhyung Kim if (st == NULL)
499407b36f6SNamhyung Kim break;
500407b36f6SNamhyung Kim
501407b36f6SNamhyung Kim st->nr_contended = data.count;
502407b36f6SNamhyung Kim st->wait_time_total = data.total_time;
503407b36f6SNamhyung Kim st->wait_time_max = data.max_time;
504407b36f6SNamhyung Kim st->wait_time_min = data.min_time;
505407b36f6SNamhyung Kim
5060fba2265SNamhyung Kim if (data.count)
5079e9c5f3cSNamhyung Kim st->avg_wait_time = data.total_time / data.count;
5089e9c5f3cSNamhyung Kim
5099e9c5f3cSNamhyung Kim if (con->aggr_mode == LOCK_AGGR_CALLER && verbose > 0) {
510a6eaf966SNamhyung Kim st->callstack = memdup(stack_trace, stack_size);
511492fef21SNamhyung Kim if (st->callstack == NULL)
51216cad1d3SNamhyung Kim break;
513fd507d3eSNamhyung Kim }
5149e9c5f3cSNamhyung Kim
51516cad1d3SNamhyung Kim next:
5169e9c5f3cSNamhyung Kim prev_key = &key;
517407b36f6SNamhyung Kim
518407b36f6SNamhyung Kim /* we're fine now, reset the error */
5199e9c5f3cSNamhyung Kim err = 0;
5209e9c5f3cSNamhyung Kim }
5219e9c5f3cSNamhyung Kim
522407b36f6SNamhyung Kim free(stack_trace);
523407b36f6SNamhyung Kim
524d0c502e4SNamhyung Kim return err;
525407b36f6SNamhyung Kim }
526407b36f6SNamhyung Kim
lock_contention_finish(struct lock_contention * con)527407b36f6SNamhyung Kim int lock_contention_finish(struct lock_contention *con)
528407b36f6SNamhyung Kim {
529407b36f6SNamhyung Kim if (skel) {
530407b36f6SNamhyung Kim skel->bss->enabled = 0;
531d0c502e4SNamhyung Kim lock_contention_bpf__destroy(skel);
532d0c502e4SNamhyung Kim }
533d0c502e4SNamhyung Kim
534d0c502e4SNamhyung Kim while (!RB_EMPTY_ROOT(&con->cgroups)) {
535d0c502e4SNamhyung Kim struct rb_node *node = rb_first(&con->cgroups);
536d0c502e4SNamhyung Kim struct cgroup *cgrp = rb_entry(node, struct cgroup, node);
537d0c502e4SNamhyung Kim
538d0c502e4SNamhyung Kim rb_erase(node, &con->cgroups);
539407b36f6SNamhyung Kim cgroup__put(cgrp);
540407b36f6SNamhyung Kim }
541
542 return 0;
543 }
544