1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/bpf.h>
3 #include <bpf/bpf_tracing.h>
4 #include <stdbool.h>
5 #include "timerlat_bpf.h"
6
7 #define nosubprog __always_inline
8 #define MAX_ENTRIES_DEFAULT 4096
9
10 char LICENSE[] SEC("license") = "GPL";
11
12 struct trace_event_raw_timerlat_sample {
13 unsigned long long timer_latency;
14 int context;
15 } __attribute__((preserve_access_index));
16
17 struct {
18 __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
19 __uint(max_entries, MAX_ENTRIES_DEFAULT);
20 __type(key, unsigned int);
21 __type(value, unsigned long long);
22 } hist_irq SEC(".maps"), hist_thread SEC(".maps"), hist_user SEC(".maps");
23
24 struct {
25 __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
26 __uint(max_entries, SUMMARY_FIELD_N);
27 __type(key, unsigned int);
28 __type(value, unsigned long long);
29 } summary_irq SEC(".maps"), summary_thread SEC(".maps"), summary_user SEC(".maps");
30
31 struct {
32 __uint(type, BPF_MAP_TYPE_RINGBUF);
33 __uint(max_entries, 1);
34 } signal_stop_tracing SEC(".maps");
35
36 /* Params to be set by rtla */
37 const volatile int bucket_size = 1;
38 const volatile int output_divisor = 1000;
39 const volatile int entries = 256;
40 const volatile int irq_threshold;
41 const volatile int thread_threshold;
42 const volatile bool aa_only;
43
44 int stop_tracing;
45
map_get(void * map,unsigned int key)46 nosubprog unsigned long long map_get(void *map,
47 unsigned int key)
48 {
49 unsigned long long *value_ptr;
50
51 value_ptr = bpf_map_lookup_elem(map, &key);
52
53 return !value_ptr ? 0 : *value_ptr;
54 }
55
map_set(void * map,unsigned int key,unsigned long long value)56 nosubprog void map_set(void *map,
57 unsigned int key,
58 unsigned long long value)
59 {
60 bpf_map_update_elem(map, &key, &value, BPF_ANY);
61 }
62
map_increment(void * map,unsigned int key)63 nosubprog void map_increment(void *map,
64 unsigned int key)
65 {
66 map_set(map, key, map_get(map, key) + 1);
67 }
68
update_main_hist(void * map,int bucket)69 nosubprog void update_main_hist(void *map,
70 int bucket)
71 {
72 if (entries == 0)
73 /* No histogram */
74 return;
75
76 if (bucket >= entries)
77 /* Overflow */
78 return;
79
80 map_increment(map, bucket);
81 }
82
update_summary(void * map,unsigned long long latency,int bucket)83 nosubprog void update_summary(void *map,
84 unsigned long long latency,
85 int bucket)
86 {
87 if (aa_only)
88 /* Auto-analysis only, nothing to be done here */
89 return;
90
91 map_set(map, SUMMARY_CURRENT, latency);
92
93 if (bucket >= entries)
94 /* Overflow */
95 map_increment(map, SUMMARY_OVERFLOW);
96
97 if (latency > map_get(map, SUMMARY_MAX))
98 map_set(map, SUMMARY_MAX, latency);
99
100 if (latency < map_get(map, SUMMARY_MIN) || map_get(map, SUMMARY_COUNT) == 0)
101 map_set(map, SUMMARY_MIN, latency);
102
103 map_increment(map, SUMMARY_COUNT);
104 map_set(map, SUMMARY_SUM, map_get(map, SUMMARY_SUM) + latency);
105 }
106
set_stop_tracing(void)107 nosubprog void set_stop_tracing(void)
108 {
109 int value = 0;
110
111 /* Suppress further sample processing */
112 stop_tracing = 1;
113
114 /* Signal to userspace */
115 bpf_ringbuf_output(&signal_stop_tracing, &value, sizeof(value), 0);
116 }
117
118 SEC("tp/osnoise/timerlat_sample")
handle_timerlat_sample(struct trace_event_raw_timerlat_sample * tp_args)119 int handle_timerlat_sample(struct trace_event_raw_timerlat_sample *tp_args)
120 {
121 unsigned long long latency, latency_us;
122 int bucket;
123
124 if (stop_tracing)
125 return 0;
126
127 latency = tp_args->timer_latency / output_divisor;
128 latency_us = tp_args->timer_latency / 1000;
129 bucket = latency / bucket_size;
130
131 if (tp_args->context == 0) {
132 update_main_hist(&hist_irq, bucket);
133 update_summary(&summary_irq, latency, bucket);
134
135 if (irq_threshold != 0 && latency_us >= irq_threshold)
136 set_stop_tracing();
137 } else if (tp_args->context == 1) {
138 update_main_hist(&hist_thread, bucket);
139 update_summary(&summary_thread, latency, bucket);
140
141 if (thread_threshold != 0 && latency_us >= thread_threshold)
142 set_stop_tracing();
143 } else {
144 update_main_hist(&hist_user, bucket);
145 update_summary(&summary_user, latency, bucket);
146 }
147
148 return 0;
149 }
150