1 // SPDX-License-Identifier: GPL-2.0
2
3 #include <vmlinux.h>
4 #include <bpf/bpf_helpers.h>
5 #include "bpf_misc.h"
6 #include "bpf_experimental.h"
7
8 /* From include/linux/filter.h */
9 #define MAX_BPF_STACK 512
10
11 #if defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64)
12
13 struct elem {
14 struct bpf_timer t;
15 char pad[256];
16 };
17
18 struct {
19 __uint(type, BPF_MAP_TYPE_ARRAY);
20 __uint(max_entries, 1);
21 __type(key, int);
22 __type(value, struct elem);
23 } array SEC(".maps");
24
25 SEC("kprobe")
26 __description("Private stack, single prog")
27 __success
28 __arch_x86_64
29 __jited(" movabsq $0x{{.*}}, %r9")
30 __jited(" addq %gs:{{.*}}, %r9")
31 __jited(" movl $0x2a, %edi")
32 __jited(" movq %rdi, -0x100(%r9)")
33 __arch_arm64
34 __jited(" stp x25, x27, [sp, {{.*}}]!")
35 __jited(" mov x27, {{.*}}")
36 __jited(" movk x27, {{.*}}, lsl #16")
37 __jited(" movk x27, {{.*}}")
38 __jited(" mrs x10, TPIDR_EL{{[0-1]}}")
39 __jited(" add x27, x27, x10")
40 __jited(" add x25, x27, {{.*}}")
41 __jited(" mov x0, #0x2a")
42 __jited(" str x0, [x27]")
43 __jited("...")
44 __jited(" ldp x25, x27, [sp], {{.*}}")
private_stack_single_prog(void)45 __naked void private_stack_single_prog(void)
46 {
47 asm volatile (" \
48 r1 = 42; \
49 *(u64 *)(r10 - 256) = r1; \
50 r0 = 0; \
51 exit; \
52 " ::: __clobber_all);
53 }
54
55 SEC("raw_tp")
56 __description("No private stack")
57 __success
58 __arch_x86_64
59 __jited(" subq $0x8, %rsp")
60 __arch_arm64
61 __jited(" mov x25, sp")
62 __jited(" sub sp, sp, #0x10")
no_private_stack_nested(void)63 __naked void no_private_stack_nested(void)
64 {
65 asm volatile (" \
66 r1 = 42; \
67 *(u64 *)(r10 - 8) = r1; \
68 r0 = 0; \
69 exit; \
70 " ::: __clobber_all);
71 }
72
73 __used
cumulative_stack_depth_subprog(void)74 __naked static void cumulative_stack_depth_subprog(void)
75 {
76 asm volatile (" \
77 r1 = 41; \
78 *(u64 *)(r10 - 32) = r1; \
79 call %[bpf_get_smp_processor_id]; \
80 exit; \
81 " :
82 : __imm(bpf_get_smp_processor_id)
83 : __clobber_all);
84 }
85
86 SEC("kprobe")
87 __description("Private stack, subtree > MAX_BPF_STACK")
88 __success
89 __log_level(4) __msg("stack depth max 512")
90 __msg("subprog 0 (private_stack_nested_1) main {{.*}} stack 512")
91 __msg("subprog 1 (cumulative_stack_depth_subprog) static {{.*}} stack 32")
92 __arch_x86_64
93 /* private stack fp for the main prog */
94 __jited(" movabsq $0x{{.*}}, %r9")
95 __jited(" addq %gs:{{.*}}, %r9")
96 __jited(" movl $0x2a, %edi")
97 __jited(" movq %rdi, -0x200(%r9)")
98 __jited(" pushq %r9")
99 __jited("...")
100 __jited(" callq 0x{{.*}}")
101 __jited(" popq %r9")
102 __jited(" xorl %eax, %eax")
103 __arch_arm64
104 __jited(" stp x25, x27, [sp, {{.*}}]!")
105 __jited(" mov x27, {{.*}}")
106 __jited(" movk x27, {{.*}}, lsl #16")
107 __jited(" movk x27, {{.*}}")
108 __jited(" mrs x10, TPIDR_EL{{[0-1]}}")
109 __jited(" add x27, x27, x10")
110 __jited(" add x25, x27, {{.*}}")
111 __jited(" mov x0, #0x2a")
112 __jited(" str x0, [x27]")
113 __jited(" bl {{.*}}")
114 __jited("...")
115 __jited(" ldp x25, x27, [sp], {{.*}}")
private_stack_nested_1(void)116 __naked void private_stack_nested_1(void)
117 {
118 asm volatile (" \
119 r1 = 42; \
120 *(u64 *)(r10 - %[max_bpf_stack]) = r1; \
121 call cumulative_stack_depth_subprog; \
122 r0 = 0; \
123 exit; \
124 " :
125 : __imm_const(max_bpf_stack, MAX_BPF_STACK)
126 : __clobber_all);
127 }
128
129 __naked __noinline __used
loop_callback(void)130 static unsigned long loop_callback(void)
131 {
132 asm volatile (" \
133 call %[bpf_get_prandom_u32]; \
134 r1 = 42; \
135 *(u64 *)(r10 - 512) = r1; \
136 call cumulative_stack_depth_subprog; \
137 r0 = 0; \
138 exit; \
139 " :
140 : __imm(bpf_get_prandom_u32)
141 : __clobber_common);
142 }
143
144 SEC("raw_tp")
145 __description("Private stack, callback")
146 __success
147 __arch_x86_64
148 /* for func loop_callback */
149 __jited("func #1")
150 __jited(" endbr64")
151 __jited(" nopl (%rax,%rax)")
152 __jited(" nopl (%rax)")
153 __jited(" pushq %rbp")
154 __jited(" movq %rsp, %rbp")
155 __jited(" endbr64")
156 __jited(" movabsq $0x{{.*}}, %r9")
157 __jited(" addq %gs:{{.*}}, %r9")
158 __jited(" pushq %r9")
159 __jited("...")
160 __jited(" callq")
161 __jited(" popq %r9")
162 __jited(" movl $0x2a, %edi")
163 __jited(" movq %rdi, -0x200(%r9)")
164 __jited(" pushq %r9")
165 __jited("...")
166 __jited(" callq")
167 __jited(" popq %r9")
168 __arch_arm64
169 __jited("func #1")
170 __jited("...")
171 __jited(" stp x25, x27, [sp, {{.*}}]!")
172 __jited(" mov x27, {{.*}}")
173 __jited(" movk x27, {{.*}}, lsl #16")
174 __jited(" movk x27, {{.*}}")
175 __jited(" mrs x10, TPIDR_EL{{[0-1]}}")
176 __jited(" add x27, x27, x10")
177 __jited(" add x25, x27, {{.*}}")
178 __jited(" bl 0x{{.*}}")
179 __jited(" mov x8, x0")
180 __jited(" mov x0, #0x2a")
181 __jited(" str x0, [x27]")
182 __jited(" bl 0x{{.*}}")
183 __jited(" mov x8, x0")
184 __jited(" mov x8, #0x0")
185 __jited(" ldp x25, x27, [sp], {{.*}}")
private_stack_callback(void)186 __naked void private_stack_callback(void)
187 {
188 asm volatile (" \
189 r1 = 1; \
190 r2 = %[loop_callback]; \
191 r3 = 0; \
192 r4 = 0; \
193 call %[bpf_loop]; \
194 r0 = 0; \
195 exit; \
196 " :
197 : __imm_ptr(loop_callback),
198 __imm(bpf_loop)
199 : __clobber_common);
200 }
201
202 SEC("fentry/bpf_fentry_test9")
203 __description("Private stack, exception in main prog")
204 __success __retval(0)
205 __arch_x86_64
206 __jited(" pushq %r9")
207 __jited("...")
208 __jited(" callq")
209 __jited(" popq %r9")
210 __arch_arm64
211 __jited(" stp x29, x30, [sp, #-0x10]!")
212 __jited(" mov x29, sp")
213 __jited(" stp xzr, x26, [sp, #-0x10]!")
214 __jited(" mov x26, sp")
215 __jited(" stp x19, x20, [sp, #-0x10]!")
216 __jited(" stp x21, x22, [sp, #-0x10]!")
217 __jited(" stp x23, x24, [sp, #-0x10]!")
218 __jited(" stp x25, x26, [sp, #-0x10]!")
219 __jited(" stp x27, x28, [sp, #-0x10]!")
220 __jited(" mov x27, {{.*}}")
221 __jited(" movk x27, {{.*}}, lsl #16")
222 __jited(" movk x27, {{.*}}")
223 __jited(" mrs x10, TPIDR_EL{{[0-1]}}")
224 __jited(" add x27, x27, x10")
225 __jited(" add x25, x27, {{.*}}")
226 __jited(" mov x0, #0x2a")
227 __jited(" str x0, [x27]")
228 __jited(" mov x0, #0x0")
229 __jited(" bl 0x{{.*}}")
230 __jited(" mov x8, x0")
231 __jited(" ldp x27, x28, [sp], #0x10")
private_stack_exception_main_prog(void)232 int private_stack_exception_main_prog(void)
233 {
234 asm volatile (" \
235 r1 = 42; \
236 *(u64 *)(r10 - 512) = r1; \
237 " ::: __clobber_common);
238
239 bpf_throw(0);
240 return 0;
241 }
242
subprog_exception(void)243 __used static int subprog_exception(void)
244 {
245 bpf_throw(0);
246 return 0;
247 }
248
249 SEC("fentry/bpf_fentry_test9")
250 __description("Private stack, exception in subprog")
251 __success __retval(0)
252 __arch_x86_64
253 __jited(" movq %rdi, -0x200(%r9)")
254 __jited(" pushq %r9")
255 __jited("...")
256 __jited(" callq")
257 __jited(" popq %r9")
258 __arch_arm64
259 __jited(" stp x27, x28, [sp, #-0x10]!")
260 __jited(" mov x27, {{.*}}")
261 __jited(" movk x27, {{.*}}, lsl #16")
262 __jited(" movk x27, {{.*}}")
263 __jited(" mrs x10, TPIDR_EL{{[0-1]}}")
264 __jited(" add x27, x27, x10")
265 __jited(" add x25, x27, {{.*}}")
266 __jited(" mov x0, #0x2a")
267 __jited(" str x0, [x27]")
268 __jited(" bl 0x{{.*}}")
269 __jited(" mov x8, x0")
270 __jited(" ldp x27, x28, [sp], #0x10")
private_stack_exception_sub_prog(void)271 int private_stack_exception_sub_prog(void)
272 {
273 asm volatile (" \
274 r1 = 42; \
275 *(u64 *)(r10 - 512) = r1; \
276 call subprog_exception; \
277 " ::: __clobber_common);
278
279 return 0;
280 }
281
282 int glob;
subprog2(int * val)283 __noinline static void subprog2(int *val)
284 {
285 glob += val[0] * 2;
286 }
287
subprog1(int * val)288 __noinline static void subprog1(int *val)
289 {
290 int tmp[64] = {};
291
292 tmp[0] = *val;
293 subprog2(tmp);
294 }
295
timer_cb1(void * map,int * key,struct bpf_timer * timer)296 __noinline static int timer_cb1(void *map, int *key, struct bpf_timer *timer)
297 {
298 subprog1(key);
299 return 0;
300 }
301
timer_cb2(void * map,int * key,struct bpf_timer * timer)302 __noinline static int timer_cb2(void *map, int *key, struct bpf_timer *timer)
303 {
304 return 0;
305 }
306
307 SEC("fentry/bpf_fentry_test9")
308 __description("Private stack, async callback, not nested")
309 __success __retval(0)
310 __arch_x86_64
311 __jited(" movabsq $0x{{.*}}, %r9")
312 __arch_arm64
313 __jited(" mrs x10, TPIDR_EL{{[0-1]}}")
314 __jited(" add x27, x27, x10")
315 __jited(" add x25, x27, {{.*}}")
private_stack_async_callback_1(void)316 int private_stack_async_callback_1(void)
317 {
318 struct bpf_timer *arr_timer;
319 int array_key = 0;
320
321 arr_timer = bpf_map_lookup_elem(&array, &array_key);
322 if (!arr_timer)
323 return 0;
324
325 bpf_timer_init(arr_timer, &array, 1);
326 bpf_timer_set_callback(arr_timer, timer_cb2);
327 bpf_timer_start(arr_timer, 0, 0);
328 subprog1(&array_key);
329 return 0;
330 }
331
332 SEC("fentry/bpf_fentry_test9")
333 __description("Private stack, async callback, potential nesting")
334 __success __retval(0)
__load_if_JITed()335 __load_if_JITed()
336 __log_level(4) __msg("stack depth max 272")
337 __msg("subprog 0 (private_stack_async_callback_2) main {{.*}} stack 8")
338 __msg("subprog 1 (timer_cb1) static {{.*}} stack 0")
339 __msg("subprog 2 (subprog1) static {{.*}} stack 256")
340 __msg("subprog 3 (subprog2) static {{.*}} stack 0")
341 __arch_x86_64
342 __jited(" subq $0x100, %rsp")
343 __arch_arm64
344 __jited(" sub sp, sp, #0x100")
345 int private_stack_async_callback_2(void)
346 {
347 struct bpf_timer *arr_timer;
348 int array_key = 0;
349
350 arr_timer = bpf_map_lookup_elem(&array, &array_key);
351 if (!arr_timer)
352 return 0;
353
354 bpf_timer_init(arr_timer, &array, 1);
355 bpf_timer_set_callback(arr_timer, timer_cb1);
356 bpf_timer_start(arr_timer, 0, 0);
357 subprog1(&array_key);
358 return 0;
359 }
360
361 SEC("fentry/bpf_fentry_test9")
362 __description("private stack, max stack depth is private stack")
363 __success
364 __log_level(4) __msg("stack depth max 256")
365 __msg("subprog 0 (private_stack_max_depth) main {{.*}} stack 8")
366 __msg("subprog 1 (subprog1) static insns_self {{[0-9]+}} insns_total {{[0-9]+}} stack 256")
367 __msg("subprog 2 (subprog2) static insns_self {{[0-9]+}} insns_total {{[0-9]+}} stack 0")
private_stack_max_depth(void)368 int private_stack_max_depth(void)
369 {
370 int x = 0;
371
372 subprog1(&x);
373 return 0;
374 }
375
376 #else
377
378 SEC("kprobe")
379 __description("private stack is not supported, use a dummy test")
380 __success
dummy_test(void)381 int dummy_test(void)
382 {
383 return 0;
384 }
385
386 #endif
387
388 char _license[] SEC("license") = "GPL";
389