xref: /linux/tools/testing/selftests/bpf/progs/irq.c (revision 4f9786035f9e519db41375818e1d0b5f20da2f10) 
1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
3 #include <vmlinux.h>
4 #include <bpf/bpf_helpers.h>
5 #include "bpf_misc.h"
6 #include "bpf_experimental.h"
7 
8 unsigned long global_flags;
9 
10 extern void bpf_local_irq_save(unsigned long *) __weak __ksym;
11 extern void bpf_local_irq_restore(unsigned long *) __weak __ksym;
12 extern int bpf_copy_from_user_str(void *dst, u32 dst__sz, const void *unsafe_ptr__ign, u64 flags) __weak __ksym;
13 
14 struct bpf_res_spin_lock lockA __hidden SEC(".data.A");
15 struct bpf_res_spin_lock lockB __hidden SEC(".data.B");
16 
17 SEC("?tc")
18 __failure __msg("arg#0 doesn't point to an irq flag on stack")
19 int irq_save_bad_arg(struct __sk_buff *ctx)
20 {
21 	bpf_local_irq_save(&global_flags);
22 	return 0;
23 }
24 
25 SEC("?tc")
26 __failure __msg("arg#0 doesn't point to an irq flag on stack")
27 int irq_restore_bad_arg(struct __sk_buff *ctx)
28 {
29 	bpf_local_irq_restore(&global_flags);
30 	return 0;
31 }
32 
33 SEC("?tc")
34 __failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region")
35 int irq_restore_missing_2(struct __sk_buff *ctx)
36 {
37 	unsigned long flags1;
38 	unsigned long flags2;
39 
40 	bpf_local_irq_save(&flags1);
41 	bpf_local_irq_save(&flags2);
42 	return 0;
43 }
44 
45 SEC("?tc")
46 __failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region")
47 int irq_restore_missing_3(struct __sk_buff *ctx)
48 {
49 	unsigned long flags1;
50 	unsigned long flags2;
51 	unsigned long flags3;
52 
53 	bpf_local_irq_save(&flags1);
54 	bpf_local_irq_save(&flags2);
55 	bpf_local_irq_save(&flags3);
56 	return 0;
57 }
58 
59 SEC("?tc")
60 __failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region")
61 int irq_restore_missing_3_minus_2(struct __sk_buff *ctx)
62 {
63 	unsigned long flags1;
64 	unsigned long flags2;
65 	unsigned long flags3;
66 
67 	bpf_local_irq_save(&flags1);
68 	bpf_local_irq_save(&flags2);
69 	bpf_local_irq_save(&flags3);
70 	bpf_local_irq_restore(&flags3);
71 	bpf_local_irq_restore(&flags2);
72 	return 0;
73 }
74 
75 static __noinline void local_irq_save(unsigned long *flags)
76 {
77 	bpf_local_irq_save(flags);
78 }
79 
80 static __noinline void local_irq_restore(unsigned long *flags)
81 {
82 	bpf_local_irq_restore(flags);
83 }
84 
85 SEC("?tc")
86 __failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region")
87 int irq_restore_missing_1_subprog(struct __sk_buff *ctx)
88 {
89 	unsigned long flags;
90 
91 	local_irq_save(&flags);
92 	return 0;
93 }
94 
95 SEC("?tc")
96 __failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region")
97 int irq_restore_missing_2_subprog(struct __sk_buff *ctx)
98 {
99 	unsigned long flags1;
100 	unsigned long flags2;
101 
102 	local_irq_save(&flags1);
103 	local_irq_save(&flags2);
104 	return 0;
105 }
106 
107 SEC("?tc")
108 __failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region")
109 int irq_restore_missing_3_subprog(struct __sk_buff *ctx)
110 {
111 	unsigned long flags1;
112 	unsigned long flags2;
113 	unsigned long flags3;
114 
115 	local_irq_save(&flags1);
116 	local_irq_save(&flags2);
117 	local_irq_save(&flags3);
118 	return 0;
119 }
120 
121 SEC("?tc")
122 __failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region")
123 int irq_restore_missing_3_minus_2_subprog(struct __sk_buff *ctx)
124 {
125 	unsigned long flags1;
126 	unsigned long flags2;
127 	unsigned long flags3;
128 
129 	local_irq_save(&flags1);
130 	local_irq_save(&flags2);
131 	local_irq_save(&flags3);
132 	local_irq_restore(&flags3);
133 	local_irq_restore(&flags2);
134 	return 0;
135 }
136 
137 SEC("?tc")
138 __success
139 int irq_balance(struct __sk_buff *ctx)
140 {
141 	unsigned long flags;
142 
143 	local_irq_save(&flags);
144 	local_irq_restore(&flags);
145 	return 0;
146 }
147 
148 SEC("?tc")
149 __success
150 int irq_balance_n(struct __sk_buff *ctx)
151 {
152 	unsigned long flags1;
153 	unsigned long flags2;
154 	unsigned long flags3;
155 
156 	local_irq_save(&flags1);
157 	local_irq_save(&flags2);
158 	local_irq_save(&flags3);
159 	local_irq_restore(&flags3);
160 	local_irq_restore(&flags2);
161 	local_irq_restore(&flags1);
162 	return 0;
163 }
164 
165 static __noinline void local_irq_balance(void)
166 {
167 	unsigned long flags;
168 
169 	local_irq_save(&flags);
170 	local_irq_restore(&flags);
171 }
172 
173 static __noinline void local_irq_balance_n(void)
174 {
175 	unsigned long flags1;
176 	unsigned long flags2;
177 	unsigned long flags3;
178 
179 	local_irq_save(&flags1);
180 	local_irq_save(&flags2);
181 	local_irq_save(&flags3);
182 	local_irq_restore(&flags3);
183 	local_irq_restore(&flags2);
184 	local_irq_restore(&flags1);
185 }
186 
187 SEC("?tc")
188 __success
189 int irq_balance_subprog(struct __sk_buff *ctx)
190 {
191 	local_irq_balance();
192 	return 0;
193 }
194 
195 SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
196 __failure __msg("sleepable helper bpf_copy_from_user#")
197 int irq_sleepable_helper(void *ctx)
198 {
199 	unsigned long flags;
200 	u32 data;
201 
202 	local_irq_save(&flags);
203 	bpf_copy_from_user(&data, sizeof(data), NULL);
204 	local_irq_restore(&flags);
205 	return 0;
206 }
207 
208 SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
209 __failure __msg("kernel func bpf_copy_from_user_str is sleepable within IRQ-disabled region")
210 int irq_sleepable_kfunc(void *ctx)
211 {
212 	unsigned long flags;
213 	u32 data;
214 
215 	local_irq_save(&flags);
216 	bpf_copy_from_user_str(&data, sizeof(data), NULL, 0);
217 	local_irq_restore(&flags);
218 	return 0;
219 }
220 
221 int __noinline global_local_irq_balance(void)
222 {
223 	local_irq_balance_n();
224 	return 0;
225 }
226 
227 SEC("?tc")
228 __success
229 int irq_global_subprog(struct __sk_buff *ctx)
230 {
231 	unsigned long flags;
232 
233 	bpf_local_irq_save(&flags);
234 	global_local_irq_balance();
235 	bpf_local_irq_restore(&flags);
236 	return 0;
237 }
238 
239 SEC("?tc")
240 __failure __msg("cannot restore irq state out of order")
241 int irq_restore_ooo(struct __sk_buff *ctx)
242 {
243 	unsigned long flags1;
244 	unsigned long flags2;
245 
246 	bpf_local_irq_save(&flags1);
247 	bpf_local_irq_save(&flags2);
248 	bpf_local_irq_restore(&flags1);
249 	bpf_local_irq_restore(&flags2);
250 	return 0;
251 }
252 
253 SEC("?tc")
254 __failure __msg("cannot restore irq state out of order")
255 int irq_restore_ooo_3(struct __sk_buff *ctx)
256 {
257 	unsigned long flags1;
258 	unsigned long flags2;
259 	unsigned long flags3;
260 
261 	bpf_local_irq_save(&flags1);
262 	bpf_local_irq_save(&flags2);
263 	bpf_local_irq_restore(&flags2);
264 	bpf_local_irq_save(&flags3);
265 	bpf_local_irq_restore(&flags1);
266 	bpf_local_irq_restore(&flags3);
267 	return 0;
268 }
269 
270 static __noinline void local_irq_save_3(unsigned long *flags1, unsigned long *flags2,
271 					unsigned long *flags3)
272 {
273 	local_irq_save(flags1);
274 	local_irq_save(flags2);
275 	local_irq_save(flags3);
276 }
277 
278 SEC("?tc")
279 __success
280 int irq_restore_3_subprog(struct __sk_buff *ctx)
281 {
282 	unsigned long flags1;
283 	unsigned long flags2;
284 	unsigned long flags3;
285 
286 	local_irq_save_3(&flags1, &flags2, &flags3);
287 	bpf_local_irq_restore(&flags3);
288 	bpf_local_irq_restore(&flags2);
289 	bpf_local_irq_restore(&flags1);
290 	return 0;
291 }
292 
293 SEC("?tc")
294 __failure __msg("cannot restore irq state out of order")
295 int irq_restore_4_subprog(struct __sk_buff *ctx)
296 {
297 	unsigned long flags1;
298 	unsigned long flags2;
299 	unsigned long flags3;
300 	unsigned long flags4;
301 
302 	local_irq_save_3(&flags1, &flags2, &flags3);
303 	bpf_local_irq_restore(&flags3);
304 	bpf_local_irq_save(&flags4);
305 	bpf_local_irq_restore(&flags4);
306 	bpf_local_irq_restore(&flags1);
307 	return 0;
308 }
309 
310 SEC("?tc")
311 __failure __msg("cannot restore irq state out of order")
312 int irq_restore_ooo_3_subprog(struct __sk_buff *ctx)
313 {
314 	unsigned long flags1;
315 	unsigned long flags2;
316 	unsigned long flags3;
317 
318 	local_irq_save_3(&flags1, &flags2, &flags3);
319 	bpf_local_irq_restore(&flags3);
320 	bpf_local_irq_restore(&flags2);
321 	bpf_local_irq_save(&flags3);
322 	bpf_local_irq_restore(&flags1);
323 	return 0;
324 }
325 
326 SEC("?tc")
327 __failure __msg("expected an initialized")
328 int irq_restore_invalid(struct __sk_buff *ctx)
329 {
330 	unsigned long flags1;
331 	unsigned long flags = 0xfaceb00c;
332 
333 	bpf_local_irq_save(&flags1);
334 	bpf_local_irq_restore(&flags);
335 	return 0;
336 }
337 
338 SEC("?tc")
339 __failure __msg("expected uninitialized")
340 int irq_save_invalid(struct __sk_buff *ctx)
341 {
342 	unsigned long flags1;
343 
344 	bpf_local_irq_save(&flags1);
345 	bpf_local_irq_save(&flags1);
346 	return 0;
347 }
348 
349 SEC("?tc")
350 __failure __msg("expected an initialized")
351 int irq_restore_iter(struct __sk_buff *ctx)
352 {
353 	struct bpf_iter_num it;
354 
355 	bpf_iter_num_new(&it, 0, 42);
356 	bpf_local_irq_restore((unsigned long *)&it);
357 	return 0;
358 }
359 
360 SEC("?tc")
361 __failure __msg("Unreleased reference id=1")
362 int irq_save_iter(struct __sk_buff *ctx)
363 {
364 	struct bpf_iter_num it;
365 
366 	/* Ensure same sized slot has st->ref_obj_id set, so we reject based on
367 	 * slot_type != STACK_IRQ_FLAG...
368 	 */
369 	_Static_assert(sizeof(it) == sizeof(unsigned long), "broken iterator size");
370 
371 	bpf_iter_num_new(&it, 0, 42);
372 	bpf_local_irq_save((unsigned long *)&it);
373 	bpf_local_irq_restore((unsigned long *)&it);
374 	return 0;
375 }
376 
377 SEC("?tc")
378 __failure __msg("expected an initialized")
379 int irq_flag_overwrite(struct __sk_buff *ctx)
380 {
381 	unsigned long flags;
382 
383 	bpf_local_irq_save(&flags);
384 	flags = 0xdeadbeef;
385 	bpf_local_irq_restore(&flags);
386 	return 0;
387 }
388 
389 SEC("?tc")
390 __failure __msg("expected an initialized")
391 int irq_flag_overwrite_partial(struct __sk_buff *ctx)
392 {
393 	unsigned long flags;
394 
395 	bpf_local_irq_save(&flags);
396 	*(((char *)&flags) + 1) = 0xff;
397 	bpf_local_irq_restore(&flags);
398 	return 0;
399 }
400 
401 SEC("?tc")
402 __failure __msg("cannot restore irq state out of order")
403 int irq_ooo_refs_array(struct __sk_buff *ctx)
404 {
405 	unsigned long flags[4];
406 	struct { int i; } *p;
407 
408 	/* refs=1 */
409 	bpf_local_irq_save(&flags[0]);
410 
411 	/* refs=1,2 */
412 	p = bpf_obj_new(typeof(*p));
413 	if (!p) {
414 		bpf_local_irq_restore(&flags[0]);
415 		return 0;
416 	}
417 
418 	/* refs=1,2,3 */
419 	bpf_local_irq_save(&flags[1]);
420 
421 	/* refs=1,2,3,4 */
422 	bpf_local_irq_save(&flags[2]);
423 
424 	/* Now when we remove ref=2, the verifier must not break the ordering in
425 	 * the refs array between 1,3,4. With an older implementation, the
426 	 * verifier would swap the last element with the removed element, but to
427 	 * maintain the stack property we need to use memmove.
428 	 */
429 	bpf_obj_drop(p);
430 
431 	/* Save and restore to reset active_irq_id to 3, as the ordering is now
432 	 * refs=1,4,3. When restoring the linear scan will find prev_id in order
433 	 * as 3 instead of 4.
434 	 */
435 	bpf_local_irq_save(&flags[3]);
436 	bpf_local_irq_restore(&flags[3]);
437 
438 	/* With the incorrect implementation, we can release flags[1], flags[2],
439 	 * and flags[0], i.e. in the wrong order.
440 	 */
441 	bpf_local_irq_restore(&flags[1]);
442 	bpf_local_irq_restore(&flags[2]);
443 	bpf_local_irq_restore(&flags[0]);
444 	return 0;
445 }
446 
447 int __noinline
448 global_subprog(int i)
449 {
450 	if (i)
451 		bpf_printk("%p", &i);
452 	return i;
453 }
454 
455 int __noinline
456 global_sleepable_helper_subprog(int i)
457 {
458 	if (i)
459 		bpf_copy_from_user(&i, sizeof(i), NULL);
460 	return i;
461 }
462 
463 int __noinline
464 global_sleepable_kfunc_subprog(int i)
465 {
466 	if (i)
467 		bpf_copy_from_user_str(&i, sizeof(i), NULL, 0);
468 	global_subprog(i);
469 	return i;
470 }
471 
472 int __noinline
473 global_subprog_calling_sleepable_global(int i)
474 {
475 	if (!i)
476 		global_sleepable_kfunc_subprog(i);
477 	return i;
478 }
479 
480 SEC("?syscall")
481 __success
482 int irq_non_sleepable_global_subprog(void *ctx)
483 {
484 	unsigned long flags;
485 
486 	bpf_local_irq_save(&flags);
487 	global_subprog(0);
488 	bpf_local_irq_restore(&flags);
489 	return 0;
490 }
491 
492 SEC("?syscall")
493 __failure __msg("global functions that may sleep are not allowed in non-sleepable context")
494 int irq_sleepable_helper_global_subprog(void *ctx)
495 {
496 	unsigned long flags;
497 
498 	bpf_local_irq_save(&flags);
499 	global_sleepable_helper_subprog(0);
500 	bpf_local_irq_restore(&flags);
501 	return 0;
502 }
503 
504 SEC("?syscall")
505 __failure __msg("global functions that may sleep are not allowed in non-sleepable context")
506 int irq_sleepable_global_subprog_indirect(void *ctx)
507 {
508 	unsigned long flags;
509 
510 	bpf_local_irq_save(&flags);
511 	global_subprog_calling_sleepable_global(0);
512 	bpf_local_irq_restore(&flags);
513 	return 0;
514 }
515 
516 SEC("?tc")
517 __failure __msg("cannot restore irq state out of order")
518 int irq_ooo_lock_cond_inv(struct __sk_buff *ctx)
519 {
520 	unsigned long flags1, flags2;
521 
522 	if (bpf_res_spin_lock_irqsave(&lockA, &flags1))
523 		return 0;
524 	if (bpf_res_spin_lock_irqsave(&lockB, &flags2)) {
525 		bpf_res_spin_unlock_irqrestore(&lockA, &flags1);
526 		return 0;
527 	}
528 
529 	bpf_res_spin_unlock_irqrestore(&lockB, &flags1);
530 	bpf_res_spin_unlock_irqrestore(&lockA, &flags2);
531 	return 0;
532 }
533 
534 SEC("?tc")
535 __failure __msg("function calls are not allowed")
536 int irq_wrong_kfunc_class_1(struct __sk_buff *ctx)
537 {
538 	unsigned long flags1;
539 
540 	if (bpf_res_spin_lock_irqsave(&lockA, &flags1))
541 		return 0;
542 	/* For now, bpf_local_irq_restore is not allowed in critical section,
543 	 * but this test ensures error will be caught with kfunc_class when it's
544 	 * opened up. Tested by temporarily permitting this kfunc in critical
545 	 * section.
546 	 */
547 	bpf_local_irq_restore(&flags1);
548 	bpf_res_spin_unlock_irqrestore(&lockA, &flags1);
549 	return 0;
550 }
551 
552 SEC("?tc")
553 __failure __msg("function calls are not allowed")
554 int irq_wrong_kfunc_class_2(struct __sk_buff *ctx)
555 {
556 	unsigned long flags1, flags2;
557 
558 	bpf_local_irq_save(&flags1);
559 	if (bpf_res_spin_lock_irqsave(&lockA, &flags2))
560 		return 0;
561 	bpf_local_irq_restore(&flags2);
562 	bpf_res_spin_unlock_irqrestore(&lockA, &flags1);
563 	return 0;
564 }
565 
566 char _license[] SEC("license") = "GPL";
567