1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */
3 #include <vmlinux.h>
4 #include <bpf/bpf_helpers.h>
5 #include "bpf_misc.h"
6 #include "bpf_experimental.h"
7
8 unsigned long global_flags;
9
10 extern void bpf_local_irq_save(unsigned long *) __weak __ksym;
11 extern void bpf_local_irq_restore(unsigned long *) __weak __ksym;
12 extern int bpf_copy_from_user_str(void *dst, u32 dst__sz, const void *unsafe_ptr__ign, u64 flags) __weak __ksym;
13
14 SEC("?tc")
15 __failure __msg("arg#0 doesn't point to an irq flag on stack")
irq_save_bad_arg(struct __sk_buff * ctx)16 int irq_save_bad_arg(struct __sk_buff *ctx)
17 {
18 bpf_local_irq_save(&global_flags);
19 return 0;
20 }
21
22 SEC("?tc")
23 __failure __msg("arg#0 doesn't point to an irq flag on stack")
irq_restore_bad_arg(struct __sk_buff * ctx)24 int irq_restore_bad_arg(struct __sk_buff *ctx)
25 {
26 bpf_local_irq_restore(&global_flags);
27 return 0;
28 }
29
30 SEC("?tc")
31 __failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region")
irq_restore_missing_2(struct __sk_buff * ctx)32 int irq_restore_missing_2(struct __sk_buff *ctx)
33 {
34 unsigned long flags1;
35 unsigned long flags2;
36
37 bpf_local_irq_save(&flags1);
38 bpf_local_irq_save(&flags2);
39 return 0;
40 }
41
42 SEC("?tc")
43 __failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region")
irq_restore_missing_3(struct __sk_buff * ctx)44 int irq_restore_missing_3(struct __sk_buff *ctx)
45 {
46 unsigned long flags1;
47 unsigned long flags2;
48 unsigned long flags3;
49
50 bpf_local_irq_save(&flags1);
51 bpf_local_irq_save(&flags2);
52 bpf_local_irq_save(&flags3);
53 return 0;
54 }
55
56 SEC("?tc")
57 __failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region")
irq_restore_missing_3_minus_2(struct __sk_buff * ctx)58 int irq_restore_missing_3_minus_2(struct __sk_buff *ctx)
59 {
60 unsigned long flags1;
61 unsigned long flags2;
62 unsigned long flags3;
63
64 bpf_local_irq_save(&flags1);
65 bpf_local_irq_save(&flags2);
66 bpf_local_irq_save(&flags3);
67 bpf_local_irq_restore(&flags3);
68 bpf_local_irq_restore(&flags2);
69 return 0;
70 }
71
local_irq_save(unsigned long * flags)72 static __noinline void local_irq_save(unsigned long *flags)
73 {
74 bpf_local_irq_save(flags);
75 }
76
local_irq_restore(unsigned long * flags)77 static __noinline void local_irq_restore(unsigned long *flags)
78 {
79 bpf_local_irq_restore(flags);
80 }
81
82 SEC("?tc")
83 __failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region")
irq_restore_missing_1_subprog(struct __sk_buff * ctx)84 int irq_restore_missing_1_subprog(struct __sk_buff *ctx)
85 {
86 unsigned long flags;
87
88 local_irq_save(&flags);
89 return 0;
90 }
91
92 SEC("?tc")
93 __failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region")
irq_restore_missing_2_subprog(struct __sk_buff * ctx)94 int irq_restore_missing_2_subprog(struct __sk_buff *ctx)
95 {
96 unsigned long flags1;
97 unsigned long flags2;
98
99 local_irq_save(&flags1);
100 local_irq_save(&flags2);
101 return 0;
102 }
103
104 SEC("?tc")
105 __failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region")
irq_restore_missing_3_subprog(struct __sk_buff * ctx)106 int irq_restore_missing_3_subprog(struct __sk_buff *ctx)
107 {
108 unsigned long flags1;
109 unsigned long flags2;
110 unsigned long flags3;
111
112 local_irq_save(&flags1);
113 local_irq_save(&flags2);
114 local_irq_save(&flags3);
115 return 0;
116 }
117
118 SEC("?tc")
119 __failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region")
irq_restore_missing_3_minus_2_subprog(struct __sk_buff * ctx)120 int irq_restore_missing_3_minus_2_subprog(struct __sk_buff *ctx)
121 {
122 unsigned long flags1;
123 unsigned long flags2;
124 unsigned long flags3;
125
126 local_irq_save(&flags1);
127 local_irq_save(&flags2);
128 local_irq_save(&flags3);
129 local_irq_restore(&flags3);
130 local_irq_restore(&flags2);
131 return 0;
132 }
133
134 SEC("?tc")
135 __success
irq_balance(struct __sk_buff * ctx)136 int irq_balance(struct __sk_buff *ctx)
137 {
138 unsigned long flags;
139
140 local_irq_save(&flags);
141 local_irq_restore(&flags);
142 return 0;
143 }
144
145 SEC("?tc")
146 __success
irq_balance_n(struct __sk_buff * ctx)147 int irq_balance_n(struct __sk_buff *ctx)
148 {
149 unsigned long flags1;
150 unsigned long flags2;
151 unsigned long flags3;
152
153 local_irq_save(&flags1);
154 local_irq_save(&flags2);
155 local_irq_save(&flags3);
156 local_irq_restore(&flags3);
157 local_irq_restore(&flags2);
158 local_irq_restore(&flags1);
159 return 0;
160 }
161
local_irq_balance(void)162 static __noinline void local_irq_balance(void)
163 {
164 unsigned long flags;
165
166 local_irq_save(&flags);
167 local_irq_restore(&flags);
168 }
169
local_irq_balance_n(void)170 static __noinline void local_irq_balance_n(void)
171 {
172 unsigned long flags1;
173 unsigned long flags2;
174 unsigned long flags3;
175
176 local_irq_save(&flags1);
177 local_irq_save(&flags2);
178 local_irq_save(&flags3);
179 local_irq_restore(&flags3);
180 local_irq_restore(&flags2);
181 local_irq_restore(&flags1);
182 }
183
184 SEC("?tc")
185 __success
irq_balance_subprog(struct __sk_buff * ctx)186 int irq_balance_subprog(struct __sk_buff *ctx)
187 {
188 local_irq_balance();
189 return 0;
190 }
191
192 SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
193 __failure __msg("sleepable helper bpf_copy_from_user#")
irq_sleepable_helper(void * ctx)194 int irq_sleepable_helper(void *ctx)
195 {
196 unsigned long flags;
197 u32 data;
198
199 local_irq_save(&flags);
200 bpf_copy_from_user(&data, sizeof(data), NULL);
201 local_irq_restore(&flags);
202 return 0;
203 }
204
205 SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
206 __failure __msg("kernel func bpf_copy_from_user_str is sleepable within IRQ-disabled region")
irq_sleepable_kfunc(void * ctx)207 int irq_sleepable_kfunc(void *ctx)
208 {
209 unsigned long flags;
210 u32 data;
211
212 local_irq_save(&flags);
213 bpf_copy_from_user_str(&data, sizeof(data), NULL, 0);
214 local_irq_restore(&flags);
215 return 0;
216 }
217
global_local_irq_balance(void)218 int __noinline global_local_irq_balance(void)
219 {
220 local_irq_balance_n();
221 return 0;
222 }
223
224 SEC("?tc")
225 __failure __msg("global function calls are not allowed with IRQs disabled")
irq_global_subprog(struct __sk_buff * ctx)226 int irq_global_subprog(struct __sk_buff *ctx)
227 {
228 unsigned long flags;
229
230 bpf_local_irq_save(&flags);
231 global_local_irq_balance();
232 bpf_local_irq_restore(&flags);
233 return 0;
234 }
235
236 SEC("?tc")
237 __failure __msg("cannot restore irq state out of order")
irq_restore_ooo(struct __sk_buff * ctx)238 int irq_restore_ooo(struct __sk_buff *ctx)
239 {
240 unsigned long flags1;
241 unsigned long flags2;
242
243 bpf_local_irq_save(&flags1);
244 bpf_local_irq_save(&flags2);
245 bpf_local_irq_restore(&flags1);
246 bpf_local_irq_restore(&flags2);
247 return 0;
248 }
249
250 SEC("?tc")
251 __failure __msg("cannot restore irq state out of order")
irq_restore_ooo_3(struct __sk_buff * ctx)252 int irq_restore_ooo_3(struct __sk_buff *ctx)
253 {
254 unsigned long flags1;
255 unsigned long flags2;
256 unsigned long flags3;
257
258 bpf_local_irq_save(&flags1);
259 bpf_local_irq_save(&flags2);
260 bpf_local_irq_restore(&flags2);
261 bpf_local_irq_save(&flags3);
262 bpf_local_irq_restore(&flags1);
263 bpf_local_irq_restore(&flags3);
264 return 0;
265 }
266
local_irq_save_3(unsigned long * flags1,unsigned long * flags2,unsigned long * flags3)267 static __noinline void local_irq_save_3(unsigned long *flags1, unsigned long *flags2,
268 unsigned long *flags3)
269 {
270 local_irq_save(flags1);
271 local_irq_save(flags2);
272 local_irq_save(flags3);
273 }
274
275 SEC("?tc")
276 __success
irq_restore_3_subprog(struct __sk_buff * ctx)277 int irq_restore_3_subprog(struct __sk_buff *ctx)
278 {
279 unsigned long flags1;
280 unsigned long flags2;
281 unsigned long flags3;
282
283 local_irq_save_3(&flags1, &flags2, &flags3);
284 bpf_local_irq_restore(&flags3);
285 bpf_local_irq_restore(&flags2);
286 bpf_local_irq_restore(&flags1);
287 return 0;
288 }
289
290 SEC("?tc")
291 __failure __msg("cannot restore irq state out of order")
irq_restore_4_subprog(struct __sk_buff * ctx)292 int irq_restore_4_subprog(struct __sk_buff *ctx)
293 {
294 unsigned long flags1;
295 unsigned long flags2;
296 unsigned long flags3;
297 unsigned long flags4;
298
299 local_irq_save_3(&flags1, &flags2, &flags3);
300 bpf_local_irq_restore(&flags3);
301 bpf_local_irq_save(&flags4);
302 bpf_local_irq_restore(&flags4);
303 bpf_local_irq_restore(&flags1);
304 return 0;
305 }
306
307 SEC("?tc")
308 __failure __msg("cannot restore irq state out of order")
irq_restore_ooo_3_subprog(struct __sk_buff * ctx)309 int irq_restore_ooo_3_subprog(struct __sk_buff *ctx)
310 {
311 unsigned long flags1;
312 unsigned long flags2;
313 unsigned long flags3;
314
315 local_irq_save_3(&flags1, &flags2, &flags3);
316 bpf_local_irq_restore(&flags3);
317 bpf_local_irq_restore(&flags2);
318 bpf_local_irq_save(&flags3);
319 bpf_local_irq_restore(&flags1);
320 return 0;
321 }
322
323 SEC("?tc")
324 __failure __msg("expected an initialized")
irq_restore_invalid(struct __sk_buff * ctx)325 int irq_restore_invalid(struct __sk_buff *ctx)
326 {
327 unsigned long flags1;
328 unsigned long flags = 0xfaceb00c;
329
330 bpf_local_irq_save(&flags1);
331 bpf_local_irq_restore(&flags);
332 return 0;
333 }
334
335 SEC("?tc")
336 __failure __msg("expected uninitialized")
irq_save_invalid(struct __sk_buff * ctx)337 int irq_save_invalid(struct __sk_buff *ctx)
338 {
339 unsigned long flags1;
340
341 bpf_local_irq_save(&flags1);
342 bpf_local_irq_save(&flags1);
343 return 0;
344 }
345
346 SEC("?tc")
347 __failure __msg("expected an initialized")
irq_restore_iter(struct __sk_buff * ctx)348 int irq_restore_iter(struct __sk_buff *ctx)
349 {
350 struct bpf_iter_num it;
351
352 bpf_iter_num_new(&it, 0, 42);
353 bpf_local_irq_restore((unsigned long *)&it);
354 return 0;
355 }
356
357 SEC("?tc")
358 __failure __msg("Unreleased reference id=1")
irq_save_iter(struct __sk_buff * ctx)359 int irq_save_iter(struct __sk_buff *ctx)
360 {
361 struct bpf_iter_num it;
362
363 /* Ensure same sized slot has st->ref_obj_id set, so we reject based on
364 * slot_type != STACK_IRQ_FLAG...
365 */
366 _Static_assert(sizeof(it) == sizeof(unsigned long), "broken iterator size");
367
368 bpf_iter_num_new(&it, 0, 42);
369 bpf_local_irq_save((unsigned long *)&it);
370 bpf_local_irq_restore((unsigned long *)&it);
371 return 0;
372 }
373
374 SEC("?tc")
375 __failure __msg("expected an initialized")
irq_flag_overwrite(struct __sk_buff * ctx)376 int irq_flag_overwrite(struct __sk_buff *ctx)
377 {
378 unsigned long flags;
379
380 bpf_local_irq_save(&flags);
381 flags = 0xdeadbeef;
382 bpf_local_irq_restore(&flags);
383 return 0;
384 }
385
386 SEC("?tc")
387 __failure __msg("expected an initialized")
irq_flag_overwrite_partial(struct __sk_buff * ctx)388 int irq_flag_overwrite_partial(struct __sk_buff *ctx)
389 {
390 unsigned long flags;
391
392 bpf_local_irq_save(&flags);
393 *(((char *)&flags) + 1) = 0xff;
394 bpf_local_irq_restore(&flags);
395 return 0;
396 }
397
398 SEC("?tc")
399 __failure __msg("cannot restore irq state out of order")
irq_ooo_refs_array(struct __sk_buff * ctx)400 int irq_ooo_refs_array(struct __sk_buff *ctx)
401 {
402 unsigned long flags[4];
403 struct { int i; } *p;
404
405 /* refs=1 */
406 bpf_local_irq_save(&flags[0]);
407
408 /* refs=1,2 */
409 p = bpf_obj_new(typeof(*p));
410 if (!p) {
411 bpf_local_irq_restore(&flags[0]);
412 return 0;
413 }
414
415 /* refs=1,2,3 */
416 bpf_local_irq_save(&flags[1]);
417
418 /* refs=1,2,3,4 */
419 bpf_local_irq_save(&flags[2]);
420
421 /* Now when we remove ref=2, the verifier must not break the ordering in
422 * the refs array between 1,3,4. With an older implementation, the
423 * verifier would swap the last element with the removed element, but to
424 * maintain the stack property we need to use memmove.
425 */
426 bpf_obj_drop(p);
427
428 /* Save and restore to reset active_irq_id to 3, as the ordering is now
429 * refs=1,4,3. When restoring the linear scan will find prev_id in order
430 * as 3 instead of 4.
431 */
432 bpf_local_irq_save(&flags[3]);
433 bpf_local_irq_restore(&flags[3]);
434
435 /* With the incorrect implementation, we can release flags[1], flags[2],
436 * and flags[0], i.e. in the wrong order.
437 */
438 bpf_local_irq_restore(&flags[1]);
439 bpf_local_irq_restore(&flags[2]);
440 bpf_local_irq_restore(&flags[0]);
441 return 0;
442 }
443
444 char _license[] SEC("license") = "GPL";
445