1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2024 Meta Platforms, Inc. and affiliates. */ 3 #include <vmlinux.h> 4 #include <bpf/bpf_helpers.h> 5 #include "bpf_misc.h" 6 #include "bpf_experimental.h" 7 8 unsigned long global_flags; 9 10 extern void bpf_local_irq_save(unsigned long *) __weak __ksym; 11 extern void bpf_local_irq_restore(unsigned long *) __weak __ksym; 12 extern int bpf_copy_from_user_str(void *dst, u32 dst__sz, const void *unsafe_ptr__ign, u64 flags) __weak __ksym; 13 14 SEC("?tc") 15 __failure __msg("arg#0 doesn't point to an irq flag on stack") 16 int irq_save_bad_arg(struct __sk_buff *ctx) 17 { 18 bpf_local_irq_save(&global_flags); 19 return 0; 20 } 21 22 SEC("?tc") 23 __failure __msg("arg#0 doesn't point to an irq flag on stack") 24 int irq_restore_bad_arg(struct __sk_buff *ctx) 25 { 26 bpf_local_irq_restore(&global_flags); 27 return 0; 28 } 29 30 SEC("?tc") 31 __failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region") 32 int irq_restore_missing_2(struct __sk_buff *ctx) 33 { 34 unsigned long flags1; 35 unsigned long flags2; 36 37 bpf_local_irq_save(&flags1); 38 bpf_local_irq_save(&flags2); 39 return 0; 40 } 41 42 SEC("?tc") 43 __failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region") 44 int irq_restore_missing_3(struct __sk_buff *ctx) 45 { 46 unsigned long flags1; 47 unsigned long flags2; 48 unsigned long flags3; 49 50 bpf_local_irq_save(&flags1); 51 bpf_local_irq_save(&flags2); 52 bpf_local_irq_save(&flags3); 53 return 0; 54 } 55 56 SEC("?tc") 57 __failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region") 58 int irq_restore_missing_3_minus_2(struct __sk_buff *ctx) 59 { 60 unsigned long flags1; 61 unsigned long flags2; 62 unsigned long flags3; 63 64 bpf_local_irq_save(&flags1); 65 bpf_local_irq_save(&flags2); 66 bpf_local_irq_save(&flags3); 67 bpf_local_irq_restore(&flags3); 68 bpf_local_irq_restore(&flags2); 69 return 0; 70 } 71 72 static __noinline void local_irq_save(unsigned long *flags) 73 { 74 bpf_local_irq_save(flags); 75 } 76 77 static __noinline void local_irq_restore(unsigned long *flags) 78 { 79 bpf_local_irq_restore(flags); 80 } 81 82 SEC("?tc") 83 __failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region") 84 int irq_restore_missing_1_subprog(struct __sk_buff *ctx) 85 { 86 unsigned long flags; 87 88 local_irq_save(&flags); 89 return 0; 90 } 91 92 SEC("?tc") 93 __failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region") 94 int irq_restore_missing_2_subprog(struct __sk_buff *ctx) 95 { 96 unsigned long flags1; 97 unsigned long flags2; 98 99 local_irq_save(&flags1); 100 local_irq_save(&flags2); 101 return 0; 102 } 103 104 SEC("?tc") 105 __failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region") 106 int irq_restore_missing_3_subprog(struct __sk_buff *ctx) 107 { 108 unsigned long flags1; 109 unsigned long flags2; 110 unsigned long flags3; 111 112 local_irq_save(&flags1); 113 local_irq_save(&flags2); 114 local_irq_save(&flags3); 115 return 0; 116 } 117 118 SEC("?tc") 119 __failure __msg("BPF_EXIT instruction in main prog cannot be used inside bpf_local_irq_save-ed region") 120 int irq_restore_missing_3_minus_2_subprog(struct __sk_buff *ctx) 121 { 122 unsigned long flags1; 123 unsigned long flags2; 124 unsigned long flags3; 125 126 local_irq_save(&flags1); 127 local_irq_save(&flags2); 128 local_irq_save(&flags3); 129 local_irq_restore(&flags3); 130 local_irq_restore(&flags2); 131 return 0; 132 } 133 134 SEC("?tc") 135 __success 136 int irq_balance(struct __sk_buff *ctx) 137 { 138 unsigned long flags; 139 140 local_irq_save(&flags); 141 local_irq_restore(&flags); 142 return 0; 143 } 144 145 SEC("?tc") 146 __success 147 int irq_balance_n(struct __sk_buff *ctx) 148 { 149 unsigned long flags1; 150 unsigned long flags2; 151 unsigned long flags3; 152 153 local_irq_save(&flags1); 154 local_irq_save(&flags2); 155 local_irq_save(&flags3); 156 local_irq_restore(&flags3); 157 local_irq_restore(&flags2); 158 local_irq_restore(&flags1); 159 return 0; 160 } 161 162 static __noinline void local_irq_balance(void) 163 { 164 unsigned long flags; 165 166 local_irq_save(&flags); 167 local_irq_restore(&flags); 168 } 169 170 static __noinline void local_irq_balance_n(void) 171 { 172 unsigned long flags1; 173 unsigned long flags2; 174 unsigned long flags3; 175 176 local_irq_save(&flags1); 177 local_irq_save(&flags2); 178 local_irq_save(&flags3); 179 local_irq_restore(&flags3); 180 local_irq_restore(&flags2); 181 local_irq_restore(&flags1); 182 } 183 184 SEC("?tc") 185 __success 186 int irq_balance_subprog(struct __sk_buff *ctx) 187 { 188 local_irq_balance(); 189 return 0; 190 } 191 192 SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") 193 __failure __msg("sleepable helper bpf_copy_from_user#") 194 int irq_sleepable_helper(void *ctx) 195 { 196 unsigned long flags; 197 u32 data; 198 199 local_irq_save(&flags); 200 bpf_copy_from_user(&data, sizeof(data), NULL); 201 local_irq_restore(&flags); 202 return 0; 203 } 204 205 SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") 206 __failure __msg("kernel func bpf_copy_from_user_str is sleepable within IRQ-disabled region") 207 int irq_sleepable_kfunc(void *ctx) 208 { 209 unsigned long flags; 210 u32 data; 211 212 local_irq_save(&flags); 213 bpf_copy_from_user_str(&data, sizeof(data), NULL, 0); 214 local_irq_restore(&flags); 215 return 0; 216 } 217 218 int __noinline global_local_irq_balance(void) 219 { 220 local_irq_balance_n(); 221 return 0; 222 } 223 224 SEC("?tc") 225 __failure __msg("global function calls are not allowed with IRQs disabled") 226 int irq_global_subprog(struct __sk_buff *ctx) 227 { 228 unsigned long flags; 229 230 bpf_local_irq_save(&flags); 231 global_local_irq_balance(); 232 bpf_local_irq_restore(&flags); 233 return 0; 234 } 235 236 SEC("?tc") 237 __failure __msg("cannot restore irq state out of order") 238 int irq_restore_ooo(struct __sk_buff *ctx) 239 { 240 unsigned long flags1; 241 unsigned long flags2; 242 243 bpf_local_irq_save(&flags1); 244 bpf_local_irq_save(&flags2); 245 bpf_local_irq_restore(&flags1); 246 bpf_local_irq_restore(&flags2); 247 return 0; 248 } 249 250 SEC("?tc") 251 __failure __msg("cannot restore irq state out of order") 252 int irq_restore_ooo_3(struct __sk_buff *ctx) 253 { 254 unsigned long flags1; 255 unsigned long flags2; 256 unsigned long flags3; 257 258 bpf_local_irq_save(&flags1); 259 bpf_local_irq_save(&flags2); 260 bpf_local_irq_restore(&flags2); 261 bpf_local_irq_save(&flags3); 262 bpf_local_irq_restore(&flags1); 263 bpf_local_irq_restore(&flags3); 264 return 0; 265 } 266 267 static __noinline void local_irq_save_3(unsigned long *flags1, unsigned long *flags2, 268 unsigned long *flags3) 269 { 270 local_irq_save(flags1); 271 local_irq_save(flags2); 272 local_irq_save(flags3); 273 } 274 275 SEC("?tc") 276 __success 277 int irq_restore_3_subprog(struct __sk_buff *ctx) 278 { 279 unsigned long flags1; 280 unsigned long flags2; 281 unsigned long flags3; 282 283 local_irq_save_3(&flags1, &flags2, &flags3); 284 bpf_local_irq_restore(&flags3); 285 bpf_local_irq_restore(&flags2); 286 bpf_local_irq_restore(&flags1); 287 return 0; 288 } 289 290 SEC("?tc") 291 __failure __msg("cannot restore irq state out of order") 292 int irq_restore_4_subprog(struct __sk_buff *ctx) 293 { 294 unsigned long flags1; 295 unsigned long flags2; 296 unsigned long flags3; 297 unsigned long flags4; 298 299 local_irq_save_3(&flags1, &flags2, &flags3); 300 bpf_local_irq_restore(&flags3); 301 bpf_local_irq_save(&flags4); 302 bpf_local_irq_restore(&flags4); 303 bpf_local_irq_restore(&flags1); 304 return 0; 305 } 306 307 SEC("?tc") 308 __failure __msg("cannot restore irq state out of order") 309 int irq_restore_ooo_3_subprog(struct __sk_buff *ctx) 310 { 311 unsigned long flags1; 312 unsigned long flags2; 313 unsigned long flags3; 314 315 local_irq_save_3(&flags1, &flags2, &flags3); 316 bpf_local_irq_restore(&flags3); 317 bpf_local_irq_restore(&flags2); 318 bpf_local_irq_save(&flags3); 319 bpf_local_irq_restore(&flags1); 320 return 0; 321 } 322 323 SEC("?tc") 324 __failure __msg("expected an initialized") 325 int irq_restore_invalid(struct __sk_buff *ctx) 326 { 327 unsigned long flags1; 328 unsigned long flags = 0xfaceb00c; 329 330 bpf_local_irq_save(&flags1); 331 bpf_local_irq_restore(&flags); 332 return 0; 333 } 334 335 SEC("?tc") 336 __failure __msg("expected uninitialized") 337 int irq_save_invalid(struct __sk_buff *ctx) 338 { 339 unsigned long flags1; 340 341 bpf_local_irq_save(&flags1); 342 bpf_local_irq_save(&flags1); 343 return 0; 344 } 345 346 SEC("?tc") 347 __failure __msg("expected an initialized") 348 int irq_restore_iter(struct __sk_buff *ctx) 349 { 350 struct bpf_iter_num it; 351 352 bpf_iter_num_new(&it, 0, 42); 353 bpf_local_irq_restore((unsigned long *)&it); 354 return 0; 355 } 356 357 SEC("?tc") 358 __failure __msg("Unreleased reference id=1") 359 int irq_save_iter(struct __sk_buff *ctx) 360 { 361 struct bpf_iter_num it; 362 363 /* Ensure same sized slot has st->ref_obj_id set, so we reject based on 364 * slot_type != STACK_IRQ_FLAG... 365 */ 366 _Static_assert(sizeof(it) == sizeof(unsigned long), "broken iterator size"); 367 368 bpf_iter_num_new(&it, 0, 42); 369 bpf_local_irq_save((unsigned long *)&it); 370 bpf_local_irq_restore((unsigned long *)&it); 371 return 0; 372 } 373 374 SEC("?tc") 375 __failure __msg("expected an initialized") 376 int irq_flag_overwrite(struct __sk_buff *ctx) 377 { 378 unsigned long flags; 379 380 bpf_local_irq_save(&flags); 381 flags = 0xdeadbeef; 382 bpf_local_irq_restore(&flags); 383 return 0; 384 } 385 386 SEC("?tc") 387 __failure __msg("expected an initialized") 388 int irq_flag_overwrite_partial(struct __sk_buff *ctx) 389 { 390 unsigned long flags; 391 392 bpf_local_irq_save(&flags); 393 *(((char *)&flags) + 1) = 0xff; 394 bpf_local_irq_restore(&flags); 395 return 0; 396 } 397 398 SEC("?tc") 399 __failure __msg("cannot restore irq state out of order") 400 int irq_ooo_refs_array(struct __sk_buff *ctx) 401 { 402 unsigned long flags[4]; 403 struct { int i; } *p; 404 405 /* refs=1 */ 406 bpf_local_irq_save(&flags[0]); 407 408 /* refs=1,2 */ 409 p = bpf_obj_new(typeof(*p)); 410 if (!p) { 411 bpf_local_irq_restore(&flags[0]); 412 return 0; 413 } 414 415 /* refs=1,2,3 */ 416 bpf_local_irq_save(&flags[1]); 417 418 /* refs=1,2,3,4 */ 419 bpf_local_irq_save(&flags[2]); 420 421 /* Now when we remove ref=2, the verifier must not break the ordering in 422 * the refs array between 1,3,4. With an older implementation, the 423 * verifier would swap the last element with the removed element, but to 424 * maintain the stack property we need to use memmove. 425 */ 426 bpf_obj_drop(p); 427 428 /* Save and restore to reset active_irq_id to 3, as the ordering is now 429 * refs=1,4,3. When restoring the linear scan will find prev_id in order 430 * as 3 instead of 4. 431 */ 432 bpf_local_irq_save(&flags[3]); 433 bpf_local_irq_restore(&flags[3]); 434 435 /* With the incorrect implementation, we can release flags[1], flags[2], 436 * and flags[0], i.e. in the wrong order. 437 */ 438 bpf_local_irq_restore(&flags[1]); 439 bpf_local_irq_restore(&flags[2]); 440 bpf_local_irq_restore(&flags[0]); 441 return 0; 442 } 443 444 char _license[] SEC("license") = "GPL"; 445