1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <vmlinux.h> 4 #include <bpf/bpf_helpers.h> 5 #include "bpf_misc.h" 6 #include "bpf_experimental.h" 7 8 /* From include/linux/filter.h */ 9 #define MAX_BPF_STACK 512 10 11 #if defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) 12 13 struct elem { 14 struct bpf_timer t; 15 char pad[256]; 16 }; 17 18 struct { 19 __uint(type, BPF_MAP_TYPE_ARRAY); 20 __uint(max_entries, 1); 21 __type(key, int); 22 __type(value, struct elem); 23 } array SEC(".maps"); 24 25 SEC("kprobe") 26 __description("Private stack, single prog") 27 __success 28 __arch_x86_64 29 __jited(" movabsq $0x{{.*}}, %r9") 30 __jited(" addq %gs:{{.*}}, %r9") 31 __jited(" movl $0x2a, %edi") 32 __jited(" movq %rdi, -0x100(%r9)") 33 __arch_arm64 34 __jited(" stp x25, x27, [sp, {{.*}}]!") 35 __jited(" mov x27, {{.*}}") 36 __jited(" movk x27, {{.*}}, lsl #16") 37 __jited(" movk x27, {{.*}}") 38 __jited(" mrs x10, TPIDR_EL{{[0-1]}}") 39 __jited(" add x27, x27, x10") 40 __jited(" add x25, x27, {{.*}}") 41 __jited(" mov x0, #0x2a") 42 __jited(" str x0, [x27]") 43 __jited("...") 44 __jited(" ldp x25, x27, [sp], {{.*}}") 45 __naked void private_stack_single_prog(void) 46 { 47 asm volatile (" \ 48 r1 = 42; \ 49 *(u64 *)(r10 - 256) = r1; \ 50 r0 = 0; \ 51 exit; \ 52 " ::: __clobber_all); 53 } 54 55 SEC("raw_tp") 56 __description("No private stack") 57 __success 58 __arch_x86_64 59 __jited(" subq $0x8, %rsp") 60 __arch_arm64 61 __jited(" mov x25, sp") 62 __jited(" sub sp, sp, #0x10") 63 __naked void no_private_stack_nested(void) 64 { 65 asm volatile (" \ 66 r1 = 42; \ 67 *(u64 *)(r10 - 8) = r1; \ 68 r0 = 0; \ 69 exit; \ 70 " ::: __clobber_all); 71 } 72 73 __used 74 __naked static void cumulative_stack_depth_subprog(void) 75 { 76 asm volatile (" \ 77 r1 = 41; \ 78 *(u64 *)(r10 - 32) = r1; \ 79 call %[bpf_get_smp_processor_id]; \ 80 exit; \ 81 " : 82 : __imm(bpf_get_smp_processor_id) 83 : __clobber_all); 84 } 85 86 SEC("kprobe") 87 __description("Private stack, subtree > MAX_BPF_STACK") 88 __success 89 __log_level(4) __msg("stack depth max 512") 90 __msg("subprog 0 (private_stack_nested_1) main {{.*}} stack 512") 91 __msg("subprog 1 (cumulative_stack_depth_subprog) static {{.*}} stack 32") 92 __arch_x86_64 93 /* private stack fp for the main prog */ 94 __jited(" movabsq $0x{{.*}}, %r9") 95 __jited(" addq %gs:{{.*}}, %r9") 96 __jited(" movl $0x2a, %edi") 97 __jited(" movq %rdi, -0x200(%r9)") 98 __jited(" pushq %r9") 99 __jited("...") 100 __jited(" callq 0x{{.*}}") 101 __jited(" popq %r9") 102 __jited(" xorl %eax, %eax") 103 __arch_arm64 104 __jited(" stp x25, x27, [sp, {{.*}}]!") 105 __jited(" mov x27, {{.*}}") 106 __jited(" movk x27, {{.*}}, lsl #16") 107 __jited(" movk x27, {{.*}}") 108 __jited(" mrs x10, TPIDR_EL{{[0-1]}}") 109 __jited(" add x27, x27, x10") 110 __jited(" add x25, x27, {{.*}}") 111 __jited(" mov x0, #0x2a") 112 __jited(" str x0, [x27]") 113 __jited(" bl {{.*}}") 114 __jited("...") 115 __jited(" ldp x25, x27, [sp], {{.*}}") 116 __naked void private_stack_nested_1(void) 117 { 118 asm volatile (" \ 119 r1 = 42; \ 120 *(u64 *)(r10 - %[max_bpf_stack]) = r1; \ 121 call cumulative_stack_depth_subprog; \ 122 r0 = 0; \ 123 exit; \ 124 " : 125 : __imm_const(max_bpf_stack, MAX_BPF_STACK) 126 : __clobber_all); 127 } 128 129 __naked __noinline __used 130 static unsigned long loop_callback(void) 131 { 132 asm volatile (" \ 133 call %[bpf_get_prandom_u32]; \ 134 r1 = 42; \ 135 *(u64 *)(r10 - 512) = r1; \ 136 call cumulative_stack_depth_subprog; \ 137 r0 = 0; \ 138 exit; \ 139 " : 140 : __imm(bpf_get_prandom_u32) 141 : __clobber_common); 142 } 143 144 SEC("raw_tp") 145 __description("Private stack, callback") 146 __success 147 __arch_x86_64 148 /* for func loop_callback */ 149 __jited("func #1") 150 __jited(" endbr64") 151 __jited(" nopl (%rax,%rax)") 152 __jited(" nopl (%rax)") 153 __jited(" pushq %rbp") 154 __jited(" movq %rsp, %rbp") 155 __jited(" endbr64") 156 __jited(" movabsq $0x{{.*}}, %r9") 157 __jited(" addq %gs:{{.*}}, %r9") 158 __jited(" pushq %r9") 159 __jited("...") 160 __jited(" callq") 161 __jited(" popq %r9") 162 __jited(" movl $0x2a, %edi") 163 __jited(" movq %rdi, -0x200(%r9)") 164 __jited(" pushq %r9") 165 __jited("...") 166 __jited(" callq") 167 __jited(" popq %r9") 168 __arch_arm64 169 __jited("func #1") 170 __jited("...") 171 __jited(" stp x25, x27, [sp, {{.*}}]!") 172 __jited(" mov x27, {{.*}}") 173 __jited(" movk x27, {{.*}}, lsl #16") 174 __jited(" movk x27, {{.*}}") 175 __jited(" mrs x10, TPIDR_EL{{[0-1]}}") 176 __jited(" add x27, x27, x10") 177 __jited(" add x25, x27, {{.*}}") 178 __jited(" bl 0x{{.*}}") 179 __jited(" mov x8, x0") 180 __jited(" mov x0, #0x2a") 181 __jited(" str x0, [x27]") 182 __jited(" bl 0x{{.*}}") 183 __jited(" mov x8, x0") 184 __jited(" mov x8, #0x0") 185 __jited(" ldp x25, x27, [sp], {{.*}}") 186 __naked void private_stack_callback(void) 187 { 188 asm volatile (" \ 189 r1 = 1; \ 190 r2 = %[loop_callback]; \ 191 r3 = 0; \ 192 r4 = 0; \ 193 call %[bpf_loop]; \ 194 r0 = 0; \ 195 exit; \ 196 " : 197 : __imm_ptr(loop_callback), 198 __imm(bpf_loop) 199 : __clobber_common); 200 } 201 202 SEC("fentry/bpf_fentry_test9") 203 __description("Private stack, exception in main prog") 204 __success __retval(0) 205 __arch_x86_64 206 __jited(" pushq %r9") 207 __jited("...") 208 __jited(" callq") 209 __jited(" popq %r9") 210 __arch_arm64 211 __jited(" stp x29, x30, [sp, #-0x10]!") 212 __jited(" mov x29, sp") 213 __jited(" stp xzr, x26, [sp, #-0x10]!") 214 __jited(" mov x26, sp") 215 __jited(" stp x19, x20, [sp, #-0x10]!") 216 __jited(" stp x21, x22, [sp, #-0x10]!") 217 __jited(" stp x23, x24, [sp, #-0x10]!") 218 __jited(" stp x25, x26, [sp, #-0x10]!") 219 __jited(" stp x27, x28, [sp, #-0x10]!") 220 __jited(" mov x27, {{.*}}") 221 __jited(" movk x27, {{.*}}, lsl #16") 222 __jited(" movk x27, {{.*}}") 223 __jited(" mrs x10, TPIDR_EL{{[0-1]}}") 224 __jited(" add x27, x27, x10") 225 __jited(" add x25, x27, {{.*}}") 226 __jited(" mov x0, #0x2a") 227 __jited(" str x0, [x27]") 228 __jited(" mov x0, #0x0") 229 __jited(" bl 0x{{.*}}") 230 __jited(" mov x8, x0") 231 __jited(" ldp x27, x28, [sp], #0x10") 232 int private_stack_exception_main_prog(void) 233 { 234 asm volatile (" \ 235 r1 = 42; \ 236 *(u64 *)(r10 - 512) = r1; \ 237 " ::: __clobber_common); 238 239 bpf_throw(0); 240 return 0; 241 } 242 243 __used static int subprog_exception(void) 244 { 245 bpf_throw(0); 246 return 0; 247 } 248 249 SEC("fentry/bpf_fentry_test9") 250 __description("Private stack, exception in subprog") 251 __success __retval(0) 252 __arch_x86_64 253 __jited(" movq %rdi, -0x200(%r9)") 254 __jited(" pushq %r9") 255 __jited("...") 256 __jited(" callq") 257 __jited(" popq %r9") 258 __arch_arm64 259 __jited(" stp x27, x28, [sp, #-0x10]!") 260 __jited(" mov x27, {{.*}}") 261 __jited(" movk x27, {{.*}}, lsl #16") 262 __jited(" movk x27, {{.*}}") 263 __jited(" mrs x10, TPIDR_EL{{[0-1]}}") 264 __jited(" add x27, x27, x10") 265 __jited(" add x25, x27, {{.*}}") 266 __jited(" mov x0, #0x2a") 267 __jited(" str x0, [x27]") 268 __jited(" bl 0x{{.*}}") 269 __jited(" mov x8, x0") 270 __jited(" ldp x27, x28, [sp], #0x10") 271 int private_stack_exception_sub_prog(void) 272 { 273 asm volatile (" \ 274 r1 = 42; \ 275 *(u64 *)(r10 - 512) = r1; \ 276 call subprog_exception; \ 277 " ::: __clobber_common); 278 279 return 0; 280 } 281 282 int glob; 283 __noinline static void subprog2(int *val) 284 { 285 glob += val[0] * 2; 286 } 287 288 __noinline static void subprog1(int *val) 289 { 290 int tmp[64] = {}; 291 292 tmp[0] = *val; 293 subprog2(tmp); 294 } 295 296 __noinline static int timer_cb1(void *map, int *key, struct bpf_timer *timer) 297 { 298 subprog1(key); 299 return 0; 300 } 301 302 __noinline static int timer_cb2(void *map, int *key, struct bpf_timer *timer) 303 { 304 return 0; 305 } 306 307 SEC("fentry/bpf_fentry_test9") 308 __description("Private stack, async callback, not nested") 309 __success __retval(0) 310 __arch_x86_64 311 __jited(" movabsq $0x{{.*}}, %r9") 312 __arch_arm64 313 __jited(" mrs x10, TPIDR_EL{{[0-1]}}") 314 __jited(" add x27, x27, x10") 315 __jited(" add x25, x27, {{.*}}") 316 int private_stack_async_callback_1(void) 317 { 318 struct bpf_timer *arr_timer; 319 int array_key = 0; 320 321 arr_timer = bpf_map_lookup_elem(&array, &array_key); 322 if (!arr_timer) 323 return 0; 324 325 bpf_timer_init(arr_timer, &array, 1); 326 bpf_timer_set_callback(arr_timer, timer_cb2); 327 bpf_timer_start(arr_timer, 0, 0); 328 subprog1(&array_key); 329 return 0; 330 } 331 332 SEC("fentry/bpf_fentry_test9") 333 __description("Private stack, async callback, potential nesting") 334 __success __retval(0) 335 __load_if_JITed() 336 __log_level(4) __msg("stack depth max 272") 337 __msg("subprog 0 (private_stack_async_callback_2) main {{.*}} stack 8") 338 __msg("subprog 1 (timer_cb1) static {{.*}} stack 0") 339 __msg("subprog 2 (subprog1) static {{.*}} stack 256") 340 __msg("subprog 3 (subprog2) static {{.*}} stack 0") 341 __arch_x86_64 342 __jited(" subq $0x100, %rsp") 343 __arch_arm64 344 __jited(" sub sp, sp, #0x100") 345 int private_stack_async_callback_2(void) 346 { 347 struct bpf_timer *arr_timer; 348 int array_key = 0; 349 350 arr_timer = bpf_map_lookup_elem(&array, &array_key); 351 if (!arr_timer) 352 return 0; 353 354 bpf_timer_init(arr_timer, &array, 1); 355 bpf_timer_set_callback(arr_timer, timer_cb1); 356 bpf_timer_start(arr_timer, 0, 0); 357 subprog1(&array_key); 358 return 0; 359 } 360 361 SEC("fentry/bpf_fentry_test9") 362 __description("private stack, max stack depth is private stack") 363 __success 364 __log_level(4) __msg("stack depth max 256") 365 __msg("subprog 0 (private_stack_max_depth) main {{.*}} stack 8") 366 __msg("subprog 1 (subprog1) static insns_self {{[0-9]+}} insns_total {{[0-9]+}} stack 256") 367 __msg("subprog 2 (subprog2) static insns_self {{[0-9]+}} insns_total {{[0-9]+}} stack 0") 368 int private_stack_max_depth(void) 369 { 370 int x = 0; 371 372 subprog1(&x); 373 return 0; 374 } 375 376 #else 377 378 SEC("kprobe") 379 __description("private stack is not supported, use a dummy test") 380 __success 381 int dummy_test(void) 382 { 383 return 0; 384 } 385 386 #endif 387 388 char _license[] SEC("license") = "GPL"; 389