1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <vmlinux.h> 4 #include <bpf/bpf_helpers.h> 5 #include "bpf_misc.h" 6 #include "bpf_experimental.h" 7 8 /* From include/linux/filter.h */ 9 #define MAX_BPF_STACK 512 10 11 #if defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) 12 13 struct elem { 14 struct bpf_timer t; 15 char pad[256]; 16 }; 17 18 struct { 19 __uint(type, BPF_MAP_TYPE_ARRAY); 20 __uint(max_entries, 1); 21 __type(key, int); 22 __type(value, struct elem); 23 } array SEC(".maps"); 24 25 SEC("kprobe") 26 __description("Private stack, single prog") 27 __success 28 __arch_x86_64 29 __jited(" movabsq $0x{{.*}}, %r9") 30 __jited(" addq %gs:{{.*}}, %r9") 31 __jited(" movl $0x2a, %edi") 32 __jited(" movq %rdi, -0x100(%r9)") 33 __arch_arm64 34 __jited(" stp x25, x27, [sp, {{.*}}]!") 35 __jited(" mov x27, {{.*}}") 36 __jited(" movk x27, {{.*}}, lsl #16") 37 __jited(" movk x27, {{.*}}") 38 __jited(" mrs x10, TPIDR_EL{{[0-1]}}") 39 __jited(" add x27, x27, x10") 40 __jited(" add x25, x27, {{.*}}") 41 __jited(" mov x0, #0x2a") 42 __jited(" str x0, [x27]") 43 __jited("...") 44 __jited(" ldp x25, x27, [sp], {{.*}}") 45 __naked void private_stack_single_prog(void) 46 { 47 asm volatile (" \ 48 r1 = 42; \ 49 *(u64 *)(r10 - 256) = r1; \ 50 r0 = 0; \ 51 exit; \ 52 " ::: __clobber_all); 53 } 54 55 SEC("raw_tp") 56 __description("No private stack") 57 __success 58 __arch_x86_64 59 __jited(" subq $0x8, %rsp") 60 __arch_arm64 61 __jited(" mov x25, sp") 62 __jited(" sub sp, sp, #0x10") 63 __naked void no_private_stack_nested(void) 64 { 65 asm volatile (" \ 66 r1 = 42; \ 67 *(u64 *)(r10 - 8) = r1; \ 68 r0 = 0; \ 69 exit; \ 70 " ::: __clobber_all); 71 } 72 73 __used 74 __naked static void cumulative_stack_depth_subprog(void) 75 { 76 asm volatile (" \ 77 r1 = 41; \ 78 *(u64 *)(r10 - 32) = r1; \ 79 call %[bpf_get_smp_processor_id]; \ 80 exit; \ 81 " : 82 : __imm(bpf_get_smp_processor_id) 83 : __clobber_all); 84 } 85 86 SEC("kprobe") 87 __description("Private stack, subtree > MAX_BPF_STACK") 88 __success 89 __log_level(4) __msg("stack depth 512+32 max 512") 90 __arch_x86_64 91 /* private stack fp for the main prog */ 92 __jited(" movabsq $0x{{.*}}, %r9") 93 __jited(" addq %gs:{{.*}}, %r9") 94 __jited(" movl $0x2a, %edi") 95 __jited(" movq %rdi, -0x200(%r9)") 96 __jited(" pushq %r9") 97 __jited("...") 98 __jited(" callq 0x{{.*}}") 99 __jited(" popq %r9") 100 __jited(" xorl %eax, %eax") 101 __arch_arm64 102 __jited(" stp x25, x27, [sp, {{.*}}]!") 103 __jited(" mov x27, {{.*}}") 104 __jited(" movk x27, {{.*}}, lsl #16") 105 __jited(" movk x27, {{.*}}") 106 __jited(" mrs x10, TPIDR_EL{{[0-1]}}") 107 __jited(" add x27, x27, x10") 108 __jited(" add x25, x27, {{.*}}") 109 __jited(" mov x0, #0x2a") 110 __jited(" str x0, [x27]") 111 __jited(" bl {{.*}}") 112 __jited("...") 113 __jited(" ldp x25, x27, [sp], {{.*}}") 114 __naked void private_stack_nested_1(void) 115 { 116 asm volatile (" \ 117 r1 = 42; \ 118 *(u64 *)(r10 - %[max_bpf_stack]) = r1; \ 119 call cumulative_stack_depth_subprog; \ 120 r0 = 0; \ 121 exit; \ 122 " : 123 : __imm_const(max_bpf_stack, MAX_BPF_STACK) 124 : __clobber_all); 125 } 126 127 __naked __noinline __used 128 static unsigned long loop_callback(void) 129 { 130 asm volatile (" \ 131 call %[bpf_get_prandom_u32]; \ 132 r1 = 42; \ 133 *(u64 *)(r10 - 512) = r1; \ 134 call cumulative_stack_depth_subprog; \ 135 r0 = 0; \ 136 exit; \ 137 " : 138 : __imm(bpf_get_prandom_u32) 139 : __clobber_common); 140 } 141 142 SEC("raw_tp") 143 __description("Private stack, callback") 144 __success 145 __arch_x86_64 146 /* for func loop_callback */ 147 __jited("func #1") 148 __jited(" endbr64") 149 __jited(" nopl (%rax,%rax)") 150 __jited(" nopl (%rax)") 151 __jited(" pushq %rbp") 152 __jited(" movq %rsp, %rbp") 153 __jited(" endbr64") 154 __jited(" movabsq $0x{{.*}}, %r9") 155 __jited(" addq %gs:{{.*}}, %r9") 156 __jited(" pushq %r9") 157 __jited("...") 158 __jited(" callq") 159 __jited(" popq %r9") 160 __jited(" movl $0x2a, %edi") 161 __jited(" movq %rdi, -0x200(%r9)") 162 __jited(" pushq %r9") 163 __jited("...") 164 __jited(" callq") 165 __jited(" popq %r9") 166 __arch_arm64 167 __jited("func #1") 168 __jited("...") 169 __jited(" stp x25, x27, [sp, {{.*}}]!") 170 __jited(" mov x27, {{.*}}") 171 __jited(" movk x27, {{.*}}, lsl #16") 172 __jited(" movk x27, {{.*}}") 173 __jited(" mrs x10, TPIDR_EL{{[0-1]}}") 174 __jited(" add x27, x27, x10") 175 __jited(" add x25, x27, {{.*}}") 176 __jited(" bl 0x{{.*}}") 177 __jited(" mov x8, x0") 178 __jited(" mov x0, #0x2a") 179 __jited(" str x0, [x27]") 180 __jited(" bl 0x{{.*}}") 181 __jited(" mov x8, x0") 182 __jited(" mov x8, #0x0") 183 __jited(" ldp x25, x27, [sp], {{.*}}") 184 __naked void private_stack_callback(void) 185 { 186 asm volatile (" \ 187 r1 = 1; \ 188 r2 = %[loop_callback]; \ 189 r3 = 0; \ 190 r4 = 0; \ 191 call %[bpf_loop]; \ 192 r0 = 0; \ 193 exit; \ 194 " : 195 : __imm_ptr(loop_callback), 196 __imm(bpf_loop) 197 : __clobber_common); 198 } 199 200 SEC("fentry/bpf_fentry_test9") 201 __description("Private stack, exception in main prog") 202 __success __retval(0) 203 __arch_x86_64 204 __jited(" pushq %r9") 205 __jited("...") 206 __jited(" callq") 207 __jited(" popq %r9") 208 __arch_arm64 209 __jited(" stp x29, x30, [sp, #-0x10]!") 210 __jited(" mov x29, sp") 211 __jited(" stp xzr, x26, [sp, #-0x10]!") 212 __jited(" mov x26, sp") 213 __jited(" stp x19, x20, [sp, #-0x10]!") 214 __jited(" stp x21, x22, [sp, #-0x10]!") 215 __jited(" stp x23, x24, [sp, #-0x10]!") 216 __jited(" stp x25, x26, [sp, #-0x10]!") 217 __jited(" stp x27, x28, [sp, #-0x10]!") 218 __jited(" mov x27, {{.*}}") 219 __jited(" movk x27, {{.*}}, lsl #16") 220 __jited(" movk x27, {{.*}}") 221 __jited(" mrs x10, TPIDR_EL{{[0-1]}}") 222 __jited(" add x27, x27, x10") 223 __jited(" add x25, x27, {{.*}}") 224 __jited(" mov x0, #0x2a") 225 __jited(" str x0, [x27]") 226 __jited(" mov x0, #0x0") 227 __jited(" bl 0x{{.*}}") 228 __jited(" mov x8, x0") 229 __jited(" ldp x27, x28, [sp], #0x10") 230 int private_stack_exception_main_prog(void) 231 { 232 asm volatile (" \ 233 r1 = 42; \ 234 *(u64 *)(r10 - 512) = r1; \ 235 " ::: __clobber_common); 236 237 bpf_throw(0); 238 return 0; 239 } 240 241 __used static int subprog_exception(void) 242 { 243 bpf_throw(0); 244 return 0; 245 } 246 247 SEC("fentry/bpf_fentry_test9") 248 __description("Private stack, exception in subprog") 249 __success __retval(0) 250 __arch_x86_64 251 __jited(" movq %rdi, -0x200(%r9)") 252 __jited(" pushq %r9") 253 __jited("...") 254 __jited(" callq") 255 __jited(" popq %r9") 256 __arch_arm64 257 __jited(" stp x27, x28, [sp, #-0x10]!") 258 __jited(" mov x27, {{.*}}") 259 __jited(" movk x27, {{.*}}, lsl #16") 260 __jited(" movk x27, {{.*}}") 261 __jited(" mrs x10, TPIDR_EL{{[0-1]}}") 262 __jited(" add x27, x27, x10") 263 __jited(" add x25, x27, {{.*}}") 264 __jited(" mov x0, #0x2a") 265 __jited(" str x0, [x27]") 266 __jited(" bl 0x{{.*}}") 267 __jited(" mov x8, x0") 268 __jited(" ldp x27, x28, [sp], #0x10") 269 int private_stack_exception_sub_prog(void) 270 { 271 asm volatile (" \ 272 r1 = 42; \ 273 *(u64 *)(r10 - 512) = r1; \ 274 call subprog_exception; \ 275 " ::: __clobber_common); 276 277 return 0; 278 } 279 280 int glob; 281 __noinline static void subprog2(int *val) 282 { 283 glob += val[0] * 2; 284 } 285 286 __noinline static void subprog1(int *val) 287 { 288 int tmp[64] = {}; 289 290 tmp[0] = *val; 291 subprog2(tmp); 292 } 293 294 __noinline static int timer_cb1(void *map, int *key, struct bpf_timer *timer) 295 { 296 subprog1(key); 297 return 0; 298 } 299 300 __noinline static int timer_cb2(void *map, int *key, struct bpf_timer *timer) 301 { 302 return 0; 303 } 304 305 SEC("fentry/bpf_fentry_test9") 306 __description("Private stack, async callback, not nested") 307 __success __retval(0) 308 __arch_x86_64 309 __jited(" movabsq $0x{{.*}}, %r9") 310 __arch_arm64 311 __jited(" mrs x10, TPIDR_EL{{[0-1]}}") 312 __jited(" add x27, x27, x10") 313 __jited(" add x25, x27, {{.*}}") 314 int private_stack_async_callback_1(void) 315 { 316 struct bpf_timer *arr_timer; 317 int array_key = 0; 318 319 arr_timer = bpf_map_lookup_elem(&array, &array_key); 320 if (!arr_timer) 321 return 0; 322 323 bpf_timer_init(arr_timer, &array, 1); 324 bpf_timer_set_callback(arr_timer, timer_cb2); 325 bpf_timer_start(arr_timer, 0, 0); 326 subprog1(&array_key); 327 return 0; 328 } 329 330 SEC("fentry/bpf_fentry_test9") 331 __description("Private stack, async callback, potential nesting") 332 __success __retval(0) 333 __load_if_JITed() 334 __log_level(4) __msg("stack depth 8+0+256+0 max 272") 335 __arch_x86_64 336 __jited(" subq $0x100, %rsp") 337 __arch_arm64 338 __jited(" sub sp, sp, #0x100") 339 int private_stack_async_callback_2(void) 340 { 341 struct bpf_timer *arr_timer; 342 int array_key = 0; 343 344 arr_timer = bpf_map_lookup_elem(&array, &array_key); 345 if (!arr_timer) 346 return 0; 347 348 bpf_timer_init(arr_timer, &array, 1); 349 bpf_timer_set_callback(arr_timer, timer_cb1); 350 bpf_timer_start(arr_timer, 0, 0); 351 subprog1(&array_key); 352 return 0; 353 } 354 355 SEC("fentry/bpf_fentry_test9") 356 __description("private stack, max stack depth is private stack") 357 __success 358 __log_level(4) __msg("stack depth 8+256+0 max 256") 359 int private_stack_max_depth(void) 360 { 361 int x = 0; 362 363 subprog1(&x); 364 return 0; 365 } 366 367 #else 368 369 SEC("kprobe") 370 __description("private stack is not supported, use a dummy test") 371 __success 372 int dummy_test(void) 373 { 374 return 0; 375 } 376 377 #endif 378 379 char _license[] SEC("license") = "GPL"; 380