1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <vmlinux.h> 4 #include <bpf/bpf_helpers.h> 5 #include "bpf_misc.h" 6 #include "bpf_experimental.h" 7 8 /* From include/linux/filter.h */ 9 #define MAX_BPF_STACK 512 10 11 #if defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) 12 13 struct elem { 14 struct bpf_timer t; 15 char pad[256]; 16 }; 17 18 struct { 19 __uint(type, BPF_MAP_TYPE_ARRAY); 20 __uint(max_entries, 1); 21 __type(key, int); 22 __type(value, struct elem); 23 } array SEC(".maps"); 24 25 SEC("kprobe") 26 __description("Private stack, single prog") 27 __success 28 __arch_x86_64 29 __jited(" movabsq $0x{{.*}}, %r9") 30 __jited(" addq %gs:{{.*}}, %r9") 31 __jited(" movl $0x2a, %edi") 32 __jited(" movq %rdi, -0x100(%r9)") 33 __arch_arm64 34 __jited(" stp x25, x27, [sp, {{.*}}]!") 35 __jited(" mov x27, {{.*}}") 36 __jited(" movk x27, {{.*}}, lsl #16") 37 __jited(" movk x27, {{.*}}") 38 __jited(" mrs x10, TPIDR_EL{{[0-1]}}") 39 __jited(" add x27, x27, x10") 40 __jited(" add x25, x27, {{.*}}") 41 __jited(" mov x0, #0x2a") 42 __jited(" str x0, [x27]") 43 __jited("...") 44 __jited(" ldp x25, x27, [sp], {{.*}}") 45 __naked void private_stack_single_prog(void) 46 { 47 asm volatile (" \ 48 r1 = 42; \ 49 *(u64 *)(r10 - 256) = r1; \ 50 r0 = 0; \ 51 exit; \ 52 " ::: __clobber_all); 53 } 54 55 SEC("raw_tp") 56 __description("No private stack") 57 __success 58 __arch_x86_64 59 __jited(" subq $0x8, %rsp") 60 __arch_arm64 61 __jited(" mov x25, sp") 62 __jited(" sub sp, sp, #0x10") 63 __naked void no_private_stack_nested(void) 64 { 65 asm volatile (" \ 66 r1 = 42; \ 67 *(u64 *)(r10 - 8) = r1; \ 68 r0 = 0; \ 69 exit; \ 70 " ::: __clobber_all); 71 } 72 73 __used 74 __naked static void cumulative_stack_depth_subprog(void) 75 { 76 asm volatile (" \ 77 r1 = 41; \ 78 *(u64 *)(r10 - 32) = r1; \ 79 call %[bpf_get_smp_processor_id]; \ 80 exit; \ 81 " : 82 : __imm(bpf_get_smp_processor_id) 83 : __clobber_all); 84 } 85 86 SEC("kprobe") 87 __description("Private stack, subtree > MAX_BPF_STACK") 88 __success 89 __log_level(4) __msg("stack depth 512+32 max 512") 90 __arch_x86_64 91 /* private stack fp for the main prog */ 92 __jited(" movabsq $0x{{.*}}, %r9") 93 __jited(" addq %gs:{{.*}}, %r9") 94 __jited(" movl $0x2a, %edi") 95 __jited(" movq %rdi, -0x200(%r9)") 96 __jited(" pushq %r9") 97 __jited(" callq 0x{{.*}}") 98 __jited(" popq %r9") 99 __jited(" xorl %eax, %eax") 100 __arch_arm64 101 __jited(" stp x25, x27, [sp, {{.*}}]!") 102 __jited(" mov x27, {{.*}}") 103 __jited(" movk x27, {{.*}}, lsl #16") 104 __jited(" movk x27, {{.*}}") 105 __jited(" mrs x10, TPIDR_EL{{[0-1]}}") 106 __jited(" add x27, x27, x10") 107 __jited(" add x25, x27, {{.*}}") 108 __jited(" mov x0, #0x2a") 109 __jited(" str x0, [x27]") 110 __jited(" bl {{.*}}") 111 __jited("...") 112 __jited(" ldp x25, x27, [sp], {{.*}}") 113 __naked void private_stack_nested_1(void) 114 { 115 asm volatile (" \ 116 r1 = 42; \ 117 *(u64 *)(r10 - %[max_bpf_stack]) = r1; \ 118 call cumulative_stack_depth_subprog; \ 119 r0 = 0; \ 120 exit; \ 121 " : 122 : __imm_const(max_bpf_stack, MAX_BPF_STACK) 123 : __clobber_all); 124 } 125 126 __naked __noinline __used 127 static unsigned long loop_callback(void) 128 { 129 asm volatile (" \ 130 call %[bpf_get_prandom_u32]; \ 131 r1 = 42; \ 132 *(u64 *)(r10 - 512) = r1; \ 133 call cumulative_stack_depth_subprog; \ 134 r0 = 0; \ 135 exit; \ 136 " : 137 : __imm(bpf_get_prandom_u32) 138 : __clobber_common); 139 } 140 141 SEC("raw_tp") 142 __description("Private stack, callback") 143 __success 144 __arch_x86_64 145 /* for func loop_callback */ 146 __jited("func #1") 147 __jited(" endbr64") 148 __jited(" nopl (%rax,%rax)") 149 __jited(" nopl (%rax)") 150 __jited(" pushq %rbp") 151 __jited(" movq %rsp, %rbp") 152 __jited(" endbr64") 153 __jited(" movabsq $0x{{.*}}, %r9") 154 __jited(" addq %gs:{{.*}}, %r9") 155 __jited(" pushq %r9") 156 __jited(" callq") 157 __jited(" popq %r9") 158 __jited(" movl $0x2a, %edi") 159 __jited(" movq %rdi, -0x200(%r9)") 160 __jited(" pushq %r9") 161 __jited(" callq") 162 __jited(" popq %r9") 163 __arch_arm64 164 __jited("func #1") 165 __jited("...") 166 __jited(" stp x25, x27, [sp, {{.*}}]!") 167 __jited(" mov x27, {{.*}}") 168 __jited(" movk x27, {{.*}}, lsl #16") 169 __jited(" movk x27, {{.*}}") 170 __jited(" mrs x10, TPIDR_EL{{[0-1]}}") 171 __jited(" add x27, x27, x10") 172 __jited(" add x25, x27, {{.*}}") 173 __jited(" bl 0x{{.*}}") 174 __jited(" mov x8, x0") 175 __jited(" mov x0, #0x2a") 176 __jited(" str x0, [x27]") 177 __jited(" bl 0x{{.*}}") 178 __jited(" mov x8, x0") 179 __jited(" mov x8, #0x0") 180 __jited(" ldp x25, x27, [sp], {{.*}}") 181 __naked void private_stack_callback(void) 182 { 183 asm volatile (" \ 184 r1 = 1; \ 185 r2 = %[loop_callback]; \ 186 r3 = 0; \ 187 r4 = 0; \ 188 call %[bpf_loop]; \ 189 r0 = 0; \ 190 exit; \ 191 " : 192 : __imm_ptr(loop_callback), 193 __imm(bpf_loop) 194 : __clobber_common); 195 } 196 197 SEC("fentry/bpf_fentry_test9") 198 __description("Private stack, exception in main prog") 199 __success __retval(0) 200 __arch_x86_64 201 __jited(" pushq %r9") 202 __jited(" callq") 203 __jited(" popq %r9") 204 __arch_arm64 205 __jited(" stp x29, x30, [sp, #-0x10]!") 206 __jited(" mov x29, sp") 207 __jited(" stp xzr, x26, [sp, #-0x10]!") 208 __jited(" mov x26, sp") 209 __jited(" stp x19, x20, [sp, #-0x10]!") 210 __jited(" stp x21, x22, [sp, #-0x10]!") 211 __jited(" stp x23, x24, [sp, #-0x10]!") 212 __jited(" stp x25, x26, [sp, #-0x10]!") 213 __jited(" stp x27, x28, [sp, #-0x10]!") 214 __jited(" mov x27, {{.*}}") 215 __jited(" movk x27, {{.*}}, lsl #16") 216 __jited(" movk x27, {{.*}}") 217 __jited(" mrs x10, TPIDR_EL{{[0-1]}}") 218 __jited(" add x27, x27, x10") 219 __jited(" add x25, x27, {{.*}}") 220 __jited(" mov x0, #0x2a") 221 __jited(" str x0, [x27]") 222 __jited(" mov x0, #0x0") 223 __jited(" bl 0x{{.*}}") 224 __jited(" mov x8, x0") 225 __jited(" ldp x27, x28, [sp], #0x10") 226 int private_stack_exception_main_prog(void) 227 { 228 asm volatile (" \ 229 r1 = 42; \ 230 *(u64 *)(r10 - 512) = r1; \ 231 " ::: __clobber_common); 232 233 bpf_throw(0); 234 return 0; 235 } 236 237 __used static int subprog_exception(void) 238 { 239 bpf_throw(0); 240 return 0; 241 } 242 243 SEC("fentry/bpf_fentry_test9") 244 __description("Private stack, exception in subprog") 245 __success __retval(0) 246 __arch_x86_64 247 __jited(" movq %rdi, -0x200(%r9)") 248 __jited(" pushq %r9") 249 __jited(" callq") 250 __jited(" popq %r9") 251 __arch_arm64 252 __jited(" stp x27, x28, [sp, #-0x10]!") 253 __jited(" mov x27, {{.*}}") 254 __jited(" movk x27, {{.*}}, lsl #16") 255 __jited(" movk x27, {{.*}}") 256 __jited(" mrs x10, TPIDR_EL{{[0-1]}}") 257 __jited(" add x27, x27, x10") 258 __jited(" add x25, x27, {{.*}}") 259 __jited(" mov x0, #0x2a") 260 __jited(" str x0, [x27]") 261 __jited(" bl 0x{{.*}}") 262 __jited(" mov x8, x0") 263 __jited(" ldp x27, x28, [sp], #0x10") 264 int private_stack_exception_sub_prog(void) 265 { 266 asm volatile (" \ 267 r1 = 42; \ 268 *(u64 *)(r10 - 512) = r1; \ 269 call subprog_exception; \ 270 " ::: __clobber_common); 271 272 return 0; 273 } 274 275 int glob; 276 __noinline static void subprog2(int *val) 277 { 278 glob += val[0] * 2; 279 } 280 281 __noinline static void subprog1(int *val) 282 { 283 int tmp[64] = {}; 284 285 tmp[0] = *val; 286 subprog2(tmp); 287 } 288 289 __noinline static int timer_cb1(void *map, int *key, struct bpf_timer *timer) 290 { 291 subprog1(key); 292 return 0; 293 } 294 295 __noinline static int timer_cb2(void *map, int *key, struct bpf_timer *timer) 296 { 297 return 0; 298 } 299 300 SEC("fentry/bpf_fentry_test9") 301 __description("Private stack, async callback, not nested") 302 __success __retval(0) 303 __arch_x86_64 304 __jited(" movabsq $0x{{.*}}, %r9") 305 __arch_arm64 306 __jited(" mrs x10, TPIDR_EL{{[0-1]}}") 307 __jited(" add x27, x27, x10") 308 __jited(" add x25, x27, {{.*}}") 309 int private_stack_async_callback_1(void) 310 { 311 struct bpf_timer *arr_timer; 312 int array_key = 0; 313 314 arr_timer = bpf_map_lookup_elem(&array, &array_key); 315 if (!arr_timer) 316 return 0; 317 318 bpf_timer_init(arr_timer, &array, 1); 319 bpf_timer_set_callback(arr_timer, timer_cb2); 320 bpf_timer_start(arr_timer, 0, 0); 321 subprog1(&array_key); 322 return 0; 323 } 324 325 SEC("fentry/bpf_fentry_test9") 326 __description("Private stack, async callback, potential nesting") 327 __success __retval(0) 328 __load_if_JITed() 329 __log_level(4) __msg("stack depth 8+0+256+0 max 272") 330 __arch_x86_64 331 __jited(" subq $0x100, %rsp") 332 __arch_arm64 333 __jited(" sub sp, sp, #0x100") 334 int private_stack_async_callback_2(void) 335 { 336 struct bpf_timer *arr_timer; 337 int array_key = 0; 338 339 arr_timer = bpf_map_lookup_elem(&array, &array_key); 340 if (!arr_timer) 341 return 0; 342 343 bpf_timer_init(arr_timer, &array, 1); 344 bpf_timer_set_callback(arr_timer, timer_cb1); 345 bpf_timer_start(arr_timer, 0, 0); 346 subprog1(&array_key); 347 return 0; 348 } 349 350 SEC("fentry/bpf_fentry_test9") 351 __description("private stack, max stack depth is private stack") 352 __success 353 __log_level(4) __msg("stack depth 8+256+0 max 256") 354 int private_stack_max_depth(void) 355 { 356 int x = 0; 357 358 subprog1(&x); 359 return 0; 360 } 361 362 #else 363 364 SEC("kprobe") 365 __description("private stack is not supported, use a dummy test") 366 __success 367 int dummy_test(void) 368 { 369 return 0; 370 } 371 372 #endif 373 374 char _license[] SEC("license") = "GPL"; 375