1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ 3 #include <vmlinux.h> 4 #include <bpf/bpf_helpers.h> 5 #include "../test_kmods/bpf_testmod_kfunc.h" 6 #include "bpf_misc.h" 7 8 #ifdef __SIZEOF_INT128__ 9 typedef unsigned __int128 u128; 10 #endif 11 12 #define MIX_A 0xdeadbeefcafef00dULL 13 #define MIX_B 0x0123456789abcdefULL 14 15 #if defined(__clang__) 16 17 SEC("tc") 18 __arch_x86_64 __arch_arm64 19 __load_if_JITed() 20 __success __retval(0) 21 int aggregate_arg_kfunc_struct(struct __sk_buff *skb) 22 { 23 __u64 a = skb->len ^ MIX_A; 24 __u64 b = skb->len ^ MIX_B; 25 struct prog_test_pair_arg s = { .lo = a, .hi = b }; 26 27 if (bpf_kfunc_call_test_pair_arg(1, s, 2) != 2 * a + 3 * b + 9) 28 return 1; 29 30 return 0; 31 } 32 33 #endif 34 35 #ifdef __SIZEOF_INT128__ 36 37 SEC("tc") 38 __arch_x86_64 __arch_arm64 39 __load_if_JITed() 40 __success __retval(0) 41 int aggregate_arg_kfunc_int128(struct __sk_buff *skb) 42 { 43 __u64 a = skb->len ^ MIX_A; 44 __u64 b = skb->len ^ MIX_B; 45 u128 v = ((u128)a << 64) | b; 46 47 if (bpf_kfunc_call_test_i128_arg(1, 2, v) != 4 * a + 3 * b + 5) 48 return 1; 49 50 return 0; 51 } 52 53 /* 54 * arm64 rounds the register number up to an even one for an argument 55 * aligned to 16 bytes, so it wants this __int128 in x2 and x3, leaving x1 56 * empty, and the last argument in x4. The x86-64 ABI has no such rule. 57 */ 58 SEC("tc") 59 __arch_x86_64 __arch_arm64 60 __load_if_JITed() 61 __success __retval(0) 62 int aggregate_arg_kfunc_int128_odd(struct __sk_buff *skb) 63 { 64 __u64 a = skb->len ^ MIX_A; 65 __u64 b = skb->len ^ MIX_B; 66 u128 v = ((u128)a << 64) | b; 67 68 if (bpf_kfunc_call_test_i128_arg_odd(1, v, 2) != 1 + 2 * b + 3 * a + 4 * 2) 69 return 1; 70 71 return 0; 72 } 73 74 #endif /* __SIZEOF_INT128__ */ 75 76 #if defined(__clang__) && defined(__BPF_FEATURE_STACK_ARGUMENT) 77 78 SEC("tc") 79 __arch_x86_64 __arch_arm64 80 __load_if_JITed() 81 __success __retval(0) 82 int aggregate_arg_kfunc_last_regs(struct __sk_buff *skb) 83 { 84 __u64 a = skb->len ^ MIX_A; 85 __u64 b = skb->len ^ MIX_B; 86 struct prog_test_pair_arg s = { .lo = a, .hi = b }; 87 88 if (bpf_kfunc_call_test_pair_arg_nofit(1, 2, 3, 4, s) != 5 * a + 6 * b + 30) 89 return 1; 90 91 return 0; 92 } 93 94 /* 95 * The x86-64 ABI moves an argument its six remaining registers cannot hold 96 * wholly onto the stack. arm64, with eight argument registers, still has a 97 * pair for it. 98 */ 99 SEC("tc") 100 __arch_x86_64 __arch_arm64 101 __load_if_JITed() 102 __success __retval(0) 103 int aggregate_arg_kfunc_straddle(struct __sk_buff *skb) 104 { 105 __u64 a = skb->len ^ MIX_A; 106 __u64 b = skb->len ^ MIX_B; 107 struct prog_test_big_arg s = { .a = a, .b = b }; 108 109 if (bpf_kfunc_call_stack_arg_big(1, 2, 3, 4, 5, s) != 6 * a + 7 * b + 55) 110 return 1; 111 112 return 0; 113 } 114 115 /* The same, with an argument after the struct to take the eighth register. */ 116 SEC("tc") 117 __arch_x86_64 __arch_arm64 118 __load_if_JITed() 119 __success __retval(0) 120 int aggregate_arg_kfunc_tail(struct __sk_buff *skb) 121 { 122 __u64 a = skb->len ^ MIX_A; 123 __u64 b = skb->len ^ MIX_B; 124 struct prog_test_pair_arg s = { .lo = a, .hi = b }; 125 126 if (bpf_kfunc_call_test_pair_arg_tail(1, 2, 3, 4, 5, s, 6) != 6 * a + 7 * b + 103) 127 return 1; 128 129 return 0; 130 } 131 132 /* 133 * arm64 gives no register to an argument its eight registers cannot hold, 134 * nor to anything after it. Past its six registers the x86-64 ABI has both 135 * eightbytes on the stack either way. 136 */ 137 SEC("tc") 138 __arch_x86_64 __arch_arm64 139 __load_if_JITed() 140 __success __retval(0) 141 int aggregate_arg_kfunc_split8(struct __sk_buff *skb) 142 { 143 __u64 a = skb->len ^ MIX_A; 144 __u64 b = skb->len ^ MIX_B; 145 struct prog_test_pair_arg s = { .lo = a, .hi = b }; 146 147 if (bpf_kfunc_call_test_pair_arg_split8(1, 2, 3, 4, 5, 6, 7, s) != 8 * a + 9 * b + 140) 148 return 1; 149 150 return 0; 151 } 152 153 #ifdef __SIZEOF_INT128__ 154 155 /* 156 * The hole of aggregate_arg_kfunc_int128_odd, with arguments after it that 157 * arm64 shifts through x5 to x7, where its JIT keeps the BPF stack slots, 158 * and onto its stack. The second __int128 then leaves the second arm64 159 * stack slot empty. x86-64 packs the eightbytes like the BPF convention, 160 * so nothing moves there. 161 */ 162 SEC("tc") 163 __arch_x86_64 __arch_arm64 164 __load_if_JITed() 165 __success __retval(0) 166 int aggregate_arg_kfunc_int128_odd_many(struct __sk_buff *skb) 167 { 168 __u64 a = skb->len ^ MIX_A; 169 __u64 b = skb->len ^ MIX_B; 170 u128 v = ((u128)a << 64) | b; 171 u128 w = ((u128)~a << 64) | ~b; 172 173 if (bpf_kfunc_call_test_i128_arg_odd_many(1, v, 2, 3, 4, 5, 6, w) != 174 1 + 2 * b + 3 * a + 4 * 2 + 5 * 3 + 6 * 4 + 7 * 5 + 8 * 6 + 175 9 * ~b + 10 * ~a) 176 return 1; 177 178 return 0; 179 } 180 181 /* 182 * x86-64 moves s to the stack and gives R9 back to the seventh argument. 183 * w, aligned to 16 bytes by the __int128 it holds, then leaves a stack slot 184 * empty before it: the fourth on x86-64, the second on arm64. 185 */ 186 SEC("tc") 187 __arch_x86_64 __arch_arm64 188 __load_if_JITed() 189 __success __retval(0) 190 int aggregate_arg_kfunc_int128_stack_hole(struct __sk_buff *skb) 191 { 192 __u64 a = skb->len ^ MIX_A; 193 __u64 b = skb->len ^ MIX_B; 194 struct prog_test_pair_arg s = { .lo = a, .hi = b }; 195 struct prog_test_i128_arg w = { .v = ((u128)~a << 64) | ~b }; 196 197 if (bpf_kfunc_call_test_i128_arg_stack_hole(1, 2, 3, 4, 5, s, 6, 7, w) != 198 1 + 2 * 2 + 3 * 3 + 4 * 4 + 5 * 5 + 6 * a + 7 * b + 8 * 6 + 9 * 7 + 199 10 * ~b + 11 * ~a) 200 return 1; 201 202 return 0; 203 } 204 205 /* 206 * Both conventions pad the stack to align this __int128, and the BPF 207 * convention pads for neither, so both JITs move it up an eightbyte. 208 */ 209 SEC("tc") 210 __arch_x86_64 __arch_arm64 211 __load_if_JITed() 212 __success __retval(0) 213 int aggregate_arg_kfunc_int128_pad(struct __sk_buff *skb) 214 { 215 __u64 a = skb->len ^ MIX_A; 216 __u64 b = skb->len ^ MIX_B; 217 u128 v = ((u128)a << 64) | b; 218 219 if (bpf_kfunc_call_test_i128_arg_pad(1, 2, 3, 4, 5, 6, 7, v) != 9 * a + 8 * b + 140) 220 return 1; 221 222 return 0; 223 } 224 225 #endif /* __SIZEOF_INT128__ */ 226 227 #endif 228 229 SEC("tc") 230 __arch_x86_64 __arch_arm64 231 __failure __msg("R1 type STRUCT is not composed of scalars") 232 int aggregate_arg_kfunc_ptr_member(struct __sk_buff *skb) 233 { 234 struct prog_test_ptr_arg s = { .p = skb, .x = 1 }; 235 236 return bpf_kfunc_call_test_ptr_arg(s); 237 } 238 239 char _license[] SEC("license") = "GPL"; 240