1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <linux/bpf.h> 4 #include <bpf/bpf_helpers.h> 5 #include "../../../include/linux/filter.h" 6 #include <bpf_arena_common.h> 7 #include <bpf/bpf_core_read.h> 8 #include "bpf_misc.h" 9 10 struct { 11 __uint(type, BPF_MAP_TYPE_ARENA); 12 __uint(map_flags, BPF_F_MMAPABLE | BPF_F_NO_USER_CONV); 13 __uint(max_entries, 1); 14 } arena SEC(".maps"); 15 16 extern long bpf_kfunc_call_test4(signed char a, short b, int c, long d) __ksym; 17 18 /* to retain debug info for BTF generation */ 19 void __kfunc_btf_root(void) 20 { 21 bpf_kfunc_call_test4(0, 0, 0, 0); 22 bpf_arena_alloc_pages(0, 0, 0, 0, 0); 23 bpf_rdonly_cast(0, 0); 24 } 25 26 SEC("socket") 27 __flag(BPF_F_TEST_STATE_FREQ) 28 __flag(BPF_F_TEST_RND_HI32) 29 __success __retval(0) 30 __naked void zext_lost_across_checkpoint(void) 31 { 32 asm volatile (" \ 33 call %[bpf_ktime_get_ns]; \ 34 r8 = r0; \ 35 r6 = 0xdeadbeefcafebabe ll; /* inject some value for r6's upper half */ \ 36 if r8 != 0 goto 1f; /* fall-through cached first, branch pruned */ \ 37 r6 = 32; /* full 64-bit def */ \ 38 goto 2f; \ 39 1: w6 = 32; /* 32-bit def, zext mark lost */ \ 40 2: r0 = r6; /* buggy verifier believed upper 32 bits are 0 */ \ 41 /* and thus did not zero extended w6 = 32. */ \ 42 r0 >>= 32; \ 43 exit; \ 44 " : 45 : __imm(bpf_ktime_get_ns) 46 : __clobber_all); 47 } 48 49 /* 32-bit ALU result read as 64-bit -> zext */ 50 SEC("socket") 51 __success __log_level(2) 52 __msg("w1 = w0{{ +}}; zext") 53 __naked void zext_alu32_hi_used(void) 54 { 55 asm volatile (" \ 56 call %[bpf_get_prandom_u32]; \ 57 w1 = w0; \ 58 r0 = r1; \ 59 exit; \ 60 " : 61 : __imm(bpf_get_prandom_u32) 62 : __clobber_all); 63 } 64 65 /* 32-bit ALU result read only as 32-bit -> no zext */ 66 SEC("socket") 67 __success __log_level(2) 68 __not_msg("; zext") 69 __naked void no_zext_alu32_hi_unused(void) 70 { 71 asm volatile (" \ 72 call %[bpf_get_prandom_u32]; \ 73 w1 = w0; /* MOV */ \ 74 w2 = w1; \ 75 w2 += w1; /* ALU32, BPF_X */ \ 76 w2 += 1; /* ALU32, BPF_K */ \ 77 w2 = w2; /* keep w2 alive for previous instruction */ \ 78 r0 = 0; \ 79 exit; \ 80 " : 81 : __imm(bpf_get_prandom_u32) 82 : __clobber_all); 83 } 84 85 /* 64-bit definition is never zero extended */ 86 SEC("socket") 87 __success __log_level(2) 88 __not_msg("r1 = r0{{.*}}; zext") 89 __naked void no_zext_mov64(void) 90 { 91 asm volatile (" \ 92 call %[bpf_get_prandom_u32]; \ 93 r1 = r0; \ 94 r0 = r1; \ 95 exit; \ 96 " : 97 : __imm(bpf_get_prandom_u32) 98 : __clobber_all); 99 } 100 101 /* Narrow load result read as 64-bit -> zext */ 102 SEC("socket") 103 __success __log_level(2) 104 __msg("r1 = *(u32 *)(r10 -8){{ +}}; zext") 105 __naked void zext_narrow_load_hi_used(void) 106 { 107 asm volatile (" \ 108 r0 = 0; \ 109 *(u64 *)(r10 - 8) = r0; \ 110 r1 = *(u32 *)(r10 - 8); \ 111 r0 = r1; \ 112 exit; \ 113 " ::: __clobber_all); 114 } 115 116 /* 32-bit atomic fetch result read as 64-bit -> zext */ 117 SEC("socket") 118 __success __log_level(2) 119 __msg("r1 = atomic_fetch_add((u32 *)(r10 -8), r1){{ +}}; zext") 120 __naked void zext_atomic_fetch32_hi_used(void) 121 { 122 asm volatile (" \ 123 r1 = 0; \ 124 *(u64 *)(r10 - 8) = r1; \ 125 w1 = 1; \ 126 .8byte %[fetch_add32]; \ 127 r0 = r1; \ 128 exit; \ 129 " : 130 : __imm_insn(fetch_add32, 131 BPF_ATOMIC_OP(BPF_W, BPF_ADD | BPF_FETCH, BPF_REG_10, BPF_REG_1, -8)) 132 : __clobber_all); 133 } 134 135 /* 32-bit atomic cmpxchg result (r0) read as 64-bit -> zext */ 136 SEC("socket") 137 __success __log_level(2) 138 __msg("r0 = atomic_cmpxchg((u32 *)(r10 -8), r0, r1){{ +}}; zext") 139 __naked void zext_cmpxchg32_hi_used(void) 140 { 141 asm volatile (" \ 142 r1 = 0; \ 143 *(u64 *)(r10 - 8) = r1; \ 144 w0 = 0; \ 145 w1 = 1; \ 146 .8byte %[cmpxchg32]; \ 147 r2 = r0; \ 148 r0 = r2; \ 149 exit; \ 150 " : 151 : __imm_insn(cmpxchg32, 152 BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, BPF_REG_10, BPF_REG_1, -8)) 153 : __clobber_all); 154 } 155 156 /* 32-bit def before a branch, upper half used on one branch -> zext */ 157 SEC("socket") 158 __success __log_level(2) 159 __msg("w6 = 32{{ +}}; zext") 160 __naked void zext_cfg_hi_used_one_branch(void) 161 { 162 asm volatile (" \ 163 call %[bpf_get_prandom_u32]; \ 164 w6 = 32; \ 165 if r0 == 0 goto 1f; \ 166 r0 = r6; \ 167 exit; \ 168 1: r0 = 0; \ 169 exit; \ 170 " : 171 : __imm(bpf_get_prandom_u32) 172 : __clobber_all); 173 } 174 175 /* r1's upper half is dead, so 'w1 = 1' must NOT be marked for zero extension. */ 176 SEC("socket") 177 __success __log_level(2) 178 __not_msg("w1 = 1{{.*}}; zext") 179 __naked void no_zext_other_reg_hi_used(void) 180 { 181 asm volatile (" \ 182 call %[bpf_get_prandom_u32]; \ 183 r6 = r0; \ 184 r6 <<= 32; \ 185 w1 = 1; \ 186 r0 = r6; \ 187 exit; \ 188 " : 189 : __imm(bpf_get_prandom_u32) 190 : __clobber_all); 191 } 192 193 /* LD_ABS defines r0; when r0 is read as 64-bit it must be zero extended */ 194 SEC("socket") 195 __success __log_level(2) 196 __msg("r0 = *(u8 *)skb[0]{{.*}}; zext") 197 __naked void zext_ld_abs_hi_used(void) 198 { 199 asm volatile (" \ 200 r6 = r1; \ 201 r0 = *(u8 *)skb[0]; \ 202 r7 = r0; \ 203 r0 = r7; \ 204 exit; \ 205 " ::: __clobber_all); 206 } 207 208 /* Helper parameters are read as 64-bit (call_use_mask() fallback) */ 209 SEC("socket") 210 __success __log_level(2) 211 __msg("w2 = 1{{ +}}; zext") 212 __naked void helper_param_read_as_64bit(void) 213 { 214 asm volatile (" \ 215 r1 = r10; \ 216 r1 += -8; \ 217 w2 = 1; \ 218 call %[bpf_trace_printk]; \ 219 r0 = 0; \ 220 exit; \ 221 " : 222 : __imm(bpf_trace_printk) 223 : __clobber_all); 224 } 225 226 static __used __naked int subprog_reads_arg_as_64bit(void) 227 { 228 asm volatile (" \ 229 r0 = r1; \ 230 exit; \ 231 " ::: __clobber_all); 232 } 233 234 /* subprogram parameters are conservatively read as 64-bit */ 235 SEC("socket") 236 __success __log_level(2) 237 __msg("w1 = w0{{ +}}; zext") 238 __naked void subprog_param_read_as_64bit(void) 239 { 240 asm volatile (" \ 241 call %[bpf_get_prandom_u32]; \ 242 w1 = w0; \ 243 call subprog_reads_arg_as_64bit; \ 244 r0 = 0; \ 245 exit; \ 246 " : 247 : __imm(bpf_get_prandom_u32) 248 : __clobber_all); 249 } 250 251 /* kfunc parameters are zero extended */ 252 SEC("tc") 253 __success __log_level(2) 254 __msg("w1 = 1{{ +}}; zext") 255 __msg("w2 = 1{{ +}}; zext") 256 __msg("w3 = 1{{ +}}; zext") 257 __msg("w4 = 1{{ +}}; zext") 258 __naked void kfunc_param_read_per_btf(void) 259 { 260 asm volatile (" \ 261 w1 = 1; \ 262 w2 = 1; \ 263 w3 = 1; \ 264 w4 = 1; \ 265 call bpf_kfunc_call_test4; \ 266 r0 = 0; \ 267 exit; \ 268 " ::: __clobber_all); 269 } 270 271 SEC("socket") 272 __success __log_level(2) 273 __not_msg("; zext") 274 __naked void alu32_and_32bit_conditional(void) 275 { 276 asm volatile (" \ 277 call %[bpf_get_prandom_u32]; \ 278 w1 = w0; \ 279 if w1 > 42 goto 1f; /* BPF_K */ \ 280 w2 = 28; \ 281 if w2 > w1 goto 1f; /* BPF_X */ \ 282 r0 = 0; \ 283 1: exit; \ 284 " : 285 : __imm(bpf_get_prandom_u32) 286 : __clobber_all); 287 } 288 289 SEC("socket") 290 __success __log_level(2) 291 __msg("w1 = w0{{ +}}; zext") 292 __naked void alu32_and_64bit_conditional(void) 293 { 294 asm volatile (" \ 295 call %[bpf_get_prandom_u32]; \ 296 w1 = w0; \ 297 if r1 > 42 goto 1f; /* BPF_K */ \ 298 r2 = 28; \ 299 if r2 > r1 goto 1f; /* BPF_X */ \ 300 r0 = 0; \ 301 1: exit; \ 302 " : 303 : __imm(bpf_get_prandom_u32) 304 : __clobber_all); 305 } 306 307 SEC("socket") 308 __success __log_level(2) 309 __not_msg("; zext") 310 __naked void alu64_and_conditionals(void) 311 { 312 asm volatile (" \ 313 call %[bpf_get_prandom_u32]; \ 314 r1 = r0; \ 315 if w1 > 42 goto 1f; /* BPF_K */ \ 316 if r1 > 42 goto 1f; /* BPF_K */ \ 317 r2 = 28; \ 318 if w2 > w1 goto 1f; /* BPF_X */ \ 319 if r2 > r1 goto 1f; /* BPF_X */ \ 320 r0 = 0; \ 321 1: exit; \ 322 " : 323 : __imm(bpf_get_prandom_u32) 324 : __clobber_all); 325 } 326 327 #ifdef __BPF_FEATURE_ADDR_SPACE_CAST 328 329 SEC("?fentry.s/" SYS_PREFIX "sys_getpgid") 330 __arch_s390x 331 __xlated("7: w1 = w0") 332 __xlated("8: w1 = w1") 333 __xlated("9: w1 += 8") 334 __xlated("10: w1 = w1") 335 __xlated("11: w2 = w1") 336 __xlated("12: w2 = w2") 337 __xlated("13: *(u64 *)(r1 +0) = r2") 338 __naked void arena_ptr(void) 339 { 340 asm volatile (" \ 341 r1 = %[arena] ll; \ 342 r2 = 0; \ 343 r3 = 1; \ 344 r4 = 0; \ 345 r5 = 0; \ 346 call %[bpf_arena_alloc_pages]; \ 347 r1 = addr_space_cast(r0, 0, 1); /* needs zext */ \ 348 r1 += 8; /* needs zext */ \ 349 r2 = addr_space_cast(r1, 1, 0); /* needs zext because of BPF_F_NO_USER_CONV */ \ 350 *(u64 *)(r1 +0) = r2; \ 351 r0 = 0; \ 352 exit; \ 353 " : 354 : __imm(bpf_arena_alloc_pages), 355 __imm_addr(arena) 356 : __clobber_all); 357 } 358 359 #endif 360 361 /* Check if probe mem loads keep their zero extension. */ 362 SEC("socket") 363 __success __log_level(2) 364 __arch_s390x 365 __xlated("3: r1 = *(u64 *)(r0 +0)") 366 __xlated("4: r2 = *(u32 *)(r0 +0)") 367 __xlated("5: w2 = w2") 368 __xlated("6: r3 = *(u16 *)(r0 +0)") 369 __xlated("7: w3 = w3") 370 __xlated("8: r4 = *(u8 *)(r0 +0)") 371 __xlated("9: w4 = w4") 372 __naked void probe_mem(void) 373 { 374 asm volatile (" \ 375 r1 = 0; \ 376 r2 = 0; \ 377 call %[bpf_rdonly_cast]; \ 378 r1 = *(u64 *)(r0 + 0); /* BPF_PROBE_MEM */ \ 379 r2 = *(u32 *)(r0 + 0); /* BPF_PROBE_MEM */ \ 380 r3 = *(u16 *)(r0 + 0); /* BPF_PROBE_MEM */ \ 381 r4 = *(u8 *)(r0 + 0); /* BPF_PROBE_MEM */ \ 382 r0 = r1; /* make the registers used */ \ 383 r0 += r2; \ 384 r0 += r3; \ 385 r0 += r4; \ 386 1: exit; \ 387 " : 388 : __imm(bpf_rdonly_cast) 389 : __clobber_all); 390 } 391 392 char _license[] SEC("license") = "GPL"; 393