1 // SPDX-License-Identifier: GPL-2.0
2
3 #include <linux/bpf.h>
4 #include <bpf/bpf_helpers.h>
5 #include "../../../include/linux/filter.h"
6 #include <bpf_arena_common.h>
7 #include <bpf/bpf_core_read.h>
8 #include "bpf_misc.h"
9
10 struct {
11 __uint(type, BPF_MAP_TYPE_ARENA);
12 __uint(map_flags, BPF_F_MMAPABLE | BPF_F_NO_USER_CONV);
13 __uint(max_entries, 1);
14 } arena SEC(".maps");
15
16 extern long bpf_kfunc_call_test4(signed char a, short b, int c, long d) __ksym;
17
18 /* to retain debug info for BTF generation */
__kfunc_btf_root(void)19 void __kfunc_btf_root(void)
20 {
21 bpf_kfunc_call_test4(0, 0, 0, 0);
22 bpf_arena_alloc_pages(0, 0, 0, 0, 0);
23 bpf_rdonly_cast(0, 0);
24 }
25
26 SEC("socket")
__flag(BPF_F_TEST_STATE_FREQ)27 __flag(BPF_F_TEST_STATE_FREQ)
28 __flag(BPF_F_TEST_RND_HI32)
29 __success __retval(0)
30 __naked void zext_lost_across_checkpoint(void)
31 {
32 asm volatile (" \
33 call %[bpf_ktime_get_ns]; \
34 r8 = r0; \
35 r6 = 0xdeadbeefcafebabe ll; /* inject some value for r6's upper half */ \
36 if r8 != 0 goto 1f; /* fall-through cached first, branch pruned */ \
37 r6 = 32; /* full 64-bit def */ \
38 goto 2f; \
39 1: w6 = 32; /* 32-bit def, zext mark lost */ \
40 2: r0 = r6; /* buggy verifier believed upper 32 bits are 0 */ \
41 /* and thus did not zero extended w6 = 32. */ \
42 r0 >>= 32; \
43 exit; \
44 " :
45 : __imm(bpf_ktime_get_ns)
46 : __clobber_all);
47 }
48
49 /* 32-bit ALU result read as 64-bit -> zext */
50 SEC("socket")
51 __success __log_level(2)
52 __msg("w1 = w0{{ +}}; zext")
zext_alu32_hi_used(void)53 __naked void zext_alu32_hi_used(void)
54 {
55 asm volatile (" \
56 call %[bpf_get_prandom_u32]; \
57 w1 = w0; \
58 r0 = r1; \
59 exit; \
60 " :
61 : __imm(bpf_get_prandom_u32)
62 : __clobber_all);
63 }
64
65 /* 32-bit ALU result read only as 32-bit -> no zext */
66 SEC("socket")
67 __success __log_level(2)
68 __not_msg("; zext")
no_zext_alu32_hi_unused(void)69 __naked void no_zext_alu32_hi_unused(void)
70 {
71 asm volatile (" \
72 call %[bpf_get_prandom_u32]; \
73 w1 = w0; /* MOV */ \
74 w2 = w1; \
75 w2 += w1; /* ALU32, BPF_X */ \
76 w2 += 1; /* ALU32, BPF_K */ \
77 w2 = w2; /* keep w2 alive for previous instruction */ \
78 r0 = 0; \
79 exit; \
80 " :
81 : __imm(bpf_get_prandom_u32)
82 : __clobber_all);
83 }
84
85 /* 64-bit definition is never zero extended */
86 SEC("socket")
87 __success __log_level(2)
88 __not_msg("r1 = r0{{.*}}; zext")
no_zext_mov64(void)89 __naked void no_zext_mov64(void)
90 {
91 asm volatile (" \
92 call %[bpf_get_prandom_u32]; \
93 r1 = r0; \
94 r0 = r1; \
95 exit; \
96 " :
97 : __imm(bpf_get_prandom_u32)
98 : __clobber_all);
99 }
100
101 /* Narrow load result read as 64-bit -> zext */
102 SEC("socket")
103 __success __log_level(2)
104 __msg("r1 = *(u32 *)(r10 -8){{ +}}; zext")
zext_narrow_load_hi_used(void)105 __naked void zext_narrow_load_hi_used(void)
106 {
107 asm volatile (" \
108 r0 = 0; \
109 *(u64 *)(r10 - 8) = r0; \
110 r1 = *(u32 *)(r10 - 8); \
111 r0 = r1; \
112 exit; \
113 " ::: __clobber_all);
114 }
115
116 /* 32-bit atomic fetch result read as 64-bit -> zext */
117 SEC("socket")
118 __success __log_level(2)
119 __msg("r1 = atomic_fetch_add((u32 *)(r10 -8), r1){{ +}}; zext")
zext_atomic_fetch32_hi_used(void)120 __naked void zext_atomic_fetch32_hi_used(void)
121 {
122 asm volatile (" \
123 r1 = 0; \
124 *(u64 *)(r10 - 8) = r1; \
125 w1 = 1; \
126 .8byte %[fetch_add32]; \
127 r0 = r1; \
128 exit; \
129 " :
130 : __imm_insn(fetch_add32,
131 BPF_ATOMIC_OP(BPF_W, BPF_ADD | BPF_FETCH, BPF_REG_10, BPF_REG_1, -8))
132 : __clobber_all);
133 }
134
135 /* 32-bit atomic cmpxchg result (r0) read as 64-bit -> zext */
136 SEC("socket")
137 __success __log_level(2)
138 __msg("r0 = atomic_cmpxchg((u32 *)(r10 -8), r0, r1){{ +}}; zext")
zext_cmpxchg32_hi_used(void)139 __naked void zext_cmpxchg32_hi_used(void)
140 {
141 asm volatile (" \
142 r1 = 0; \
143 *(u64 *)(r10 - 8) = r1; \
144 w0 = 0; \
145 w1 = 1; \
146 .8byte %[cmpxchg32]; \
147 r2 = r0; \
148 r0 = r2; \
149 exit; \
150 " :
151 : __imm_insn(cmpxchg32,
152 BPF_ATOMIC_OP(BPF_W, BPF_CMPXCHG, BPF_REG_10, BPF_REG_1, -8))
153 : __clobber_all);
154 }
155
156 /* 32-bit def before a branch, upper half used on one branch -> zext */
157 SEC("socket")
158 __success __log_level(2)
159 __msg("w6 = 32{{ +}}; zext")
zext_cfg_hi_used_one_branch(void)160 __naked void zext_cfg_hi_used_one_branch(void)
161 {
162 asm volatile (" \
163 call %[bpf_get_prandom_u32]; \
164 w6 = 32; \
165 if r0 == 0 goto 1f; \
166 r0 = r6; \
167 exit; \
168 1: r0 = 0; \
169 exit; \
170 " :
171 : __imm(bpf_get_prandom_u32)
172 : __clobber_all);
173 }
174
175 /* r1's upper half is dead, so 'w1 = 1' must NOT be marked for zero extension. */
176 SEC("socket")
177 __success __log_level(2)
178 __not_msg("w1 = 1{{.*}}; zext")
no_zext_other_reg_hi_used(void)179 __naked void no_zext_other_reg_hi_used(void)
180 {
181 asm volatile (" \
182 call %[bpf_get_prandom_u32]; \
183 r6 = r0; \
184 r6 <<= 32; \
185 w1 = 1; \
186 r0 = r6; \
187 exit; \
188 " :
189 : __imm(bpf_get_prandom_u32)
190 : __clobber_all);
191 }
192
193 /* LD_ABS defines r0; when r0 is read as 64-bit it must be zero extended */
194 SEC("socket")
195 __success __log_level(2)
196 __msg("r0 = *(u8 *)skb[0]{{.*}}; zext")
zext_ld_abs_hi_used(void)197 __naked void zext_ld_abs_hi_used(void)
198 {
199 asm volatile (" \
200 r6 = r1; \
201 r0 = *(u8 *)skb[0]; \
202 r7 = r0; \
203 r0 = r7; \
204 exit; \
205 " ::: __clobber_all);
206 }
207
208 /* Helper parameters are read as 64-bit (call_use_mask() fallback) */
209 SEC("socket")
210 __success __log_level(2)
211 __msg("w2 = 1{{ +}}; zext")
helper_param_read_as_64bit(void)212 __naked void helper_param_read_as_64bit(void)
213 {
214 asm volatile (" \
215 r1 = r10; \
216 r1 += -8; \
217 w2 = 1; \
218 call %[bpf_trace_printk]; \
219 r0 = 0; \
220 exit; \
221 " :
222 : __imm(bpf_trace_printk)
223 : __clobber_all);
224 }
225
subprog_reads_arg_as_64bit(void)226 static __used __naked int subprog_reads_arg_as_64bit(void)
227 {
228 asm volatile (" \
229 r0 = r1; \
230 exit; \
231 " ::: __clobber_all);
232 }
233
234 /* subprogram parameters are conservatively read as 64-bit */
235 SEC("socket")
236 __success __log_level(2)
237 __msg("w1 = w0{{ +}}; zext")
subprog_param_read_as_64bit(void)238 __naked void subprog_param_read_as_64bit(void)
239 {
240 asm volatile (" \
241 call %[bpf_get_prandom_u32]; \
242 w1 = w0; \
243 call subprog_reads_arg_as_64bit; \
244 r0 = 0; \
245 exit; \
246 " :
247 : __imm(bpf_get_prandom_u32)
248 : __clobber_all);
249 }
250
251 /* kfunc parameters are zero extended */
252 SEC("tc")
253 __success __log_level(2)
254 __msg("w1 = 1{{ +}}; zext")
255 __msg("w2 = 1{{ +}}; zext")
256 __msg("w3 = 1{{ +}}; zext")
257 __msg("w4 = 1{{ +}}; zext")
kfunc_param_read_per_btf(void)258 __naked void kfunc_param_read_per_btf(void)
259 {
260 asm volatile (" \
261 w1 = 1; \
262 w2 = 1; \
263 w3 = 1; \
264 w4 = 1; \
265 call bpf_kfunc_call_test4; \
266 r0 = 0; \
267 exit; \
268 " ::: __clobber_all);
269 }
270
271 SEC("socket")
272 __success __log_level(2)
273 __not_msg("; zext")
alu32_and_32bit_conditional(void)274 __naked void alu32_and_32bit_conditional(void)
275 {
276 asm volatile (" \
277 call %[bpf_get_prandom_u32]; \
278 w1 = w0; \
279 if w1 > 42 goto 1f; /* BPF_K */ \
280 w2 = 28; \
281 if w2 > w1 goto 1f; /* BPF_X */ \
282 r0 = 0; \
283 1: exit; \
284 " :
285 : __imm(bpf_get_prandom_u32)
286 : __clobber_all);
287 }
288
289 SEC("socket")
290 __success __log_level(2)
291 __msg("w1 = w0{{ +}}; zext")
alu32_and_64bit_conditional(void)292 __naked void alu32_and_64bit_conditional(void)
293 {
294 asm volatile (" \
295 call %[bpf_get_prandom_u32]; \
296 w1 = w0; \
297 if r1 > 42 goto 1f; /* BPF_K */ \
298 r2 = 28; \
299 if r2 > r1 goto 1f; /* BPF_X */ \
300 r0 = 0; \
301 1: exit; \
302 " :
303 : __imm(bpf_get_prandom_u32)
304 : __clobber_all);
305 }
306
307 SEC("socket")
308 __success __log_level(2)
309 __not_msg("; zext")
alu64_and_conditionals(void)310 __naked void alu64_and_conditionals(void)
311 {
312 asm volatile (" \
313 call %[bpf_get_prandom_u32]; \
314 r1 = r0; \
315 if w1 > 42 goto 1f; /* BPF_K */ \
316 if r1 > 42 goto 1f; /* BPF_K */ \
317 r2 = 28; \
318 if w2 > w1 goto 1f; /* BPF_X */ \
319 if r2 > r1 goto 1f; /* BPF_X */ \
320 r0 = 0; \
321 1: exit; \
322 " :
323 : __imm(bpf_get_prandom_u32)
324 : __clobber_all);
325 }
326
327 #ifdef __BPF_FEATURE_ADDR_SPACE_CAST
328
329 SEC("?fentry.s/" SYS_PREFIX "sys_getpgid")
330 __arch_s390x
331 __xlated("7: w1 = w0")
332 __xlated("8: w1 = w1")
333 __xlated("9: w1 += 8")
334 __xlated("10: w1 = w1")
335 __xlated("11: w2 = w1")
336 __xlated("12: w2 = w2")
337 __xlated("13: *(u64 *)(r1 +0) = r2")
arena_ptr(void)338 __naked void arena_ptr(void)
339 {
340 asm volatile (" \
341 r1 = %[arena] ll; \
342 r2 = 0; \
343 r3 = 1; \
344 r4 = 0; \
345 r5 = 0; \
346 call %[bpf_arena_alloc_pages]; \
347 r1 = addr_space_cast(r0, 0, 1); /* needs zext */ \
348 r1 += 8; /* needs zext */ \
349 r2 = addr_space_cast(r1, 1, 0); /* needs zext because of BPF_F_NO_USER_CONV */ \
350 *(u64 *)(r1 +0) = r2; \
351 r0 = 0; \
352 exit; \
353 " :
354 : __imm(bpf_arena_alloc_pages),
355 __imm_addr(arena)
356 : __clobber_all);
357 }
358
359 #endif
360
361 /* Check if probe mem loads keep their zero extension. */
362 SEC("socket")
363 __success __log_level(2)
364 __arch_s390x
365 __xlated("3: r1 = *(u64 *)(r0 +0)")
366 __xlated("4: r2 = *(u32 *)(r0 +0)")
367 __xlated("5: w2 = w2")
368 __xlated("6: r3 = *(u16 *)(r0 +0)")
369 __xlated("7: w3 = w3")
370 __xlated("8: r4 = *(u8 *)(r0 +0)")
371 __xlated("9: w4 = w4")
probe_mem(void)372 __naked void probe_mem(void)
373 {
374 asm volatile (" \
375 r1 = 0; \
376 r2 = 0; \
377 call %[bpf_rdonly_cast]; \
378 r1 = *(u64 *)(r0 + 0); /* BPF_PROBE_MEM */ \
379 r2 = *(u32 *)(r0 + 0); /* BPF_PROBE_MEM */ \
380 r3 = *(u16 *)(r0 + 0); /* BPF_PROBE_MEM */ \
381 r4 = *(u8 *)(r0 + 0); /* BPF_PROBE_MEM */ \
382 r0 = r1; /* make the registers used */ \
383 r0 += r2; \
384 r0 += r3; \
385 r0 += r4; \
386 1: exit; \
387 " :
388 : __imm(bpf_rdonly_cast)
389 : __clobber_all);
390 }
391
392 char _license[] SEC("license") = "GPL";
393