1*b2441318SGreg Kroah-Hartman/* SPDX-License-Identifier: GPL-2.0 */ 26b3d4eecSDavid S. Miller#include <asm/ptrace.h> 36b3d4eecSDavid S. Miller 46b3d4eecSDavid S. Miller#include "bpf_jit_32.h" 56b3d4eecSDavid S. Miller 66b3d4eecSDavid S. Miller#define SAVE_SZ 96 76b3d4eecSDavid S. Miller#define SCRATCH_OFF 72 86b3d4eecSDavid S. Miller#define BE_PTR(label) be label 96b3d4eecSDavid S. Miller#define SIGN_EXTEND(reg) 106b3d4eecSDavid S. Miller 116b3d4eecSDavid S. Miller#define SKF_MAX_NEG_OFF (-0x200000) /* SKF_LL_OFF from filter.h */ 126b3d4eecSDavid S. Miller 136b3d4eecSDavid S. Miller .text 146b3d4eecSDavid S. Miller .globl bpf_jit_load_word 156b3d4eecSDavid S. Millerbpf_jit_load_word: 166b3d4eecSDavid S. Miller cmp r_OFF, 0 176b3d4eecSDavid S. Miller bl bpf_slow_path_word_neg 186b3d4eecSDavid S. Miller nop 196b3d4eecSDavid S. Miller .globl bpf_jit_load_word_positive_offset 206b3d4eecSDavid S. Millerbpf_jit_load_word_positive_offset: 216b3d4eecSDavid S. Miller sub r_HEADLEN, r_OFF, r_TMP 226b3d4eecSDavid S. Miller cmp r_TMP, 3 236b3d4eecSDavid S. Miller ble bpf_slow_path_word 246b3d4eecSDavid S. Miller add r_SKB_DATA, r_OFF, r_TMP 256b3d4eecSDavid S. Miller andcc r_TMP, 3, %g0 266b3d4eecSDavid S. Miller bne load_word_unaligned 276b3d4eecSDavid S. Miller nop 286b3d4eecSDavid S. Miller retl 296b3d4eecSDavid S. Miller ld [r_TMP], r_A 306b3d4eecSDavid S. Millerload_word_unaligned: 316b3d4eecSDavid S. Miller ldub [r_TMP + 0x0], r_OFF 326b3d4eecSDavid S. Miller ldub [r_TMP + 0x1], r_TMP2 336b3d4eecSDavid S. Miller sll r_OFF, 8, r_OFF 346b3d4eecSDavid S. Miller or r_OFF, r_TMP2, r_OFF 356b3d4eecSDavid S. Miller ldub [r_TMP + 0x2], r_TMP2 366b3d4eecSDavid S. Miller sll r_OFF, 8, r_OFF 376b3d4eecSDavid S. Miller or r_OFF, r_TMP2, r_OFF 386b3d4eecSDavid S. Miller ldub [r_TMP + 0x3], r_TMP2 396b3d4eecSDavid S. Miller sll r_OFF, 8, r_OFF 406b3d4eecSDavid S. Miller retl 416b3d4eecSDavid S. Miller or r_OFF, r_TMP2, r_A 426b3d4eecSDavid S. Miller 436b3d4eecSDavid S. Miller .globl bpf_jit_load_half 446b3d4eecSDavid S. Millerbpf_jit_load_half: 456b3d4eecSDavid S. Miller cmp r_OFF, 0 466b3d4eecSDavid S. Miller bl bpf_slow_path_half_neg 476b3d4eecSDavid S. Miller nop 486b3d4eecSDavid S. Miller .globl bpf_jit_load_half_positive_offset 496b3d4eecSDavid S. Millerbpf_jit_load_half_positive_offset: 506b3d4eecSDavid S. Miller sub r_HEADLEN, r_OFF, r_TMP 516b3d4eecSDavid S. Miller cmp r_TMP, 1 526b3d4eecSDavid S. Miller ble bpf_slow_path_half 536b3d4eecSDavid S. Miller add r_SKB_DATA, r_OFF, r_TMP 546b3d4eecSDavid S. Miller andcc r_TMP, 1, %g0 556b3d4eecSDavid S. Miller bne load_half_unaligned 566b3d4eecSDavid S. Miller nop 576b3d4eecSDavid S. Miller retl 586b3d4eecSDavid S. Miller lduh [r_TMP], r_A 596b3d4eecSDavid S. Millerload_half_unaligned: 606b3d4eecSDavid S. Miller ldub [r_TMP + 0x0], r_OFF 616b3d4eecSDavid S. Miller ldub [r_TMP + 0x1], r_TMP2 626b3d4eecSDavid S. Miller sll r_OFF, 8, r_OFF 636b3d4eecSDavid S. Miller retl 646b3d4eecSDavid S. Miller or r_OFF, r_TMP2, r_A 656b3d4eecSDavid S. Miller 666b3d4eecSDavid S. Miller .globl bpf_jit_load_byte 676b3d4eecSDavid S. Millerbpf_jit_load_byte: 686b3d4eecSDavid S. Miller cmp r_OFF, 0 696b3d4eecSDavid S. Miller bl bpf_slow_path_byte_neg 706b3d4eecSDavid S. Miller nop 716b3d4eecSDavid S. Miller .globl bpf_jit_load_byte_positive_offset 726b3d4eecSDavid S. Millerbpf_jit_load_byte_positive_offset: 736b3d4eecSDavid S. Miller cmp r_OFF, r_HEADLEN 746b3d4eecSDavid S. Miller bge bpf_slow_path_byte 756b3d4eecSDavid S. Miller nop 766b3d4eecSDavid S. Miller retl 776b3d4eecSDavid S. Miller ldub [r_SKB_DATA + r_OFF], r_A 786b3d4eecSDavid S. Miller 796b3d4eecSDavid S. Miller .globl bpf_jit_load_byte_msh 806b3d4eecSDavid S. Millerbpf_jit_load_byte_msh: 816b3d4eecSDavid S. Miller cmp r_OFF, 0 826b3d4eecSDavid S. Miller bl bpf_slow_path_byte_msh_neg 836b3d4eecSDavid S. Miller nop 846b3d4eecSDavid S. Miller .globl bpf_jit_load_byte_msh_positive_offset 856b3d4eecSDavid S. Millerbpf_jit_load_byte_msh_positive_offset: 866b3d4eecSDavid S. Miller cmp r_OFF, r_HEADLEN 876b3d4eecSDavid S. Miller bge bpf_slow_path_byte_msh 886b3d4eecSDavid S. Miller nop 896b3d4eecSDavid S. Miller ldub [r_SKB_DATA + r_OFF], r_OFF 906b3d4eecSDavid S. Miller and r_OFF, 0xf, r_OFF 916b3d4eecSDavid S. Miller retl 926b3d4eecSDavid S. Miller sll r_OFF, 2, r_X 936b3d4eecSDavid S. Miller 946b3d4eecSDavid S. Miller#define bpf_slow_path_common(LEN) \ 956b3d4eecSDavid S. Miller save %sp, -SAVE_SZ, %sp; \ 966b3d4eecSDavid S. Miller mov %i0, %o0; \ 976b3d4eecSDavid S. Miller mov r_OFF, %o1; \ 986b3d4eecSDavid S. Miller add %fp, SCRATCH_OFF, %o2; \ 996b3d4eecSDavid S. Miller call skb_copy_bits; \ 1006b3d4eecSDavid S. Miller mov (LEN), %o3; \ 1016b3d4eecSDavid S. Miller cmp %o0, 0; \ 1026b3d4eecSDavid S. Miller restore; 1036b3d4eecSDavid S. Miller 1046b3d4eecSDavid S. Millerbpf_slow_path_word: 1056b3d4eecSDavid S. Miller bpf_slow_path_common(4) 1066b3d4eecSDavid S. Miller bl bpf_error 1076b3d4eecSDavid S. Miller ld [%sp + SCRATCH_OFF], r_A 1086b3d4eecSDavid S. Miller retl 1096b3d4eecSDavid S. Miller nop 1106b3d4eecSDavid S. Millerbpf_slow_path_half: 1116b3d4eecSDavid S. Miller bpf_slow_path_common(2) 1126b3d4eecSDavid S. Miller bl bpf_error 1136b3d4eecSDavid S. Miller lduh [%sp + SCRATCH_OFF], r_A 1146b3d4eecSDavid S. Miller retl 1156b3d4eecSDavid S. Miller nop 1166b3d4eecSDavid S. Millerbpf_slow_path_byte: 1176b3d4eecSDavid S. Miller bpf_slow_path_common(1) 1186b3d4eecSDavid S. Miller bl bpf_error 1196b3d4eecSDavid S. Miller ldub [%sp + SCRATCH_OFF], r_A 1206b3d4eecSDavid S. Miller retl 1216b3d4eecSDavid S. Miller nop 1226b3d4eecSDavid S. Millerbpf_slow_path_byte_msh: 1236b3d4eecSDavid S. Miller bpf_slow_path_common(1) 1246b3d4eecSDavid S. Miller bl bpf_error 1256b3d4eecSDavid S. Miller ldub [%sp + SCRATCH_OFF], r_A 1266b3d4eecSDavid S. Miller and r_OFF, 0xf, r_OFF 1276b3d4eecSDavid S. Miller retl 1286b3d4eecSDavid S. Miller sll r_OFF, 2, r_X 1296b3d4eecSDavid S. Miller 1306b3d4eecSDavid S. Miller#define bpf_negative_common(LEN) \ 1316b3d4eecSDavid S. Miller save %sp, -SAVE_SZ, %sp; \ 1326b3d4eecSDavid S. Miller mov %i0, %o0; \ 1336b3d4eecSDavid S. Miller mov r_OFF, %o1; \ 1346b3d4eecSDavid S. Miller SIGN_EXTEND(%o1); \ 1356b3d4eecSDavid S. Miller call bpf_internal_load_pointer_neg_helper; \ 1366b3d4eecSDavid S. Miller mov (LEN), %o2; \ 1376b3d4eecSDavid S. Miller mov %o0, r_TMP; \ 1386b3d4eecSDavid S. Miller cmp %o0, 0; \ 1396b3d4eecSDavid S. Miller BE_PTR(bpf_error); \ 1406b3d4eecSDavid S. Miller restore; 1416b3d4eecSDavid S. Miller 1426b3d4eecSDavid S. Millerbpf_slow_path_word_neg: 1436b3d4eecSDavid S. Miller sethi %hi(SKF_MAX_NEG_OFF), r_TMP 1446b3d4eecSDavid S. Miller cmp r_OFF, r_TMP 1456b3d4eecSDavid S. Miller bl bpf_error 1466b3d4eecSDavid S. Miller nop 1476b3d4eecSDavid S. Miller .globl bpf_jit_load_word_negative_offset 1486b3d4eecSDavid S. Millerbpf_jit_load_word_negative_offset: 1496b3d4eecSDavid S. Miller bpf_negative_common(4) 1506b3d4eecSDavid S. Miller andcc r_TMP, 3, %g0 1516b3d4eecSDavid S. Miller bne load_word_unaligned 1526b3d4eecSDavid S. Miller nop 1536b3d4eecSDavid S. Miller retl 1546b3d4eecSDavid S. Miller ld [r_TMP], r_A 1556b3d4eecSDavid S. Miller 1566b3d4eecSDavid S. Millerbpf_slow_path_half_neg: 1576b3d4eecSDavid S. Miller sethi %hi(SKF_MAX_NEG_OFF), r_TMP 1586b3d4eecSDavid S. Miller cmp r_OFF, r_TMP 1596b3d4eecSDavid S. Miller bl bpf_error 1606b3d4eecSDavid S. Miller nop 1616b3d4eecSDavid S. Miller .globl bpf_jit_load_half_negative_offset 1626b3d4eecSDavid S. Millerbpf_jit_load_half_negative_offset: 1636b3d4eecSDavid S. Miller bpf_negative_common(2) 1646b3d4eecSDavid S. Miller andcc r_TMP, 1, %g0 1656b3d4eecSDavid S. Miller bne load_half_unaligned 1666b3d4eecSDavid S. Miller nop 1676b3d4eecSDavid S. Miller retl 1686b3d4eecSDavid S. Miller lduh [r_TMP], r_A 1696b3d4eecSDavid S. Miller 1706b3d4eecSDavid S. Millerbpf_slow_path_byte_neg: 1716b3d4eecSDavid S. Miller sethi %hi(SKF_MAX_NEG_OFF), r_TMP 1726b3d4eecSDavid S. Miller cmp r_OFF, r_TMP 1736b3d4eecSDavid S. Miller bl bpf_error 1746b3d4eecSDavid S. Miller nop 1756b3d4eecSDavid S. Miller .globl bpf_jit_load_byte_negative_offset 1766b3d4eecSDavid S. Millerbpf_jit_load_byte_negative_offset: 1776b3d4eecSDavid S. Miller bpf_negative_common(1) 1786b3d4eecSDavid S. Miller retl 1796b3d4eecSDavid S. Miller ldub [r_TMP], r_A 1806b3d4eecSDavid S. Miller 1816b3d4eecSDavid S. Millerbpf_slow_path_byte_msh_neg: 1826b3d4eecSDavid S. Miller sethi %hi(SKF_MAX_NEG_OFF), r_TMP 1836b3d4eecSDavid S. Miller cmp r_OFF, r_TMP 1846b3d4eecSDavid S. Miller bl bpf_error 1856b3d4eecSDavid S. Miller nop 1866b3d4eecSDavid S. Miller .globl bpf_jit_load_byte_msh_negative_offset 1876b3d4eecSDavid S. Millerbpf_jit_load_byte_msh_negative_offset: 1886b3d4eecSDavid S. Miller bpf_negative_common(1) 1896b3d4eecSDavid S. Miller ldub [r_TMP], r_OFF 1906b3d4eecSDavid S. Miller and r_OFF, 0xf, r_OFF 1916b3d4eecSDavid S. Miller retl 1926b3d4eecSDavid S. Miller sll r_OFF, 2, r_X 1936b3d4eecSDavid S. Miller 1946b3d4eecSDavid S. Millerbpf_error: 1956b3d4eecSDavid S. Miller /* Make the JIT program return zero. The JIT epilogue 1966b3d4eecSDavid S. Miller * stores away the original %o7 into r_saved_O7. The 1976b3d4eecSDavid S. Miller * normal leaf function return is to use "retl" which 1986b3d4eecSDavid S. Miller * would evalute to "jmpl %o7 + 8, %g0" but we want to 1996b3d4eecSDavid S. Miller * use the saved value thus the sequence you see here. 2006b3d4eecSDavid S. Miller */ 2016b3d4eecSDavid S. Miller jmpl r_saved_O7 + 8, %g0 2026b3d4eecSDavid S. Miller clr %o0 203