1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * BPF JIT compiler for LoongArch 4 * 5 * Copyright (C) 2022 Loongson Technology Corporation Limited 6 */ 7 #include <linux/bitfield.h> 8 #include <linux/bpf.h> 9 #include <linux/filter.h> 10 #include <asm/cacheflush.h> 11 #include <asm/inst.h> 12 13 struct jit_ctx { 14 const struct bpf_prog *prog; 15 unsigned int idx; 16 unsigned int flags; 17 unsigned int epilogue_offset; 18 u32 *offset; 19 int num_exentries; 20 union loongarch_instruction *image; 21 u32 stack_size; 22 }; 23 24 struct jit_data { 25 struct bpf_binary_header *header; 26 u8 *image; 27 struct jit_ctx ctx; 28 }; 29 30 static inline void emit_nop(union loongarch_instruction *insn) 31 { 32 insn->word = INSN_NOP; 33 } 34 35 #define emit_insn(ctx, func, ...) \ 36 do { \ 37 if (ctx->image != NULL) { \ 38 union loongarch_instruction *insn = &ctx->image[ctx->idx]; \ 39 emit_##func(insn, ##__VA_ARGS__); \ 40 } \ 41 ctx->idx++; \ 42 } while (0) 43 44 #define is_signed_imm12(val) signed_imm_check(val, 12) 45 #define is_signed_imm14(val) signed_imm_check(val, 14) 46 #define is_signed_imm16(val) signed_imm_check(val, 16) 47 #define is_signed_imm26(val) signed_imm_check(val, 26) 48 #define is_signed_imm32(val) signed_imm_check(val, 32) 49 #define is_signed_imm52(val) signed_imm_check(val, 52) 50 #define is_unsigned_imm12(val) unsigned_imm_check(val, 12) 51 52 static inline int bpf2la_offset(int bpf_insn, int off, const struct jit_ctx *ctx) 53 { 54 /* BPF JMP offset is relative to the next instruction */ 55 bpf_insn++; 56 /* 57 * Whereas LoongArch branch instructions encode the offset 58 * from the branch itself, so we must subtract 1 from the 59 * instruction offset. 60 */ 61 return (ctx->offset[bpf_insn + off] - (ctx->offset[bpf_insn] - 1)); 62 } 63 64 static inline int epilogue_offset(const struct jit_ctx *ctx) 65 { 66 int from = ctx->idx; 67 int to = ctx->epilogue_offset; 68 69 return (to - from); 70 } 71 72 /* Zero-extend 32 bits into 64 bits */ 73 static inline void emit_zext_32(struct jit_ctx *ctx, enum loongarch_gpr reg, bool is32) 74 { 75 if (!is32) 76 return; 77 78 emit_insn(ctx, lu32id, reg, 0); 79 } 80 81 /* Signed-extend 32 bits into 64 bits */ 82 static inline void emit_sext_32(struct jit_ctx *ctx, enum loongarch_gpr reg, bool is32) 83 { 84 if (!is32) 85 return; 86 87 emit_insn(ctx, addiw, reg, reg, 0); 88 } 89 90 static inline void move_addr(struct jit_ctx *ctx, enum loongarch_gpr rd, u64 addr) 91 { 92 u64 imm_11_0, imm_31_12, imm_51_32, imm_63_52; 93 94 /* lu12iw rd, imm_31_12 */ 95 imm_31_12 = (addr >> 12) & 0xfffff; 96 emit_insn(ctx, lu12iw, rd, imm_31_12); 97 98 /* ori rd, rd, imm_11_0 */ 99 imm_11_0 = addr & 0xfff; 100 emit_insn(ctx, ori, rd, rd, imm_11_0); 101 102 /* lu32id rd, imm_51_32 */ 103 imm_51_32 = (addr >> 32) & 0xfffff; 104 emit_insn(ctx, lu32id, rd, imm_51_32); 105 106 /* lu52id rd, rd, imm_63_52 */ 107 imm_63_52 = (addr >> 52) & 0xfff; 108 emit_insn(ctx, lu52id, rd, rd, imm_63_52); 109 } 110 111 static inline void move_imm(struct jit_ctx *ctx, enum loongarch_gpr rd, long imm, bool is32) 112 { 113 long imm_11_0, imm_31_12, imm_51_32, imm_63_52, imm_51_0, imm_51_31; 114 115 /* or rd, $zero, $zero */ 116 if (imm == 0) { 117 emit_insn(ctx, or, rd, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_ZERO); 118 return; 119 } 120 121 /* addiw rd, $zero, imm_11_0 */ 122 if (is_signed_imm12(imm)) { 123 emit_insn(ctx, addiw, rd, LOONGARCH_GPR_ZERO, imm); 124 goto zext; 125 } 126 127 /* ori rd, $zero, imm_11_0 */ 128 if (is_unsigned_imm12(imm)) { 129 emit_insn(ctx, ori, rd, LOONGARCH_GPR_ZERO, imm); 130 goto zext; 131 } 132 133 /* lu52id rd, $zero, imm_63_52 */ 134 imm_63_52 = (imm >> 52) & 0xfff; 135 imm_51_0 = imm & 0xfffffffffffff; 136 if (imm_63_52 != 0 && imm_51_0 == 0) { 137 emit_insn(ctx, lu52id, rd, LOONGARCH_GPR_ZERO, imm_63_52); 138 return; 139 } 140 141 /* lu12iw rd, imm_31_12 */ 142 imm_31_12 = (imm >> 12) & 0xfffff; 143 emit_insn(ctx, lu12iw, rd, imm_31_12); 144 145 /* ori rd, rd, imm_11_0 */ 146 imm_11_0 = imm & 0xfff; 147 if (imm_11_0 != 0) 148 emit_insn(ctx, ori, rd, rd, imm_11_0); 149 150 if (!is_signed_imm32(imm)) { 151 if (imm_51_0 != 0) { 152 /* 153 * If bit[51:31] is all 0 or all 1, 154 * it means bit[51:32] is sign extended by lu12iw, 155 * no need to call lu32id to do a new filled operation. 156 */ 157 imm_51_31 = (imm >> 31) & 0x1fffff; 158 if (imm_51_31 != 0 && imm_51_31 != 0x1fffff) { 159 /* lu32id rd, imm_51_32 */ 160 imm_51_32 = (imm >> 32) & 0xfffff; 161 emit_insn(ctx, lu32id, rd, imm_51_32); 162 } 163 } 164 165 /* lu52id rd, rd, imm_63_52 */ 166 if (!is_signed_imm52(imm)) 167 emit_insn(ctx, lu52id, rd, rd, imm_63_52); 168 } 169 170 zext: 171 emit_zext_32(ctx, rd, is32); 172 } 173 174 static inline void move_reg(struct jit_ctx *ctx, enum loongarch_gpr rd, 175 enum loongarch_gpr rj) 176 { 177 emit_insn(ctx, or, rd, rj, LOONGARCH_GPR_ZERO); 178 } 179 180 static inline int invert_jmp_cond(u8 cond) 181 { 182 switch (cond) { 183 case BPF_JEQ: 184 return BPF_JNE; 185 case BPF_JNE: 186 case BPF_JSET: 187 return BPF_JEQ; 188 case BPF_JGT: 189 return BPF_JLE; 190 case BPF_JGE: 191 return BPF_JLT; 192 case BPF_JLT: 193 return BPF_JGE; 194 case BPF_JLE: 195 return BPF_JGT; 196 case BPF_JSGT: 197 return BPF_JSLE; 198 case BPF_JSGE: 199 return BPF_JSLT; 200 case BPF_JSLT: 201 return BPF_JSGE; 202 case BPF_JSLE: 203 return BPF_JSGT; 204 } 205 return -1; 206 } 207 208 static inline void cond_jmp_offset(struct jit_ctx *ctx, u8 cond, enum loongarch_gpr rj, 209 enum loongarch_gpr rd, int jmp_offset) 210 { 211 switch (cond) { 212 case BPF_JEQ: 213 /* PC += jmp_offset if rj == rd */ 214 emit_insn(ctx, beq, rj, rd, jmp_offset); 215 return; 216 case BPF_JNE: 217 case BPF_JSET: 218 /* PC += jmp_offset if rj != rd */ 219 emit_insn(ctx, bne, rj, rd, jmp_offset); 220 return; 221 case BPF_JGT: 222 /* PC += jmp_offset if rj > rd (unsigned) */ 223 emit_insn(ctx, bltu, rd, rj, jmp_offset); 224 return; 225 case BPF_JLT: 226 /* PC += jmp_offset if rj < rd (unsigned) */ 227 emit_insn(ctx, bltu, rj, rd, jmp_offset); 228 return; 229 case BPF_JGE: 230 /* PC += jmp_offset if rj >= rd (unsigned) */ 231 emit_insn(ctx, bgeu, rj, rd, jmp_offset); 232 return; 233 case BPF_JLE: 234 /* PC += jmp_offset if rj <= rd (unsigned) */ 235 emit_insn(ctx, bgeu, rd, rj, jmp_offset); 236 return; 237 case BPF_JSGT: 238 /* PC += jmp_offset if rj > rd (signed) */ 239 emit_insn(ctx, blt, rd, rj, jmp_offset); 240 return; 241 case BPF_JSLT: 242 /* PC += jmp_offset if rj < rd (signed) */ 243 emit_insn(ctx, blt, rj, rd, jmp_offset); 244 return; 245 case BPF_JSGE: 246 /* PC += jmp_offset if rj >= rd (signed) */ 247 emit_insn(ctx, bge, rj, rd, jmp_offset); 248 return; 249 case BPF_JSLE: 250 /* PC += jmp_offset if rj <= rd (signed) */ 251 emit_insn(ctx, bge, rd, rj, jmp_offset); 252 return; 253 } 254 } 255 256 static inline void cond_jmp_offs26(struct jit_ctx *ctx, u8 cond, enum loongarch_gpr rj, 257 enum loongarch_gpr rd, int jmp_offset) 258 { 259 cond = invert_jmp_cond(cond); 260 cond_jmp_offset(ctx, cond, rj, rd, 2); 261 emit_insn(ctx, b, jmp_offset); 262 } 263 264 static inline void uncond_jmp_offs26(struct jit_ctx *ctx, int jmp_offset) 265 { 266 emit_insn(ctx, b, jmp_offset); 267 } 268 269 static inline int emit_cond_jmp(struct jit_ctx *ctx, u8 cond, enum loongarch_gpr rj, 270 enum loongarch_gpr rd, int jmp_offset) 271 { 272 /* 273 * A large PC-relative jump offset may overflow the immediate field of 274 * the native conditional branch instruction, triggering a conversion 275 * to use an absolute jump instead, this jump sequence is particularly 276 * nasty. For now, use cond_jmp_offs26() directly to keep it simple. 277 * In the future, maybe we can add support for far branching, the branch 278 * relaxation requires more than two passes to converge, the code seems 279 * too complex to understand, not quite sure whether it is necessary and 280 * worth the extra pain. Anyway, just leave it as it is to enhance code 281 * readability now. 282 */ 283 if (is_signed_imm26(jmp_offset)) { 284 cond_jmp_offs26(ctx, cond, rj, rd, jmp_offset); 285 return 0; 286 } 287 288 return -EINVAL; 289 } 290 291 static inline int emit_uncond_jmp(struct jit_ctx *ctx, int jmp_offset) 292 { 293 if (is_signed_imm26(jmp_offset)) { 294 uncond_jmp_offs26(ctx, jmp_offset); 295 return 0; 296 } 297 298 return -EINVAL; 299 } 300 301 static inline int emit_tailcall_jmp(struct jit_ctx *ctx, u8 cond, enum loongarch_gpr rj, 302 enum loongarch_gpr rd, int jmp_offset) 303 { 304 if (is_signed_imm16(jmp_offset)) { 305 cond_jmp_offset(ctx, cond, rj, rd, jmp_offset); 306 return 0; 307 } 308 309 return -EINVAL; 310 } 311