1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * BPF JIT compiler for LoongArch 4 * 5 * Copyright (C) 2022 Loongson Technology Corporation Limited 6 */ 7 #include <linux/bitfield.h> 8 #include <linux/bpf.h> 9 #include <linux/filter.h> 10 #include <asm/cacheflush.h> 11 #include <asm/inst.h> 12 13 struct jit_ctx { 14 const struct bpf_prog *prog; 15 unsigned int idx; 16 unsigned int flags; 17 unsigned int epilogue_offset; 18 u32 *offset; 19 int num_exentries; 20 union loongarch_instruction *image; 21 union loongarch_instruction *ro_image; 22 u32 stack_size; 23 }; 24 25 struct jit_data { 26 struct bpf_binary_header *header; 27 u8 *image; 28 struct jit_ctx ctx; 29 }; 30 31 static inline void emit_nop(union loongarch_instruction *insn) 32 { 33 insn->word = INSN_NOP; 34 } 35 36 #define emit_insn(ctx, func, ...) \ 37 do { \ 38 if (ctx->image != NULL) { \ 39 union loongarch_instruction *insn = &ctx->image[ctx->idx]; \ 40 emit_##func(insn, ##__VA_ARGS__); \ 41 } \ 42 ctx->idx++; \ 43 } while (0) 44 45 #define is_signed_imm12(val) signed_imm_check(val, 12) 46 #define is_signed_imm14(val) signed_imm_check(val, 14) 47 #define is_signed_imm16(val) signed_imm_check(val, 16) 48 #define is_signed_imm26(val) signed_imm_check(val, 26) 49 #define is_signed_imm32(val) signed_imm_check(val, 32) 50 #define is_signed_imm52(val) signed_imm_check(val, 52) 51 #define is_unsigned_imm12(val) unsigned_imm_check(val, 12) 52 53 static inline int bpf2la_offset(int bpf_insn, int off, const struct jit_ctx *ctx) 54 { 55 /* BPF JMP offset is relative to the next instruction */ 56 bpf_insn++; 57 /* 58 * Whereas LoongArch branch instructions encode the offset 59 * from the branch itself, so we must subtract 1 from the 60 * instruction offset. 61 */ 62 return (ctx->offset[bpf_insn + off] - (ctx->offset[bpf_insn] - 1)); 63 } 64 65 static inline int epilogue_offset(const struct jit_ctx *ctx) 66 { 67 int from = ctx->idx; 68 int to = ctx->epilogue_offset; 69 70 return (to - from); 71 } 72 73 /* Zero-extend 32 bits into 64 bits */ 74 static inline void emit_zext_32(struct jit_ctx *ctx, enum loongarch_gpr reg, bool is32) 75 { 76 if (!is32) 77 return; 78 79 emit_insn(ctx, lu32id, reg, 0); 80 } 81 82 /* Signed-extend 32 bits into 64 bits */ 83 static inline void emit_sext_32(struct jit_ctx *ctx, enum loongarch_gpr reg, bool is32) 84 { 85 if (!is32) 86 return; 87 88 emit_insn(ctx, addiw, reg, reg, 0); 89 } 90 91 static inline void move_addr(struct jit_ctx *ctx, enum loongarch_gpr rd, u64 addr) 92 { 93 u64 imm_11_0, imm_31_12, imm_51_32, imm_63_52; 94 95 /* lu12iw rd, imm_31_12 */ 96 imm_31_12 = (addr >> 12) & 0xfffff; 97 emit_insn(ctx, lu12iw, rd, imm_31_12); 98 99 /* ori rd, rd, imm_11_0 */ 100 imm_11_0 = addr & 0xfff; 101 emit_insn(ctx, ori, rd, rd, imm_11_0); 102 103 /* lu32id rd, imm_51_32 */ 104 imm_51_32 = (addr >> 32) & 0xfffff; 105 emit_insn(ctx, lu32id, rd, imm_51_32); 106 107 /* lu52id rd, rd, imm_63_52 */ 108 imm_63_52 = (addr >> 52) & 0xfff; 109 emit_insn(ctx, lu52id, rd, rd, imm_63_52); 110 } 111 112 static inline void move_imm(struct jit_ctx *ctx, enum loongarch_gpr rd, long imm, bool is32) 113 { 114 long imm_11_0, imm_31_12, imm_51_32, imm_63_52, imm_51_0, imm_51_31; 115 116 /* or rd, $zero, $zero */ 117 if (imm == 0) { 118 emit_insn(ctx, or, rd, LOONGARCH_GPR_ZERO, LOONGARCH_GPR_ZERO); 119 return; 120 } 121 122 /* addiw rd, $zero, imm_11_0 */ 123 if (is_signed_imm12(imm)) { 124 emit_insn(ctx, addiw, rd, LOONGARCH_GPR_ZERO, imm); 125 goto zext; 126 } 127 128 /* ori rd, $zero, imm_11_0 */ 129 if (is_unsigned_imm12(imm)) { 130 emit_insn(ctx, ori, rd, LOONGARCH_GPR_ZERO, imm); 131 goto zext; 132 } 133 134 /* lu52id rd, $zero, imm_63_52 */ 135 imm_63_52 = (imm >> 52) & 0xfff; 136 imm_51_0 = imm & 0xfffffffffffff; 137 if (imm_63_52 != 0 && imm_51_0 == 0) { 138 emit_insn(ctx, lu52id, rd, LOONGARCH_GPR_ZERO, imm_63_52); 139 return; 140 } 141 142 /* lu12iw rd, imm_31_12 */ 143 imm_31_12 = (imm >> 12) & 0xfffff; 144 emit_insn(ctx, lu12iw, rd, imm_31_12); 145 146 /* ori rd, rd, imm_11_0 */ 147 imm_11_0 = imm & 0xfff; 148 if (imm_11_0 != 0) 149 emit_insn(ctx, ori, rd, rd, imm_11_0); 150 151 if (!is_signed_imm32(imm)) { 152 if (imm_51_0 != 0) { 153 /* 154 * If bit[51:31] is all 0 or all 1, 155 * it means bit[51:32] is sign extended by lu12iw, 156 * no need to call lu32id to do a new filled operation. 157 */ 158 imm_51_31 = (imm >> 31) & 0x1fffff; 159 if (imm_51_31 != 0 && imm_51_31 != 0x1fffff) { 160 /* lu32id rd, imm_51_32 */ 161 imm_51_32 = (imm >> 32) & 0xfffff; 162 emit_insn(ctx, lu32id, rd, imm_51_32); 163 } 164 } 165 166 /* lu52id rd, rd, imm_63_52 */ 167 if (!is_signed_imm52(imm)) 168 emit_insn(ctx, lu52id, rd, rd, imm_63_52); 169 } 170 171 zext: 172 emit_zext_32(ctx, rd, is32); 173 } 174 175 static inline void move_reg(struct jit_ctx *ctx, enum loongarch_gpr rd, 176 enum loongarch_gpr rj) 177 { 178 emit_insn(ctx, or, rd, rj, LOONGARCH_GPR_ZERO); 179 } 180 181 static inline int invert_jmp_cond(u8 cond) 182 { 183 switch (cond) { 184 case BPF_JEQ: 185 return BPF_JNE; 186 case BPF_JNE: 187 case BPF_JSET: 188 return BPF_JEQ; 189 case BPF_JGT: 190 return BPF_JLE; 191 case BPF_JGE: 192 return BPF_JLT; 193 case BPF_JLT: 194 return BPF_JGE; 195 case BPF_JLE: 196 return BPF_JGT; 197 case BPF_JSGT: 198 return BPF_JSLE; 199 case BPF_JSGE: 200 return BPF_JSLT; 201 case BPF_JSLT: 202 return BPF_JSGE; 203 case BPF_JSLE: 204 return BPF_JSGT; 205 } 206 return -1; 207 } 208 209 static inline void cond_jmp_offset(struct jit_ctx *ctx, u8 cond, enum loongarch_gpr rj, 210 enum loongarch_gpr rd, int jmp_offset) 211 { 212 switch (cond) { 213 case BPF_JEQ: 214 /* PC += jmp_offset if rj == rd */ 215 emit_insn(ctx, beq, rj, rd, jmp_offset); 216 return; 217 case BPF_JNE: 218 case BPF_JSET: 219 /* PC += jmp_offset if rj != rd */ 220 emit_insn(ctx, bne, rj, rd, jmp_offset); 221 return; 222 case BPF_JGT: 223 /* PC += jmp_offset if rj > rd (unsigned) */ 224 emit_insn(ctx, bltu, rd, rj, jmp_offset); 225 return; 226 case BPF_JLT: 227 /* PC += jmp_offset if rj < rd (unsigned) */ 228 emit_insn(ctx, bltu, rj, rd, jmp_offset); 229 return; 230 case BPF_JGE: 231 /* PC += jmp_offset if rj >= rd (unsigned) */ 232 emit_insn(ctx, bgeu, rj, rd, jmp_offset); 233 return; 234 case BPF_JLE: 235 /* PC += jmp_offset if rj <= rd (unsigned) */ 236 emit_insn(ctx, bgeu, rd, rj, jmp_offset); 237 return; 238 case BPF_JSGT: 239 /* PC += jmp_offset if rj > rd (signed) */ 240 emit_insn(ctx, blt, rd, rj, jmp_offset); 241 return; 242 case BPF_JSLT: 243 /* PC += jmp_offset if rj < rd (signed) */ 244 emit_insn(ctx, blt, rj, rd, jmp_offset); 245 return; 246 case BPF_JSGE: 247 /* PC += jmp_offset if rj >= rd (signed) */ 248 emit_insn(ctx, bge, rj, rd, jmp_offset); 249 return; 250 case BPF_JSLE: 251 /* PC += jmp_offset if rj <= rd (signed) */ 252 emit_insn(ctx, bge, rd, rj, jmp_offset); 253 return; 254 } 255 } 256 257 static inline void cond_jmp_offs26(struct jit_ctx *ctx, u8 cond, enum loongarch_gpr rj, 258 enum loongarch_gpr rd, int jmp_offset) 259 { 260 cond = invert_jmp_cond(cond); 261 cond_jmp_offset(ctx, cond, rj, rd, 2); 262 emit_insn(ctx, b, jmp_offset); 263 } 264 265 static inline void uncond_jmp_offs26(struct jit_ctx *ctx, int jmp_offset) 266 { 267 emit_insn(ctx, b, jmp_offset); 268 } 269 270 static inline int emit_cond_jmp(struct jit_ctx *ctx, u8 cond, enum loongarch_gpr rj, 271 enum loongarch_gpr rd, int jmp_offset) 272 { 273 /* 274 * A large PC-relative jump offset may overflow the immediate field of 275 * the native conditional branch instruction, triggering a conversion 276 * to use an absolute jump instead, this jump sequence is particularly 277 * nasty. For now, use cond_jmp_offs26() directly to keep it simple. 278 * In the future, maybe we can add support for far branching, the branch 279 * relaxation requires more than two passes to converge, the code seems 280 * too complex to understand, not quite sure whether it is necessary and 281 * worth the extra pain. Anyway, just leave it as it is to enhance code 282 * readability now. 283 */ 284 if (is_signed_imm26(jmp_offset)) { 285 cond_jmp_offs26(ctx, cond, rj, rd, jmp_offset); 286 return 0; 287 } 288 289 return -EINVAL; 290 } 291 292 static inline int emit_uncond_jmp(struct jit_ctx *ctx, int jmp_offset) 293 { 294 if (is_signed_imm26(jmp_offset)) { 295 uncond_jmp_offs26(ctx, jmp_offset); 296 return 0; 297 } 298 299 return -EINVAL; 300 } 301 302 static inline int emit_tailcall_jmp(struct jit_ctx *ctx, u8 cond, enum loongarch_gpr rj, 303 enum loongarch_gpr rd, int jmp_offset) 304 { 305 if (is_signed_imm16(jmp_offset)) { 306 cond_jmp_offset(ctx, cond, rj, rd, jmp_offset); 307 return 0; 308 } 309 310 return -EINVAL; 311 } 312 313 static inline void bpf_flush_icache(void *start, void *end) 314 { 315 flush_icache_range((unsigned long)start, (unsigned long)end); 316 } 317