1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Just-In-Time compiler for eBPF bytecode on MIPS. 4 * Implementation of JIT functions for 64-bit CPUs. 5 * 6 * Copyright (c) 2021 Anyfi Networks AB. 7 * Author: Johan Almbladh <johan.almbladh@gmail.com> 8 * 9 * Based on code and ideas from 10 * Copyright (c) 2017 Cavium, Inc. 11 * Copyright (c) 2017 Shubham Bansal <illusionist.neo@gmail.com> 12 * Copyright (c) 2011 Mircea Gherzan <mgherzan@gmail.com> 13 */ 14 15 #include <linux/errno.h> 16 #include <linux/filter.h> 17 #include <linux/bpf.h> 18 #include <asm/cpu-features.h> 19 #include <asm/isa-rev.h> 20 #include <asm/uasm.h> 21 22 #include "bpf_jit_comp.h" 23 24 /* MIPS t0-t3 are not available in the n64 ABI */ 25 #undef MIPS_R_T0 26 #undef MIPS_R_T1 27 #undef MIPS_R_T2 28 #undef MIPS_R_T3 29 30 /* Stack is 16-byte aligned in n64 ABI */ 31 #define MIPS_STACK_ALIGNMENT 16 32 33 /* Extra 64-bit eBPF registers used by JIT */ 34 #define JIT_REG_TC (MAX_BPF_JIT_REG + 0) 35 #define JIT_REG_ZX (MAX_BPF_JIT_REG + 1) 36 37 /* Number of prologue bytes to skip when doing a tail call */ 38 #define JIT_TCALL_SKIP 4 39 40 /* Callee-saved CPU registers that the JIT must preserve */ 41 #define JIT_CALLEE_REGS \ 42 (BIT(MIPS_R_S0) | \ 43 BIT(MIPS_R_S1) | \ 44 BIT(MIPS_R_S2) | \ 45 BIT(MIPS_R_S3) | \ 46 BIT(MIPS_R_S4) | \ 47 BIT(MIPS_R_S5) | \ 48 BIT(MIPS_R_S6) | \ 49 BIT(MIPS_R_S7) | \ 50 BIT(MIPS_R_GP) | \ 51 BIT(MIPS_R_FP) | \ 52 BIT(MIPS_R_RA)) 53 54 /* Caller-saved CPU registers available for JIT use */ 55 #define JIT_CALLER_REGS \ 56 (BIT(MIPS_R_A5) | \ 57 BIT(MIPS_R_A6) | \ 58 BIT(MIPS_R_A7)) 59 /* 60 * Mapping of 64-bit eBPF registers to 64-bit native MIPS registers. 61 * MIPS registers t4 - t7 may be used by the JIT as temporary registers. 62 * MIPS registers t8 - t9 are reserved for single-register common functions. 63 */ 64 static const u8 bpf2mips64[] = { 65 /* Return value from in-kernel function, and exit value from eBPF */ 66 [BPF_REG_0] = MIPS_R_V0, 67 /* Arguments from eBPF program to in-kernel function */ 68 [BPF_REG_1] = MIPS_R_A0, 69 [BPF_REG_2] = MIPS_R_A1, 70 [BPF_REG_3] = MIPS_R_A2, 71 [BPF_REG_4] = MIPS_R_A3, 72 [BPF_REG_5] = MIPS_R_A4, 73 /* Callee-saved registers that in-kernel function will preserve */ 74 [BPF_REG_6] = MIPS_R_S0, 75 [BPF_REG_7] = MIPS_R_S1, 76 [BPF_REG_8] = MIPS_R_S2, 77 [BPF_REG_9] = MIPS_R_S3, 78 /* Read-only frame pointer to access the eBPF stack */ 79 [BPF_REG_FP] = MIPS_R_FP, 80 /* Temporary register for blinding constants */ 81 [BPF_REG_AX] = MIPS_R_AT, 82 /* Tail call count register, caller-saved */ 83 [JIT_REG_TC] = MIPS_R_A5, 84 /* Constant for register zero-extension */ 85 [JIT_REG_ZX] = MIPS_R_V1, 86 }; 87 88 /* 89 * MIPS 32-bit operations on 64-bit registers generate a sign-extended 90 * result. However, the eBPF ISA mandates zero-extension, so we rely on the 91 * verifier to add that for us (emit_zext_ver). In addition, ALU arithmetic 92 * operations, right shift and byte swap require properly sign-extended 93 * operands or the result is unpredictable. We emit explicit sign-extensions 94 * in those cases. 95 */ 96 97 /* Sign extension */ 98 static void emit_sext(struct jit_context *ctx, u8 dst, u8 src) 99 { 100 emit(ctx, sll, dst, src, 0); 101 clobber_reg(ctx, dst); 102 } 103 104 /* Zero extension */ 105 static void emit_zext(struct jit_context *ctx, u8 dst) 106 { 107 if (cpu_has_mips64r2 || cpu_has_mips64r6) { 108 emit(ctx, dinsu, dst, MIPS_R_ZERO, 32, 32); 109 } else { 110 emit(ctx, and, dst, dst, bpf2mips64[JIT_REG_ZX]); 111 access_reg(ctx, JIT_REG_ZX); /* We need the ZX register */ 112 } 113 clobber_reg(ctx, dst); 114 } 115 116 /* Zero extension, if verifier does not do it for us */ 117 static void emit_zext_ver(struct jit_context *ctx, u8 dst) 118 { 119 if (!ctx->program->aux->verifier_zext) 120 emit_zext(ctx, dst); 121 } 122 123 /* dst = imm (64-bit) */ 124 static void emit_mov_i64(struct jit_context *ctx, u8 dst, u64 imm64) 125 { 126 if (imm64 >= 0xffffffffffff8000ULL || imm64 < 0x8000ULL) { 127 emit(ctx, daddiu, dst, MIPS_R_ZERO, (s16)imm64); 128 } else if (imm64 >= 0xffffffff80000000ULL || 129 (imm64 < 0x80000000 && imm64 > 0xffff)) { 130 emit(ctx, lui, dst, (s16)(imm64 >> 16)); 131 emit(ctx, ori, dst, dst, (u16)imm64 & 0xffff); 132 } else { 133 u8 acc = MIPS_R_ZERO; 134 int k; 135 136 for (k = 0; k < 4; k++) { 137 u16 half = imm64 >> (48 - 16 * k); 138 139 if (acc == dst) 140 emit(ctx, dsll, dst, dst, 16); 141 142 if (half) { 143 emit(ctx, ori, dst, acc, half); 144 acc = dst; 145 } 146 } 147 } 148 clobber_reg(ctx, dst); 149 } 150 151 /* ALU immediate operation (64-bit) */ 152 static void emit_alu_i64(struct jit_context *ctx, u8 dst, s32 imm, u8 op) 153 { 154 switch (BPF_OP(op)) { 155 /* dst = dst | imm */ 156 case BPF_OR: 157 emit(ctx, ori, dst, dst, (u16)imm); 158 break; 159 /* dst = dst ^ imm */ 160 case BPF_XOR: 161 emit(ctx, xori, dst, dst, (u16)imm); 162 break; 163 /* dst = -dst */ 164 case BPF_NEG: 165 emit(ctx, dsubu, dst, MIPS_R_ZERO, dst); 166 break; 167 /* dst = dst << imm */ 168 case BPF_LSH: 169 emit(ctx, dsll_safe, dst, dst, imm); 170 break; 171 /* dst = dst >> imm */ 172 case BPF_RSH: 173 emit(ctx, dsrl_safe, dst, dst, imm); 174 break; 175 /* dst = dst >> imm (arithmetic) */ 176 case BPF_ARSH: 177 emit(ctx, dsra_safe, dst, dst, imm); 178 break; 179 /* dst = dst + imm */ 180 case BPF_ADD: 181 emit(ctx, daddiu, dst, dst, imm); 182 break; 183 /* dst = dst - imm */ 184 case BPF_SUB: 185 emit(ctx, daddiu, dst, dst, -imm); 186 break; 187 default: 188 /* Width-generic operations */ 189 emit_alu_i(ctx, dst, imm, op); 190 } 191 clobber_reg(ctx, dst); 192 } 193 194 /* ALU register operation (64-bit) */ 195 static void emit_alu_r64(struct jit_context *ctx, u8 dst, u8 src, u8 op) 196 { 197 switch (BPF_OP(op)) { 198 /* dst = dst << src */ 199 case BPF_LSH: 200 emit(ctx, dsllv, dst, dst, src); 201 break; 202 /* dst = dst >> src */ 203 case BPF_RSH: 204 emit(ctx, dsrlv, dst, dst, src); 205 break; 206 /* dst = dst >> src (arithmetic) */ 207 case BPF_ARSH: 208 emit(ctx, dsrav, dst, dst, src); 209 break; 210 /* dst = dst + src */ 211 case BPF_ADD: 212 emit(ctx, daddu, dst, dst, src); 213 break; 214 /* dst = dst - src */ 215 case BPF_SUB: 216 emit(ctx, dsubu, dst, dst, src); 217 break; 218 /* dst = dst * src */ 219 case BPF_MUL: 220 if (cpu_has_mips64r6) { 221 emit(ctx, dmulu, dst, dst, src); 222 } else { 223 emit(ctx, dmultu, dst, src); 224 emit(ctx, mflo, dst); 225 } 226 break; 227 /* dst = dst / src */ 228 case BPF_DIV: 229 if (cpu_has_mips64r6) { 230 emit(ctx, ddivu_r6, dst, dst, src); 231 } else { 232 emit(ctx, ddivu, dst, src); 233 emit(ctx, mflo, dst); 234 } 235 break; 236 /* dst = dst % src */ 237 case BPF_MOD: 238 if (cpu_has_mips64r6) { 239 emit(ctx, dmodu, dst, dst, src); 240 } else { 241 emit(ctx, ddivu, dst, src); 242 emit(ctx, mfhi, dst); 243 } 244 break; 245 default: 246 /* Width-generic operations */ 247 emit_alu_r(ctx, dst, src, op); 248 } 249 clobber_reg(ctx, dst); 250 } 251 252 /* Swap sub words in a register double word */ 253 static void emit_swap_r64(struct jit_context *ctx, u8 dst, u8 mask, u32 bits) 254 { 255 u8 tmp = MIPS_R_T9; 256 257 emit(ctx, and, tmp, dst, mask); /* tmp = dst & mask */ 258 emit(ctx, dsll, tmp, tmp, bits); /* tmp = tmp << bits */ 259 emit(ctx, dsrl, dst, dst, bits); /* dst = dst >> bits */ 260 emit(ctx, and, dst, dst, mask); /* dst = dst & mask */ 261 emit(ctx, or, dst, dst, tmp); /* dst = dst | tmp */ 262 } 263 264 /* Swap bytes and truncate a register double word, word or half word */ 265 static void emit_bswap_r64(struct jit_context *ctx, u8 dst, u32 width) 266 { 267 switch (width) { 268 /* Swap bytes in a double word */ 269 case 64: 270 if (cpu_has_mips64r2 || cpu_has_mips64r6) { 271 emit(ctx, dsbh, dst, dst); 272 emit(ctx, dshd, dst, dst); 273 } else { 274 u8 t1 = MIPS_R_T6; 275 u8 t2 = MIPS_R_T7; 276 277 emit(ctx, dsll32, t2, dst, 0); /* t2 = dst << 32 */ 278 emit(ctx, dsrl32, dst, dst, 0); /* dst = dst >> 32 */ 279 emit(ctx, or, dst, dst, t2); /* dst = dst | t2 */ 280 281 emit(ctx, ori, t2, MIPS_R_ZERO, 0xffff); 282 emit(ctx, dsll32, t1, t2, 0); /* t1 = t2 << 32 */ 283 emit(ctx, or, t1, t1, t2); /* t1 = t1 | t2 */ 284 emit_swap_r64(ctx, dst, t1, 16);/* dst = swap16(dst) */ 285 286 emit(ctx, lui, t2, 0xff); /* t2 = 0x00ff0000 */ 287 emit(ctx, ori, t2, t2, 0xff); /* t2 = t2 | 0x00ff */ 288 emit(ctx, dsll32, t1, t2, 0); /* t1 = t2 << 32 */ 289 emit(ctx, or, t1, t1, t2); /* t1 = t1 | t2 */ 290 emit_swap_r64(ctx, dst, t1, 8); /* dst = swap8(dst) */ 291 } 292 break; 293 /* Swap bytes in a half word */ 294 /* Swap bytes in a word */ 295 case 32: 296 case 16: 297 emit_sext(ctx, dst, dst); 298 emit_bswap_r(ctx, dst, width); 299 if (cpu_has_mips64r2 || cpu_has_mips64r6) 300 emit_zext(ctx, dst); 301 break; 302 } 303 clobber_reg(ctx, dst); 304 } 305 306 /* Truncate a register double word, word or half word */ 307 static void emit_trunc_r64(struct jit_context *ctx, u8 dst, u32 width) 308 { 309 switch (width) { 310 case 64: 311 break; 312 /* Zero-extend a word */ 313 case 32: 314 emit_zext(ctx, dst); 315 break; 316 /* Zero-extend a half word */ 317 case 16: 318 emit(ctx, andi, dst, dst, 0xffff); 319 break; 320 } 321 clobber_reg(ctx, dst); 322 } 323 324 /* Load operation: dst = *(size*)(src + off) */ 325 static void emit_ldx(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 size) 326 { 327 switch (size) { 328 /* Load a byte */ 329 case BPF_B: 330 emit(ctx, lbu, dst, off, src); 331 break; 332 /* Load a half word */ 333 case BPF_H: 334 emit(ctx, lhu, dst, off, src); 335 break; 336 /* Load a word */ 337 case BPF_W: 338 emit(ctx, lwu, dst, off, src); 339 break; 340 /* Load a double word */ 341 case BPF_DW: 342 emit(ctx, ld, dst, off, src); 343 break; 344 } 345 clobber_reg(ctx, dst); 346 } 347 348 /* Store operation: *(size *)(dst + off) = src */ 349 static void emit_stx(struct jit_context *ctx, u8 dst, u8 src, s16 off, u8 size) 350 { 351 switch (size) { 352 /* Store a byte */ 353 case BPF_B: 354 emit(ctx, sb, src, off, dst); 355 break; 356 /* Store a half word */ 357 case BPF_H: 358 emit(ctx, sh, src, off, dst); 359 break; 360 /* Store a word */ 361 case BPF_W: 362 emit(ctx, sw, src, off, dst); 363 break; 364 /* Store a double word */ 365 case BPF_DW: 366 emit(ctx, sd, src, off, dst); 367 break; 368 } 369 } 370 371 /* Atomic read-modify-write */ 372 static void emit_atomic_r64(struct jit_context *ctx, 373 u8 dst, u8 src, s16 off, u8 code) 374 { 375 u8 t1 = MIPS_R_T6; 376 u8 t2 = MIPS_R_T7; 377 378 LLSC_sync(ctx); 379 emit(ctx, lld, t1, off, dst); 380 switch (code) { 381 case BPF_ADD: 382 case BPF_ADD | BPF_FETCH: 383 emit(ctx, daddu, t2, t1, src); 384 break; 385 case BPF_AND: 386 case BPF_AND | BPF_FETCH: 387 emit(ctx, and, t2, t1, src); 388 break; 389 case BPF_OR: 390 case BPF_OR | BPF_FETCH: 391 emit(ctx, or, t2, t1, src); 392 break; 393 case BPF_XOR: 394 case BPF_XOR | BPF_FETCH: 395 emit(ctx, xor, t2, t1, src); 396 break; 397 case BPF_XCHG: 398 emit(ctx, move, t2, src); 399 break; 400 } 401 emit(ctx, scd, t2, off, dst); 402 emit(ctx, LLSC_beqz, t2, -16 - LLSC_offset); 403 emit(ctx, nop); /* Delay slot */ 404 405 if (code & BPF_FETCH) { 406 emit(ctx, move, src, t1); 407 clobber_reg(ctx, src); 408 } 409 } 410 411 /* Atomic compare-and-exchange */ 412 static void emit_cmpxchg_r64(struct jit_context *ctx, u8 dst, u8 src, s16 off) 413 { 414 u8 r0 = bpf2mips64[BPF_REG_0]; 415 u8 t1 = MIPS_R_T6; 416 u8 t2 = MIPS_R_T7; 417 418 LLSC_sync(ctx); 419 emit(ctx, lld, t1, off, dst); 420 emit(ctx, bne, t1, r0, 12); 421 emit(ctx, move, t2, src); /* Delay slot */ 422 emit(ctx, scd, t2, off, dst); 423 emit(ctx, LLSC_beqz, t2, -20 - LLSC_offset); 424 emit(ctx, move, r0, t1); /* Delay slot */ 425 426 clobber_reg(ctx, r0); 427 } 428 429 /* Function call */ 430 static int emit_call(struct jit_context *ctx, const struct bpf_insn *insn) 431 { 432 u8 zx = bpf2mips64[JIT_REG_ZX]; 433 u8 tmp = MIPS_R_T6; 434 bool fixed; 435 u64 addr; 436 437 /* Decode the call address */ 438 if (bpf_jit_get_func_addr(ctx->program, insn, false, 439 &addr, &fixed) < 0) 440 return -1; 441 if (!fixed) 442 return -1; 443 444 /* Push caller-saved registers on stack */ 445 push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, 0, 0); 446 447 /* Emit function call */ 448 emit_mov_i64(ctx, tmp, addr & JALR_MASK); 449 emit(ctx, jalr, MIPS_R_RA, tmp); 450 emit(ctx, nop); /* Delay slot */ 451 452 /* Restore caller-saved registers */ 453 pop_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, 0, 0); 454 455 /* Re-initialize the JIT zero-extension register if accessed */ 456 if (ctx->accessed & BIT(JIT_REG_ZX)) { 457 emit(ctx, daddiu, zx, MIPS_R_ZERO, -1); 458 emit(ctx, dsrl32, zx, zx, 0); 459 } 460 461 clobber_reg(ctx, MIPS_R_RA); 462 clobber_reg(ctx, MIPS_R_V0); 463 clobber_reg(ctx, MIPS_R_V1); 464 return 0; 465 } 466 467 /* Function tail call */ 468 static int emit_tail_call(struct jit_context *ctx) 469 { 470 u8 ary = bpf2mips64[BPF_REG_2]; 471 u8 ind = bpf2mips64[BPF_REG_3]; 472 u8 tcc = bpf2mips64[JIT_REG_TC]; 473 u8 tmp = MIPS_R_T6; 474 int off; 475 476 /* 477 * Tail call: 478 * eBPF R1 - function argument (context ptr), passed in a0-a1 479 * eBPF R2 - ptr to object with array of function entry points 480 * eBPF R3 - array index of function to be called 481 */ 482 483 /* if (ind >= ary->map.max_entries) goto out */ 484 off = offsetof(struct bpf_array, map.max_entries); 485 if (off > 0x7fff) 486 return -1; 487 emit(ctx, lwu, tmp, off, ary); /* tmp = ary->map.max_entrs*/ 488 emit(ctx, sltu, tmp, ind, tmp); /* tmp = ind < t1 */ 489 emit(ctx, beqz, tmp, get_offset(ctx, 1)); /* PC += off(1) if tmp == 0*/ 490 491 /* if (--TCC < 0) goto out */ 492 emit(ctx, daddiu, tcc, tcc, -1); /* tcc-- (delay slot) */ 493 emit(ctx, bltz, tcc, get_offset(ctx, 1)); /* PC += off(1) if tcc < 0 */ 494 /* (next insn delay slot) */ 495 /* prog = ary->ptrs[ind] */ 496 off = offsetof(struct bpf_array, ptrs); 497 if (off > 0x7fff) 498 return -1; 499 emit(ctx, dsll, tmp, ind, 3); /* tmp = ind << 3 */ 500 emit(ctx, daddu, tmp, tmp, ary); /* tmp += ary */ 501 emit(ctx, ld, tmp, off, tmp); /* tmp = *(tmp + off) */ 502 503 /* if (prog == 0) goto out */ 504 emit(ctx, beqz, tmp, get_offset(ctx, 1)); /* PC += off(1) if tmp == 0*/ 505 emit(ctx, nop); /* Delay slot */ 506 507 /* func = prog->bpf_func + 8 (prologue skip offset) */ 508 off = offsetof(struct bpf_prog, bpf_func); 509 if (off > 0x7fff) 510 return -1; 511 emit(ctx, ld, tmp, off, tmp); /* tmp = *(tmp + off) */ 512 emit(ctx, daddiu, tmp, tmp, JIT_TCALL_SKIP); /* tmp += skip (4) */ 513 514 /* goto func */ 515 build_epilogue(ctx, tmp); 516 access_reg(ctx, JIT_REG_TC); 517 return 0; 518 } 519 520 /* 521 * Stack frame layout for a JITed program (stack grows down). 522 * 523 * Higher address : Previous stack frame : 524 * +===========================+ <--- MIPS sp before call 525 * | Callee-saved registers, | 526 * | including RA and FP | 527 * +---------------------------+ <--- eBPF FP (MIPS fp) 528 * | Local eBPF variables | 529 * | allocated by program | 530 * +---------------------------+ 531 * | Reserved for caller-saved | 532 * | registers | 533 * Lower address +===========================+ <--- MIPS sp 534 */ 535 536 /* Build program prologue to set up the stack and registers */ 537 void build_prologue(struct jit_context *ctx) 538 { 539 u8 fp = bpf2mips64[BPF_REG_FP]; 540 u8 tc = bpf2mips64[JIT_REG_TC]; 541 u8 zx = bpf2mips64[JIT_REG_ZX]; 542 int stack, saved, locals, reserved; 543 544 /* 545 * The first instruction initializes the tail call count register. 546 * On a tail call, the calling function jumps into the prologue 547 * after this instruction. 548 */ 549 emit(ctx, addiu, tc, MIPS_R_ZERO, min(MAX_TAIL_CALL_CNT + 1, 0xffff)); 550 551 /* === Entry-point for tail calls === */ 552 553 /* 554 * If the eBPF frame pointer and tail call count registers were 555 * accessed they must be preserved. Mark them as clobbered here 556 * to save and restore them on the stack as needed. 557 */ 558 if (ctx->accessed & BIT(BPF_REG_FP)) 559 clobber_reg(ctx, fp); 560 if (ctx->accessed & BIT(JIT_REG_TC)) 561 clobber_reg(ctx, tc); 562 if (ctx->accessed & BIT(JIT_REG_ZX)) 563 clobber_reg(ctx, zx); 564 565 /* Compute the stack space needed for callee-saved registers */ 566 saved = hweight32(ctx->clobbered & JIT_CALLEE_REGS) * sizeof(u64); 567 saved = ALIGN(saved, MIPS_STACK_ALIGNMENT); 568 569 /* Stack space used by eBPF program local data */ 570 locals = ALIGN(ctx->program->aux->stack_depth, MIPS_STACK_ALIGNMENT); 571 572 /* 573 * If we are emitting function calls, reserve extra stack space for 574 * caller-saved registers needed by the JIT. The required space is 575 * computed automatically during resource usage discovery (pass 1). 576 */ 577 reserved = ctx->stack_used; 578 579 /* Allocate the stack frame */ 580 stack = ALIGN(saved + locals + reserved, MIPS_STACK_ALIGNMENT); 581 if (stack) 582 emit(ctx, daddiu, MIPS_R_SP, MIPS_R_SP, -stack); 583 584 /* Store callee-saved registers on stack */ 585 push_regs(ctx, ctx->clobbered & JIT_CALLEE_REGS, 0, stack - saved); 586 587 /* Initialize the eBPF frame pointer if accessed */ 588 if (ctx->accessed & BIT(BPF_REG_FP)) 589 emit(ctx, daddiu, fp, MIPS_R_SP, stack - saved); 590 591 /* Initialize the ePF JIT zero-extension register if accessed */ 592 if (ctx->accessed & BIT(JIT_REG_ZX)) { 593 emit(ctx, daddiu, zx, MIPS_R_ZERO, -1); 594 emit(ctx, dsrl32, zx, zx, 0); 595 } 596 597 ctx->saved_size = saved; 598 ctx->stack_size = stack; 599 } 600 601 /* Build the program epilogue to restore the stack and registers */ 602 void build_epilogue(struct jit_context *ctx, int dest_reg) 603 { 604 /* Restore callee-saved registers from stack */ 605 pop_regs(ctx, ctx->clobbered & JIT_CALLEE_REGS, 0, 606 ctx->stack_size - ctx->saved_size); 607 608 /* Release the stack frame */ 609 if (ctx->stack_size) 610 emit(ctx, daddiu, MIPS_R_SP, MIPS_R_SP, ctx->stack_size); 611 612 /* Jump to return address and sign-extend the 32-bit return value */ 613 emit(ctx, jr, dest_reg); 614 emit(ctx, sll, MIPS_R_V0, MIPS_R_V0, 0); /* Delay slot */ 615 } 616 617 /* Build one eBPF instruction */ 618 int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) 619 { 620 u8 dst = bpf2mips64[insn->dst_reg]; 621 u8 src = bpf2mips64[insn->src_reg]; 622 u8 res = bpf2mips64[BPF_REG_0]; 623 u8 code = insn->code; 624 s16 off = insn->off; 625 s32 imm = insn->imm; 626 s32 val, rel; 627 u8 alu, jmp; 628 629 switch (code) { 630 /* ALU operations */ 631 /* dst = imm */ 632 case BPF_ALU | BPF_MOV | BPF_K: 633 emit_mov_i(ctx, dst, imm); 634 emit_zext_ver(ctx, dst); 635 break; 636 /* dst = src */ 637 case BPF_ALU | BPF_MOV | BPF_X: 638 if (imm == 1) { 639 /* Special mov32 for zext */ 640 emit_zext(ctx, dst); 641 } else { 642 emit_mov_r(ctx, dst, src); 643 emit_zext_ver(ctx, dst); 644 } 645 break; 646 /* dst = -dst */ 647 case BPF_ALU | BPF_NEG: 648 emit_sext(ctx, dst, dst); 649 emit_alu_i(ctx, dst, 0, BPF_NEG); 650 emit_zext_ver(ctx, dst); 651 break; 652 /* dst = dst & imm */ 653 /* dst = dst | imm */ 654 /* dst = dst ^ imm */ 655 /* dst = dst << imm */ 656 case BPF_ALU | BPF_OR | BPF_K: 657 case BPF_ALU | BPF_AND | BPF_K: 658 case BPF_ALU | BPF_XOR | BPF_K: 659 case BPF_ALU | BPF_LSH | BPF_K: 660 if (!valid_alu_i(BPF_OP(code), imm)) { 661 emit_mov_i(ctx, MIPS_R_T4, imm); 662 emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code)); 663 } else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) { 664 emit_alu_i(ctx, dst, val, alu); 665 } 666 emit_zext_ver(ctx, dst); 667 break; 668 /* dst = dst >> imm */ 669 /* dst = dst >> imm (arithmetic) */ 670 /* dst = dst + imm */ 671 /* dst = dst - imm */ 672 /* dst = dst * imm */ 673 /* dst = dst / imm */ 674 /* dst = dst % imm */ 675 case BPF_ALU | BPF_RSH | BPF_K: 676 case BPF_ALU | BPF_ARSH | BPF_K: 677 case BPF_ALU | BPF_ADD | BPF_K: 678 case BPF_ALU | BPF_SUB | BPF_K: 679 case BPF_ALU | BPF_MUL | BPF_K: 680 case BPF_ALU | BPF_DIV | BPF_K: 681 case BPF_ALU | BPF_MOD | BPF_K: 682 if (!valid_alu_i(BPF_OP(code), imm)) { 683 emit_sext(ctx, dst, dst); 684 emit_mov_i(ctx, MIPS_R_T4, imm); 685 emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code)); 686 } else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) { 687 emit_sext(ctx, dst, dst); 688 emit_alu_i(ctx, dst, val, alu); 689 } 690 emit_zext_ver(ctx, dst); 691 break; 692 /* dst = dst & src */ 693 /* dst = dst | src */ 694 /* dst = dst ^ src */ 695 /* dst = dst << src */ 696 case BPF_ALU | BPF_AND | BPF_X: 697 case BPF_ALU | BPF_OR | BPF_X: 698 case BPF_ALU | BPF_XOR | BPF_X: 699 case BPF_ALU | BPF_LSH | BPF_X: 700 emit_alu_r(ctx, dst, src, BPF_OP(code)); 701 emit_zext_ver(ctx, dst); 702 break; 703 /* dst = dst >> src */ 704 /* dst = dst >> src (arithmetic) */ 705 /* dst = dst + src */ 706 /* dst = dst - src */ 707 /* dst = dst * src */ 708 /* dst = dst / src */ 709 /* dst = dst % src */ 710 case BPF_ALU | BPF_RSH | BPF_X: 711 case BPF_ALU | BPF_ARSH | BPF_X: 712 case BPF_ALU | BPF_ADD | BPF_X: 713 case BPF_ALU | BPF_SUB | BPF_X: 714 case BPF_ALU | BPF_MUL | BPF_X: 715 case BPF_ALU | BPF_DIV | BPF_X: 716 case BPF_ALU | BPF_MOD | BPF_X: 717 emit_sext(ctx, dst, dst); 718 emit_sext(ctx, MIPS_R_T4, src); 719 emit_alu_r(ctx, dst, MIPS_R_T4, BPF_OP(code)); 720 emit_zext_ver(ctx, dst); 721 break; 722 /* dst = imm (64-bit) */ 723 case BPF_ALU64 | BPF_MOV | BPF_K: 724 emit_mov_i(ctx, dst, imm); 725 break; 726 /* dst = src (64-bit) */ 727 case BPF_ALU64 | BPF_MOV | BPF_X: 728 emit_mov_r(ctx, dst, src); 729 break; 730 /* dst = -dst (64-bit) */ 731 case BPF_ALU64 | BPF_NEG: 732 emit_alu_i64(ctx, dst, 0, BPF_NEG); 733 break; 734 /* dst = dst & imm (64-bit) */ 735 /* dst = dst | imm (64-bit) */ 736 /* dst = dst ^ imm (64-bit) */ 737 /* dst = dst << imm (64-bit) */ 738 /* dst = dst >> imm (64-bit) */ 739 /* dst = dst >> imm ((64-bit, arithmetic) */ 740 /* dst = dst + imm (64-bit) */ 741 /* dst = dst - imm (64-bit) */ 742 /* dst = dst * imm (64-bit) */ 743 /* dst = dst / imm (64-bit) */ 744 /* dst = dst % imm (64-bit) */ 745 case BPF_ALU64 | BPF_AND | BPF_K: 746 case BPF_ALU64 | BPF_OR | BPF_K: 747 case BPF_ALU64 | BPF_XOR | BPF_K: 748 case BPF_ALU64 | BPF_LSH | BPF_K: 749 case BPF_ALU64 | BPF_RSH | BPF_K: 750 case BPF_ALU64 | BPF_ARSH | BPF_K: 751 case BPF_ALU64 | BPF_ADD | BPF_K: 752 case BPF_ALU64 | BPF_SUB | BPF_K: 753 case BPF_ALU64 | BPF_MUL | BPF_K: 754 case BPF_ALU64 | BPF_DIV | BPF_K: 755 case BPF_ALU64 | BPF_MOD | BPF_K: 756 if (!valid_alu_i(BPF_OP(code), imm)) { 757 emit_mov_i(ctx, MIPS_R_T4, imm); 758 emit_alu_r64(ctx, dst, MIPS_R_T4, BPF_OP(code)); 759 } else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) { 760 emit_alu_i64(ctx, dst, val, alu); 761 } 762 break; 763 /* dst = dst & src (64-bit) */ 764 /* dst = dst | src (64-bit) */ 765 /* dst = dst ^ src (64-bit) */ 766 /* dst = dst << src (64-bit) */ 767 /* dst = dst >> src (64-bit) */ 768 /* dst = dst >> src (64-bit, arithmetic) */ 769 /* dst = dst + src (64-bit) */ 770 /* dst = dst - src (64-bit) */ 771 /* dst = dst * src (64-bit) */ 772 /* dst = dst / src (64-bit) */ 773 /* dst = dst % src (64-bit) */ 774 case BPF_ALU64 | BPF_AND | BPF_X: 775 case BPF_ALU64 | BPF_OR | BPF_X: 776 case BPF_ALU64 | BPF_XOR | BPF_X: 777 case BPF_ALU64 | BPF_LSH | BPF_X: 778 case BPF_ALU64 | BPF_RSH | BPF_X: 779 case BPF_ALU64 | BPF_ARSH | BPF_X: 780 case BPF_ALU64 | BPF_ADD | BPF_X: 781 case BPF_ALU64 | BPF_SUB | BPF_X: 782 case BPF_ALU64 | BPF_MUL | BPF_X: 783 case BPF_ALU64 | BPF_DIV | BPF_X: 784 case BPF_ALU64 | BPF_MOD | BPF_X: 785 emit_alu_r64(ctx, dst, src, BPF_OP(code)); 786 break; 787 /* dst = htole(dst) */ 788 /* dst = htobe(dst) */ 789 case BPF_ALU | BPF_END | BPF_FROM_LE: 790 case BPF_ALU | BPF_END | BPF_FROM_BE: 791 if (BPF_SRC(code) == 792 #ifdef __BIG_ENDIAN 793 BPF_FROM_LE 794 #else 795 BPF_FROM_BE 796 #endif 797 ) 798 emit_bswap_r64(ctx, dst, imm); 799 else 800 emit_trunc_r64(ctx, dst, imm); 801 break; 802 /* dst = imm64 */ 803 case BPF_LD | BPF_IMM | BPF_DW: 804 emit_mov_i64(ctx, dst, (u32)imm | ((u64)insn[1].imm << 32)); 805 return 1; 806 /* LDX: dst = *(size *)(src + off) */ 807 case BPF_LDX | BPF_MEM | BPF_W: 808 case BPF_LDX | BPF_MEM | BPF_H: 809 case BPF_LDX | BPF_MEM | BPF_B: 810 case BPF_LDX | BPF_MEM | BPF_DW: 811 emit_ldx(ctx, dst, src, off, BPF_SIZE(code)); 812 break; 813 /* ST: *(size *)(dst + off) = imm */ 814 case BPF_ST | BPF_MEM | BPF_W: 815 case BPF_ST | BPF_MEM | BPF_H: 816 case BPF_ST | BPF_MEM | BPF_B: 817 case BPF_ST | BPF_MEM | BPF_DW: 818 emit_mov_i(ctx, MIPS_R_T4, imm); 819 emit_stx(ctx, dst, MIPS_R_T4, off, BPF_SIZE(code)); 820 break; 821 /* STX: *(size *)(dst + off) = src */ 822 case BPF_STX | BPF_MEM | BPF_W: 823 case BPF_STX | BPF_MEM | BPF_H: 824 case BPF_STX | BPF_MEM | BPF_B: 825 case BPF_STX | BPF_MEM | BPF_DW: 826 emit_stx(ctx, dst, src, off, BPF_SIZE(code)); 827 break; 828 /* Speculation barrier */ 829 case BPF_ST | BPF_NOSPEC: 830 break; 831 /* Atomics */ 832 case BPF_STX | BPF_ATOMIC | BPF_W: 833 case BPF_STX | BPF_ATOMIC | BPF_DW: 834 switch (imm) { 835 case BPF_ADD: 836 case BPF_ADD | BPF_FETCH: 837 case BPF_AND: 838 case BPF_AND | BPF_FETCH: 839 case BPF_OR: 840 case BPF_OR | BPF_FETCH: 841 case BPF_XOR: 842 case BPF_XOR | BPF_FETCH: 843 case BPF_XCHG: 844 if (BPF_SIZE(code) == BPF_DW) { 845 emit_atomic_r64(ctx, dst, src, off, imm); 846 } else if (imm & BPF_FETCH) { 847 u8 tmp = dst; 848 849 if (src == dst) { /* Don't overwrite dst */ 850 emit_mov_r(ctx, MIPS_R_T4, dst); 851 tmp = MIPS_R_T4; 852 } 853 emit_sext(ctx, src, src); 854 emit_atomic_r(ctx, tmp, src, off, imm); 855 emit_zext_ver(ctx, src); 856 } else { /* 32-bit, no fetch */ 857 emit_sext(ctx, MIPS_R_T4, src); 858 emit_atomic_r(ctx, dst, MIPS_R_T4, off, imm); 859 } 860 break; 861 case BPF_CMPXCHG: 862 if (BPF_SIZE(code) == BPF_DW) { 863 emit_cmpxchg_r64(ctx, dst, src, off); 864 } else { 865 u8 tmp = res; 866 867 if (res == dst) /* Don't overwrite dst */ 868 tmp = MIPS_R_T4; 869 emit_sext(ctx, tmp, res); 870 emit_sext(ctx, MIPS_R_T5, src); 871 emit_cmpxchg_r(ctx, dst, MIPS_R_T5, tmp, off); 872 if (res == dst) /* Restore result */ 873 emit_mov_r(ctx, res, MIPS_R_T4); 874 /* Result zext inserted by verifier */ 875 } 876 break; 877 default: 878 goto notyet; 879 } 880 break; 881 /* PC += off if dst == src */ 882 /* PC += off if dst != src */ 883 /* PC += off if dst & src */ 884 /* PC += off if dst > src */ 885 /* PC += off if dst >= src */ 886 /* PC += off if dst < src */ 887 /* PC += off if dst <= src */ 888 /* PC += off if dst > src (signed) */ 889 /* PC += off if dst >= src (signed) */ 890 /* PC += off if dst < src (signed) */ 891 /* PC += off if dst <= src (signed) */ 892 case BPF_JMP32 | BPF_JEQ | BPF_X: 893 case BPF_JMP32 | BPF_JNE | BPF_X: 894 case BPF_JMP32 | BPF_JSET | BPF_X: 895 case BPF_JMP32 | BPF_JGT | BPF_X: 896 case BPF_JMP32 | BPF_JGE | BPF_X: 897 case BPF_JMP32 | BPF_JLT | BPF_X: 898 case BPF_JMP32 | BPF_JLE | BPF_X: 899 case BPF_JMP32 | BPF_JSGT | BPF_X: 900 case BPF_JMP32 | BPF_JSGE | BPF_X: 901 case BPF_JMP32 | BPF_JSLT | BPF_X: 902 case BPF_JMP32 | BPF_JSLE | BPF_X: 903 if (off == 0) 904 break; 905 setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel); 906 emit_sext(ctx, MIPS_R_T4, dst); /* Sign-extended dst */ 907 emit_sext(ctx, MIPS_R_T5, src); /* Sign-extended src */ 908 emit_jmp_r(ctx, MIPS_R_T4, MIPS_R_T5, rel, jmp); 909 if (finish_jmp(ctx, jmp, off) < 0) 910 goto toofar; 911 break; 912 /* PC += off if dst == imm */ 913 /* PC += off if dst != imm */ 914 /* PC += off if dst & imm */ 915 /* PC += off if dst > imm */ 916 /* PC += off if dst >= imm */ 917 /* PC += off if dst < imm */ 918 /* PC += off if dst <= imm */ 919 /* PC += off if dst > imm (signed) */ 920 /* PC += off if dst >= imm (signed) */ 921 /* PC += off if dst < imm (signed) */ 922 /* PC += off if dst <= imm (signed) */ 923 case BPF_JMP32 | BPF_JEQ | BPF_K: 924 case BPF_JMP32 | BPF_JNE | BPF_K: 925 case BPF_JMP32 | BPF_JSET | BPF_K: 926 case BPF_JMP32 | BPF_JGT | BPF_K: 927 case BPF_JMP32 | BPF_JGE | BPF_K: 928 case BPF_JMP32 | BPF_JLT | BPF_K: 929 case BPF_JMP32 | BPF_JLE | BPF_K: 930 case BPF_JMP32 | BPF_JSGT | BPF_K: 931 case BPF_JMP32 | BPF_JSGE | BPF_K: 932 case BPF_JMP32 | BPF_JSLT | BPF_K: 933 case BPF_JMP32 | BPF_JSLE | BPF_K: 934 if (off == 0) 935 break; 936 setup_jmp_i(ctx, imm, 32, BPF_OP(code), off, &jmp, &rel); 937 emit_sext(ctx, MIPS_R_T4, dst); /* Sign-extended dst */ 938 if (valid_jmp_i(jmp, imm)) { 939 emit_jmp_i(ctx, MIPS_R_T4, imm, rel, jmp); 940 } else { 941 /* Move large immediate to register, sign-extended */ 942 emit_mov_i(ctx, MIPS_R_T5, imm); 943 emit_jmp_r(ctx, MIPS_R_T4, MIPS_R_T5, rel, jmp); 944 } 945 if (finish_jmp(ctx, jmp, off) < 0) 946 goto toofar; 947 break; 948 /* PC += off if dst == src */ 949 /* PC += off if dst != src */ 950 /* PC += off if dst & src */ 951 /* PC += off if dst > src */ 952 /* PC += off if dst >= src */ 953 /* PC += off if dst < src */ 954 /* PC += off if dst <= src */ 955 /* PC += off if dst > src (signed) */ 956 /* PC += off if dst >= src (signed) */ 957 /* PC += off if dst < src (signed) */ 958 /* PC += off if dst <= src (signed) */ 959 case BPF_JMP | BPF_JEQ | BPF_X: 960 case BPF_JMP | BPF_JNE | BPF_X: 961 case BPF_JMP | BPF_JSET | BPF_X: 962 case BPF_JMP | BPF_JGT | BPF_X: 963 case BPF_JMP | BPF_JGE | BPF_X: 964 case BPF_JMP | BPF_JLT | BPF_X: 965 case BPF_JMP | BPF_JLE | BPF_X: 966 case BPF_JMP | BPF_JSGT | BPF_X: 967 case BPF_JMP | BPF_JSGE | BPF_X: 968 case BPF_JMP | BPF_JSLT | BPF_X: 969 case BPF_JMP | BPF_JSLE | BPF_X: 970 if (off == 0) 971 break; 972 setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel); 973 emit_jmp_r(ctx, dst, src, rel, jmp); 974 if (finish_jmp(ctx, jmp, off) < 0) 975 goto toofar; 976 break; 977 /* PC += off if dst == imm */ 978 /* PC += off if dst != imm */ 979 /* PC += off if dst & imm */ 980 /* PC += off if dst > imm */ 981 /* PC += off if dst >= imm */ 982 /* PC += off if dst < imm */ 983 /* PC += off if dst <= imm */ 984 /* PC += off if dst > imm (signed) */ 985 /* PC += off if dst >= imm (signed) */ 986 /* PC += off if dst < imm (signed) */ 987 /* PC += off if dst <= imm (signed) */ 988 case BPF_JMP | BPF_JEQ | BPF_K: 989 case BPF_JMP | BPF_JNE | BPF_K: 990 case BPF_JMP | BPF_JSET | BPF_K: 991 case BPF_JMP | BPF_JGT | BPF_K: 992 case BPF_JMP | BPF_JGE | BPF_K: 993 case BPF_JMP | BPF_JLT | BPF_K: 994 case BPF_JMP | BPF_JLE | BPF_K: 995 case BPF_JMP | BPF_JSGT | BPF_K: 996 case BPF_JMP | BPF_JSGE | BPF_K: 997 case BPF_JMP | BPF_JSLT | BPF_K: 998 case BPF_JMP | BPF_JSLE | BPF_K: 999 if (off == 0) 1000 break; 1001 setup_jmp_i(ctx, imm, 64, BPF_OP(code), off, &jmp, &rel); 1002 if (valid_jmp_i(jmp, imm)) { 1003 emit_jmp_i(ctx, dst, imm, rel, jmp); 1004 } else { 1005 /* Move large immediate to register */ 1006 emit_mov_i(ctx, MIPS_R_T4, imm); 1007 emit_jmp_r(ctx, dst, MIPS_R_T4, rel, jmp); 1008 } 1009 if (finish_jmp(ctx, jmp, off) < 0) 1010 goto toofar; 1011 break; 1012 /* PC += off */ 1013 case BPF_JMP | BPF_JA: 1014 if (off == 0) 1015 break; 1016 if (emit_ja(ctx, off) < 0) 1017 goto toofar; 1018 break; 1019 /* Tail call */ 1020 case BPF_JMP | BPF_TAIL_CALL: 1021 if (emit_tail_call(ctx) < 0) 1022 goto invalid; 1023 break; 1024 /* Function call */ 1025 case BPF_JMP | BPF_CALL: 1026 if (emit_call(ctx, insn) < 0) 1027 goto invalid; 1028 break; 1029 /* Function return */ 1030 case BPF_JMP | BPF_EXIT: 1031 /* 1032 * Optimization: when last instruction is EXIT 1033 * simply continue to epilogue. 1034 */ 1035 if (ctx->bpf_index == ctx->program->len - 1) 1036 break; 1037 if (emit_exit(ctx) < 0) 1038 goto toofar; 1039 break; 1040 1041 default: 1042 invalid: 1043 pr_err_once("unknown opcode %02x\n", code); 1044 return -EINVAL; 1045 notyet: 1046 pr_info_once("*** NOT YET: opcode %02x ***\n", code); 1047 return -EFAULT; 1048 toofar: 1049 pr_info_once("*** TOO FAR: jump at %u opcode %02x ***\n", 1050 ctx->bpf_index, code); 1051 return -E2BIG; 1052 } 1053 return 0; 1054 } 1055