1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Just-In-Time compiler for eBPF bytecode on MIPS. 4 * Implementation of JIT functions for 32-bit CPUs. 5 * 6 * Copyright (c) 2021 Anyfi Networks AB. 7 * Author: Johan Almbladh <johan.almbladh@gmail.com> 8 * 9 * Based on code and ideas from 10 * Copyright (c) 2017 Cavium, Inc. 11 * Copyright (c) 2017 Shubham Bansal <illusionist.neo@gmail.com> 12 * Copyright (c) 2011 Mircea Gherzan <mgherzan@gmail.com> 13 */ 14 15 #include <linux/math64.h> 16 #include <linux/errno.h> 17 #include <linux/filter.h> 18 #include <linux/bpf.h> 19 #include <asm/cpu-features.h> 20 #include <asm/isa-rev.h> 21 #include <asm/uasm.h> 22 23 #include "bpf_jit_comp.h" 24 25 /* MIPS a4-a7 are not available in the o32 ABI */ 26 #undef MIPS_R_A4 27 #undef MIPS_R_A5 28 #undef MIPS_R_A6 29 #undef MIPS_R_A7 30 31 /* Stack is 8-byte aligned in o32 ABI */ 32 #define MIPS_STACK_ALIGNMENT 8 33 34 /* 35 * The top 16 bytes of a stack frame is reserved for the callee in O32 ABI. 36 * This corresponds to stack space for register arguments a0-a3. 37 */ 38 #define JIT_RESERVED_STACK 16 39 40 /* Temporary 64-bit register used by JIT */ 41 #define JIT_REG_TMP MAX_BPF_JIT_REG 42 43 /* 44 * Number of prologue bytes to skip when doing a tail call. 45 * Tail call count (TCC) initialization (8 bytes) always, plus 46 * R0-to-v0 assignment (4 bytes) if big endian. 47 */ 48 #ifdef __BIG_ENDIAN 49 #define JIT_TCALL_SKIP 12 50 #else 51 #define JIT_TCALL_SKIP 8 52 #endif 53 54 /* CPU registers holding the callee return value */ 55 #define JIT_RETURN_REGS \ 56 (BIT(MIPS_R_V0) | \ 57 BIT(MIPS_R_V1)) 58 59 /* CPU registers arguments passed to callee directly */ 60 #define JIT_ARG_REGS \ 61 (BIT(MIPS_R_A0) | \ 62 BIT(MIPS_R_A1) | \ 63 BIT(MIPS_R_A2) | \ 64 BIT(MIPS_R_A3)) 65 66 /* CPU register arguments passed to callee on stack */ 67 #define JIT_STACK_REGS \ 68 (BIT(MIPS_R_T0) | \ 69 BIT(MIPS_R_T1) | \ 70 BIT(MIPS_R_T2) | \ 71 BIT(MIPS_R_T3) | \ 72 BIT(MIPS_R_T4) | \ 73 BIT(MIPS_R_T5)) 74 75 /* Caller-saved CPU registers */ 76 #define JIT_CALLER_REGS \ 77 (JIT_RETURN_REGS | \ 78 JIT_ARG_REGS | \ 79 JIT_STACK_REGS) 80 81 /* Callee-saved CPU registers */ 82 #define JIT_CALLEE_REGS \ 83 (BIT(MIPS_R_S0) | \ 84 BIT(MIPS_R_S1) | \ 85 BIT(MIPS_R_S2) | \ 86 BIT(MIPS_R_S3) | \ 87 BIT(MIPS_R_S4) | \ 88 BIT(MIPS_R_S5) | \ 89 BIT(MIPS_R_S6) | \ 90 BIT(MIPS_R_S7) | \ 91 BIT(MIPS_R_GP) | \ 92 BIT(MIPS_R_FP) | \ 93 BIT(MIPS_R_RA)) 94 95 /* 96 * Mapping of 64-bit eBPF registers to 32-bit native MIPS registers. 97 * 98 * 1) Native register pairs are ordered according to CPU endianness, following 99 * the MIPS convention for passing 64-bit arguments and return values. 100 * 2) The eBPF return value, arguments and callee-saved registers are mapped 101 * to their native MIPS equivalents. 102 * 3) Since the 32 highest bits in the eBPF FP register are always zero, 103 * only one general-purpose register is actually needed for the mapping. 104 * We use the fp register for this purpose, and map the highest bits to 105 * the MIPS register r0 (zero). 106 * 4) We use the MIPS gp and at registers as internal temporary registers 107 * for constant blinding. The gp register is callee-saved. 108 * 5) One 64-bit temporary register is mapped for use when sign-extending 109 * immediate operands. MIPS registers t6-t9 are available to the JIT 110 * for as temporaries when implementing complex 64-bit operations. 111 * 112 * With this scheme all eBPF registers are being mapped to native MIPS 113 * registers without having to use any stack scratch space. The direct 114 * register mapping (2) simplifies the handling of function calls. 115 */ 116 static const u8 bpf2mips32[][2] = { 117 /* Return value from in-kernel function, and exit value from eBPF */ 118 [BPF_REG_0] = {MIPS_R_V1, MIPS_R_V0}, 119 /* Arguments from eBPF program to in-kernel function */ 120 [BPF_REG_1] = {MIPS_R_A1, MIPS_R_A0}, 121 [BPF_REG_2] = {MIPS_R_A3, MIPS_R_A2}, 122 /* Remaining arguments, to be passed on the stack per O32 ABI */ 123 [BPF_REG_3] = {MIPS_R_T1, MIPS_R_T0}, 124 [BPF_REG_4] = {MIPS_R_T3, MIPS_R_T2}, 125 [BPF_REG_5] = {MIPS_R_T5, MIPS_R_T4}, 126 /* Callee-saved registers that in-kernel function will preserve */ 127 [BPF_REG_6] = {MIPS_R_S1, MIPS_R_S0}, 128 [BPF_REG_7] = {MIPS_R_S3, MIPS_R_S2}, 129 [BPF_REG_8] = {MIPS_R_S5, MIPS_R_S4}, 130 [BPF_REG_9] = {MIPS_R_S7, MIPS_R_S6}, 131 /* Read-only frame pointer to access the eBPF stack */ 132 #ifdef __BIG_ENDIAN 133 [BPF_REG_FP] = {MIPS_R_FP, MIPS_R_ZERO}, 134 #else 135 [BPF_REG_FP] = {MIPS_R_ZERO, MIPS_R_FP}, 136 #endif 137 /* Temporary register for blinding constants */ 138 [BPF_REG_AX] = {MIPS_R_GP, MIPS_R_AT}, 139 /* Temporary register for internal JIT use */ 140 [JIT_REG_TMP] = {MIPS_R_T7, MIPS_R_T6}, 141 }; 142 143 /* Get low CPU register for a 64-bit eBPF register mapping */ 144 static inline u8 lo(const u8 reg[]) 145 { 146 #ifdef __BIG_ENDIAN 147 return reg[0]; 148 #else 149 return reg[1]; 150 #endif 151 } 152 153 /* Get high CPU register for a 64-bit eBPF register mapping */ 154 static inline u8 hi(const u8 reg[]) 155 { 156 #ifdef __BIG_ENDIAN 157 return reg[1]; 158 #else 159 return reg[0]; 160 #endif 161 } 162 163 /* 164 * Mark a 64-bit CPU register pair as clobbered, it needs to be 165 * saved/restored by the program if callee-saved. 166 */ 167 static void clobber_reg64(struct jit_context *ctx, const u8 reg[]) 168 { 169 clobber_reg(ctx, reg[0]); 170 clobber_reg(ctx, reg[1]); 171 } 172 173 /* dst = imm (sign-extended) */ 174 static void emit_mov_se_i64(struct jit_context *ctx, const u8 dst[], s32 imm) 175 { 176 emit_mov_i(ctx, lo(dst), imm); 177 if (imm < 0) 178 emit(ctx, addiu, hi(dst), MIPS_R_ZERO, -1); 179 else 180 emit(ctx, move, hi(dst), MIPS_R_ZERO); 181 clobber_reg64(ctx, dst); 182 } 183 184 /* Zero extension, if verifier does not do it for us */ 185 static void emit_zext_ver(struct jit_context *ctx, const u8 dst[]) 186 { 187 if (!ctx->program->aux->verifier_zext) { 188 emit(ctx, move, hi(dst), MIPS_R_ZERO); 189 clobber_reg(ctx, hi(dst)); 190 } 191 } 192 193 /* Sign-extend an 8- or 16-bit field into a native register. */ 194 static void emit_movsx_r(struct jit_context *ctx, u8 dst, u8 src, s16 off) 195 { 196 int shift; 197 198 if (cpu_has_mips32r2 || cpu_has_mips32r6) { 199 if (off == 8) 200 emit(ctx, seb, dst, src); 201 else 202 emit(ctx, seh, dst, src); 203 } else { 204 shift = 32 - off; 205 emit(ctx, sll, dst, src, shift); 206 emit(ctx, sra, dst, dst, shift); 207 } 208 } 209 210 /* Register move operation (32-bit), optionally with sign extension */ 211 static void emit_mov_r32(struct jit_context *ctx, const u8 dst[], 212 const u8 src[], s16 off) 213 { 214 switch (off) { 215 case 8: 216 case 16: 217 emit_movsx_r(ctx, lo(dst), lo(src), off); 218 break; 219 default: 220 /* off == 0 is MOV; the verifier rejects other offsets. */ 221 emit_mov_r(ctx, lo(dst), lo(src)); 222 break; 223 } 224 clobber_reg(ctx, lo(dst)); 225 emit_zext_ver(ctx, dst); 226 } 227 228 /* Register move operation (64-bit), optionally with sign extension */ 229 static void emit_mov_r64(struct jit_context *ctx, const u8 dst[], 230 const u8 src[], s16 off) 231 { 232 switch (off) { 233 case 8: 234 case 16: 235 emit_movsx_r(ctx, lo(dst), lo(src), off); 236 emit(ctx, sra, hi(dst), lo(dst), 31); 237 break; 238 case 32: 239 emit(ctx, move, lo(dst), lo(src)); 240 emit(ctx, sra, hi(dst), lo(dst), 31); 241 break; 242 default: 243 /* 244 * off == 0 is ordinary MOV. The verifier rejects other 245 * offsets; defined exceptions require 246 * bpf_jit_supports_percpu_insn() or bpf_jit_supports_arena(), 247 * neither implemented by MIPS. 248 */ 249 emit_mov_r(ctx, lo(dst), lo(src)); 250 emit_mov_r(ctx, hi(dst), hi(src)); 251 break; 252 } 253 clobber_reg64(ctx, dst); 254 } 255 256 /* Load delay slot, if ISA mandates it */ 257 static void emit_load_delay(struct jit_context *ctx) 258 { 259 if (!cpu_has_mips_2_3_4_5_r) 260 emit(ctx, nop); 261 } 262 263 /* ALU immediate operation (64-bit) */ 264 static void emit_alu_i64(struct jit_context *ctx, 265 const u8 dst[], s32 imm, u8 op) 266 { 267 u8 src = MIPS_R_T6; 268 269 /* 270 * ADD/SUB with all but the max negative imm can be handled by 271 * inverting the operation and the imm value, saving one insn. 272 */ 273 if (imm > S32_MIN && imm < 0) 274 switch (op) { 275 case BPF_ADD: 276 op = BPF_SUB; 277 imm = -imm; 278 break; 279 case BPF_SUB: 280 op = BPF_ADD; 281 imm = -imm; 282 break; 283 } 284 285 /* Move immediate to temporary register */ 286 emit_mov_i(ctx, src, imm); 287 288 switch (op) { 289 /* dst = dst + imm */ 290 case BPF_ADD: 291 emit(ctx, addu, lo(dst), lo(dst), src); 292 emit(ctx, sltu, MIPS_R_T9, lo(dst), src); 293 emit(ctx, addu, hi(dst), hi(dst), MIPS_R_T9); 294 if (imm < 0) 295 emit(ctx, addiu, hi(dst), hi(dst), -1); 296 break; 297 /* dst = dst - imm */ 298 case BPF_SUB: 299 emit(ctx, sltu, MIPS_R_T9, lo(dst), src); 300 emit(ctx, subu, lo(dst), lo(dst), src); 301 emit(ctx, subu, hi(dst), hi(dst), MIPS_R_T9); 302 if (imm < 0) 303 emit(ctx, addiu, hi(dst), hi(dst), 1); 304 break; 305 /* dst = dst | imm */ 306 case BPF_OR: 307 emit(ctx, or, lo(dst), lo(dst), src); 308 if (imm < 0) 309 emit(ctx, addiu, hi(dst), MIPS_R_ZERO, -1); 310 break; 311 /* dst = dst & imm */ 312 case BPF_AND: 313 emit(ctx, and, lo(dst), lo(dst), src); 314 if (imm >= 0) 315 emit(ctx, move, hi(dst), MIPS_R_ZERO); 316 break; 317 /* dst = dst ^ imm */ 318 case BPF_XOR: 319 emit(ctx, xor, lo(dst), lo(dst), src); 320 if (imm < 0) { 321 emit(ctx, subu, hi(dst), MIPS_R_ZERO, hi(dst)); 322 emit(ctx, addiu, hi(dst), hi(dst), -1); 323 } 324 break; 325 } 326 clobber_reg64(ctx, dst); 327 } 328 329 /* ALU register operation (64-bit) */ 330 static void emit_alu_r64(struct jit_context *ctx, 331 const u8 dst[], const u8 src[], u8 op) 332 { 333 switch (BPF_OP(op)) { 334 /* dst = dst + src */ 335 case BPF_ADD: 336 if (src == dst) { 337 emit(ctx, srl, MIPS_R_T9, lo(dst), 31); 338 emit(ctx, addu, lo(dst), lo(dst), lo(dst)); 339 } else { 340 emit(ctx, addu, lo(dst), lo(dst), lo(src)); 341 emit(ctx, sltu, MIPS_R_T9, lo(dst), lo(src)); 342 } 343 emit(ctx, addu, hi(dst), hi(dst), hi(src)); 344 emit(ctx, addu, hi(dst), hi(dst), MIPS_R_T9); 345 break; 346 /* dst = dst - src */ 347 case BPF_SUB: 348 emit(ctx, sltu, MIPS_R_T9, lo(dst), lo(src)); 349 emit(ctx, subu, lo(dst), lo(dst), lo(src)); 350 emit(ctx, subu, hi(dst), hi(dst), hi(src)); 351 emit(ctx, subu, hi(dst), hi(dst), MIPS_R_T9); 352 break; 353 /* dst = dst | src */ 354 case BPF_OR: 355 emit(ctx, or, lo(dst), lo(dst), lo(src)); 356 emit(ctx, or, hi(dst), hi(dst), hi(src)); 357 break; 358 /* dst = dst & src */ 359 case BPF_AND: 360 emit(ctx, and, lo(dst), lo(dst), lo(src)); 361 emit(ctx, and, hi(dst), hi(dst), hi(src)); 362 break; 363 /* dst = dst ^ src */ 364 case BPF_XOR: 365 emit(ctx, xor, lo(dst), lo(dst), lo(src)); 366 emit(ctx, xor, hi(dst), hi(dst), hi(src)); 367 break; 368 } 369 clobber_reg64(ctx, dst); 370 } 371 372 /* ALU invert (64-bit) */ 373 static void emit_neg_i64(struct jit_context *ctx, const u8 dst[]) 374 { 375 emit(ctx, sltu, MIPS_R_T9, MIPS_R_ZERO, lo(dst)); 376 emit(ctx, subu, lo(dst), MIPS_R_ZERO, lo(dst)); 377 emit(ctx, subu, hi(dst), MIPS_R_ZERO, hi(dst)); 378 emit(ctx, subu, hi(dst), hi(dst), MIPS_R_T9); 379 380 clobber_reg64(ctx, dst); 381 } 382 383 /* ALU shift immediate (64-bit) */ 384 static void emit_shift_i64(struct jit_context *ctx, 385 const u8 dst[], u32 imm, u8 op) 386 { 387 switch (BPF_OP(op)) { 388 /* dst = dst << imm */ 389 case BPF_LSH: 390 if (imm < 32) { 391 emit(ctx, srl, MIPS_R_T9, lo(dst), 32 - imm); 392 emit(ctx, sll, lo(dst), lo(dst), imm); 393 emit(ctx, sll, hi(dst), hi(dst), imm); 394 emit(ctx, or, hi(dst), hi(dst), MIPS_R_T9); 395 } else { 396 emit(ctx, sll, hi(dst), lo(dst), imm - 32); 397 emit(ctx, move, lo(dst), MIPS_R_ZERO); 398 } 399 break; 400 /* dst = dst >> imm */ 401 case BPF_RSH: 402 if (imm < 32) { 403 emit(ctx, sll, MIPS_R_T9, hi(dst), 32 - imm); 404 emit(ctx, srl, lo(dst), lo(dst), imm); 405 emit(ctx, srl, hi(dst), hi(dst), imm); 406 emit(ctx, or, lo(dst), lo(dst), MIPS_R_T9); 407 } else { 408 emit(ctx, srl, lo(dst), hi(dst), imm - 32); 409 emit(ctx, move, hi(dst), MIPS_R_ZERO); 410 } 411 break; 412 /* dst = dst >> imm (arithmetic) */ 413 case BPF_ARSH: 414 if (imm < 32) { 415 emit(ctx, sll, MIPS_R_T9, hi(dst), 32 - imm); 416 emit(ctx, srl, lo(dst), lo(dst), imm); 417 emit(ctx, sra, hi(dst), hi(dst), imm); 418 emit(ctx, or, lo(dst), lo(dst), MIPS_R_T9); 419 } else { 420 emit(ctx, sra, lo(dst), hi(dst), imm - 32); 421 emit(ctx, sra, hi(dst), hi(dst), 31); 422 } 423 break; 424 } 425 clobber_reg64(ctx, dst); 426 } 427 428 /* ALU shift register (64-bit) */ 429 static void emit_shift_r64(struct jit_context *ctx, 430 const u8 dst[], u8 src, u8 op) 431 { 432 u8 t1 = MIPS_R_T8; 433 u8 t2 = MIPS_R_T9; 434 435 emit(ctx, andi, t1, src, 32); /* t1 = src & 32 */ 436 emit(ctx, beqz, t1, 16); /* PC += 16 if t1 == 0 */ 437 emit(ctx, nor, t2, src, MIPS_R_ZERO); /* t2 = ~src (delay slot) */ 438 439 switch (BPF_OP(op)) { 440 /* dst = dst << src */ 441 case BPF_LSH: 442 /* Next: shift >= 32 */ 443 emit(ctx, sllv, hi(dst), lo(dst), src); /* dh = dl << src */ 444 emit(ctx, move, lo(dst), MIPS_R_ZERO); /* dl = 0 */ 445 emit(ctx, b, 20); /* PC += 20 */ 446 /* +16: shift < 32 */ 447 emit(ctx, srl, t1, lo(dst), 1); /* t1 = dl >> 1 */ 448 emit(ctx, srlv, t1, t1, t2); /* t1 = t1 >> t2 */ 449 emit(ctx, sllv, lo(dst), lo(dst), src); /* dl = dl << src */ 450 emit(ctx, sllv, hi(dst), hi(dst), src); /* dh = dh << src */ 451 emit(ctx, or, hi(dst), hi(dst), t1); /* dh = dh | t1 */ 452 break; 453 /* dst = dst >> src */ 454 case BPF_RSH: 455 /* Next: shift >= 32 */ 456 emit(ctx, srlv, lo(dst), hi(dst), src); /* dl = dh >> src */ 457 emit(ctx, move, hi(dst), MIPS_R_ZERO); /* dh = 0 */ 458 emit(ctx, b, 20); /* PC += 20 */ 459 /* +16: shift < 32 */ 460 emit(ctx, sll, t1, hi(dst), 1); /* t1 = dl << 1 */ 461 emit(ctx, sllv, t1, t1, t2); /* t1 = t1 << t2 */ 462 emit(ctx, srlv, lo(dst), lo(dst), src); /* dl = dl >> src */ 463 emit(ctx, srlv, hi(dst), hi(dst), src); /* dh = dh >> src */ 464 emit(ctx, or, lo(dst), lo(dst), t1); /* dl = dl | t1 */ 465 break; 466 /* dst = dst >> src (arithmetic) */ 467 case BPF_ARSH: 468 /* Next: shift >= 32 */ 469 emit(ctx, srav, lo(dst), hi(dst), src); /* dl = dh >>a src */ 470 emit(ctx, sra, hi(dst), hi(dst), 31); /* dh = dh >>a 31 */ 471 emit(ctx, b, 20); /* PC += 20 */ 472 /* +16: shift < 32 */ 473 emit(ctx, sll, t1, hi(dst), 1); /* t1 = dl << 1 */ 474 emit(ctx, sllv, t1, t1, t2); /* t1 = t1 << t2 */ 475 emit(ctx, srlv, lo(dst), lo(dst), src); /* dl = dl >>a src */ 476 emit(ctx, srav, hi(dst), hi(dst), src); /* dh = dh >> src */ 477 emit(ctx, or, lo(dst), lo(dst), t1); /* dl = dl | t1 */ 478 break; 479 } 480 481 /* +20: Done */ 482 clobber_reg64(ctx, dst); 483 } 484 485 /* ALU mul immediate (64x32-bit) */ 486 static void emit_mul_i64(struct jit_context *ctx, const u8 dst[], s32 imm) 487 { 488 u8 src = MIPS_R_T6; 489 u8 tmp = MIPS_R_T9; 490 491 switch (imm) { 492 /* dst = dst * 1 is a no-op */ 493 case 1: 494 break; 495 /* dst = dst * -1 */ 496 case -1: 497 emit_neg_i64(ctx, dst); 498 break; 499 case 0: 500 emit_mov_r(ctx, lo(dst), MIPS_R_ZERO); 501 emit_mov_r(ctx, hi(dst), MIPS_R_ZERO); 502 break; 503 /* Full 64x32 multiply */ 504 default: 505 /* hi(dst) = hi(dst) * src(imm) */ 506 emit_mov_i(ctx, src, imm); 507 if (cpu_has_mips32r1 || cpu_has_mips32r6) { 508 emit(ctx, mul, hi(dst), hi(dst), src); 509 } else { 510 emit(ctx, multu, hi(dst), src); 511 emit(ctx, mflo, hi(dst)); 512 } 513 514 /* hi(dst) = hi(dst) - lo(dst) */ 515 if (imm < 0) 516 emit(ctx, subu, hi(dst), hi(dst), lo(dst)); 517 518 /* tmp = lo(dst) * src(imm) >> 32 */ 519 /* lo(dst) = lo(dst) * src(imm) */ 520 if (cpu_has_mips32r6) { 521 emit(ctx, muhu, tmp, lo(dst), src); 522 emit(ctx, mulu, lo(dst), lo(dst), src); 523 } else { 524 emit(ctx, multu, lo(dst), src); 525 emit(ctx, mflo, lo(dst)); 526 emit(ctx, mfhi, tmp); 527 } 528 529 /* hi(dst) += tmp */ 530 emit(ctx, addu, hi(dst), hi(dst), tmp); 531 clobber_reg64(ctx, dst); 532 break; 533 } 534 } 535 536 /* ALU mul register (64x64-bit) */ 537 static void emit_mul_r64(struct jit_context *ctx, 538 const u8 dst[], const u8 src[]) 539 { 540 u8 acc = MIPS_R_T8; 541 u8 tmp = MIPS_R_T9; 542 543 /* acc = hi(dst) * lo(src) */ 544 if (cpu_has_mips32r1 || cpu_has_mips32r6) { 545 emit(ctx, mul, acc, hi(dst), lo(src)); 546 } else { 547 emit(ctx, multu, hi(dst), lo(src)); 548 emit(ctx, mflo, acc); 549 } 550 551 /* tmp = lo(dst) * hi(src) */ 552 if (cpu_has_mips32r1 || cpu_has_mips32r6) { 553 emit(ctx, mul, tmp, lo(dst), hi(src)); 554 } else { 555 emit(ctx, multu, lo(dst), hi(src)); 556 emit(ctx, mflo, tmp); 557 } 558 559 /* acc += tmp */ 560 emit(ctx, addu, acc, acc, tmp); 561 562 /* tmp = lo(dst) * lo(src) >> 32 */ 563 /* lo(dst) = lo(dst) * lo(src) */ 564 if (cpu_has_mips32r6) { 565 emit(ctx, muhu, tmp, lo(dst), lo(src)); 566 emit(ctx, mulu, lo(dst), lo(dst), lo(src)); 567 } else { 568 emit(ctx, multu, lo(dst), lo(src)); 569 emit(ctx, mflo, lo(dst)); 570 emit(ctx, mfhi, tmp); 571 } 572 573 /* hi(dst) = acc + tmp */ 574 emit(ctx, addu, hi(dst), acc, tmp); 575 clobber_reg64(ctx, dst); 576 } 577 578 /* Helper function for unsigned 64-bit modulo */ 579 static u64 jit_mod64(u64 a, u64 b) 580 { 581 u64 rem; 582 583 div64_u64_rem(a, b, &rem); 584 return rem; 585 } 586 587 /* Helper function for signed 64-bit modulo */ 588 static s64 jit_smod64(s64 a, s64 b) 589 { 590 s64 quot = div64_s64(a, b); 591 592 return a - quot * b; 593 } 594 595 /* ALU div/mod register (64-bit) */ 596 static void emit_divmod_r64(struct jit_context *ctx, const u8 dst[], 597 const u8 src[], u8 op, s16 off) 598 { 599 const u8 *r0 = bpf2mips32[BPF_REG_0]; /* Mapped to v0-v1 */ 600 const u8 *r1 = bpf2mips32[BPF_REG_1]; /* Mapped to a0-a1 */ 601 const u8 *r2 = bpf2mips32[BPF_REG_2]; /* Mapped to a2-a3 */ 602 bool is_signed = off == 1; 603 int exclude, k; 604 u32 addr = 0; 605 606 /* Push caller-saved registers on stack */ 607 push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, 608 0, JIT_RESERVED_STACK); 609 610 /* Put 64-bit arguments 1 and 2 in registers a0-a3 */ 611 for (k = 0; k < 2; k++) { 612 emit(ctx, move, MIPS_R_T9, src[k]); 613 emit(ctx, move, r1[k], dst[k]); 614 emit(ctx, move, r2[k], MIPS_R_T9); 615 } 616 617 /* Emit function call */ 618 switch (BPF_OP(op)) { 619 /* dst = dst / src */ 620 case BPF_DIV: 621 addr = is_signed ? (u32)&div64_s64 : (u32)&div64_u64; 622 break; 623 /* dst = dst % src */ 624 case BPF_MOD: 625 addr = is_signed ? (u32)&jit_smod64 : (u32)&jit_mod64; 626 break; 627 } 628 emit_mov_i(ctx, MIPS_R_T9, addr); 629 emit(ctx, jalr, MIPS_R_RA, MIPS_R_T9); 630 emit(ctx, nop); /* Delay slot */ 631 632 /* Store the 64-bit result in dst */ 633 emit(ctx, move, dst[0], r0[0]); 634 emit(ctx, move, dst[1], r0[1]); 635 636 /* Restore caller-saved registers, excluding the computed result */ 637 exclude = BIT(lo(dst)) | BIT(hi(dst)); 638 pop_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, 639 exclude, JIT_RESERVED_STACK); 640 emit_load_delay(ctx); 641 642 clobber_reg64(ctx, dst); 643 clobber_reg(ctx, MIPS_R_V0); 644 clobber_reg(ctx, MIPS_R_V1); 645 clobber_reg(ctx, MIPS_R_RA); 646 } 647 648 /* Swap bytes in a register word */ 649 static void emit_swap8_r(struct jit_context *ctx, u8 dst, u8 src, u8 mask) 650 { 651 u8 tmp = MIPS_R_T9; 652 653 emit(ctx, and, tmp, src, mask); /* tmp = src & 0x00ff00ff */ 654 emit(ctx, sll, tmp, tmp, 8); /* tmp = tmp << 8 */ 655 emit(ctx, srl, dst, src, 8); /* dst = src >> 8 */ 656 emit(ctx, and, dst, dst, mask); /* dst = dst & 0x00ff00ff */ 657 emit(ctx, or, dst, dst, tmp); /* dst = dst | tmp */ 658 } 659 660 /* Swap half words in a register word */ 661 static void emit_swap16_r(struct jit_context *ctx, u8 dst, u8 src) 662 { 663 u8 tmp = MIPS_R_T9; 664 665 emit(ctx, sll, tmp, src, 16); /* tmp = src << 16 */ 666 emit(ctx, srl, dst, src, 16); /* dst = src >> 16 */ 667 emit(ctx, or, dst, dst, tmp); /* dst = dst | tmp */ 668 } 669 670 /* Swap bytes and truncate a register double word, word or half word */ 671 static void emit_bswap_r64(struct jit_context *ctx, const u8 dst[], u32 width) 672 { 673 u8 tmp = MIPS_R_T8; 674 675 switch (width) { 676 /* Swap bytes in a double word */ 677 case 64: 678 if (cpu_has_mips32r2 || cpu_has_mips32r6) { 679 emit(ctx, rotr, tmp, hi(dst), 16); 680 emit(ctx, rotr, hi(dst), lo(dst), 16); 681 emit(ctx, wsbh, lo(dst), tmp); 682 emit(ctx, wsbh, hi(dst), hi(dst)); 683 } else { 684 emit_swap16_r(ctx, tmp, lo(dst)); 685 emit_swap16_r(ctx, lo(dst), hi(dst)); 686 emit(ctx, move, hi(dst), tmp); 687 688 emit(ctx, lui, tmp, 0xff); /* tmp = 0x00ff0000 */ 689 emit(ctx, ori, tmp, tmp, 0xff); /* tmp = 0x00ff00ff */ 690 emit_swap8_r(ctx, lo(dst), lo(dst), tmp); 691 emit_swap8_r(ctx, hi(dst), hi(dst), tmp); 692 } 693 break; 694 /* Swap bytes in a word */ 695 /* Swap bytes in a half word */ 696 case 32: 697 case 16: 698 emit_bswap_r(ctx, lo(dst), width); 699 emit(ctx, move, hi(dst), MIPS_R_ZERO); 700 break; 701 } 702 clobber_reg64(ctx, dst); 703 } 704 705 /* Truncate a register double word, word or half word */ 706 static void emit_trunc_r64(struct jit_context *ctx, const u8 dst[], u32 width) 707 { 708 switch (width) { 709 case 64: 710 break; 711 /* Zero-extend a word */ 712 case 32: 713 emit(ctx, move, hi(dst), MIPS_R_ZERO); 714 clobber_reg(ctx, hi(dst)); 715 break; 716 /* Zero-extend a half word */ 717 case 16: 718 emit(ctx, move, hi(dst), MIPS_R_ZERO); 719 emit(ctx, andi, lo(dst), lo(dst), 0xffff); 720 clobber_reg64(ctx, dst); 721 break; 722 } 723 } 724 725 /* Load operation: dst = *(size*)(src + off) */ 726 static void emit_ldx(struct jit_context *ctx, 727 const u8 dst[], u8 src, s16 off, u8 size) 728 { 729 switch (size) { 730 /* Load a byte */ 731 case BPF_B: 732 emit(ctx, lbu, lo(dst), off, src); 733 emit(ctx, move, hi(dst), MIPS_R_ZERO); 734 break; 735 /* Load a half word */ 736 case BPF_H: 737 emit(ctx, lhu, lo(dst), off, src); 738 emit(ctx, move, hi(dst), MIPS_R_ZERO); 739 break; 740 /* Load a word */ 741 case BPF_W: 742 emit(ctx, lw, lo(dst), off, src); 743 emit(ctx, move, hi(dst), MIPS_R_ZERO); 744 break; 745 /* Load a double word */ 746 case BPF_DW: 747 if (dst[1] == src) { 748 emit(ctx, lw, dst[0], off + 4, src); 749 emit(ctx, lw, dst[1], off, src); 750 } else { 751 emit(ctx, lw, dst[1], off, src); 752 emit(ctx, lw, dst[0], off + 4, src); 753 } 754 emit_load_delay(ctx); 755 break; 756 } 757 clobber_reg64(ctx, dst); 758 } 759 760 /* Store operation: *(size *)(dst + off) = src */ 761 static void emit_stx(struct jit_context *ctx, 762 const u8 dst, const u8 src[], s16 off, u8 size) 763 { 764 switch (size) { 765 /* Store a byte */ 766 case BPF_B: 767 emit(ctx, sb, lo(src), off, dst); 768 break; 769 /* Store a half word */ 770 case BPF_H: 771 emit(ctx, sh, lo(src), off, dst); 772 break; 773 /* Store a word */ 774 case BPF_W: 775 emit(ctx, sw, lo(src), off, dst); 776 break; 777 /* Store a double word */ 778 case BPF_DW: 779 emit(ctx, sw, src[1], off, dst); 780 emit(ctx, sw, src[0], off + 4, dst); 781 break; 782 } 783 } 784 785 /* Atomic read-modify-write (32-bit, non-ll/sc fallback) */ 786 static void emit_atomic_r32(struct jit_context *ctx, 787 u8 dst, u8 src, s16 off, u8 code) 788 { 789 u32 exclude = 0; 790 u32 addr = 0; 791 792 /* Push caller-saved registers on stack */ 793 push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, 794 0, JIT_RESERVED_STACK); 795 /* 796 * Argument 1: dst+off if xchg, otherwise src, passed in register a0 797 * Argument 2: src if xchg, otherwise dst+off, passed in register a1 798 */ 799 emit(ctx, move, MIPS_R_T9, dst); 800 if (code == BPF_XCHG) { 801 emit(ctx, move, MIPS_R_A1, src); 802 emit(ctx, addiu, MIPS_R_A0, MIPS_R_T9, off); 803 } else { 804 emit(ctx, move, MIPS_R_A0, src); 805 emit(ctx, addiu, MIPS_R_A1, MIPS_R_T9, off); 806 } 807 808 /* Emit function call */ 809 switch (code) { 810 case BPF_ADD: 811 addr = (u32)&atomic_add; 812 break; 813 case BPF_ADD | BPF_FETCH: 814 addr = (u32)&atomic_fetch_add; 815 break; 816 case BPF_SUB: 817 addr = (u32)&atomic_sub; 818 break; 819 case BPF_SUB | BPF_FETCH: 820 addr = (u32)&atomic_fetch_sub; 821 break; 822 case BPF_OR: 823 addr = (u32)&atomic_or; 824 break; 825 case BPF_OR | BPF_FETCH: 826 addr = (u32)&atomic_fetch_or; 827 break; 828 case BPF_AND: 829 addr = (u32)&atomic_and; 830 break; 831 case BPF_AND | BPF_FETCH: 832 addr = (u32)&atomic_fetch_and; 833 break; 834 case BPF_XOR: 835 addr = (u32)&atomic_xor; 836 break; 837 case BPF_XOR | BPF_FETCH: 838 addr = (u32)&atomic_fetch_xor; 839 break; 840 case BPF_XCHG: 841 addr = (u32)&atomic_xchg; 842 break; 843 } 844 emit_mov_i(ctx, MIPS_R_T9, addr); 845 emit(ctx, jalr, MIPS_R_RA, MIPS_R_T9); 846 emit(ctx, nop); /* Delay slot */ 847 848 /* Update src register with old value, if specified */ 849 if (code & BPF_FETCH) { 850 emit(ctx, move, src, MIPS_R_V0); 851 exclude = BIT(src); 852 clobber_reg(ctx, src); 853 } 854 855 /* Restore caller-saved registers, except any fetched value */ 856 pop_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, 857 exclude, JIT_RESERVED_STACK); 858 emit_load_delay(ctx); 859 clobber_reg(ctx, MIPS_R_RA); 860 } 861 862 /* Helper function for 64-bit atomic exchange */ 863 static s64 jit_xchg64(s64 a, atomic64_t *v) 864 { 865 return atomic64_xchg(v, a); 866 } 867 868 /* Atomic read-modify-write (64-bit) */ 869 static void emit_atomic_r64(struct jit_context *ctx, 870 u8 dst, const u8 src[], s16 off, u8 code) 871 { 872 const u8 *r0 = bpf2mips32[BPF_REG_0]; /* Mapped to v0-v1 */ 873 const u8 *r1 = bpf2mips32[BPF_REG_1]; /* Mapped to a0-a1 */ 874 u32 exclude = 0; 875 u32 addr = 0; 876 877 /* Push caller-saved registers on stack */ 878 push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, 879 0, JIT_RESERVED_STACK); 880 /* 881 * Argument 1: 64-bit src, passed in registers a0-a1 882 * Argument 2: 32-bit dst+off, passed in register a2 883 */ 884 emit(ctx, move, MIPS_R_T9, dst); 885 emit(ctx, move, r1[0], src[0]); 886 emit(ctx, move, r1[1], src[1]); 887 emit(ctx, addiu, MIPS_R_A2, MIPS_R_T9, off); 888 889 /* Emit function call */ 890 switch (code) { 891 case BPF_ADD: 892 addr = (u32)&atomic64_add; 893 break; 894 case BPF_ADD | BPF_FETCH: 895 addr = (u32)&atomic64_fetch_add; 896 break; 897 case BPF_SUB: 898 addr = (u32)&atomic64_sub; 899 break; 900 case BPF_SUB | BPF_FETCH: 901 addr = (u32)&atomic64_fetch_sub; 902 break; 903 case BPF_OR: 904 addr = (u32)&atomic64_or; 905 break; 906 case BPF_OR | BPF_FETCH: 907 addr = (u32)&atomic64_fetch_or; 908 break; 909 case BPF_AND: 910 addr = (u32)&atomic64_and; 911 break; 912 case BPF_AND | BPF_FETCH: 913 addr = (u32)&atomic64_fetch_and; 914 break; 915 case BPF_XOR: 916 addr = (u32)&atomic64_xor; 917 break; 918 case BPF_XOR | BPF_FETCH: 919 addr = (u32)&atomic64_fetch_xor; 920 break; 921 case BPF_XCHG: 922 addr = (u32)&jit_xchg64; 923 break; 924 } 925 emit_mov_i(ctx, MIPS_R_T9, addr); 926 emit(ctx, jalr, MIPS_R_RA, MIPS_R_T9); 927 emit(ctx, nop); /* Delay slot */ 928 929 /* Update src register with old value, if specified */ 930 if (code & BPF_FETCH) { 931 emit(ctx, move, lo(src), lo(r0)); 932 emit(ctx, move, hi(src), hi(r0)); 933 exclude = BIT(src[0]) | BIT(src[1]); 934 clobber_reg64(ctx, src); 935 } 936 937 /* Restore caller-saved registers, except any fetched value */ 938 pop_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, 939 exclude, JIT_RESERVED_STACK); 940 emit_load_delay(ctx); 941 clobber_reg(ctx, MIPS_R_RA); 942 } 943 944 /* Atomic compare-and-exchange (32-bit, non-ll/sc fallback) */ 945 static void emit_cmpxchg_r32(struct jit_context *ctx, u8 dst, u8 src, s16 off) 946 { 947 const u8 *r0 = bpf2mips32[BPF_REG_0]; 948 949 /* Push caller-saved registers on stack */ 950 push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, 951 JIT_RETURN_REGS, JIT_RESERVED_STACK + 2 * sizeof(u32)); 952 /* 953 * Argument 1: 32-bit dst+off, passed in register a0 954 * Argument 2: 32-bit r0, passed in register a1 955 * Argument 3: 32-bit src, passed in register a2 956 */ 957 emit(ctx, addiu, MIPS_R_T9, dst, off); 958 emit(ctx, move, MIPS_R_T8, src); 959 emit(ctx, move, MIPS_R_A1, lo(r0)); 960 emit(ctx, move, MIPS_R_A0, MIPS_R_T9); 961 emit(ctx, move, MIPS_R_A2, MIPS_R_T8); 962 963 /* Emit function call */ 964 emit_mov_i(ctx, MIPS_R_T9, (u32)&atomic_cmpxchg); 965 emit(ctx, jalr, MIPS_R_RA, MIPS_R_T9); 966 emit(ctx, nop); /* Delay slot */ 967 968 #ifdef __BIG_ENDIAN 969 emit(ctx, move, lo(r0), MIPS_R_V0); 970 #endif 971 /* Restore caller-saved registers, except the return value */ 972 pop_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, 973 JIT_RETURN_REGS, JIT_RESERVED_STACK + 2 * sizeof(u32)); 974 emit_load_delay(ctx); 975 clobber_reg(ctx, MIPS_R_V0); 976 clobber_reg(ctx, MIPS_R_V1); 977 clobber_reg(ctx, MIPS_R_RA); 978 } 979 980 /* Atomic compare-and-exchange (64-bit) */ 981 static void emit_cmpxchg_r64(struct jit_context *ctx, 982 u8 dst, const u8 src[], s16 off) 983 { 984 const u8 *r0 = bpf2mips32[BPF_REG_0]; 985 const u8 *r2 = bpf2mips32[BPF_REG_2]; 986 987 /* Push caller-saved registers on stack */ 988 push_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, 989 JIT_RETURN_REGS, JIT_RESERVED_STACK + 2 * sizeof(u32)); 990 /* 991 * Argument 1: 32-bit dst+off, passed in register a0 (a1 unused) 992 * Argument 2: 64-bit r0, passed in registers a2-a3 993 * Argument 3: 64-bit src, passed on stack 994 */ 995 push_regs(ctx, BIT(src[0]) | BIT(src[1]), 0, JIT_RESERVED_STACK); 996 emit(ctx, addiu, MIPS_R_T9, dst, off); 997 emit(ctx, move, r2[0], r0[0]); 998 emit(ctx, move, r2[1], r0[1]); 999 emit(ctx, move, MIPS_R_A0, MIPS_R_T9); 1000 1001 /* Emit function call */ 1002 emit_mov_i(ctx, MIPS_R_T9, (u32)&atomic64_cmpxchg); 1003 emit(ctx, jalr, MIPS_R_RA, MIPS_R_T9); 1004 emit(ctx, nop); /* Delay slot */ 1005 1006 /* Restore caller-saved registers, except the return value */ 1007 pop_regs(ctx, ctx->clobbered & JIT_CALLER_REGS, 1008 JIT_RETURN_REGS, JIT_RESERVED_STACK + 2 * sizeof(u32)); 1009 emit_load_delay(ctx); 1010 clobber_reg(ctx, MIPS_R_V0); 1011 clobber_reg(ctx, MIPS_R_V1); 1012 clobber_reg(ctx, MIPS_R_RA); 1013 } 1014 1015 /* 1016 * Conditional movz or an emulated equivalent. 1017 * Note that the rs register may be modified. 1018 */ 1019 static void emit_movz_r(struct jit_context *ctx, u8 rd, u8 rs, u8 rt) 1020 { 1021 if (cpu_has_mips_2) { 1022 emit(ctx, movz, rd, rs, rt); /* rd = rt ? rd : rs */ 1023 } else if (cpu_has_mips32r6) { 1024 if (rs != MIPS_R_ZERO) 1025 emit(ctx, seleqz, rs, rs, rt); /* rs = 0 if rt == 0 */ 1026 emit(ctx, selnez, rd, rd, rt); /* rd = 0 if rt != 0 */ 1027 if (rs != MIPS_R_ZERO) 1028 emit(ctx, or, rd, rd, rs); /* rd = rd | rs */ 1029 } else { 1030 emit(ctx, bnez, rt, 8); /* PC += 8 if rd != 0 */ 1031 emit(ctx, nop); /* +0: delay slot */ 1032 emit(ctx, or, rd, rs, MIPS_R_ZERO); /* +4: rd = rs */ 1033 } 1034 clobber_reg(ctx, rd); 1035 clobber_reg(ctx, rs); 1036 } 1037 1038 /* 1039 * Conditional movn or an emulated equivalent. 1040 * Note that the rs register may be modified. 1041 */ 1042 static void emit_movn_r(struct jit_context *ctx, u8 rd, u8 rs, u8 rt) 1043 { 1044 if (cpu_has_mips_2) { 1045 emit(ctx, movn, rd, rs, rt); /* rd = rt ? rs : rd */ 1046 } else if (cpu_has_mips32r6) { 1047 if (rs != MIPS_R_ZERO) 1048 emit(ctx, selnez, rs, rs, rt); /* rs = 0 if rt == 0 */ 1049 emit(ctx, seleqz, rd, rd, rt); /* rd = 0 if rt != 0 */ 1050 if (rs != MIPS_R_ZERO) 1051 emit(ctx, or, rd, rd, rs); /* rd = rd | rs */ 1052 } else { 1053 emit(ctx, beqz, rt, 8); /* PC += 8 if rd == 0 */ 1054 emit(ctx, nop); /* +0: delay slot */ 1055 emit(ctx, or, rd, rs, MIPS_R_ZERO); /* +4: rd = rs */ 1056 } 1057 clobber_reg(ctx, rd); 1058 clobber_reg(ctx, rs); 1059 } 1060 1061 /* Emulation of 64-bit sltiu rd, rs, imm, where imm may be S32_MAX + 1 */ 1062 static void emit_sltiu_r64(struct jit_context *ctx, u8 rd, 1063 const u8 rs[], s64 imm) 1064 { 1065 u8 tmp = MIPS_R_T9; 1066 1067 if (imm < 0) { 1068 emit_mov_i(ctx, rd, imm); /* rd = imm */ 1069 emit(ctx, sltu, rd, lo(rs), rd); /* rd = rsl < rd */ 1070 emit(ctx, sltiu, tmp, hi(rs), -1); /* tmp = rsh < ~0U */ 1071 emit(ctx, or, rd, rd, tmp); /* rd = rd | tmp */ 1072 } else { /* imm >= 0 */ 1073 if (imm > 0x7fff) { 1074 emit_mov_i(ctx, rd, (s32)imm); /* rd = imm */ 1075 emit(ctx, sltu, rd, lo(rs), rd); /* rd = rsl < rd */ 1076 } else { 1077 emit(ctx, sltiu, rd, lo(rs), imm); /* rd = rsl < imm */ 1078 } 1079 emit_movn_r(ctx, rd, MIPS_R_ZERO, hi(rs)); /* rd = 0 if rsh */ 1080 } 1081 } 1082 1083 /* Emulation of 64-bit sltu rd, rs, rt */ 1084 static void emit_sltu_r64(struct jit_context *ctx, u8 rd, 1085 const u8 rs[], const u8 rt[]) 1086 { 1087 u8 tmp = MIPS_R_T9; 1088 1089 emit(ctx, sltu, rd, lo(rs), lo(rt)); /* rd = rsl < rtl */ 1090 emit(ctx, subu, tmp, hi(rs), hi(rt)); /* tmp = rsh - rth */ 1091 emit_movn_r(ctx, rd, MIPS_R_ZERO, tmp); /* rd = 0 if tmp != 0 */ 1092 emit(ctx, sltu, tmp, hi(rs), hi(rt)); /* tmp = rsh < rth */ 1093 emit(ctx, or, rd, rd, tmp); /* rd = rd | tmp */ 1094 } 1095 1096 /* Emulation of 64-bit slti rd, rs, imm, where imm may be S32_MAX + 1 */ 1097 static void emit_slti_r64(struct jit_context *ctx, u8 rd, 1098 const u8 rs[], s64 imm) 1099 { 1100 u8 t1 = MIPS_R_T8; 1101 u8 t2 = MIPS_R_T9; 1102 u8 cmp; 1103 1104 /* 1105 * if ((rs < 0) ^ (imm < 0)) t1 = imm >u rsl 1106 * else t1 = rsl <u imm 1107 */ 1108 emit_mov_i(ctx, rd, (s32)imm); 1109 emit(ctx, sltu, t1, lo(rs), rd); /* t1 = rsl <u imm */ 1110 emit(ctx, sltu, t2, rd, lo(rs)); /* t2 = imm <u rsl */ 1111 emit(ctx, srl, rd, hi(rs), 31); /* rd = rsh >> 31 */ 1112 if (imm < 0) 1113 emit_movz_r(ctx, t1, t2, rd); /* t1 = rd ? t1 : t2 */ 1114 else 1115 emit_movn_r(ctx, t1, t2, rd); /* t1 = rd ? t2 : t1 */ 1116 /* 1117 * if ((imm < 0 && rsh != 0xffffffff) || 1118 * (imm >= 0 && rsh != 0)) 1119 * t1 = 0 1120 */ 1121 if (imm < 0) { 1122 emit(ctx, addiu, rd, hi(rs), 1); /* rd = rsh + 1 */ 1123 cmp = rd; 1124 } else { /* imm >= 0 */ 1125 cmp = hi(rs); 1126 } 1127 emit_movn_r(ctx, t1, MIPS_R_ZERO, cmp); /* t1 = 0 if cmp != 0 */ 1128 1129 /* 1130 * if (imm < 0) rd = rsh < -1 1131 * else rd = rsh != 0 1132 * rd = rd | t1 1133 */ 1134 emit(ctx, slti, rd, hi(rs), imm < 0 ? -1 : 0); /* rd = rsh < hi(imm) */ 1135 emit(ctx, or, rd, rd, t1); /* rd = rd | t1 */ 1136 } 1137 1138 /* Emulation of 64-bit(slt rd, rs, rt) */ 1139 static void emit_slt_r64(struct jit_context *ctx, u8 rd, 1140 const u8 rs[], const u8 rt[]) 1141 { 1142 u8 t1 = MIPS_R_T7; 1143 u8 t2 = MIPS_R_T8; 1144 u8 t3 = MIPS_R_T9; 1145 1146 /* 1147 * if ((rs < 0) ^ (rt < 0)) t1 = rtl <u rsl 1148 * else t1 = rsl <u rtl 1149 * if (rsh == rth) t1 = 0 1150 */ 1151 emit(ctx, sltu, t1, lo(rs), lo(rt)); /* t1 = rsl <u rtl */ 1152 emit(ctx, sltu, t2, lo(rt), lo(rs)); /* t2 = rtl <u rsl */ 1153 emit(ctx, xor, t3, hi(rs), hi(rt)); /* t3 = rlh ^ rth */ 1154 emit(ctx, srl, rd, t3, 31); /* rd = t3 >> 31 */ 1155 emit_movn_r(ctx, t1, t2, rd); /* t1 = rd ? t2 : t1 */ 1156 emit_movn_r(ctx, t1, MIPS_R_ZERO, t3); /* t1 = 0 if t3 != 0 */ 1157 1158 /* rd = (rsh < rth) | t1 */ 1159 emit(ctx, slt, rd, hi(rs), hi(rt)); /* rd = rsh <s rth */ 1160 emit(ctx, or, rd, rd, t1); /* rd = rd | t1 */ 1161 } 1162 1163 /* Jump immediate (64-bit) */ 1164 static void emit_jmp_i64(struct jit_context *ctx, 1165 const u8 dst[], s32 imm, s32 off, u8 op) 1166 { 1167 u8 tmp = MIPS_R_T6; 1168 1169 switch (op) { 1170 /* No-op, used internally for branch optimization */ 1171 case JIT_JNOP: 1172 break; 1173 /* PC += off if dst == imm */ 1174 /* PC += off if dst != imm */ 1175 case BPF_JEQ: 1176 case BPF_JNE: 1177 if (imm >= -0x7fff && imm <= 0x8000) { 1178 emit(ctx, addiu, tmp, lo(dst), -imm); 1179 } else if ((u32)imm <= 0xffff) { 1180 emit(ctx, xori, tmp, lo(dst), imm); 1181 } else { /* Register fallback */ 1182 emit_mov_i(ctx, tmp, imm); 1183 emit(ctx, xor, tmp, lo(dst), tmp); 1184 } 1185 if (imm < 0) { /* Compare sign extension */ 1186 emit(ctx, addiu, MIPS_R_T9, hi(dst), 1); 1187 emit(ctx, or, tmp, tmp, MIPS_R_T9); 1188 } else { /* Compare zero extension */ 1189 emit(ctx, or, tmp, tmp, hi(dst)); 1190 } 1191 if (op == BPF_JEQ) 1192 emit(ctx, beqz, tmp, off); 1193 else /* BPF_JNE */ 1194 emit(ctx, bnez, tmp, off); 1195 break; 1196 /* PC += off if dst & imm */ 1197 /* PC += off if (dst & imm) == 0 (not in BPF, used for long jumps) */ 1198 case BPF_JSET: 1199 case JIT_JNSET: 1200 if ((u32)imm <= 0xffff) { 1201 emit(ctx, andi, tmp, lo(dst), imm); 1202 } else { /* Register fallback */ 1203 emit_mov_i(ctx, tmp, imm); 1204 emit(ctx, and, tmp, lo(dst), tmp); 1205 } 1206 if (imm < 0) /* Sign-extension pulls in high word */ 1207 emit(ctx, or, tmp, tmp, hi(dst)); 1208 if (op == BPF_JSET) 1209 emit(ctx, bnez, tmp, off); 1210 else /* JIT_JNSET */ 1211 emit(ctx, beqz, tmp, off); 1212 break; 1213 /* PC += off if dst > imm */ 1214 case BPF_JGT: 1215 emit_sltiu_r64(ctx, tmp, dst, (s64)imm + 1); 1216 emit(ctx, beqz, tmp, off); 1217 break; 1218 /* PC += off if dst >= imm */ 1219 case BPF_JGE: 1220 emit_sltiu_r64(ctx, tmp, dst, imm); 1221 emit(ctx, beqz, tmp, off); 1222 break; 1223 /* PC += off if dst < imm */ 1224 case BPF_JLT: 1225 emit_sltiu_r64(ctx, tmp, dst, imm); 1226 emit(ctx, bnez, tmp, off); 1227 break; 1228 /* PC += off if dst <= imm */ 1229 case BPF_JLE: 1230 emit_sltiu_r64(ctx, tmp, dst, (s64)imm + 1); 1231 emit(ctx, bnez, tmp, off); 1232 break; 1233 /* PC += off if dst > imm (signed) */ 1234 case BPF_JSGT: 1235 emit_slti_r64(ctx, tmp, dst, (s64)imm + 1); 1236 emit(ctx, beqz, tmp, off); 1237 break; 1238 /* PC += off if dst >= imm (signed) */ 1239 case BPF_JSGE: 1240 emit_slti_r64(ctx, tmp, dst, imm); 1241 emit(ctx, beqz, tmp, off); 1242 break; 1243 /* PC += off if dst < imm (signed) */ 1244 case BPF_JSLT: 1245 emit_slti_r64(ctx, tmp, dst, imm); 1246 emit(ctx, bnez, tmp, off); 1247 break; 1248 /* PC += off if dst <= imm (signed) */ 1249 case BPF_JSLE: 1250 emit_slti_r64(ctx, tmp, dst, (s64)imm + 1); 1251 emit(ctx, bnez, tmp, off); 1252 break; 1253 } 1254 } 1255 1256 /* Jump register (64-bit) */ 1257 static void emit_jmp_r64(struct jit_context *ctx, 1258 const u8 dst[], const u8 src[], s32 off, u8 op) 1259 { 1260 u8 t1 = MIPS_R_T6; 1261 u8 t2 = MIPS_R_T7; 1262 1263 switch (op) { 1264 /* No-op, used internally for branch optimization */ 1265 case JIT_JNOP: 1266 break; 1267 /* PC += off if dst == src */ 1268 /* PC += off if dst != src */ 1269 case BPF_JEQ: 1270 case BPF_JNE: 1271 emit(ctx, subu, t1, lo(dst), lo(src)); 1272 emit(ctx, subu, t2, hi(dst), hi(src)); 1273 emit(ctx, or, t1, t1, t2); 1274 if (op == BPF_JEQ) 1275 emit(ctx, beqz, t1, off); 1276 else /* BPF_JNE */ 1277 emit(ctx, bnez, t1, off); 1278 break; 1279 /* PC += off if dst & src */ 1280 /* PC += off if (dst & imm) == 0 (not in BPF, used for long jumps) */ 1281 case BPF_JSET: 1282 case JIT_JNSET: 1283 emit(ctx, and, t1, lo(dst), lo(src)); 1284 emit(ctx, and, t2, hi(dst), hi(src)); 1285 emit(ctx, or, t1, t1, t2); 1286 if (op == BPF_JSET) 1287 emit(ctx, bnez, t1, off); 1288 else /* JIT_JNSET */ 1289 emit(ctx, beqz, t1, off); 1290 break; 1291 /* PC += off if dst > src */ 1292 case BPF_JGT: 1293 emit_sltu_r64(ctx, t1, src, dst); 1294 emit(ctx, bnez, t1, off); 1295 break; 1296 /* PC += off if dst >= src */ 1297 case BPF_JGE: 1298 emit_sltu_r64(ctx, t1, dst, src); 1299 emit(ctx, beqz, t1, off); 1300 break; 1301 /* PC += off if dst < src */ 1302 case BPF_JLT: 1303 emit_sltu_r64(ctx, t1, dst, src); 1304 emit(ctx, bnez, t1, off); 1305 break; 1306 /* PC += off if dst <= src */ 1307 case BPF_JLE: 1308 emit_sltu_r64(ctx, t1, src, dst); 1309 emit(ctx, beqz, t1, off); 1310 break; 1311 /* PC += off if dst > src (signed) */ 1312 case BPF_JSGT: 1313 emit_slt_r64(ctx, t1, src, dst); 1314 emit(ctx, bnez, t1, off); 1315 break; 1316 /* PC += off if dst >= src (signed) */ 1317 case BPF_JSGE: 1318 emit_slt_r64(ctx, t1, dst, src); 1319 emit(ctx, beqz, t1, off); 1320 break; 1321 /* PC += off if dst < src (signed) */ 1322 case BPF_JSLT: 1323 emit_slt_r64(ctx, t1, dst, src); 1324 emit(ctx, bnez, t1, off); 1325 break; 1326 /* PC += off if dst <= src (signed) */ 1327 case BPF_JSLE: 1328 emit_slt_r64(ctx, t1, src, dst); 1329 emit(ctx, beqz, t1, off); 1330 break; 1331 } 1332 } 1333 1334 /* Function call */ 1335 static int emit_call(struct jit_context *ctx, const struct bpf_insn *insn) 1336 { 1337 bool fixed; 1338 u64 addr; 1339 1340 /* Decode the call address */ 1341 if (bpf_jit_get_func_addr(ctx->program, insn, false, 1342 &addr, &fixed) < 0) 1343 return -1; 1344 if (!fixed) 1345 return -1; 1346 1347 /* Push stack arguments */ 1348 push_regs(ctx, JIT_STACK_REGS, 0, JIT_RESERVED_STACK); 1349 1350 /* Emit function call */ 1351 emit_mov_i(ctx, MIPS_R_T9, addr); 1352 emit(ctx, jalr, MIPS_R_RA, MIPS_R_T9); 1353 emit(ctx, nop); /* Delay slot */ 1354 1355 clobber_reg(ctx, MIPS_R_RA); 1356 clobber_reg(ctx, MIPS_R_V0); 1357 clobber_reg(ctx, MIPS_R_V1); 1358 return 0; 1359 } 1360 1361 /* Function tail call */ 1362 static int emit_tail_call(struct jit_context *ctx) 1363 { 1364 u8 ary = lo(bpf2mips32[BPF_REG_2]); 1365 u8 ind = lo(bpf2mips32[BPF_REG_3]); 1366 u8 t1 = MIPS_R_T8; 1367 u8 t2 = MIPS_R_T9; 1368 int off; 1369 1370 /* 1371 * Tail call: 1372 * eBPF R1 - function argument (context ptr), passed in a0-a1 1373 * eBPF R2 - ptr to object with array of function entry points 1374 * eBPF R3 - array index of function to be called 1375 * stack[sz] - remaining tail call count, initialized in prologue 1376 */ 1377 1378 /* if (ind >= ary->map.max_entries) goto out */ 1379 off = offsetof(struct bpf_array, map.max_entries); 1380 if (off > 0x7fff) 1381 return -1; 1382 emit(ctx, lw, t1, off, ary); /* t1 = ary->map.max_entries*/ 1383 emit_load_delay(ctx); /* Load delay slot */ 1384 emit(ctx, sltu, t1, ind, t1); /* t1 = ind < t1 */ 1385 emit(ctx, beqz, t1, get_offset(ctx, 1)); /* PC += off(1) if t1 == 0 */ 1386 /* (next insn delay slot) */ 1387 /* if (TCC-- <= 0) goto out */ 1388 emit(ctx, lw, t2, ctx->stack_size, MIPS_R_SP); /* t2 = *(SP + size) */ 1389 emit_load_delay(ctx); /* Load delay slot */ 1390 emit(ctx, blez, t2, get_offset(ctx, 1)); /* PC += off(1) if t2 <= 0 */ 1391 emit(ctx, addiu, t2, t2, -1); /* t2-- (delay slot) */ 1392 emit(ctx, sw, t2, ctx->stack_size, MIPS_R_SP); /* *(SP + size) = t2 */ 1393 1394 /* prog = ary->ptrs[ind] */ 1395 off = offsetof(struct bpf_array, ptrs); 1396 if (off > 0x7fff) 1397 return -1; 1398 emit(ctx, sll, t1, ind, 2); /* t1 = ind << 2 */ 1399 emit(ctx, addu, t1, t1, ary); /* t1 += ary */ 1400 emit(ctx, lw, t2, off, t1); /* t2 = *(t1 + off) */ 1401 emit_load_delay(ctx); /* Load delay slot */ 1402 1403 /* if (prog == 0) goto out */ 1404 emit(ctx, beqz, t2, get_offset(ctx, 1)); /* PC += off(1) if t2 == 0 */ 1405 emit(ctx, nop); /* Delay slot */ 1406 1407 /* func = prog->bpf_func + 8 (prologue skip offset) */ 1408 off = offsetof(struct bpf_prog, bpf_func); 1409 if (off > 0x7fff) 1410 return -1; 1411 emit(ctx, lw, t1, off, t2); /* t1 = *(t2 + off) */ 1412 emit_load_delay(ctx); /* Load delay slot */ 1413 emit(ctx, addiu, t1, t1, JIT_TCALL_SKIP); /* t1 += skip (8 or 12) */ 1414 1415 /* goto func */ 1416 build_epilogue(ctx, t1); 1417 return 0; 1418 } 1419 1420 /* 1421 * Stack frame layout for a JITed program (stack grows down). 1422 * 1423 * Higher address : Caller's stack frame : 1424 * :----------------------------: 1425 * : 64-bit eBPF args r3-r5 : 1426 * :----------------------------: 1427 * : Reserved / tail call count : 1428 * +============================+ <--- MIPS sp before call 1429 * | Callee-saved registers, | 1430 * | including RA and FP | 1431 * +----------------------------+ <--- eBPF FP (MIPS zero,fp) 1432 * | Local eBPF variables | 1433 * | allocated by program | 1434 * +----------------------------+ 1435 * | Reserved for caller-saved | 1436 * | registers | 1437 * +----------------------------+ 1438 * | Reserved for 64-bit eBPF | 1439 * | args r3-r5 & args passed | 1440 * | on stack in kernel calls | 1441 * Lower address +============================+ <--- MIPS sp 1442 */ 1443 1444 /* Build program prologue to set up the stack and registers */ 1445 void build_prologue(struct jit_context *ctx) 1446 { 1447 const u8 *r1 = bpf2mips32[BPF_REG_1]; 1448 const u8 *fp = bpf2mips32[BPF_REG_FP]; 1449 int stack, saved, locals, reserved; 1450 1451 /* 1452 * In the unlikely event that the TCC limit is raised to more 1453 * than 16 bits, it is clamped to the maximum value allowed for 1454 * the generated code (0xffff). It is better fail to compile 1455 * instead of degrading gracefully. 1456 */ 1457 BUILD_BUG_ON(MAX_TAIL_CALL_CNT > 0xffff); 1458 1459 /* 1460 * The first two instructions initialize TCC in the reserved (for us) 1461 * 16-byte area in the parent's stack frame. On a tail call, the 1462 * calling function jumps into the prologue after these instructions. 1463 */ 1464 emit(ctx, ori, MIPS_R_T9, MIPS_R_ZERO, MAX_TAIL_CALL_CNT); 1465 emit(ctx, sw, MIPS_R_T9, 0, MIPS_R_SP); 1466 1467 /* 1468 * Register eBPF R1 contains the 32-bit context pointer argument. 1469 * A 32-bit argument is always passed in MIPS register a0, regardless 1470 * of CPU endianness. Initialize R1 accordingly and zero-extend. 1471 */ 1472 #ifdef __BIG_ENDIAN 1473 emit(ctx, move, lo(r1), MIPS_R_A0); 1474 #endif 1475 1476 /* === Entry-point for tail calls === */ 1477 1478 /* Zero-extend the 32-bit argument */ 1479 emit(ctx, move, hi(r1), MIPS_R_ZERO); 1480 1481 /* If the eBPF frame pointer was accessed it must be saved */ 1482 if (ctx->accessed & BIT(BPF_REG_FP)) 1483 clobber_reg64(ctx, fp); 1484 1485 /* Compute the stack space needed for callee-saved registers */ 1486 saved = hweight32(ctx->clobbered & JIT_CALLEE_REGS) * sizeof(u32); 1487 saved = ALIGN(saved, MIPS_STACK_ALIGNMENT); 1488 1489 /* Stack space used by eBPF program local data */ 1490 locals = ALIGN(ctx->program->aux->stack_depth, MIPS_STACK_ALIGNMENT); 1491 1492 /* 1493 * If we are emitting function calls, reserve extra stack space for 1494 * caller-saved registers and function arguments passed on the stack. 1495 * The required space is computed automatically during resource 1496 * usage discovery (pass 1). 1497 */ 1498 reserved = ctx->stack_used; 1499 1500 /* Allocate the stack frame */ 1501 stack = ALIGN(saved + locals + reserved, MIPS_STACK_ALIGNMENT); 1502 emit(ctx, addiu, MIPS_R_SP, MIPS_R_SP, -stack); 1503 1504 /* Store callee-saved registers on stack */ 1505 push_regs(ctx, ctx->clobbered & JIT_CALLEE_REGS, 0, stack - saved); 1506 1507 /* Initialize the eBPF frame pointer if accessed */ 1508 if (ctx->accessed & BIT(BPF_REG_FP)) 1509 emit(ctx, addiu, lo(fp), MIPS_R_SP, stack - saved); 1510 1511 ctx->saved_size = saved; 1512 ctx->stack_size = stack; 1513 } 1514 1515 /* Build the program epilogue to restore the stack and registers */ 1516 void build_epilogue(struct jit_context *ctx, int dest_reg) 1517 { 1518 /* Restore callee-saved registers from stack */ 1519 pop_regs(ctx, ctx->clobbered & JIT_CALLEE_REGS, 0, 1520 ctx->stack_size - ctx->saved_size); 1521 /* 1522 * A 32-bit return value is always passed in MIPS register v0, 1523 * but on big-endian targets the low part of R0 is mapped to v1. 1524 */ 1525 #ifdef __BIG_ENDIAN 1526 emit(ctx, move, MIPS_R_V0, MIPS_R_V1); 1527 #endif 1528 1529 /* Jump to the return address and adjust the stack pointer */ 1530 emit(ctx, jr, dest_reg); 1531 emit(ctx, addiu, MIPS_R_SP, MIPS_R_SP, ctx->stack_size); 1532 } 1533 1534 /* Build one eBPF instruction */ 1535 int build_insn(const struct bpf_insn *insn, struct jit_context *ctx) 1536 { 1537 const u8 *dst = bpf2mips32[insn->dst_reg]; 1538 const u8 *src = bpf2mips32[insn->src_reg]; 1539 const u8 *res = bpf2mips32[BPF_REG_0]; 1540 const u8 *tmp = bpf2mips32[JIT_REG_TMP]; 1541 u8 code = insn->code; 1542 s16 off = insn->off; 1543 s32 imm = insn->imm; 1544 s32 val, rel; 1545 u8 alu, jmp; 1546 1547 switch (code) { 1548 /* ALU operations */ 1549 /* dst = imm */ 1550 case BPF_ALU | BPF_MOV | BPF_K: 1551 emit_mov_i(ctx, lo(dst), imm); 1552 emit_zext_ver(ctx, dst); 1553 break; 1554 /* dst = src */ 1555 case BPF_ALU | BPF_MOV | BPF_X: 1556 if (imm == 1) { 1557 /* Special mov32 for zext */ 1558 emit_mov_i(ctx, hi(dst), 0); 1559 } else { 1560 emit_mov_r32(ctx, dst, src, off); 1561 } 1562 break; 1563 /* dst = -dst */ 1564 case BPF_ALU | BPF_NEG: 1565 emit_alu_i(ctx, lo(dst), 0, BPF_NEG); 1566 emit_zext_ver(ctx, dst); 1567 break; 1568 /* dst = dst & imm */ 1569 /* dst = dst | imm */ 1570 /* dst = dst ^ imm */ 1571 /* dst = dst << imm */ 1572 /* dst = dst >> imm */ 1573 /* dst = dst >> imm (arithmetic) */ 1574 /* dst = dst + imm */ 1575 /* dst = dst - imm */ 1576 /* dst = dst * imm */ 1577 /* dst = dst / imm */ 1578 /* dst = dst % imm */ 1579 case BPF_ALU | BPF_OR | BPF_K: 1580 case BPF_ALU | BPF_AND | BPF_K: 1581 case BPF_ALU | BPF_XOR | BPF_K: 1582 case BPF_ALU | BPF_LSH | BPF_K: 1583 case BPF_ALU | BPF_RSH | BPF_K: 1584 case BPF_ALU | BPF_ARSH | BPF_K: 1585 case BPF_ALU | BPF_ADD | BPF_K: 1586 case BPF_ALU | BPF_SUB | BPF_K: 1587 case BPF_ALU | BPF_MUL | BPF_K: 1588 case BPF_ALU | BPF_DIV | BPF_K: 1589 case BPF_ALU | BPF_MOD | BPF_K: 1590 if (!valid_alu_i(BPF_OP(code), imm, off)) { 1591 emit_mov_i(ctx, MIPS_R_T6, imm); 1592 emit_alu_r(ctx, lo(dst), MIPS_R_T6, BPF_OP(code), off); 1593 } else if (rewrite_alu_i(BPF_OP(code), imm, &alu, &val)) { 1594 emit_alu_i(ctx, lo(dst), val, alu); 1595 } 1596 emit_zext_ver(ctx, dst); 1597 break; 1598 /* dst = dst & src */ 1599 /* dst = dst | src */ 1600 /* dst = dst ^ src */ 1601 /* dst = dst << src */ 1602 /* dst = dst >> src */ 1603 /* dst = dst >> src (arithmetic) */ 1604 /* dst = dst + src */ 1605 /* dst = dst - src */ 1606 /* dst = dst * src */ 1607 /* dst = dst / src */ 1608 /* dst = dst % src */ 1609 case BPF_ALU | BPF_AND | BPF_X: 1610 case BPF_ALU | BPF_OR | BPF_X: 1611 case BPF_ALU | BPF_XOR | BPF_X: 1612 case BPF_ALU | BPF_LSH | BPF_X: 1613 case BPF_ALU | BPF_RSH | BPF_X: 1614 case BPF_ALU | BPF_ARSH | BPF_X: 1615 case BPF_ALU | BPF_ADD | BPF_X: 1616 case BPF_ALU | BPF_SUB | BPF_X: 1617 case BPF_ALU | BPF_MUL | BPF_X: 1618 case BPF_ALU | BPF_DIV | BPF_X: 1619 case BPF_ALU | BPF_MOD | BPF_X: 1620 emit_alu_r(ctx, lo(dst), lo(src), BPF_OP(code), off); 1621 emit_zext_ver(ctx, dst); 1622 break; 1623 /* dst = imm (64-bit) */ 1624 case BPF_ALU64 | BPF_MOV | BPF_K: 1625 emit_mov_se_i64(ctx, dst, imm); 1626 break; 1627 /* dst = src (64-bit) */ 1628 case BPF_ALU64 | BPF_MOV | BPF_X: 1629 emit_mov_r64(ctx, dst, src, off); 1630 break; 1631 /* dst = -dst (64-bit) */ 1632 case BPF_ALU64 | BPF_NEG: 1633 emit_neg_i64(ctx, dst); 1634 break; 1635 /* dst = dst & imm (64-bit) */ 1636 case BPF_ALU64 | BPF_AND | BPF_K: 1637 emit_alu_i64(ctx, dst, imm, BPF_OP(code)); 1638 break; 1639 /* dst = dst | imm (64-bit) */ 1640 /* dst = dst ^ imm (64-bit) */ 1641 /* dst = dst + imm (64-bit) */ 1642 /* dst = dst - imm (64-bit) */ 1643 case BPF_ALU64 | BPF_OR | BPF_K: 1644 case BPF_ALU64 | BPF_XOR | BPF_K: 1645 case BPF_ALU64 | BPF_ADD | BPF_K: 1646 case BPF_ALU64 | BPF_SUB | BPF_K: 1647 if (imm) 1648 emit_alu_i64(ctx, dst, imm, BPF_OP(code)); 1649 break; 1650 /* dst = dst << imm (64-bit) */ 1651 /* dst = dst >> imm (64-bit) */ 1652 /* dst = dst >> imm (64-bit, arithmetic) */ 1653 case BPF_ALU64 | BPF_LSH | BPF_K: 1654 case BPF_ALU64 | BPF_RSH | BPF_K: 1655 case BPF_ALU64 | BPF_ARSH | BPF_K: 1656 if (imm) 1657 emit_shift_i64(ctx, dst, imm, BPF_OP(code)); 1658 break; 1659 /* dst = dst * imm (64-bit) */ 1660 case BPF_ALU64 | BPF_MUL | BPF_K: 1661 emit_mul_i64(ctx, dst, imm); 1662 break; 1663 /* dst = dst / imm (64-bit) */ 1664 /* dst = dst % imm (64-bit) */ 1665 case BPF_ALU64 | BPF_DIV | BPF_K: 1666 case BPF_ALU64 | BPF_MOD | BPF_K: 1667 /* 1668 * Sign-extend the immediate value into a temporary register, 1669 * and then do the operation on this register. 1670 */ 1671 emit_mov_se_i64(ctx, tmp, imm); 1672 emit_divmod_r64(ctx, dst, tmp, BPF_OP(code), off); 1673 break; 1674 /* dst = dst & src (64-bit) */ 1675 /* dst = dst | src (64-bit) */ 1676 /* dst = dst ^ src (64-bit) */ 1677 /* dst = dst + src (64-bit) */ 1678 /* dst = dst - src (64-bit) */ 1679 case BPF_ALU64 | BPF_AND | BPF_X: 1680 case BPF_ALU64 | BPF_OR | BPF_X: 1681 case BPF_ALU64 | BPF_XOR | BPF_X: 1682 case BPF_ALU64 | BPF_ADD | BPF_X: 1683 case BPF_ALU64 | BPF_SUB | BPF_X: 1684 emit_alu_r64(ctx, dst, src, BPF_OP(code)); 1685 break; 1686 /* dst = dst << src (64-bit) */ 1687 /* dst = dst >> src (64-bit) */ 1688 /* dst = dst >> src (64-bit, arithmetic) */ 1689 case BPF_ALU64 | BPF_LSH | BPF_X: 1690 case BPF_ALU64 | BPF_RSH | BPF_X: 1691 case BPF_ALU64 | BPF_ARSH | BPF_X: 1692 emit_shift_r64(ctx, dst, lo(src), BPF_OP(code)); 1693 break; 1694 /* dst = dst * src (64-bit) */ 1695 case BPF_ALU64 | BPF_MUL | BPF_X: 1696 emit_mul_r64(ctx, dst, src); 1697 break; 1698 /* dst = dst / src (64-bit) */ 1699 /* dst = dst % src (64-bit) */ 1700 case BPF_ALU64 | BPF_DIV | BPF_X: 1701 case BPF_ALU64 | BPF_MOD | BPF_X: 1702 emit_divmod_r64(ctx, dst, src, BPF_OP(code), off); 1703 break; 1704 /* dst = bswap(dst) */ 1705 case BPF_ALU64 | BPF_END: 1706 emit_bswap_r64(ctx, dst, imm); 1707 break; 1708 /* dst = htole(dst) */ 1709 /* dst = htobe(dst) */ 1710 case BPF_ALU | BPF_END | BPF_FROM_LE: 1711 case BPF_ALU | BPF_END | BPF_FROM_BE: 1712 if (BPF_SRC(code) == 1713 #ifdef __BIG_ENDIAN 1714 BPF_FROM_LE 1715 #else 1716 BPF_FROM_BE 1717 #endif 1718 ) 1719 emit_bswap_r64(ctx, dst, imm); 1720 else 1721 emit_trunc_r64(ctx, dst, imm); 1722 break; 1723 /* dst = imm64 */ 1724 case BPF_LD | BPF_IMM | BPF_DW: 1725 emit_mov_i(ctx, lo(dst), imm); 1726 emit_mov_i(ctx, hi(dst), insn[1].imm); 1727 return 1; 1728 /* LDX: dst = *(size *)(src + off) */ 1729 case BPF_LDX | BPF_MEM | BPF_W: 1730 case BPF_LDX | BPF_MEM | BPF_H: 1731 case BPF_LDX | BPF_MEM | BPF_B: 1732 case BPF_LDX | BPF_MEM | BPF_DW: 1733 emit_ldx(ctx, dst, lo(src), off, BPF_SIZE(code)); 1734 break; 1735 /* ST: *(size *)(dst + off) = imm */ 1736 case BPF_ST | BPF_MEM | BPF_W: 1737 case BPF_ST | BPF_MEM | BPF_H: 1738 case BPF_ST | BPF_MEM | BPF_B: 1739 case BPF_ST | BPF_MEM | BPF_DW: 1740 switch (BPF_SIZE(code)) { 1741 case BPF_DW: 1742 /* Sign-extend immediate value into temporary reg */ 1743 emit_mov_se_i64(ctx, tmp, imm); 1744 break; 1745 case BPF_W: 1746 case BPF_H: 1747 case BPF_B: 1748 emit_mov_i(ctx, lo(tmp), imm); 1749 break; 1750 } 1751 emit_stx(ctx, lo(dst), tmp, off, BPF_SIZE(code)); 1752 break; 1753 /* STX: *(size *)(dst + off) = src */ 1754 case BPF_STX | BPF_MEM | BPF_W: 1755 case BPF_STX | BPF_MEM | BPF_H: 1756 case BPF_STX | BPF_MEM | BPF_B: 1757 case BPF_STX | BPF_MEM | BPF_DW: 1758 emit_stx(ctx, lo(dst), src, off, BPF_SIZE(code)); 1759 break; 1760 /* Speculation barrier */ 1761 case BPF_ST | BPF_NOSPEC: 1762 break; 1763 /* Atomics */ 1764 case BPF_STX | BPF_ATOMIC | BPF_W: 1765 switch (imm) { 1766 case BPF_ADD: 1767 case BPF_ADD | BPF_FETCH: 1768 case BPF_AND: 1769 case BPF_AND | BPF_FETCH: 1770 case BPF_OR: 1771 case BPF_OR | BPF_FETCH: 1772 case BPF_XOR: 1773 case BPF_XOR | BPF_FETCH: 1774 case BPF_XCHG: 1775 if (cpu_has_llsc) 1776 emit_atomic_r(ctx, lo(dst), lo(src), off, imm); 1777 else /* Non-ll/sc fallback */ 1778 emit_atomic_r32(ctx, lo(dst), lo(src), 1779 off, imm); 1780 if (imm & BPF_FETCH) 1781 emit_zext_ver(ctx, src); 1782 break; 1783 case BPF_CMPXCHG: 1784 if (cpu_has_llsc) 1785 emit_cmpxchg_r(ctx, lo(dst), lo(src), 1786 lo(res), off); 1787 else /* Non-ll/sc fallback */ 1788 emit_cmpxchg_r32(ctx, lo(dst), lo(src), off); 1789 /* Result zero-extension inserted by verifier */ 1790 break; 1791 default: 1792 goto notyet; 1793 } 1794 break; 1795 /* Atomics (64-bit) */ 1796 case BPF_STX | BPF_ATOMIC | BPF_DW: 1797 switch (imm) { 1798 case BPF_ADD: 1799 case BPF_ADD | BPF_FETCH: 1800 case BPF_AND: 1801 case BPF_AND | BPF_FETCH: 1802 case BPF_OR: 1803 case BPF_OR | BPF_FETCH: 1804 case BPF_XOR: 1805 case BPF_XOR | BPF_FETCH: 1806 case BPF_XCHG: 1807 emit_atomic_r64(ctx, lo(dst), src, off, imm); 1808 break; 1809 case BPF_CMPXCHG: 1810 emit_cmpxchg_r64(ctx, lo(dst), src, off); 1811 break; 1812 default: 1813 goto notyet; 1814 } 1815 break; 1816 /* PC += off if dst == src */ 1817 /* PC += off if dst != src */ 1818 /* PC += off if dst & src */ 1819 /* PC += off if dst > src */ 1820 /* PC += off if dst >= src */ 1821 /* PC += off if dst < src */ 1822 /* PC += off if dst <= src */ 1823 /* PC += off if dst > src (signed) */ 1824 /* PC += off if dst >= src (signed) */ 1825 /* PC += off if dst < src (signed) */ 1826 /* PC += off if dst <= src (signed) */ 1827 case BPF_JMP32 | BPF_JEQ | BPF_X: 1828 case BPF_JMP32 | BPF_JNE | BPF_X: 1829 case BPF_JMP32 | BPF_JSET | BPF_X: 1830 case BPF_JMP32 | BPF_JGT | BPF_X: 1831 case BPF_JMP32 | BPF_JGE | BPF_X: 1832 case BPF_JMP32 | BPF_JLT | BPF_X: 1833 case BPF_JMP32 | BPF_JLE | BPF_X: 1834 case BPF_JMP32 | BPF_JSGT | BPF_X: 1835 case BPF_JMP32 | BPF_JSGE | BPF_X: 1836 case BPF_JMP32 | BPF_JSLT | BPF_X: 1837 case BPF_JMP32 | BPF_JSLE | BPF_X: 1838 if (off == 0) 1839 break; 1840 setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel); 1841 emit_jmp_r(ctx, lo(dst), lo(src), rel, jmp); 1842 if (finish_jmp(ctx, jmp, off) < 0) 1843 goto toofar; 1844 break; 1845 /* PC += off if dst == imm */ 1846 /* PC += off if dst != imm */ 1847 /* PC += off if dst & imm */ 1848 /* PC += off if dst > imm */ 1849 /* PC += off if dst >= imm */ 1850 /* PC += off if dst < imm */ 1851 /* PC += off if dst <= imm */ 1852 /* PC += off if dst > imm (signed) */ 1853 /* PC += off if dst >= imm (signed) */ 1854 /* PC += off if dst < imm (signed) */ 1855 /* PC += off if dst <= imm (signed) */ 1856 case BPF_JMP32 | BPF_JEQ | BPF_K: 1857 case BPF_JMP32 | BPF_JNE | BPF_K: 1858 case BPF_JMP32 | BPF_JSET | BPF_K: 1859 case BPF_JMP32 | BPF_JGT | BPF_K: 1860 case BPF_JMP32 | BPF_JGE | BPF_K: 1861 case BPF_JMP32 | BPF_JLT | BPF_K: 1862 case BPF_JMP32 | BPF_JLE | BPF_K: 1863 case BPF_JMP32 | BPF_JSGT | BPF_K: 1864 case BPF_JMP32 | BPF_JSGE | BPF_K: 1865 case BPF_JMP32 | BPF_JSLT | BPF_K: 1866 case BPF_JMP32 | BPF_JSLE | BPF_K: 1867 if (off == 0) 1868 break; 1869 setup_jmp_i(ctx, imm, 32, BPF_OP(code), off, &jmp, &rel); 1870 if (valid_jmp_i(jmp, imm)) { 1871 emit_jmp_i(ctx, lo(dst), imm, rel, jmp); 1872 } else { 1873 /* Move large immediate to register */ 1874 emit_mov_i(ctx, MIPS_R_T6, imm); 1875 emit_jmp_r(ctx, lo(dst), MIPS_R_T6, rel, jmp); 1876 } 1877 if (finish_jmp(ctx, jmp, off) < 0) 1878 goto toofar; 1879 break; 1880 /* PC += imm */ 1881 case BPF_JMP32 | BPF_JA: 1882 if (imm == 0) 1883 break; 1884 if (emit_ja(ctx, imm) < 0) 1885 goto toofar; 1886 break; 1887 /* PC += off if dst == src */ 1888 /* PC += off if dst != src */ 1889 /* PC += off if dst & src */ 1890 /* PC += off if dst > src */ 1891 /* PC += off if dst >= src */ 1892 /* PC += off if dst < src */ 1893 /* PC += off if dst <= src */ 1894 /* PC += off if dst > src (signed) */ 1895 /* PC += off if dst >= src (signed) */ 1896 /* PC += off if dst < src (signed) */ 1897 /* PC += off if dst <= src (signed) */ 1898 case BPF_JMP | BPF_JEQ | BPF_X: 1899 case BPF_JMP | BPF_JNE | BPF_X: 1900 case BPF_JMP | BPF_JSET | BPF_X: 1901 case BPF_JMP | BPF_JGT | BPF_X: 1902 case BPF_JMP | BPF_JGE | BPF_X: 1903 case BPF_JMP | BPF_JLT | BPF_X: 1904 case BPF_JMP | BPF_JLE | BPF_X: 1905 case BPF_JMP | BPF_JSGT | BPF_X: 1906 case BPF_JMP | BPF_JSGE | BPF_X: 1907 case BPF_JMP | BPF_JSLT | BPF_X: 1908 case BPF_JMP | BPF_JSLE | BPF_X: 1909 if (off == 0) 1910 break; 1911 setup_jmp_r(ctx, dst == src, BPF_OP(code), off, &jmp, &rel); 1912 emit_jmp_r64(ctx, dst, src, rel, jmp); 1913 if (finish_jmp(ctx, jmp, off) < 0) 1914 goto toofar; 1915 break; 1916 /* PC += off if dst == imm */ 1917 /* PC += off if dst != imm */ 1918 /* PC += off if dst & imm */ 1919 /* PC += off if dst > imm */ 1920 /* PC += off if dst >= imm */ 1921 /* PC += off if dst < imm */ 1922 /* PC += off if dst <= imm */ 1923 /* PC += off if dst > imm (signed) */ 1924 /* PC += off if dst >= imm (signed) */ 1925 /* PC += off if dst < imm (signed) */ 1926 /* PC += off if dst <= imm (signed) */ 1927 case BPF_JMP | BPF_JEQ | BPF_K: 1928 case BPF_JMP | BPF_JNE | BPF_K: 1929 case BPF_JMP | BPF_JSET | BPF_K: 1930 case BPF_JMP | BPF_JGT | BPF_K: 1931 case BPF_JMP | BPF_JGE | BPF_K: 1932 case BPF_JMP | BPF_JLT | BPF_K: 1933 case BPF_JMP | BPF_JLE | BPF_K: 1934 case BPF_JMP | BPF_JSGT | BPF_K: 1935 case BPF_JMP | BPF_JSGE | BPF_K: 1936 case BPF_JMP | BPF_JSLT | BPF_K: 1937 case BPF_JMP | BPF_JSLE | BPF_K: 1938 if (off == 0) 1939 break; 1940 setup_jmp_i(ctx, imm, 64, BPF_OP(code), off, &jmp, &rel); 1941 emit_jmp_i64(ctx, dst, imm, rel, jmp); 1942 if (finish_jmp(ctx, jmp, off) < 0) 1943 goto toofar; 1944 break; 1945 /* PC += off */ 1946 case BPF_JMP | BPF_JA: 1947 if (off == 0) 1948 break; 1949 if (emit_ja(ctx, off) < 0) 1950 goto toofar; 1951 break; 1952 /* Tail call */ 1953 case BPF_JMP | BPF_TAIL_CALL: 1954 if (emit_tail_call(ctx) < 0) 1955 goto invalid; 1956 break; 1957 /* Function call */ 1958 case BPF_JMP | BPF_CALL: 1959 if (emit_call(ctx, insn) < 0) 1960 goto invalid; 1961 break; 1962 /* Function return */ 1963 case BPF_JMP | BPF_EXIT: 1964 /* 1965 * Optimization: when last instruction is EXIT 1966 * simply continue to epilogue. 1967 */ 1968 if (ctx->bpf_index == ctx->program->len - 1) 1969 break; 1970 if (emit_exit(ctx) < 0) 1971 goto toofar; 1972 break; 1973 1974 default: 1975 invalid: 1976 pr_err_once("unknown opcode %02x\n", code); 1977 return -EINVAL; 1978 notyet: 1979 pr_info_once("*** NOT YET: opcode %02x ***\n", code); 1980 return -EFAULT; 1981 toofar: 1982 pr_info_once("*** TOO FAR: jump at %u opcode %02x ***\n", 1983 ctx->bpf_index, code); 1984 return -E2BIG; 1985 } 1986 return 0; 1987 } 1988