1 /* 2 * BPF Jit compiler for s390. 3 * 4 * Copyright IBM Corp. 2012 5 * 6 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> 7 */ 8 #include <linux/moduleloader.h> 9 #include <linux/netdevice.h> 10 #include <linux/if_vlan.h> 11 #include <linux/filter.h> 12 #include <asm/cacheflush.h> 13 #include <asm/processor.h> 14 #include <asm/facility.h> 15 16 /* 17 * Conventions: 18 * %r2 = skb pointer 19 * %r3 = offset parameter 20 * %r4 = scratch register / length parameter 21 * %r5 = BPF A accumulator 22 * %r8 = return address 23 * %r9 = save register for skb pointer 24 * %r10 = skb->data 25 * %r11 = skb->len - skb->data_len (headlen) 26 * %r12 = BPF X accumulator 27 * %r13 = literal pool pointer 28 * 0(%r15) - 63(%r15) scratch memory array with BPF_MEMWORDS 29 */ 30 int bpf_jit_enable __read_mostly; 31 32 /* 33 * assembly code in arch/x86/net/bpf_jit.S 34 */ 35 extern u8 sk_load_word[], sk_load_half[], sk_load_byte[], sk_load_byte_msh[]; 36 extern u8 sk_load_word_ind[], sk_load_half_ind[], sk_load_byte_ind[]; 37 38 struct bpf_jit { 39 unsigned int seen; 40 u8 *start; 41 u8 *prg; 42 u8 *mid; 43 u8 *lit; 44 u8 *end; 45 u8 *base_ip; 46 u8 *ret0_ip; 47 u8 *exit_ip; 48 unsigned int off_load_word; 49 unsigned int off_load_half; 50 unsigned int off_load_byte; 51 unsigned int off_load_bmsh; 52 unsigned int off_load_iword; 53 unsigned int off_load_ihalf; 54 unsigned int off_load_ibyte; 55 }; 56 57 #define BPF_SIZE_MAX 4096 /* Max size for program */ 58 59 #define SEEN_DATAREF 1 /* might call external helpers */ 60 #define SEEN_XREG 2 /* ebx is used */ 61 #define SEEN_MEM 4 /* use mem[] for temporary storage */ 62 #define SEEN_RET0 8 /* pc_ret0 points to a valid return 0 */ 63 #define SEEN_LITERAL 16 /* code uses literals */ 64 #define SEEN_LOAD_WORD 32 /* code uses sk_load_word */ 65 #define SEEN_LOAD_HALF 64 /* code uses sk_load_half */ 66 #define SEEN_LOAD_BYTE 128 /* code uses sk_load_byte */ 67 #define SEEN_LOAD_BMSH 256 /* code uses sk_load_byte_msh */ 68 #define SEEN_LOAD_IWORD 512 /* code uses sk_load_word_ind */ 69 #define SEEN_LOAD_IHALF 1024 /* code uses sk_load_half_ind */ 70 #define SEEN_LOAD_IBYTE 2048 /* code uses sk_load_byte_ind */ 71 72 #define EMIT2(op) \ 73 ({ \ 74 if (jit->prg + 2 <= jit->mid) \ 75 *(u16 *) jit->prg = op; \ 76 jit->prg += 2; \ 77 }) 78 79 #define EMIT4(op) \ 80 ({ \ 81 if (jit->prg + 4 <= jit->mid) \ 82 *(u32 *) jit->prg = op; \ 83 jit->prg += 4; \ 84 }) 85 86 #define EMIT4_DISP(op, disp) \ 87 ({ \ 88 unsigned int __disp = (disp) & 0xfff; \ 89 EMIT4(op | __disp); \ 90 }) 91 92 #define EMIT4_IMM(op, imm) \ 93 ({ \ 94 unsigned int __imm = (imm) & 0xffff; \ 95 EMIT4(op | __imm); \ 96 }) 97 98 #define EMIT4_PCREL(op, pcrel) \ 99 ({ \ 100 long __pcrel = ((pcrel) >> 1) & 0xffff; \ 101 EMIT4(op | __pcrel); \ 102 }) 103 104 #define EMIT6(op1, op2) \ 105 ({ \ 106 if (jit->prg + 6 <= jit->mid) { \ 107 *(u32 *) jit->prg = op1; \ 108 *(u16 *) (jit->prg + 4) = op2; \ 109 } \ 110 jit->prg += 6; \ 111 }) 112 113 #define EMIT6_DISP(op1, op2, disp) \ 114 ({ \ 115 unsigned int __disp = (disp) & 0xfff; \ 116 EMIT6(op1 | __disp, op2); \ 117 }) 118 119 #define EMIT6_IMM(op, imm) \ 120 ({ \ 121 unsigned int __imm = (imm); \ 122 EMIT6(op | (__imm >> 16), __imm & 0xffff); \ 123 }) 124 125 #define EMIT_CONST(val) \ 126 ({ \ 127 unsigned int ret; \ 128 ret = (unsigned int) (jit->lit - jit->base_ip); \ 129 jit->seen |= SEEN_LITERAL; \ 130 if (jit->lit + 4 <= jit->end) \ 131 *(u32 *) jit->lit = val; \ 132 jit->lit += 4; \ 133 ret; \ 134 }) 135 136 #define EMIT_FN_CONST(bit, fn) \ 137 ({ \ 138 unsigned int ret; \ 139 ret = (unsigned int) (jit->lit - jit->base_ip); \ 140 if (jit->seen & bit) { \ 141 jit->seen |= SEEN_LITERAL; \ 142 if (jit->lit + 8 <= jit->end) \ 143 *(void **) jit->lit = fn; \ 144 jit->lit += 8; \ 145 } \ 146 ret; \ 147 }) 148 149 static void bpf_jit_prologue(struct bpf_jit *jit) 150 { 151 /* Save registers and create stack frame if necessary */ 152 if (jit->seen & SEEN_DATAREF) { 153 /* stmg %r8,%r15,88(%r15) */ 154 EMIT6(0xeb8ff058, 0x0024); 155 /* lgr %r14,%r15 */ 156 EMIT4(0xb90400ef); 157 /* ahi %r15,<offset> */ 158 EMIT4_IMM(0xa7fa0000, (jit->seen & SEEN_MEM) ? -112 : -80); 159 /* stg %r14,152(%r15) */ 160 EMIT6(0xe3e0f098, 0x0024); 161 } else if ((jit->seen & SEEN_XREG) && (jit->seen & SEEN_LITERAL)) 162 /* stmg %r12,%r13,120(%r15) */ 163 EMIT6(0xebcdf078, 0x0024); 164 else if (jit->seen & SEEN_XREG) 165 /* stg %r12,120(%r15) */ 166 EMIT6(0xe3c0f078, 0x0024); 167 else if (jit->seen & SEEN_LITERAL) 168 /* stg %r13,128(%r15) */ 169 EMIT6(0xe3d0f080, 0x0024); 170 171 /* Setup literal pool */ 172 if (jit->seen & SEEN_LITERAL) { 173 /* basr %r13,0 */ 174 EMIT2(0x0dd0); 175 jit->base_ip = jit->prg; 176 } 177 jit->off_load_word = EMIT_FN_CONST(SEEN_LOAD_WORD, sk_load_word); 178 jit->off_load_half = EMIT_FN_CONST(SEEN_LOAD_HALF, sk_load_half); 179 jit->off_load_byte = EMIT_FN_CONST(SEEN_LOAD_BYTE, sk_load_byte); 180 jit->off_load_bmsh = EMIT_FN_CONST(SEEN_LOAD_BMSH, sk_load_byte_msh); 181 jit->off_load_iword = EMIT_FN_CONST(SEEN_LOAD_IWORD, sk_load_word_ind); 182 jit->off_load_ihalf = EMIT_FN_CONST(SEEN_LOAD_IHALF, sk_load_half_ind); 183 jit->off_load_ibyte = EMIT_FN_CONST(SEEN_LOAD_IBYTE, sk_load_byte_ind); 184 185 /* Filter needs to access skb data */ 186 if (jit->seen & SEEN_DATAREF) { 187 /* l %r11,<len>(%r2) */ 188 EMIT4_DISP(0x58b02000, offsetof(struct sk_buff, len)); 189 /* s %r11,<data_len>(%r2) */ 190 EMIT4_DISP(0x5bb02000, offsetof(struct sk_buff, data_len)); 191 /* lg %r10,<data>(%r2) */ 192 EMIT6_DISP(0xe3a02000, 0x0004, 193 offsetof(struct sk_buff, data)); 194 } 195 } 196 197 static void bpf_jit_epilogue(struct bpf_jit *jit) 198 { 199 /* Return 0 */ 200 if (jit->seen & SEEN_RET0) { 201 jit->ret0_ip = jit->prg; 202 /* lghi %r2,0 */ 203 EMIT4(0xa7290000); 204 } 205 jit->exit_ip = jit->prg; 206 /* Restore registers */ 207 if (jit->seen & SEEN_DATAREF) 208 /* lmg %r8,%r15,<offset>(%r15) */ 209 EMIT6_DISP(0xeb8ff000, 0x0004, 210 (jit->seen & SEEN_MEM) ? 200 : 168); 211 else if ((jit->seen & SEEN_XREG) && (jit->seen & SEEN_LITERAL)) 212 /* lmg %r12,%r13,120(%r15) */ 213 EMIT6(0xebcdf078, 0x0004); 214 else if (jit->seen & SEEN_XREG) 215 /* lg %r12,120(%r15) */ 216 EMIT6(0xe3c0f078, 0x0004); 217 else if (jit->seen & SEEN_LITERAL) 218 /* lg %r13,128(%r15) */ 219 EMIT6(0xe3d0f080, 0x0004); 220 /* br %r14 */ 221 EMIT2(0x07fe); 222 } 223 224 /* 225 * make sure we dont leak kernel information to user 226 */ 227 static void bpf_jit_noleaks(struct bpf_jit *jit, struct sock_filter *filter) 228 { 229 /* Clear temporary memory if (seen & SEEN_MEM) */ 230 if (jit->seen & SEEN_MEM) 231 /* xc 0(64,%r15),0(%r15) */ 232 EMIT6(0xd73ff000, 0xf000); 233 /* Clear X if (seen & SEEN_XREG) */ 234 if (jit->seen & SEEN_XREG) 235 /* lhi %r12,0 */ 236 EMIT4(0xa7c80000); 237 /* Clear A if the first register does not set it. */ 238 switch (filter[0].code) { 239 case BPF_S_LD_W_ABS: 240 case BPF_S_LD_H_ABS: 241 case BPF_S_LD_B_ABS: 242 case BPF_S_LD_W_LEN: 243 case BPF_S_LD_W_IND: 244 case BPF_S_LD_H_IND: 245 case BPF_S_LD_B_IND: 246 case BPF_S_LDX_B_MSH: 247 case BPF_S_LD_IMM: 248 case BPF_S_LD_MEM: 249 case BPF_S_MISC_TXA: 250 case BPF_S_ANC_PROTOCOL: 251 case BPF_S_ANC_PKTTYPE: 252 case BPF_S_ANC_IFINDEX: 253 case BPF_S_ANC_MARK: 254 case BPF_S_ANC_QUEUE: 255 case BPF_S_ANC_HATYPE: 256 case BPF_S_ANC_RXHASH: 257 case BPF_S_ANC_CPU: 258 case BPF_S_ANC_VLAN_TAG: 259 case BPF_S_ANC_VLAN_TAG_PRESENT: 260 case BPF_S_RET_K: 261 /* first instruction sets A register */ 262 break; 263 default: /* A = 0 */ 264 /* lhi %r5,0 */ 265 EMIT4(0xa7580000); 266 } 267 } 268 269 static int bpf_jit_insn(struct bpf_jit *jit, struct sock_filter *filter, 270 unsigned int *addrs, int i, int last) 271 { 272 unsigned int K; 273 int offset; 274 unsigned int mask; 275 276 K = filter->k; 277 switch (filter->code) { 278 case BPF_S_ALU_ADD_X: /* A += X */ 279 jit->seen |= SEEN_XREG; 280 /* ar %r5,%r12 */ 281 EMIT2(0x1a5c); 282 break; 283 case BPF_S_ALU_ADD_K: /* A += K */ 284 if (!K) 285 break; 286 if (K <= 16383) 287 /* ahi %r5,<K> */ 288 EMIT4_IMM(0xa75a0000, K); 289 else if (test_facility(21)) 290 /* alfi %r5,<K> */ 291 EMIT6_IMM(0xc25b0000, K); 292 else 293 /* a %r5,<d(K)>(%r13) */ 294 EMIT4_DISP(0x5a50d000, EMIT_CONST(K)); 295 break; 296 case BPF_S_ALU_SUB_X: /* A -= X */ 297 jit->seen |= SEEN_XREG; 298 /* sr %r5,%r12 */ 299 EMIT2(0x1b5c); 300 break; 301 case BPF_S_ALU_SUB_K: /* A -= K */ 302 if (!K) 303 break; 304 if (K <= 16384) 305 /* ahi %r5,-K */ 306 EMIT4_IMM(0xa75a0000, -K); 307 else if (test_facility(21)) 308 /* alfi %r5,-K */ 309 EMIT6_IMM(0xc25b0000, -K); 310 else 311 /* s %r5,<d(K)>(%r13) */ 312 EMIT4_DISP(0x5b50d000, EMIT_CONST(K)); 313 break; 314 case BPF_S_ALU_MUL_X: /* A *= X */ 315 jit->seen |= SEEN_XREG; 316 /* msr %r5,%r12 */ 317 EMIT4(0xb252005c); 318 break; 319 case BPF_S_ALU_MUL_K: /* A *= K */ 320 if (K <= 16383) 321 /* mhi %r5,K */ 322 EMIT4_IMM(0xa75c0000, K); 323 else if (test_facility(34)) 324 /* msfi %r5,<K> */ 325 EMIT6_IMM(0xc2510000, K); 326 else 327 /* ms %r5,<d(K)>(%r13) */ 328 EMIT4_DISP(0x7150d000, EMIT_CONST(K)); 329 break; 330 case BPF_S_ALU_DIV_X: /* A /= X */ 331 jit->seen |= SEEN_XREG | SEEN_RET0; 332 /* ltr %r12,%r12 */ 333 EMIT2(0x12cc); 334 /* jz <ret0> */ 335 EMIT4_PCREL(0xa7840000, (jit->ret0_ip - jit->prg)); 336 /* lhi %r4,0 */ 337 EMIT4(0xa7480000); 338 /* dr %r4,%r12 */ 339 EMIT2(0x1d4c); 340 break; 341 case BPF_S_ALU_DIV_K: /* A = reciprocal_divide(A, K) */ 342 /* m %r4,<d(K)>(%r13) */ 343 EMIT4_DISP(0x5c40d000, EMIT_CONST(K)); 344 /* lr %r5,%r4 */ 345 EMIT2(0x1854); 346 break; 347 case BPF_S_ALU_MOD_X: /* A %= X */ 348 jit->seen |= SEEN_XREG | SEEN_RET0; 349 /* ltr %r12,%r12 */ 350 EMIT2(0x12cc); 351 /* jz <ret0> */ 352 EMIT4_PCREL(0xa7840000, (jit->ret0_ip - jit->prg)); 353 /* lhi %r4,0 */ 354 EMIT4(0xa7480000); 355 /* dr %r4,%r12 */ 356 EMIT2(0x1d4c); 357 /* lr %r5,%r4 */ 358 EMIT2(0x1854); 359 break; 360 case BPF_S_ALU_MOD_K: /* A %= K */ 361 /* lhi %r4,0 */ 362 EMIT4(0xa7480000); 363 /* d %r4,<d(K)>(%r13) */ 364 EMIT4_DISP(0x5d40d000, EMIT_CONST(K)); 365 /* lr %r5,%r4 */ 366 EMIT2(0x1854); 367 break; 368 case BPF_S_ALU_AND_X: /* A &= X */ 369 jit->seen |= SEEN_XREG; 370 /* nr %r5,%r12 */ 371 EMIT2(0x145c); 372 break; 373 case BPF_S_ALU_AND_K: /* A &= K */ 374 if (test_facility(21)) 375 /* nilf %r5,<K> */ 376 EMIT6_IMM(0xc05b0000, K); 377 else 378 /* n %r5,<d(K)>(%r13) */ 379 EMIT4_DISP(0x5450d000, EMIT_CONST(K)); 380 break; 381 case BPF_S_ALU_OR_X: /* A |= X */ 382 jit->seen |= SEEN_XREG; 383 /* or %r5,%r12 */ 384 EMIT2(0x165c); 385 break; 386 case BPF_S_ALU_OR_K: /* A |= K */ 387 if (test_facility(21)) 388 /* oilf %r5,<K> */ 389 EMIT6_IMM(0xc05d0000, K); 390 else 391 /* o %r5,<d(K)>(%r13) */ 392 EMIT4_DISP(0x5650d000, EMIT_CONST(K)); 393 break; 394 case BPF_S_ANC_ALU_XOR_X: /* A ^= X; */ 395 case BPF_S_ALU_XOR_X: 396 jit->seen |= SEEN_XREG; 397 /* xr %r5,%r12 */ 398 EMIT2(0x175c); 399 break; 400 case BPF_S_ALU_XOR_K: /* A ^= K */ 401 if (!K) 402 break; 403 /* x %r5,<d(K)>(%r13) */ 404 EMIT4_DISP(0x5750d000, EMIT_CONST(K)); 405 break; 406 case BPF_S_ALU_LSH_X: /* A <<= X; */ 407 jit->seen |= SEEN_XREG; 408 /* sll %r5,0(%r12) */ 409 EMIT4(0x8950c000); 410 break; 411 case BPF_S_ALU_LSH_K: /* A <<= K */ 412 if (K == 0) 413 break; 414 /* sll %r5,K */ 415 EMIT4_DISP(0x89500000, K); 416 break; 417 case BPF_S_ALU_RSH_X: /* A >>= X; */ 418 jit->seen |= SEEN_XREG; 419 /* srl %r5,0(%r12) */ 420 EMIT4(0x8850c000); 421 break; 422 case BPF_S_ALU_RSH_K: /* A >>= K; */ 423 if (K == 0) 424 break; 425 /* srl %r5,K */ 426 EMIT4_DISP(0x88500000, K); 427 break; 428 case BPF_S_ALU_NEG: /* A = -A */ 429 /* lnr %r5,%r5 */ 430 EMIT2(0x1155); 431 break; 432 case BPF_S_JMP_JA: /* ip += K */ 433 offset = addrs[i + K] + jit->start - jit->prg; 434 EMIT4_PCREL(0xa7f40000, offset); 435 break; 436 case BPF_S_JMP_JGT_K: /* ip += (A > K) ? jt : jf */ 437 mask = 0x200000; /* jh */ 438 goto kbranch; 439 case BPF_S_JMP_JGE_K: /* ip += (A >= K) ? jt : jf */ 440 mask = 0xa00000; /* jhe */ 441 goto kbranch; 442 case BPF_S_JMP_JEQ_K: /* ip += (A == K) ? jt : jf */ 443 mask = 0x800000; /* je */ 444 kbranch: /* Emit compare if the branch targets are different */ 445 if (filter->jt != filter->jf) { 446 if (K <= 16383) 447 /* chi %r5,<K> */ 448 EMIT4_IMM(0xa75e0000, K); 449 else if (test_facility(21)) 450 /* clfi %r5,<K> */ 451 EMIT6_IMM(0xc25f0000, K); 452 else 453 /* c %r5,<d(K)>(%r13) */ 454 EMIT4_DISP(0x5950d000, EMIT_CONST(K)); 455 } 456 branch: if (filter->jt == filter->jf) { 457 if (filter->jt == 0) 458 break; 459 /* j <jt> */ 460 offset = addrs[i + filter->jt] + jit->start - jit->prg; 461 EMIT4_PCREL(0xa7f40000, offset); 462 break; 463 } 464 if (filter->jt != 0) { 465 /* brc <mask>,<jt> */ 466 offset = addrs[i + filter->jt] + jit->start - jit->prg; 467 EMIT4_PCREL(0xa7040000 | mask, offset); 468 } 469 if (filter->jf != 0) { 470 /* brc <mask^15>,<jf> */ 471 offset = addrs[i + filter->jf] + jit->start - jit->prg; 472 EMIT4_PCREL(0xa7040000 | (mask ^ 0xf00000), offset); 473 } 474 break; 475 case BPF_S_JMP_JSET_K: /* ip += (A & K) ? jt : jf */ 476 mask = 0x700000; /* jnz */ 477 /* Emit test if the branch targets are different */ 478 if (filter->jt != filter->jf) { 479 if (K > 65535) { 480 /* lr %r4,%r5 */ 481 EMIT2(0x1845); 482 /* n %r4,<d(K)>(%r13) */ 483 EMIT4_DISP(0x5440d000, EMIT_CONST(K)); 484 } else 485 /* tmll %r5,K */ 486 EMIT4_IMM(0xa7510000, K); 487 } 488 goto branch; 489 case BPF_S_JMP_JGT_X: /* ip += (A > X) ? jt : jf */ 490 mask = 0x200000; /* jh */ 491 goto xbranch; 492 case BPF_S_JMP_JGE_X: /* ip += (A >= X) ? jt : jf */ 493 mask = 0xa00000; /* jhe */ 494 goto xbranch; 495 case BPF_S_JMP_JEQ_X: /* ip += (A == X) ? jt : jf */ 496 mask = 0x800000; /* je */ 497 xbranch: /* Emit compare if the branch targets are different */ 498 if (filter->jt != filter->jf) { 499 jit->seen |= SEEN_XREG; 500 /* cr %r5,%r12 */ 501 EMIT2(0x195c); 502 } 503 goto branch; 504 case BPF_S_JMP_JSET_X: /* ip += (A & X) ? jt : jf */ 505 mask = 0x700000; /* jnz */ 506 /* Emit test if the branch targets are different */ 507 if (filter->jt != filter->jf) { 508 jit->seen |= SEEN_XREG; 509 /* lr %r4,%r5 */ 510 EMIT2(0x1845); 511 /* nr %r4,%r12 */ 512 EMIT2(0x144c); 513 } 514 goto branch; 515 case BPF_S_LD_W_ABS: /* A = *(u32 *) (skb->data+K) */ 516 jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_WORD; 517 offset = jit->off_load_word; 518 goto load_abs; 519 case BPF_S_LD_H_ABS: /* A = *(u16 *) (skb->data+K) */ 520 jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_HALF; 521 offset = jit->off_load_half; 522 goto load_abs; 523 case BPF_S_LD_B_ABS: /* A = *(u8 *) (skb->data+K) */ 524 jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_BYTE; 525 offset = jit->off_load_byte; 526 load_abs: if ((int) K < 0) 527 goto out; 528 call_fn: /* lg %r1,<d(function)>(%r13) */ 529 EMIT6_DISP(0xe310d000, 0x0004, offset); 530 /* l %r3,<d(K)>(%r13) */ 531 EMIT4_DISP(0x5830d000, EMIT_CONST(K)); 532 /* basr %r8,%r1 */ 533 EMIT2(0x0d81); 534 /* jnz <ret0> */ 535 EMIT4_PCREL(0xa7740000, (jit->ret0_ip - jit->prg)); 536 break; 537 case BPF_S_LD_W_IND: /* A = *(u32 *) (skb->data+K+X) */ 538 jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_IWORD; 539 offset = jit->off_load_iword; 540 goto call_fn; 541 case BPF_S_LD_H_IND: /* A = *(u16 *) (skb->data+K+X) */ 542 jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_IHALF; 543 offset = jit->off_load_ihalf; 544 goto call_fn; 545 case BPF_S_LD_B_IND: /* A = *(u8 *) (skb->data+K+X) */ 546 jit->seen |= SEEN_DATAREF | SEEN_RET0 | SEEN_LOAD_IBYTE; 547 offset = jit->off_load_ibyte; 548 goto call_fn; 549 case BPF_S_LDX_B_MSH: 550 /* X = (*(u8 *)(skb->data+K) & 0xf) << 2 */ 551 jit->seen |= SEEN_RET0; 552 if ((int) K < 0) { 553 /* j <ret0> */ 554 EMIT4_PCREL(0xa7f40000, (jit->ret0_ip - jit->prg)); 555 break; 556 } 557 jit->seen |= SEEN_DATAREF | SEEN_LOAD_BMSH; 558 offset = jit->off_load_bmsh; 559 goto call_fn; 560 case BPF_S_LD_W_LEN: /* A = skb->len; */ 561 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, len) != 4); 562 /* l %r5,<d(len)>(%r2) */ 563 EMIT4_DISP(0x58502000, offsetof(struct sk_buff, len)); 564 break; 565 case BPF_S_LDX_W_LEN: /* X = skb->len; */ 566 jit->seen |= SEEN_XREG; 567 /* l %r12,<d(len)>(%r2) */ 568 EMIT4_DISP(0x58c02000, offsetof(struct sk_buff, len)); 569 break; 570 case BPF_S_LD_IMM: /* A = K */ 571 if (K <= 16383) 572 /* lhi %r5,K */ 573 EMIT4_IMM(0xa7580000, K); 574 else if (test_facility(21)) 575 /* llilf %r5,<K> */ 576 EMIT6_IMM(0xc05f0000, K); 577 else 578 /* l %r5,<d(K)>(%r13) */ 579 EMIT4_DISP(0x5850d000, EMIT_CONST(K)); 580 break; 581 case BPF_S_LDX_IMM: /* X = K */ 582 jit->seen |= SEEN_XREG; 583 if (K <= 16383) 584 /* lhi %r12,<K> */ 585 EMIT4_IMM(0xa7c80000, K); 586 else if (test_facility(21)) 587 /* llilf %r12,<K> */ 588 EMIT6_IMM(0xc0cf0000, K); 589 else 590 /* l %r12,<d(K)>(%r13) */ 591 EMIT4_DISP(0x58c0d000, EMIT_CONST(K)); 592 break; 593 case BPF_S_LD_MEM: /* A = mem[K] */ 594 jit->seen |= SEEN_MEM; 595 /* l %r5,<K>(%r15) */ 596 EMIT4_DISP(0x5850f000, 597 (jit->seen & SEEN_DATAREF) ? 160 + K*4 : K*4); 598 break; 599 case BPF_S_LDX_MEM: /* X = mem[K] */ 600 jit->seen |= SEEN_XREG | SEEN_MEM; 601 /* l %r12,<K>(%r15) */ 602 EMIT4_DISP(0x58c0f000, 603 (jit->seen & SEEN_DATAREF) ? 160 + K*4 : K*4); 604 break; 605 case BPF_S_MISC_TAX: /* X = A */ 606 jit->seen |= SEEN_XREG; 607 /* lr %r12,%r5 */ 608 EMIT2(0x18c5); 609 break; 610 case BPF_S_MISC_TXA: /* A = X */ 611 jit->seen |= SEEN_XREG; 612 /* lr %r5,%r12 */ 613 EMIT2(0x185c); 614 break; 615 case BPF_S_RET_K: 616 if (K == 0) { 617 jit->seen |= SEEN_RET0; 618 if (last) 619 break; 620 /* j <ret0> */ 621 EMIT4_PCREL(0xa7f40000, jit->ret0_ip - jit->prg); 622 } else { 623 if (K <= 16383) 624 /* lghi %r2,K */ 625 EMIT4_IMM(0xa7290000, K); 626 else 627 /* llgf %r2,<K>(%r13) */ 628 EMIT6_DISP(0xe320d000, 0x0016, EMIT_CONST(K)); 629 /* j <exit> */ 630 if (last && !(jit->seen & SEEN_RET0)) 631 break; 632 EMIT4_PCREL(0xa7f40000, jit->exit_ip - jit->prg); 633 } 634 break; 635 case BPF_S_RET_A: 636 /* llgfr %r2,%r5 */ 637 EMIT4(0xb9160025); 638 /* j <exit> */ 639 EMIT4_PCREL(0xa7f40000, jit->exit_ip - jit->prg); 640 break; 641 case BPF_S_ST: /* mem[K] = A */ 642 jit->seen |= SEEN_MEM; 643 /* st %r5,<K>(%r15) */ 644 EMIT4_DISP(0x5050f000, 645 (jit->seen & SEEN_DATAREF) ? 160 + K*4 : K*4); 646 break; 647 case BPF_S_STX: /* mem[K] = X : mov %ebx,off8(%rbp) */ 648 jit->seen |= SEEN_XREG | SEEN_MEM; 649 /* st %r12,<K>(%r15) */ 650 EMIT4_DISP(0x50c0f000, 651 (jit->seen & SEEN_DATAREF) ? 160 + K*4 : K*4); 652 break; 653 case BPF_S_ANC_PROTOCOL: /* A = ntohs(skb->protocol); */ 654 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, protocol) != 2); 655 /* lhi %r5,0 */ 656 EMIT4(0xa7580000); 657 /* icm %r5,3,<d(protocol)>(%r2) */ 658 EMIT4_DISP(0xbf532000, offsetof(struct sk_buff, protocol)); 659 break; 660 case BPF_S_ANC_IFINDEX: /* if (!skb->dev) return 0; 661 * A = skb->dev->ifindex */ 662 BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, ifindex) != 4); 663 jit->seen |= SEEN_RET0; 664 /* lg %r1,<d(dev)>(%r2) */ 665 EMIT6_DISP(0xe3102000, 0x0004, offsetof(struct sk_buff, dev)); 666 /* ltgr %r1,%r1 */ 667 EMIT4(0xb9020011); 668 /* jz <ret0> */ 669 EMIT4_PCREL(0xa7840000, jit->ret0_ip - jit->prg); 670 /* l %r5,<d(ifindex)>(%r1) */ 671 EMIT4_DISP(0x58501000, offsetof(struct net_device, ifindex)); 672 break; 673 case BPF_S_ANC_MARK: /* A = skb->mark */ 674 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, mark) != 4); 675 /* l %r5,<d(mark)>(%r2) */ 676 EMIT4_DISP(0x58502000, offsetof(struct sk_buff, mark)); 677 break; 678 case BPF_S_ANC_QUEUE: /* A = skb->queue_mapping */ 679 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, queue_mapping) != 2); 680 /* lhi %r5,0 */ 681 EMIT4(0xa7580000); 682 /* icm %r5,3,<d(queue_mapping)>(%r2) */ 683 EMIT4_DISP(0xbf532000, offsetof(struct sk_buff, queue_mapping)); 684 break; 685 case BPF_S_ANC_HATYPE: /* if (!skb->dev) return 0; 686 * A = skb->dev->type */ 687 BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, type) != 2); 688 jit->seen |= SEEN_RET0; 689 /* lg %r1,<d(dev)>(%r2) */ 690 EMIT6_DISP(0xe3102000, 0x0004, offsetof(struct sk_buff, dev)); 691 /* ltgr %r1,%r1 */ 692 EMIT4(0xb9020011); 693 /* jz <ret0> */ 694 EMIT4_PCREL(0xa7840000, jit->ret0_ip - jit->prg); 695 /* lhi %r5,0 */ 696 EMIT4(0xa7580000); 697 /* icm %r5,3,<d(type)>(%r1) */ 698 EMIT4_DISP(0xbf531000, offsetof(struct net_device, type)); 699 break; 700 case BPF_S_ANC_RXHASH: /* A = skb->rxhash */ 701 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, rxhash) != 4); 702 /* l %r5,<d(rxhash)>(%r2) */ 703 EMIT4_DISP(0x58502000, offsetof(struct sk_buff, rxhash)); 704 break; 705 case BPF_S_ANC_VLAN_TAG: 706 case BPF_S_ANC_VLAN_TAG_PRESENT: 707 BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, vlan_tci) != 2); 708 BUILD_BUG_ON(VLAN_TAG_PRESENT != 0x1000); 709 /* lhi %r5,0 */ 710 EMIT4(0xa7580000); 711 /* icm %r5,3,<d(vlan_tci)>(%r2) */ 712 EMIT4_DISP(0xbf532000, offsetof(struct sk_buff, vlan_tci)); 713 if (filter->code == BPF_S_ANC_VLAN_TAG) { 714 /* nill %r5,0xefff */ 715 EMIT4_IMM(0xa5570000, ~VLAN_TAG_PRESENT); 716 } else { 717 /* nill %r5,0x1000 */ 718 EMIT4_IMM(0xa5570000, VLAN_TAG_PRESENT); 719 /* srl %r5,12 */ 720 EMIT4_DISP(0x88500000, 12); 721 } 722 break; 723 case BPF_S_ANC_CPU: /* A = smp_processor_id() */ 724 #ifdef CONFIG_SMP 725 /* l %r5,<d(cpu_nr)> */ 726 EMIT4_DISP(0x58500000, offsetof(struct _lowcore, cpu_nr)); 727 #else 728 /* lhi %r5,0 */ 729 EMIT4(0xa7580000); 730 #endif 731 break; 732 default: /* too complex, give up */ 733 goto out; 734 } 735 addrs[i] = jit->prg - jit->start; 736 return 0; 737 out: 738 return -1; 739 } 740 741 void bpf_jit_compile(struct sk_filter *fp) 742 { 743 unsigned long size, prg_len, lit_len; 744 struct bpf_jit jit, cjit; 745 unsigned int *addrs; 746 int pass, i; 747 748 if (!bpf_jit_enable) 749 return; 750 addrs = kcalloc(fp->len, sizeof(*addrs), GFP_KERNEL); 751 if (addrs == NULL) 752 return; 753 memset(&jit, 0, sizeof(cjit)); 754 memset(&cjit, 0, sizeof(cjit)); 755 756 for (pass = 0; pass < 10; pass++) { 757 jit.prg = jit.start; 758 jit.lit = jit.mid; 759 760 bpf_jit_prologue(&jit); 761 bpf_jit_noleaks(&jit, fp->insns); 762 for (i = 0; i < fp->len; i++) { 763 if (bpf_jit_insn(&jit, fp->insns + i, addrs, i, 764 i == fp->len - 1)) 765 goto out; 766 } 767 bpf_jit_epilogue(&jit); 768 if (jit.start) { 769 WARN_ON(jit.prg > cjit.prg || jit.lit > cjit.lit); 770 if (memcmp(&jit, &cjit, sizeof(jit)) == 0) 771 break; 772 } else if (jit.prg == cjit.prg && jit.lit == cjit.lit) { 773 prg_len = jit.prg - jit.start; 774 lit_len = jit.lit - jit.mid; 775 size = max_t(unsigned long, prg_len + lit_len, 776 sizeof(struct work_struct)); 777 if (size >= BPF_SIZE_MAX) 778 goto out; 779 jit.start = module_alloc(size); 780 if (!jit.start) 781 goto out; 782 jit.prg = jit.mid = jit.start + prg_len; 783 jit.lit = jit.end = jit.start + prg_len + lit_len; 784 jit.base_ip += (unsigned long) jit.start; 785 jit.exit_ip += (unsigned long) jit.start; 786 jit.ret0_ip += (unsigned long) jit.start; 787 } 788 cjit = jit; 789 } 790 if (bpf_jit_enable > 1) { 791 pr_err("flen=%d proglen=%lu pass=%d image=%p\n", 792 fp->len, jit.end - jit.start, pass, jit.start); 793 if (jit.start) { 794 printk(KERN_ERR "JIT code:\n"); 795 print_fn_code(jit.start, jit.mid - jit.start); 796 print_hex_dump(KERN_ERR, "JIT literals:\n", 797 DUMP_PREFIX_ADDRESS, 16, 1, 798 jit.mid, jit.end - jit.mid, false); 799 } 800 } 801 if (jit.start) 802 fp->bpf_func = (void *) jit.start; 803 out: 804 kfree(addrs); 805 } 806 807 static void jit_free_defer(struct work_struct *arg) 808 { 809 module_free(NULL, arg); 810 } 811 812 /* run from softirq, we must use a work_struct to call 813 * module_free() from process context 814 */ 815 void bpf_jit_free(struct sk_filter *fp) 816 { 817 struct work_struct *work; 818 819 if (fp->bpf_func == sk_run_filter) 820 return; 821 work = (struct work_struct *)fp->bpf_func; 822 INIT_WORK(work, jit_free_defer); 823 schedule_work(work); 824 } 825