1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2008-2009 Patrick McHardy <kaber@trash.net> 4 * Copyright (c) 2016 Pablo Neira Ayuso <pablo@netfilter.org> 5 * 6 * Development of this code funded by Astaro AG (http://www.astaro.com/) 7 */ 8 9 #include <linux/kernel.h> 10 #include <linux/if_vlan.h> 11 #include <linux/init.h> 12 #include <linux/module.h> 13 #include <linux/netlink.h> 14 #include <linux/netfilter.h> 15 #include <linux/netfilter/nf_tables.h> 16 #include <net/netfilter/nf_tables_core.h> 17 #include <net/netfilter/nf_tables.h> 18 #include <net/netfilter/nf_tables_offload.h> 19 /* For layer 4 checksum field offset. */ 20 #include <linux/tcp.h> 21 #include <linux/udp.h> 22 #include <net/gre.h> 23 #include <linux/icmpv6.h> 24 #include <linux/ip.h> 25 #include <linux/ipv6.h> 26 #include <net/sctp/checksum.h> 27 28 static bool nft_payload_rebuild_vlan_hdr(const struct sk_buff *skb, int mac_off, 29 struct vlan_ethhdr *veth) 30 { 31 if (skb_copy_bits(skb, mac_off, veth, ETH_HLEN)) 32 return false; 33 34 veth->h_vlan_proto = skb->vlan_proto; 35 veth->h_vlan_TCI = htons(skb_vlan_tag_get(skb)); 36 veth->h_vlan_encapsulated_proto = skb->protocol; 37 38 return true; 39 } 40 41 /* add vlan header into the user buffer for if tag was removed by offloads */ 42 static bool 43 nft_payload_copy_vlan(u32 *d, const struct sk_buff *skb, u16 offset, u8 len) 44 { 45 int mac_off = skb_mac_header(skb) - skb->data; 46 u8 *vlanh, *dst_u8 = (u8 *) d; 47 struct vlan_ethhdr veth; 48 49 vlanh = (u8 *) &veth; 50 if (offset < VLAN_ETH_HLEN) { 51 u8 ethlen = len; 52 53 if (!nft_payload_rebuild_vlan_hdr(skb, mac_off, &veth)) 54 return false; 55 56 if (offset + len > VLAN_ETH_HLEN) 57 ethlen -= offset + len - VLAN_ETH_HLEN; 58 59 memcpy(dst_u8, vlanh + offset, ethlen); 60 61 len -= ethlen; 62 if (len == 0) 63 return true; 64 65 dst_u8 += ethlen; 66 offset = ETH_HLEN; 67 } else { 68 offset -= VLAN_HLEN; 69 } 70 71 return skb_copy_bits(skb, offset + mac_off, dst_u8, len) == 0; 72 } 73 74 static int __nft_payload_inner_offset(struct nft_pktinfo *pkt) 75 { 76 unsigned int thoff = nft_thoff(pkt); 77 78 if (!(pkt->flags & NFT_PKTINFO_L4PROTO) || pkt->fragoff) 79 return -1; 80 81 switch (pkt->tprot) { 82 case IPPROTO_UDP: 83 pkt->inneroff = thoff + sizeof(struct udphdr); 84 break; 85 case IPPROTO_TCP: { 86 struct tcphdr *th, _tcph; 87 88 th = skb_header_pointer(pkt->skb, thoff, sizeof(_tcph), &_tcph); 89 if (!th) 90 return -1; 91 92 pkt->inneroff = thoff + __tcp_hdrlen(th); 93 } 94 break; 95 case IPPROTO_GRE: { 96 u32 offset = sizeof(struct gre_base_hdr); 97 struct gre_base_hdr *gre, _gre; 98 __be16 version; 99 100 gre = skb_header_pointer(pkt->skb, thoff, sizeof(_gre), &_gre); 101 if (!gre) 102 return -1; 103 104 version = gre->flags & GRE_VERSION; 105 switch (version) { 106 case GRE_VERSION_0: 107 if (gre->flags & GRE_ROUTING) 108 return -1; 109 110 if (gre->flags & GRE_CSUM) { 111 offset += sizeof_field(struct gre_full_hdr, csum) + 112 sizeof_field(struct gre_full_hdr, reserved1); 113 } 114 if (gre->flags & GRE_KEY) 115 offset += sizeof_field(struct gre_full_hdr, key); 116 117 if (gre->flags & GRE_SEQ) 118 offset += sizeof_field(struct gre_full_hdr, seq); 119 break; 120 default: 121 return -1; 122 } 123 124 pkt->inneroff = thoff + offset; 125 } 126 break; 127 case IPPROTO_IPIP: 128 pkt->inneroff = thoff; 129 break; 130 default: 131 return -1; 132 } 133 134 pkt->flags |= NFT_PKTINFO_INNER; 135 136 return 0; 137 } 138 139 int nft_payload_inner_offset(const struct nft_pktinfo *pkt) 140 { 141 if (!(pkt->flags & NFT_PKTINFO_INNER) && 142 __nft_payload_inner_offset((struct nft_pktinfo *)pkt) < 0) 143 return -1; 144 145 return pkt->inneroff; 146 } 147 148 static bool nft_payload_need_vlan_adjust(u32 offset, u32 len) 149 { 150 unsigned int boundary = offset + len; 151 152 /* data past ether src/dst requested, copy needed */ 153 if (boundary > offsetof(struct ethhdr, h_proto)) 154 return true; 155 156 return false; 157 } 158 159 void nft_payload_eval(const struct nft_expr *expr, 160 struct nft_regs *regs, 161 const struct nft_pktinfo *pkt) 162 { 163 const struct nft_payload *priv = nft_expr_priv(expr); 164 const struct sk_buff *skb = pkt->skb; 165 u32 *dest = ®s->data[priv->dreg]; 166 int offset; 167 168 if (priv->len % NFT_REG32_SIZE) 169 dest[priv->len / NFT_REG32_SIZE] = 0; 170 171 switch (priv->base) { 172 case NFT_PAYLOAD_LL_HEADER: 173 if (!skb_mac_header_was_set(skb) || skb_mac_header_len(skb) == 0) 174 goto err; 175 176 if (skb_vlan_tag_present(skb) && 177 nft_payload_need_vlan_adjust(priv->offset, priv->len)) { 178 if (!nft_payload_copy_vlan(dest, skb, 179 priv->offset, priv->len)) 180 goto err; 181 return; 182 } 183 offset = skb_mac_header(skb) - skb->data; 184 break; 185 case NFT_PAYLOAD_NETWORK_HEADER: 186 offset = skb_network_offset(skb) + pkt->nhoff; 187 break; 188 case NFT_PAYLOAD_TRANSPORT_HEADER: 189 if (!(pkt->flags & NFT_PKTINFO_L4PROTO) || pkt->fragoff) 190 goto err; 191 offset = nft_thoff(pkt); 192 break; 193 case NFT_PAYLOAD_INNER_HEADER: 194 offset = nft_payload_inner_offset(pkt); 195 if (offset < 0) 196 goto err; 197 break; 198 default: 199 DEBUG_NET_WARN_ON_ONCE(1); 200 goto err; 201 } 202 offset += priv->offset; 203 204 if (skb_copy_bits(skb, offset, dest, priv->len) < 0) 205 goto err; 206 return; 207 err: 208 regs->verdict.code = NFT_BREAK; 209 } 210 211 static const struct nla_policy nft_payload_policy[NFTA_PAYLOAD_MAX + 1] = { 212 [NFTA_PAYLOAD_SREG] = NLA_POLICY_MAX(NLA_BE32, NFT_REG32_MAX), 213 [NFTA_PAYLOAD_DREG] = NLA_POLICY_MAX(NLA_BE32, NFT_REG32_MAX), 214 [NFTA_PAYLOAD_BASE] = { .type = NLA_U32 }, 215 [NFTA_PAYLOAD_OFFSET] = { .type = NLA_BE32 }, 216 [NFTA_PAYLOAD_LEN] = NLA_POLICY_MAX(NLA_BE32, 255), 217 [NFTA_PAYLOAD_CSUM_TYPE] = { .type = NLA_U32 }, 218 [NFTA_PAYLOAD_CSUM_OFFSET] = NLA_POLICY_MAX(NLA_BE32, 255), 219 [NFTA_PAYLOAD_CSUM_FLAGS] = NLA_POLICY_MASK(NLA_BE32, NFT_PAYLOAD_L4CSUM_PSEUDOHDR), 220 }; 221 222 static int nft_payload_init(const struct nft_ctx *ctx, 223 const struct nft_expr *expr, 224 const struct nlattr * const tb[]) 225 { 226 struct nft_payload *priv = nft_expr_priv(expr); 227 u32 offset; 228 int err; 229 230 priv->base = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_BASE])); 231 priv->len = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_LEN])); 232 233 err = nft_parse_u32_check(tb[NFTA_PAYLOAD_OFFSET], U16_MAX, &offset); 234 if (err < 0) 235 return err; 236 priv->offset = offset; 237 238 return nft_parse_register_store(ctx, tb[NFTA_PAYLOAD_DREG], 239 &priv->dreg, NULL, NFT_DATA_VALUE, 240 priv->len); 241 } 242 243 static int nft_payload_dump(struct sk_buff *skb, 244 const struct nft_expr *expr, bool reset) 245 { 246 const struct nft_payload *priv = nft_expr_priv(expr); 247 248 if (nft_dump_register(skb, NFTA_PAYLOAD_DREG, priv->dreg) || 249 nla_put_be32(skb, NFTA_PAYLOAD_BASE, htonl(priv->base)) || 250 nla_put_be32(skb, NFTA_PAYLOAD_OFFSET, htonl(priv->offset)) || 251 nla_put_be32(skb, NFTA_PAYLOAD_LEN, htonl(priv->len))) 252 goto nla_put_failure; 253 return 0; 254 255 nla_put_failure: 256 return -1; 257 } 258 259 static bool nft_payload_offload_mask(struct nft_offload_reg *reg, 260 u32 priv_len, u32 field_len) 261 { 262 unsigned int remainder, delta, k; 263 struct nft_data mask = {}; 264 __be32 remainder_mask; 265 266 if (priv_len == field_len) { 267 memset(®->mask, 0xff, priv_len); 268 return true; 269 } else if (priv_len > field_len) { 270 return false; 271 } 272 273 memset(&mask, 0xff, field_len); 274 remainder = priv_len % sizeof(u32); 275 if (remainder) { 276 k = priv_len / sizeof(u32); 277 delta = field_len - priv_len; 278 remainder_mask = htonl(~((1 << (delta * BITS_PER_BYTE)) - 1)); 279 mask.data[k] = (__force u32)remainder_mask; 280 } 281 282 memcpy(®->mask, &mask, field_len); 283 284 return true; 285 } 286 287 static int nft_payload_offload_ll(struct nft_offload_ctx *ctx, 288 struct nft_flow_rule *flow, 289 const struct nft_payload *priv) 290 { 291 struct nft_offload_reg *reg = &ctx->regs[priv->dreg]; 292 293 switch (priv->offset) { 294 case offsetof(struct ethhdr, h_source): 295 if (!nft_payload_offload_mask(reg, priv->len, ETH_ALEN)) 296 return -EOPNOTSUPP; 297 298 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_ETH_ADDRS, eth_addrs, 299 src, ETH_ALEN, reg); 300 break; 301 case offsetof(struct ethhdr, h_dest): 302 if (!nft_payload_offload_mask(reg, priv->len, ETH_ALEN)) 303 return -EOPNOTSUPP; 304 305 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_ETH_ADDRS, eth_addrs, 306 dst, ETH_ALEN, reg); 307 break; 308 case offsetof(struct ethhdr, h_proto): 309 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16))) 310 return -EOPNOTSUPP; 311 312 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_BASIC, basic, 313 n_proto, sizeof(__be16), reg); 314 nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_NETWORK); 315 break; 316 case offsetof(struct vlan_ethhdr, h_vlan_TCI): 317 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16))) 318 return -EOPNOTSUPP; 319 320 NFT_OFFLOAD_MATCH_FLAGS(FLOW_DISSECTOR_KEY_VLAN, vlan, 321 vlan_tci, sizeof(__be16), reg, 322 NFT_OFFLOAD_F_NETWORK2HOST); 323 break; 324 case offsetof(struct vlan_ethhdr, h_vlan_encapsulated_proto): 325 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16))) 326 return -EOPNOTSUPP; 327 328 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_VLAN, vlan, 329 vlan_tpid, sizeof(__be16), reg); 330 nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_NETWORK); 331 break; 332 case offsetof(struct vlan_ethhdr, h_vlan_TCI) + sizeof(struct vlan_hdr): 333 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16))) 334 return -EOPNOTSUPP; 335 336 NFT_OFFLOAD_MATCH_FLAGS(FLOW_DISSECTOR_KEY_CVLAN, cvlan, 337 vlan_tci, sizeof(__be16), reg, 338 NFT_OFFLOAD_F_NETWORK2HOST); 339 break; 340 case offsetof(struct vlan_ethhdr, h_vlan_encapsulated_proto) + 341 sizeof(struct vlan_hdr): 342 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16))) 343 return -EOPNOTSUPP; 344 345 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_CVLAN, cvlan, 346 vlan_tpid, sizeof(__be16), reg); 347 nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_NETWORK); 348 break; 349 default: 350 return -EOPNOTSUPP; 351 } 352 353 return 0; 354 } 355 356 static int nft_payload_offload_ip(struct nft_offload_ctx *ctx, 357 struct nft_flow_rule *flow, 358 const struct nft_payload *priv) 359 { 360 struct nft_offload_reg *reg = &ctx->regs[priv->dreg]; 361 362 switch (priv->offset) { 363 case offsetof(struct iphdr, saddr): 364 if (!nft_payload_offload_mask(reg, priv->len, 365 sizeof(struct in_addr))) 366 return -EOPNOTSUPP; 367 368 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4, src, 369 sizeof(struct in_addr), reg); 370 nft_flow_rule_set_addr_type(flow, FLOW_DISSECTOR_KEY_IPV4_ADDRS); 371 break; 372 case offsetof(struct iphdr, daddr): 373 if (!nft_payload_offload_mask(reg, priv->len, 374 sizeof(struct in_addr))) 375 return -EOPNOTSUPP; 376 377 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4, dst, 378 sizeof(struct in_addr), reg); 379 nft_flow_rule_set_addr_type(flow, FLOW_DISSECTOR_KEY_IPV4_ADDRS); 380 break; 381 case offsetof(struct iphdr, protocol): 382 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__u8))) 383 return -EOPNOTSUPP; 384 385 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_BASIC, basic, ip_proto, 386 sizeof(__u8), reg); 387 nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_TRANSPORT); 388 break; 389 default: 390 return -EOPNOTSUPP; 391 } 392 393 return 0; 394 } 395 396 static int nft_payload_offload_ip6(struct nft_offload_ctx *ctx, 397 struct nft_flow_rule *flow, 398 const struct nft_payload *priv) 399 { 400 struct nft_offload_reg *reg = &ctx->regs[priv->dreg]; 401 402 switch (priv->offset) { 403 case offsetof(struct ipv6hdr, saddr): 404 if (!nft_payload_offload_mask(reg, priv->len, 405 sizeof(struct in6_addr))) 406 return -EOPNOTSUPP; 407 408 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV6_ADDRS, ipv6, src, 409 sizeof(struct in6_addr), reg); 410 nft_flow_rule_set_addr_type(flow, FLOW_DISSECTOR_KEY_IPV6_ADDRS); 411 break; 412 case offsetof(struct ipv6hdr, daddr): 413 if (!nft_payload_offload_mask(reg, priv->len, 414 sizeof(struct in6_addr))) 415 return -EOPNOTSUPP; 416 417 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_IPV6_ADDRS, ipv6, dst, 418 sizeof(struct in6_addr), reg); 419 nft_flow_rule_set_addr_type(flow, FLOW_DISSECTOR_KEY_IPV6_ADDRS); 420 break; 421 case offsetof(struct ipv6hdr, nexthdr): 422 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__u8))) 423 return -EOPNOTSUPP; 424 425 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_BASIC, basic, ip_proto, 426 sizeof(__u8), reg); 427 nft_offload_set_dependency(ctx, NFT_OFFLOAD_DEP_TRANSPORT); 428 break; 429 default: 430 return -EOPNOTSUPP; 431 } 432 433 return 0; 434 } 435 436 static int nft_payload_offload_nh(struct nft_offload_ctx *ctx, 437 struct nft_flow_rule *flow, 438 const struct nft_payload *priv) 439 { 440 int err; 441 442 switch (ctx->dep.l3num) { 443 case htons(ETH_P_IP): 444 err = nft_payload_offload_ip(ctx, flow, priv); 445 break; 446 case htons(ETH_P_IPV6): 447 err = nft_payload_offload_ip6(ctx, flow, priv); 448 break; 449 default: 450 return -EOPNOTSUPP; 451 } 452 453 return err; 454 } 455 456 static int nft_payload_offload_tcp(struct nft_offload_ctx *ctx, 457 struct nft_flow_rule *flow, 458 const struct nft_payload *priv) 459 { 460 struct nft_offload_reg *reg = &ctx->regs[priv->dreg]; 461 462 switch (priv->offset) { 463 case offsetof(struct tcphdr, source): 464 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16))) 465 return -EOPNOTSUPP; 466 467 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, src, 468 sizeof(__be16), reg); 469 break; 470 case offsetof(struct tcphdr, dest): 471 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16))) 472 return -EOPNOTSUPP; 473 474 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, dst, 475 sizeof(__be16), reg); 476 break; 477 default: 478 return -EOPNOTSUPP; 479 } 480 481 return 0; 482 } 483 484 static int nft_payload_offload_udp(struct nft_offload_ctx *ctx, 485 struct nft_flow_rule *flow, 486 const struct nft_payload *priv) 487 { 488 struct nft_offload_reg *reg = &ctx->regs[priv->dreg]; 489 490 switch (priv->offset) { 491 case offsetof(struct udphdr, source): 492 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16))) 493 return -EOPNOTSUPP; 494 495 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, src, 496 sizeof(__be16), reg); 497 break; 498 case offsetof(struct udphdr, dest): 499 if (!nft_payload_offload_mask(reg, priv->len, sizeof(__be16))) 500 return -EOPNOTSUPP; 501 502 NFT_OFFLOAD_MATCH(FLOW_DISSECTOR_KEY_PORTS, tp, dst, 503 sizeof(__be16), reg); 504 break; 505 default: 506 return -EOPNOTSUPP; 507 } 508 509 return 0; 510 } 511 512 static int nft_payload_offload_th(struct nft_offload_ctx *ctx, 513 struct nft_flow_rule *flow, 514 const struct nft_payload *priv) 515 { 516 int err; 517 518 switch (ctx->dep.protonum) { 519 case IPPROTO_TCP: 520 err = nft_payload_offload_tcp(ctx, flow, priv); 521 break; 522 case IPPROTO_UDP: 523 err = nft_payload_offload_udp(ctx, flow, priv); 524 break; 525 default: 526 return -EOPNOTSUPP; 527 } 528 529 return err; 530 } 531 532 static int nft_payload_offload(struct nft_offload_ctx *ctx, 533 struct nft_flow_rule *flow, 534 const struct nft_expr *expr) 535 { 536 const struct nft_payload *priv = nft_expr_priv(expr); 537 int err; 538 539 switch (priv->base) { 540 case NFT_PAYLOAD_LL_HEADER: 541 err = nft_payload_offload_ll(ctx, flow, priv); 542 break; 543 case NFT_PAYLOAD_NETWORK_HEADER: 544 err = nft_payload_offload_nh(ctx, flow, priv); 545 break; 546 case NFT_PAYLOAD_TRANSPORT_HEADER: 547 err = nft_payload_offload_th(ctx, flow, priv); 548 break; 549 default: 550 err = -EOPNOTSUPP; 551 break; 552 } 553 return err; 554 } 555 556 static const struct nft_expr_ops nft_payload_ops = { 557 .type = &nft_payload_type, 558 .size = NFT_EXPR_SIZE(sizeof(struct nft_payload)), 559 .eval = nft_payload_eval, 560 .init = nft_payload_init, 561 .dump = nft_payload_dump, 562 .offload = nft_payload_offload, 563 }; 564 565 const struct nft_expr_ops nft_payload_fast_ops = { 566 .type = &nft_payload_type, 567 .size = NFT_EXPR_SIZE(sizeof(struct nft_payload)), 568 .eval = nft_payload_eval, 569 .init = nft_payload_init, 570 .dump = nft_payload_dump, 571 .offload = nft_payload_offload, 572 }; 573 574 void nft_payload_inner_eval(const struct nft_expr *expr, struct nft_regs *regs, 575 const struct nft_pktinfo *pkt, 576 struct nft_inner_tun_ctx *tun_ctx) 577 { 578 const struct nft_payload *priv = nft_expr_priv(expr); 579 const struct sk_buff *skb = pkt->skb; 580 u32 *dest = ®s->data[priv->dreg]; 581 int offset; 582 583 if (priv->len % NFT_REG32_SIZE) 584 dest[priv->len / NFT_REG32_SIZE] = 0; 585 586 switch (priv->base) { 587 case NFT_PAYLOAD_TUN_HEADER: 588 if (!(tun_ctx->flags & NFT_PAYLOAD_CTX_INNER_TUN)) 589 goto err; 590 591 offset = tun_ctx->inner_tunoff; 592 break; 593 case NFT_PAYLOAD_LL_HEADER: 594 if (!(tun_ctx->flags & NFT_PAYLOAD_CTX_INNER_LL)) 595 goto err; 596 597 offset = tun_ctx->inner_lloff; 598 break; 599 case NFT_PAYLOAD_NETWORK_HEADER: 600 if (!(tun_ctx->flags & NFT_PAYLOAD_CTX_INNER_NH)) 601 goto err; 602 603 offset = tun_ctx->inner_nhoff; 604 break; 605 case NFT_PAYLOAD_TRANSPORT_HEADER: 606 if (!(tun_ctx->flags & NFT_PAYLOAD_CTX_INNER_TH)) 607 goto err; 608 609 offset = tun_ctx->inner_thoff; 610 break; 611 default: 612 DEBUG_NET_WARN_ON_ONCE(1); 613 goto err; 614 } 615 offset += priv->offset; 616 617 if (skb_copy_bits(skb, offset, dest, priv->len) < 0) 618 goto err; 619 620 return; 621 err: 622 regs->verdict.code = NFT_BREAK; 623 } 624 625 static int nft_payload_inner_init(const struct nft_ctx *ctx, 626 const struct nft_expr *expr, 627 const struct nlattr * const tb[]) 628 { 629 struct nft_payload *priv = nft_expr_priv(expr); 630 u32 base, offset; 631 int err; 632 633 if (!tb[NFTA_PAYLOAD_BASE] || !tb[NFTA_PAYLOAD_OFFSET] || 634 !tb[NFTA_PAYLOAD_LEN] || !tb[NFTA_PAYLOAD_DREG]) 635 return -EINVAL; 636 637 base = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_BASE])); 638 switch (base) { 639 case NFT_PAYLOAD_TUN_HEADER: 640 case NFT_PAYLOAD_LL_HEADER: 641 case NFT_PAYLOAD_NETWORK_HEADER: 642 case NFT_PAYLOAD_TRANSPORT_HEADER: 643 break; 644 default: 645 return -EOPNOTSUPP; 646 } 647 648 priv->base = base; 649 priv->len = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_LEN])); 650 err = nft_parse_u32_check(tb[NFTA_PAYLOAD_OFFSET], U16_MAX, &offset); 651 if (err < 0) 652 return err; 653 priv->offset = offset; 654 655 return nft_parse_register_store(ctx, tb[NFTA_PAYLOAD_DREG], 656 &priv->dreg, NULL, NFT_DATA_VALUE, 657 priv->len); 658 } 659 660 static const struct nft_expr_ops nft_payload_inner_ops = { 661 .type = &nft_payload_type, 662 .size = NFT_EXPR_SIZE(sizeof(struct nft_payload)), 663 .init = nft_payload_inner_init, 664 .dump = nft_payload_dump, 665 /* direct call to nft_payload_inner_eval(). */ 666 }; 667 668 static inline void nft_csum_replace(__sum16 *sum, __wsum fsum, __wsum tsum) 669 { 670 csum_replace4(sum, (__force __be32)fsum, (__force __be32)tsum); 671 if (*sum == 0) 672 *sum = CSUM_MANGLED_0; 673 } 674 675 static bool nft_payload_udp_checksum(struct sk_buff *skb, unsigned int thoff) 676 { 677 struct udphdr *uh, _uh; 678 679 uh = skb_header_pointer(skb, thoff, sizeof(_uh), &_uh); 680 if (!uh) 681 return false; 682 683 return (__force bool)uh->check; 684 } 685 686 static int nft_payload_l4csum_offset(const struct nft_pktinfo *pkt, 687 struct sk_buff *skb, 688 unsigned int *l4csum_offset) 689 { 690 if (pkt->fragoff) 691 return -1; 692 693 switch (pkt->tprot) { 694 case IPPROTO_TCP: 695 *l4csum_offset = offsetof(struct tcphdr, check); 696 break; 697 case IPPROTO_UDP: 698 if (!nft_payload_udp_checksum(skb, nft_thoff(pkt))) 699 return -1; 700 fallthrough; 701 case IPPROTO_UDPLITE: 702 *l4csum_offset = offsetof(struct udphdr, check); 703 break; 704 case IPPROTO_ICMPV6: 705 *l4csum_offset = offsetof(struct icmp6hdr, icmp6_cksum); 706 break; 707 default: 708 return -1; 709 } 710 711 *l4csum_offset += nft_thoff(pkt); 712 return 0; 713 } 714 715 static int nft_payload_csum_sctp(struct sk_buff *skb, int offset) 716 { 717 struct sctphdr *sh; 718 719 if (skb_ensure_writable(skb, offset + sizeof(*sh))) 720 return -1; 721 722 sh = (struct sctphdr *)(skb->data + offset); 723 sh->checksum = sctp_compute_cksum(skb, offset); 724 skb->ip_summed = CHECKSUM_UNNECESSARY; 725 return 0; 726 } 727 728 static int nft_payload_l4csum_update(const struct nft_pktinfo *pkt, 729 struct sk_buff *skb, 730 __wsum fsum, __wsum tsum) 731 { 732 int l4csum_offset; 733 __sum16 sum; 734 735 /* If we cannot determine layer 4 checksum offset or this packet doesn't 736 * require layer 4 checksum recalculation, skip this packet. 737 */ 738 if (nft_payload_l4csum_offset(pkt, skb, &l4csum_offset) < 0) 739 return 0; 740 741 if (skb_copy_bits(skb, l4csum_offset, &sum, sizeof(sum)) < 0) 742 return -1; 743 744 /* Checksum mangling for an arbitrary amount of bytes, based on 745 * inet_proto_csum_replace*() functions. 746 */ 747 if (skb->ip_summed != CHECKSUM_PARTIAL) { 748 nft_csum_replace(&sum, fsum, tsum); 749 if (skb->ip_summed == CHECKSUM_COMPLETE) { 750 skb->csum = ~csum_add(csum_sub(~(skb->csum), fsum), 751 tsum); 752 } 753 } else { 754 sum = ~csum_fold(csum_add(csum_sub(csum_unfold(sum), fsum), 755 tsum)); 756 } 757 758 if (skb_ensure_writable(skb, l4csum_offset + sizeof(sum)) || 759 skb_store_bits(skb, l4csum_offset, &sum, sizeof(sum)) < 0) 760 return -1; 761 762 return 0; 763 } 764 765 static int nft_payload_csum_inet(struct sk_buff *skb, const u32 *src, 766 __wsum fsum, __wsum tsum, int csum_offset) 767 { 768 __sum16 sum; 769 770 if (skb_copy_bits(skb, csum_offset, &sum, sizeof(sum)) < 0) 771 return -1; 772 773 nft_csum_replace(&sum, fsum, tsum); 774 if (skb_ensure_writable(skb, csum_offset + sizeof(sum)) || 775 skb_store_bits(skb, csum_offset, &sum, sizeof(sum)) < 0) 776 return -1; 777 778 return 0; 779 } 780 781 struct nft_payload_set { 782 enum nft_payload_bases base:8; 783 u16 offset; 784 u8 len; 785 u8 sreg; 786 u8 csum_type; 787 u8 csum_offset; 788 u8 csum_flags; 789 }; 790 791 /* This is not struct vlan_hdr. */ 792 struct nft_payload_vlan_hdr { 793 __be16 h_vlan_proto; 794 __be16 h_vlan_TCI; 795 }; 796 797 static bool 798 nft_payload_set_vlan(const u32 *src, struct sk_buff *skb, u16 offset, u8 len, 799 int *vlan_hlen) 800 { 801 struct nft_payload_vlan_hdr *vlanh; 802 __be16 vlan_proto; 803 u16 vlan_tci; 804 805 if (offset >= offsetof(struct vlan_ethhdr, h_vlan_encapsulated_proto)) { 806 *vlan_hlen = VLAN_HLEN; 807 return true; 808 } 809 810 switch (offset) { 811 case offsetof(struct vlan_ethhdr, h_vlan_proto): 812 if (len == 2) { 813 vlan_proto = nft_reg_load_be16(src); 814 skb->vlan_proto = vlan_proto; 815 } else if (len == 4) { 816 vlanh = (struct nft_payload_vlan_hdr *)src; 817 __vlan_hwaccel_put_tag(skb, vlanh->h_vlan_proto, 818 ntohs(vlanh->h_vlan_TCI)); 819 } else { 820 return false; 821 } 822 break; 823 case offsetof(struct vlan_ethhdr, h_vlan_TCI): 824 if (len != 2) 825 return false; 826 827 vlan_tci = ntohs(nft_reg_load_be16(src)); 828 skb->vlan_tci = vlan_tci; 829 break; 830 default: 831 return false; 832 } 833 834 return true; 835 } 836 837 static void nft_payload_set_eval(const struct nft_expr *expr, 838 struct nft_regs *regs, 839 const struct nft_pktinfo *pkt) 840 { 841 const struct nft_payload_set *priv = nft_expr_priv(expr); 842 const u32 *src = ®s->data[priv->sreg]; 843 int offset, csum_offset, vlan_hlen = 0; 844 struct sk_buff *skb = pkt->skb; 845 __wsum fsum, tsum; 846 847 switch (priv->base) { 848 case NFT_PAYLOAD_LL_HEADER: 849 if (!skb_mac_header_was_set(skb)) 850 goto err; 851 852 if (skb_vlan_tag_present(skb) && 853 nft_payload_need_vlan_adjust(priv->offset, priv->len)) { 854 if (!nft_payload_set_vlan(src, skb, 855 priv->offset, priv->len, 856 &vlan_hlen)) 857 goto err; 858 859 if (!vlan_hlen) 860 return; 861 } 862 863 offset = skb_mac_header(skb) - skb->data - vlan_hlen; 864 break; 865 case NFT_PAYLOAD_NETWORK_HEADER: 866 offset = skb_network_offset(skb); 867 break; 868 case NFT_PAYLOAD_TRANSPORT_HEADER: 869 if (!(pkt->flags & NFT_PKTINFO_L4PROTO) || pkt->fragoff) 870 goto err; 871 offset = nft_thoff(pkt); 872 break; 873 case NFT_PAYLOAD_INNER_HEADER: 874 offset = nft_payload_inner_offset(pkt); 875 if (offset < 0) 876 goto err; 877 break; 878 default: 879 DEBUG_NET_WARN_ON_ONCE(1); 880 goto err; 881 } 882 883 csum_offset = offset + priv->csum_offset; 884 offset += priv->offset; 885 886 if ((priv->csum_type == NFT_PAYLOAD_CSUM_INET || priv->csum_flags) && 887 ((priv->base != NFT_PAYLOAD_TRANSPORT_HEADER && 888 priv->base != NFT_PAYLOAD_INNER_HEADER) || 889 skb->ip_summed != CHECKSUM_PARTIAL)) { 890 if (offset + priv->len > skb->len) 891 goto err; 892 893 fsum = skb_checksum(skb, offset, priv->len, 0); 894 tsum = csum_partial(src, priv->len, 0); 895 896 if (priv->csum_type == NFT_PAYLOAD_CSUM_INET && 897 nft_payload_csum_inet(skb, src, fsum, tsum, csum_offset)) 898 goto err; 899 900 if (priv->csum_flags && 901 nft_payload_l4csum_update(pkt, skb, fsum, tsum) < 0) 902 goto err; 903 } 904 905 if (skb_ensure_writable(skb, max(offset + priv->len, 0)) || 906 skb_store_bits(skb, offset, src, priv->len) < 0) 907 goto err; 908 909 if (priv->csum_type == NFT_PAYLOAD_CSUM_SCTP && 910 pkt->tprot == IPPROTO_SCTP && 911 skb->ip_summed != CHECKSUM_PARTIAL) { 912 if (pkt->fragoff == 0 && 913 nft_payload_csum_sctp(skb, nft_thoff(pkt))) 914 goto err; 915 } 916 917 return; 918 err: 919 regs->verdict.code = NFT_BREAK; 920 } 921 922 static int nft_payload_set_init(const struct nft_ctx *ctx, 923 const struct nft_expr *expr, 924 const struct nlattr * const tb[]) 925 { 926 u32 csum_offset, offset, csum_type = NFT_PAYLOAD_CSUM_NONE; 927 struct nft_payload_set *priv = nft_expr_priv(expr); 928 int err; 929 930 if (ctx->net->user_ns != &init_user_ns) 931 return -EPERM; 932 933 priv->base = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_BASE])); 934 priv->len = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_LEN])); 935 936 err = nft_parse_u32_check(tb[NFTA_PAYLOAD_OFFSET], U16_MAX, &offset); 937 if (err < 0) 938 return err; 939 priv->offset = offset; 940 941 if (tb[NFTA_PAYLOAD_CSUM_TYPE]) 942 csum_type = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_CSUM_TYPE])); 943 if (tb[NFTA_PAYLOAD_CSUM_OFFSET]) { 944 err = nft_parse_u32_check(tb[NFTA_PAYLOAD_CSUM_OFFSET], U8_MAX, 945 &csum_offset); 946 if (err < 0) 947 return err; 948 949 priv->csum_offset = csum_offset; 950 } 951 if (tb[NFTA_PAYLOAD_CSUM_FLAGS]) { 952 u32 flags; 953 954 flags = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_CSUM_FLAGS])); 955 if (flags & ~NFT_PAYLOAD_L4CSUM_PSEUDOHDR) 956 return -EINVAL; 957 958 priv->csum_flags = flags; 959 } 960 961 switch (csum_type) { 962 case NFT_PAYLOAD_CSUM_NONE: 963 case NFT_PAYLOAD_CSUM_INET: 964 break; 965 case NFT_PAYLOAD_CSUM_SCTP: 966 if (priv->base != NFT_PAYLOAD_TRANSPORT_HEADER) 967 return -EINVAL; 968 969 if (priv->csum_offset != offsetof(struct sctphdr, checksum)) 970 return -EINVAL; 971 break; 972 default: 973 return -EOPNOTSUPP; 974 } 975 priv->csum_type = csum_type; 976 977 return nft_parse_register_load(ctx, tb[NFTA_PAYLOAD_SREG], &priv->sreg, 978 priv->len); 979 } 980 981 static int nft_payload_set_dump(struct sk_buff *skb, 982 const struct nft_expr *expr, bool reset) 983 { 984 const struct nft_payload_set *priv = nft_expr_priv(expr); 985 986 if (nft_dump_register(skb, NFTA_PAYLOAD_SREG, priv->sreg) || 987 nla_put_be32(skb, NFTA_PAYLOAD_BASE, htonl(priv->base)) || 988 nla_put_be32(skb, NFTA_PAYLOAD_OFFSET, htonl(priv->offset)) || 989 nla_put_be32(skb, NFTA_PAYLOAD_LEN, htonl(priv->len)) || 990 nla_put_be32(skb, NFTA_PAYLOAD_CSUM_TYPE, htonl(priv->csum_type)) || 991 nla_put_be32(skb, NFTA_PAYLOAD_CSUM_OFFSET, 992 htonl(priv->csum_offset)) || 993 nla_put_be32(skb, NFTA_PAYLOAD_CSUM_FLAGS, htonl(priv->csum_flags))) 994 goto nla_put_failure; 995 return 0; 996 997 nla_put_failure: 998 return -1; 999 } 1000 1001 static const struct nft_expr_ops nft_payload_set_ops = { 1002 .type = &nft_payload_type, 1003 .size = NFT_EXPR_SIZE(sizeof(struct nft_payload_set)), 1004 .eval = nft_payload_set_eval, 1005 .init = nft_payload_set_init, 1006 .dump = nft_payload_set_dump, 1007 }; 1008 1009 static const struct nft_expr_ops * 1010 nft_payload_select_ops(const struct nft_ctx *ctx, 1011 const struct nlattr * const tb[]) 1012 { 1013 enum nft_payload_bases base; 1014 unsigned int offset, len; 1015 int err; 1016 1017 if (tb[NFTA_PAYLOAD_BASE] == NULL || 1018 tb[NFTA_PAYLOAD_OFFSET] == NULL || 1019 tb[NFTA_PAYLOAD_LEN] == NULL) 1020 return ERR_PTR(-EINVAL); 1021 1022 base = ntohl(nla_get_be32(tb[NFTA_PAYLOAD_BASE])); 1023 switch (base) { 1024 case NFT_PAYLOAD_LL_HEADER: 1025 case NFT_PAYLOAD_NETWORK_HEADER: 1026 case NFT_PAYLOAD_TRANSPORT_HEADER: 1027 case NFT_PAYLOAD_INNER_HEADER: 1028 break; 1029 default: 1030 return ERR_PTR(-EOPNOTSUPP); 1031 } 1032 1033 if (tb[NFTA_PAYLOAD_SREG] != NULL) { 1034 if (tb[NFTA_PAYLOAD_DREG] != NULL) 1035 return ERR_PTR(-EINVAL); 1036 return &nft_payload_set_ops; 1037 } 1038 1039 if (tb[NFTA_PAYLOAD_DREG] == NULL) 1040 return ERR_PTR(-EINVAL); 1041 1042 err = nft_parse_u32_check(tb[NFTA_PAYLOAD_OFFSET], U16_MAX, &offset); 1043 if (err < 0) 1044 return ERR_PTR(err); 1045 1046 err = nft_parse_u32_check(tb[NFTA_PAYLOAD_LEN], U8_MAX, &len); 1047 if (err < 0) 1048 return ERR_PTR(err); 1049 1050 if (len <= 4 && is_power_of_2(len) && IS_ALIGNED(offset, len) && 1051 base != NFT_PAYLOAD_LL_HEADER && base != NFT_PAYLOAD_INNER_HEADER) 1052 return &nft_payload_fast_ops; 1053 else 1054 return &nft_payload_ops; 1055 } 1056 1057 struct nft_expr_type nft_payload_type __read_mostly = { 1058 .name = "payload", 1059 .select_ops = nft_payload_select_ops, 1060 .inner_ops = &nft_payload_inner_ops, 1061 .policy = nft_payload_policy, 1062 .maxattr = NFTA_PAYLOAD_MAX, 1063 .owner = THIS_MODULE, 1064 }; 1065