1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * net/sched/act_pedit.c Generic packet editor 4 * 5 * Authors: Jamal Hadi Salim (2002-4) 6 */ 7 8 #include <linux/types.h> 9 #include <linux/kernel.h> 10 #include <linux/string.h> 11 #include <linux/errno.h> 12 #include <linux/skbuff.h> 13 #include <linux/rtnetlink.h> 14 #include <linux/module.h> 15 #include <linux/init.h> 16 #include <linux/ip.h> 17 #include <linux/ipv6.h> 18 #include <linux/slab.h> 19 #include <linux/overflow.h> 20 #include <linux/unaligned.h> 21 #include <net/ip.h> 22 #include <net/ipv6.h> 23 #include <net/netlink.h> 24 #include <net/pkt_sched.h> 25 #include <linux/tc_act/tc_pedit.h> 26 #include <net/tc_act/tc_pedit.h> 27 #include <uapi/linux/tc_act/tc_pedit.h> 28 #include <net/pkt_cls.h> 29 #include <net/tc_wrapper.h> 30 31 static struct tc_action_ops act_pedit_ops; 32 33 static const struct nla_policy pedit_policy[TCA_PEDIT_MAX + 1] = { 34 [TCA_PEDIT_PARMS] = { .len = sizeof(struct tc_pedit) }, 35 [TCA_PEDIT_PARMS_EX] = { .len = sizeof(struct tc_pedit) }, 36 [TCA_PEDIT_KEYS_EX] = { .type = NLA_NESTED }, 37 }; 38 39 static const struct nla_policy pedit_key_ex_policy[TCA_PEDIT_KEY_EX_MAX + 1] = { 40 [TCA_PEDIT_KEY_EX_HTYPE] = 41 NLA_POLICY_MAX(NLA_U16, TCA_PEDIT_HDR_TYPE_MAX), 42 [TCA_PEDIT_KEY_EX_CMD] = NLA_POLICY_MAX(NLA_U16, TCA_PEDIT_CMD_MAX), 43 }; 44 45 static struct tcf_pedit_key_ex *tcf_pedit_keys_ex_parse(struct nlattr *nla, 46 u8 n, struct netlink_ext_ack *extack) 47 { 48 struct tcf_pedit_key_ex *keys_ex; 49 struct tcf_pedit_key_ex *k; 50 const struct nlattr *ka; 51 int err = -EINVAL; 52 int rem; 53 54 if (!nla) 55 return NULL; 56 57 keys_ex = kzalloc_objs(*k, n); 58 if (!keys_ex) 59 return ERR_PTR(-ENOMEM); 60 61 k = keys_ex; 62 63 nla_for_each_nested(ka, nla, rem) { 64 struct nlattr *tb[TCA_PEDIT_KEY_EX_MAX + 1]; 65 66 if (!n) { 67 NL_SET_ERR_MSG_MOD(extack, "Can't parse more extended keys than requested"); 68 err = -EINVAL; 69 goto err_out; 70 } 71 n--; 72 73 if (nla_type(ka) != TCA_PEDIT_KEY_EX) { 74 NL_SET_ERR_MSG_ATTR(extack, ka, "Unknown attribute, expected extended key"); 75 err = -EINVAL; 76 goto err_out; 77 } 78 79 err = nla_parse_nested_deprecated(tb, TCA_PEDIT_KEY_EX_MAX, 80 ka, pedit_key_ex_policy, 81 NULL); 82 if (err) 83 goto err_out; 84 85 if (NL_REQ_ATTR_CHECK(extack, nla, tb, TCA_PEDIT_KEY_EX_HTYPE)) { 86 NL_SET_ERR_MSG(extack, "Missing required attribute"); 87 err = -EINVAL; 88 goto err_out; 89 } 90 91 if (NL_REQ_ATTR_CHECK(extack, nla, tb, TCA_PEDIT_KEY_EX_CMD)) { 92 NL_SET_ERR_MSG(extack, "Missing required attribute"); 93 err = -EINVAL; 94 goto err_out; 95 } 96 97 k->htype = nla_get_u16(tb[TCA_PEDIT_KEY_EX_HTYPE]); 98 k->cmd = nla_get_u16(tb[TCA_PEDIT_KEY_EX_CMD]); 99 100 k++; 101 } 102 103 if (n) { 104 NL_SET_ERR_MSG_MOD(extack, "Not enough extended keys to parse"); 105 err = -EINVAL; 106 goto err_out; 107 } 108 109 return keys_ex; 110 111 err_out: 112 kfree(keys_ex); 113 return ERR_PTR(err); 114 } 115 116 static int tcf_pedit_key_ex_dump(struct sk_buff *skb, 117 struct tcf_pedit_key_ex *keys_ex, int n) 118 { 119 struct nlattr *keys_start = nla_nest_start_noflag(skb, 120 TCA_PEDIT_KEYS_EX); 121 122 if (!keys_start) 123 goto nla_failure; 124 for (; n > 0; n--) { 125 struct nlattr *key_start; 126 127 key_start = nla_nest_start_noflag(skb, TCA_PEDIT_KEY_EX); 128 if (!key_start) 129 goto nla_failure; 130 131 if (nla_put_u16(skb, TCA_PEDIT_KEY_EX_HTYPE, keys_ex->htype) || 132 nla_put_u16(skb, TCA_PEDIT_KEY_EX_CMD, keys_ex->cmd)) 133 goto nla_failure; 134 135 nla_nest_end(skb, key_start); 136 137 keys_ex++; 138 } 139 140 nla_nest_end(skb, keys_start); 141 142 return 0; 143 nla_failure: 144 nla_nest_cancel(skb, keys_start); 145 return -EINVAL; 146 } 147 148 static void tcf_pedit_cleanup_rcu(struct rcu_head *head) 149 { 150 struct tcf_pedit_parms *parms = 151 container_of(head, struct tcf_pedit_parms, rcu); 152 153 kfree(parms->tcfp_keys_ex); 154 kfree(parms->tcfp_keys); 155 156 kfree(parms); 157 } 158 159 static int tcf_pedit_init(struct net *net, struct nlattr *nla, 160 struct nlattr *est, struct tc_action **a, 161 struct tcf_proto *tp, u32 flags, 162 struct netlink_ext_ack *extack) 163 { 164 struct tc_action_net *tn = net_generic(net, act_pedit_ops.net_id); 165 bool bind = flags & TCA_ACT_FLAGS_BIND; 166 struct tcf_chain *goto_ch = NULL; 167 struct tcf_pedit_parms *oparms, *nparms; 168 struct nlattr *tb[TCA_PEDIT_MAX + 1]; 169 struct tc_pedit *parm; 170 struct nlattr *pattr; 171 struct tcf_pedit *p; 172 int ret = 0, err; 173 int i, ksize; 174 u32 index; 175 176 if (!nla) { 177 NL_SET_ERR_MSG_MOD(extack, "Pedit requires attributes to be passed"); 178 return -EINVAL; 179 } 180 181 err = nla_parse_nested_deprecated(tb, TCA_PEDIT_MAX, nla, 182 pedit_policy, NULL); 183 if (err < 0) 184 return err; 185 186 pattr = tb[TCA_PEDIT_PARMS]; 187 if (!pattr) 188 pattr = tb[TCA_PEDIT_PARMS_EX]; 189 if (!pattr) { 190 NL_SET_ERR_MSG_MOD(extack, "Missing required TCA_PEDIT_PARMS or TCA_PEDIT_PARMS_EX pedit attribute"); 191 return -EINVAL; 192 } 193 194 parm = nla_data(pattr); 195 196 index = parm->index; 197 err = tcf_idr_check_alloc(tn, &index, a, bind); 198 if (!err) { 199 ret = tcf_idr_create_from_flags(tn, index, est, a, 200 &act_pedit_ops, bind, flags); 201 if (ret) { 202 tcf_idr_cleanup(tn, index); 203 return ret; 204 } 205 ret = ACT_P_CREATED; 206 } else if (err > 0) { 207 if (bind) 208 return ACT_P_BOUND; 209 if (!(flags & TCA_ACT_FLAGS_REPLACE)) { 210 ret = -EEXIST; 211 goto out_release; 212 } 213 } else { 214 return err; 215 } 216 217 if (!parm->nkeys) { 218 NL_SET_ERR_MSG_MOD(extack, "Pedit requires keys to be passed"); 219 ret = -EINVAL; 220 goto out_release; 221 } 222 ksize = parm->nkeys * sizeof(struct tc_pedit_key); 223 if (nla_len(pattr) < sizeof(*parm) + ksize) { 224 NL_SET_ERR_MSG_ATTR(extack, pattr, "Length of TCA_PEDIT_PARMS or TCA_PEDIT_PARMS_EX pedit attribute is invalid"); 225 ret = -EINVAL; 226 goto out_release; 227 } 228 229 nparms = kzalloc_obj(*nparms); 230 if (!nparms) { 231 ret = -ENOMEM; 232 goto out_release; 233 } 234 235 nparms->tcfp_keys_ex = 236 tcf_pedit_keys_ex_parse(tb[TCA_PEDIT_KEYS_EX], parm->nkeys, extack); 237 if (IS_ERR(nparms->tcfp_keys_ex)) { 238 ret = PTR_ERR(nparms->tcfp_keys_ex); 239 goto out_free; 240 } 241 242 err = tcf_action_check_ctrlact(parm->action, tp, &goto_ch, extack); 243 if (err < 0) { 244 ret = err; 245 goto out_free_ex; 246 } 247 248 nparms->tcfp_flags = parm->flags; 249 nparms->tcfp_nkeys = parm->nkeys; 250 251 nparms->tcfp_keys = kmemdup(parm->keys, ksize, GFP_KERNEL); 252 if (!nparms->tcfp_keys) { 253 ret = -ENOMEM; 254 goto put_chain; 255 } 256 257 for (i = 0; i < nparms->tcfp_nkeys; ++i) { 258 u32 offmask = nparms->tcfp_keys[i].offmask; 259 u32 cur = nparms->tcfp_keys[i].off; 260 261 /* The AT option can be added to static offsets in the datapath */ 262 if (!offmask && cur % 4) { 263 NL_SET_ERR_MSG_MOD(extack, "Offsets must be on 32bit boundaries"); 264 ret = -EINVAL; 265 goto out_free_keys; 266 } 267 268 /* sanitize the shift value for any later use */ 269 nparms->tcfp_keys[i].shift = min_t(size_t, 270 BITS_PER_TYPE(int) - 1, 271 nparms->tcfp_keys[i].shift); 272 273 } 274 275 p = to_pedit(*a); 276 nparms->action = parm->action; 277 spin_lock_bh(&p->tcf_lock); 278 goto_ch = tcf_action_set_ctrlact(*a, parm->action, goto_ch); 279 oparms = rcu_replace_pointer(p->parms, nparms, 1); 280 spin_unlock_bh(&p->tcf_lock); 281 282 if (oparms) 283 call_rcu(&oparms->rcu, tcf_pedit_cleanup_rcu); 284 285 if (goto_ch) 286 tcf_chain_put_by_act(goto_ch); 287 288 return ret; 289 290 out_free_keys: 291 kfree(nparms->tcfp_keys); 292 put_chain: 293 if (goto_ch) 294 tcf_chain_put_by_act(goto_ch); 295 out_free_ex: 296 kfree(nparms->tcfp_keys_ex); 297 out_free: 298 kfree(nparms); 299 out_release: 300 tcf_idr_release(*a, bind); 301 return ret; 302 } 303 304 static void tcf_pedit_cleanup(struct tc_action *a) 305 { 306 struct tcf_pedit *p = to_pedit(a); 307 struct tcf_pedit_parms *parms; 308 309 parms = rcu_dereference_protected(p->parms, 1); 310 311 if (parms) 312 call_rcu(&parms->rcu, tcf_pedit_cleanup_rcu); 313 } 314 315 static bool offset_valid(struct sk_buff *skb, int offset, int len) 316 { 317 if (offset < -(int)skb_headroom(skb)) 318 return false; 319 320 return offset <= (int)skb->len - len; 321 } 322 323 static int pedit_l4_skb_offset(struct sk_buff *skb, int *hoffset, const int header_type) 324 { 325 const int noff = skb_network_offset(skb); 326 int ret = -EINVAL; 327 struct iphdr _iph; 328 329 switch (skb->protocol) { 330 case htons(ETH_P_IP): { 331 const struct iphdr *iph = skb_header_pointer(skb, noff, sizeof(_iph), &_iph); 332 333 if (!iph) 334 goto out; 335 if (iph->ihl < 5 || iph->protocol != header_type || 336 (iph->frag_off & htons(IP_OFFSET))) 337 goto out; 338 *hoffset = noff + iph->ihl * 4; 339 ret = 0; 340 break; 341 } 342 case htons(ETH_P_IPV6): 343 ret = ipv6_find_hdr(skb, hoffset, header_type, NULL, NULL) == header_type ? 0 : -EINVAL; 344 break; 345 } 346 out: 347 return ret; 348 } 349 350 static int pedit_skb_hdr_offset(struct sk_buff *skb, 351 enum pedit_header_type htype, int *hoffset) 352 { 353 int ret = -EINVAL; 354 /* 'htype' is validated in the netlink parsing */ 355 switch (htype) { 356 case TCA_PEDIT_KEY_EX_HDR_TYPE_ETH: 357 if (skb_mac_header_was_set(skb)) { 358 *hoffset = skb_mac_offset(skb); 359 ret = 0; 360 } 361 break; 362 case TCA_PEDIT_KEY_EX_HDR_TYPE_NETWORK: 363 case TCA_PEDIT_KEY_EX_HDR_TYPE_IP4: 364 case TCA_PEDIT_KEY_EX_HDR_TYPE_IP6: 365 *hoffset = skb_network_offset(skb); 366 ret = 0; 367 break; 368 case TCA_PEDIT_KEY_EX_HDR_TYPE_TCP: 369 ret = pedit_l4_skb_offset(skb, hoffset, IPPROTO_TCP); 370 break; 371 case TCA_PEDIT_KEY_EX_HDR_TYPE_UDP: 372 ret = pedit_l4_skb_offset(skb, hoffset, IPPROTO_UDP); 373 break; 374 default: 375 break; 376 } 377 return ret; 378 } 379 380 TC_INDIRECT_SCOPE int tcf_pedit_act(struct sk_buff *skb, 381 const struct tc_action *a, 382 struct tcf_result *res) 383 { 384 enum pedit_header_type htype = TCA_PEDIT_KEY_EX_HDR_TYPE_NETWORK; 385 enum pedit_cmd cmd = TCA_PEDIT_KEY_EX_CMD_SET; 386 struct tcf_pedit *p = to_pedit(a); 387 struct tcf_pedit_key_ex *tkey_ex; 388 struct tcf_pedit_parms *parms; 389 struct tc_pedit_key *tkey; 390 int i; 391 392 parms = rcu_dereference_bh(p->parms); 393 394 tcf_lastuse_update(&p->tcf_tm); 395 tcf_action_update_bstats(&p->common, skb); 396 397 tkey = parms->tcfp_keys; 398 tkey_ex = parms->tcfp_keys_ex; 399 400 for (i = parms->tcfp_nkeys; i > 0; i--, tkey++) { 401 int write_offset, write_len; 402 int offset = tkey->off; 403 int hoffset = 0; 404 u32 cur_val, val; 405 u32 *ptr; 406 int rc; 407 408 if (tkey_ex) { 409 htype = tkey_ex->htype; 410 cmd = tkey_ex->cmd; 411 412 tkey_ex++; 413 } 414 415 rc = pedit_skb_hdr_offset(skb, htype, &hoffset); 416 if (rc) { 417 pr_info_ratelimited("tc action pedit unable to extract header offset for header type (0x%x)\n", htype); 418 goto bad; 419 } 420 421 if (tkey->offmask) { 422 u8 *d, _d; 423 int at_offset; 424 425 if (check_add_overflow(hoffset, (int)tkey->at, &at_offset) || 426 !offset_valid(skb, at_offset, sizeof(_d))) { 427 pr_info_ratelimited("tc action pedit 'at' offset %d out of bounds\n", 428 hoffset + tkey->at); 429 goto bad; 430 } 431 d = skb_header_pointer(skb, at_offset, 432 sizeof(_d), &_d); 433 if (!d) 434 goto bad; 435 436 offset += (*d & tkey->offmask) >> tkey->shift; 437 if (offset % 4) { 438 pr_info_ratelimited("tc action pedit offset must be on 32 bit boundaries\n"); 439 goto bad; 440 } 441 } 442 443 if (check_add_overflow(hoffset, offset, &write_offset)) { 444 pr_info_ratelimited("tc action pedit offset overflow\n"); 445 goto bad; 446 } 447 448 if (!offset_valid(skb, write_offset, sizeof(*ptr))) { 449 pr_info_ratelimited("tc action pedit offset %d out of bounds\n", 450 write_offset); 451 goto bad; 452 } 453 454 if (write_offset < 0) { 455 if (skb_cow(skb, -write_offset)) 456 goto bad; 457 if (write_offset + (int)sizeof(*ptr) > 0) { 458 if (skb_ensure_writable(skb, 459 min_t(int, skb->len, 460 write_offset + (int)sizeof(*ptr)))) 461 goto bad; 462 } 463 } else { 464 if (check_add_overflow(write_offset, (int)sizeof(*ptr), 465 &write_len)) 466 goto bad; 467 if (skb_ensure_writable(skb, min_t(int, skb->len, 468 write_len))) 469 goto bad; 470 } 471 472 ptr = (u32 *)(skb->data + write_offset); 473 cur_val = get_unaligned(ptr); 474 /* just do it, baby */ 475 switch (cmd) { 476 case TCA_PEDIT_KEY_EX_CMD_SET: 477 val = tkey->val; 478 break; 479 case TCA_PEDIT_KEY_EX_CMD_ADD: 480 val = (cur_val + tkey->val) & ~tkey->mask; 481 break; 482 default: 483 pr_info_ratelimited("tc action pedit bad command (%d)\n", cmd); 484 goto bad; 485 } 486 487 put_unaligned((cur_val & tkey->mask) ^ val, ptr); 488 } 489 490 goto done; 491 492 bad: 493 tcf_action_inc_overlimit_qstats(&p->common); 494 done: 495 return parms->action; 496 } 497 498 static void tcf_pedit_stats_update(struct tc_action *a, u64 bytes, u64 packets, 499 u64 drops, u64 lastuse, bool hw) 500 { 501 struct tcf_pedit *d = to_pedit(a); 502 struct tcf_t *tm = &d->tcf_tm; 503 504 tcf_action_update_stats(a, bytes, packets, drops, hw); 505 tm->lastuse = max_t(u64, tm->lastuse, lastuse); 506 } 507 508 static int tcf_pedit_dump(struct sk_buff *skb, struct tc_action *a, 509 int bind, int ref) 510 { 511 unsigned char *b = skb_tail_pointer(skb); 512 const struct tcf_pedit *p = to_pedit(a); 513 const struct tcf_pedit_parms *parms; 514 struct tc_pedit *opt; 515 struct tcf_t t; 516 int s; 517 518 rcu_read_lock(); 519 parms = rcu_dereference(p->parms); 520 s = struct_size(opt, keys, parms->tcfp_nkeys); 521 522 opt = kzalloc(s, GFP_ATOMIC); 523 if (unlikely(!opt)) { 524 rcu_read_unlock(); 525 return -ENOBUFS; 526 } 527 opt->nkeys = parms->tcfp_nkeys; 528 529 memcpy(opt->keys, parms->tcfp_keys, 530 flex_array_size(opt, keys, parms->tcfp_nkeys)); 531 opt->index = p->tcf_index; 532 opt->flags = parms->tcfp_flags; 533 opt->action = parms->action; 534 opt->refcnt = refcount_read(&p->tcf_refcnt) - ref; 535 opt->bindcnt = atomic_read(&p->tcf_bindcnt) - bind; 536 537 if (parms->tcfp_keys_ex) { 538 if (tcf_pedit_key_ex_dump(skb, parms->tcfp_keys_ex, 539 parms->tcfp_nkeys)) 540 goto nla_put_failure; 541 542 if (nla_put(skb, TCA_PEDIT_PARMS_EX, s, opt)) 543 goto nla_put_failure; 544 } else { 545 if (nla_put(skb, TCA_PEDIT_PARMS, s, opt)) 546 goto nla_put_failure; 547 } 548 549 tcf_tm_dump(&t, &p->tcf_tm); 550 if (nla_put_64bit(skb, TCA_PEDIT_TM, sizeof(t), &t, TCA_PEDIT_PAD)) 551 goto nla_put_failure; 552 rcu_read_unlock(); 553 554 kfree(opt); 555 return skb->len; 556 557 nla_put_failure: 558 rcu_read_unlock(); 559 nlmsg_trim(skb, b); 560 kfree(opt); 561 return -1; 562 } 563 564 static int tcf_pedit_offload_act_setup(struct tc_action *act, void *entry_data, 565 u32 *index_inc, bool bind, 566 struct netlink_ext_ack *extack) 567 { 568 if (bind) { 569 struct flow_action_entry *entry = entry_data; 570 int k; 571 572 for (k = 0; k < tcf_pedit_nkeys(act); k++) { 573 switch (tcf_pedit_cmd(act, k)) { 574 case TCA_PEDIT_KEY_EX_CMD_SET: 575 entry->id = FLOW_ACTION_MANGLE; 576 break; 577 case TCA_PEDIT_KEY_EX_CMD_ADD: 578 entry->id = FLOW_ACTION_ADD; 579 break; 580 default: 581 NL_SET_ERR_MSG_MOD(extack, "Unsupported pedit command offload"); 582 return -EOPNOTSUPP; 583 } 584 entry->mangle.htype = tcf_pedit_htype(act, k); 585 entry->mangle.mask = tcf_pedit_mask(act, k); 586 entry->mangle.val = tcf_pedit_val(act, k); 587 entry->mangle.offset = tcf_pedit_offset(act, k); 588 entry->hw_stats = tc_act_hw_stats(act->hw_stats); 589 entry++; 590 } 591 *index_inc = k; 592 } else { 593 struct flow_offload_action *fl_action = entry_data; 594 u32 cmd = tcf_pedit_cmd(act, 0); 595 int k; 596 597 switch (cmd) { 598 case TCA_PEDIT_KEY_EX_CMD_SET: 599 fl_action->id = FLOW_ACTION_MANGLE; 600 break; 601 case TCA_PEDIT_KEY_EX_CMD_ADD: 602 fl_action->id = FLOW_ACTION_ADD; 603 break; 604 default: 605 NL_SET_ERR_MSG_MOD(extack, "Unsupported pedit command offload"); 606 return -EOPNOTSUPP; 607 } 608 609 for (k = 1; k < tcf_pedit_nkeys(act); k++) { 610 if (cmd != tcf_pedit_cmd(act, k)) { 611 NL_SET_ERR_MSG_MOD(extack, "Unsupported pedit command offload"); 612 return -EOPNOTSUPP; 613 } 614 } 615 } 616 617 return 0; 618 } 619 620 static struct tc_action_ops act_pedit_ops = { 621 .kind = "pedit", 622 .id = TCA_ID_PEDIT, 623 .owner = THIS_MODULE, 624 .act = tcf_pedit_act, 625 .stats_update = tcf_pedit_stats_update, 626 .dump = tcf_pedit_dump, 627 .cleanup = tcf_pedit_cleanup, 628 .init = tcf_pedit_init, 629 .offload_act_setup = tcf_pedit_offload_act_setup, 630 .size = sizeof(struct tcf_pedit), 631 }; 632 MODULE_ALIAS_NET_ACT("pedit"); 633 634 static __net_init int pedit_init_net(struct net *net) 635 { 636 struct tc_action_net *tn = net_generic(net, act_pedit_ops.net_id); 637 638 return tc_action_net_init(net, tn, &act_pedit_ops); 639 } 640 641 static void __net_exit pedit_exit_net(struct list_head *net_list) 642 { 643 tc_action_net_exit(net_list, act_pedit_ops.net_id); 644 } 645 646 static struct pernet_operations pedit_net_ops = { 647 .init = pedit_init_net, 648 .exit_batch = pedit_exit_net, 649 .id = &act_pedit_ops.net_id, 650 .size = sizeof(struct tc_action_net), 651 }; 652 653 MODULE_AUTHOR("Jamal Hadi Salim(2002-4)"); 654 MODULE_DESCRIPTION("Generic Packet Editor actions"); 655 MODULE_LICENSE("GPL"); 656 657 static int __init pedit_init_module(void) 658 { 659 return tcf_register_action(&act_pedit_ops, &pedit_net_ops); 660 } 661 662 static void __exit pedit_cleanup_module(void) 663 { 664 tcf_unregister_action(&act_pedit_ops, &pedit_net_ops); 665 } 666 667 module_init(pedit_init_module); 668 module_exit(pedit_cleanup_module); 669