1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * net/sched/act_pedit.c Generic packet editor 4 * 5 * Authors: Jamal Hadi Salim (2002-4) 6 */ 7 8 #include <linux/types.h> 9 #include <linux/kernel.h> 10 #include <linux/string.h> 11 #include <linux/errno.h> 12 #include <linux/skbuff.h> 13 #include <linux/rtnetlink.h> 14 #include <linux/module.h> 15 #include <linux/init.h> 16 #include <linux/ip.h> 17 #include <linux/ipv6.h> 18 #include <linux/slab.h> 19 #include <linux/overflow.h> 20 #include <linux/unaligned.h> 21 #include <net/ipv6.h> 22 #include <net/netlink.h> 23 #include <net/pkt_sched.h> 24 #include <linux/tc_act/tc_pedit.h> 25 #include <net/tc_act/tc_pedit.h> 26 #include <uapi/linux/tc_act/tc_pedit.h> 27 #include <net/pkt_cls.h> 28 #include <net/tc_wrapper.h> 29 30 static struct tc_action_ops act_pedit_ops; 31 32 static const struct nla_policy pedit_policy[TCA_PEDIT_MAX + 1] = { 33 [TCA_PEDIT_PARMS] = { .len = sizeof(struct tc_pedit) }, 34 [TCA_PEDIT_PARMS_EX] = { .len = sizeof(struct tc_pedit) }, 35 [TCA_PEDIT_KEYS_EX] = { .type = NLA_NESTED }, 36 }; 37 38 static const struct nla_policy pedit_key_ex_policy[TCA_PEDIT_KEY_EX_MAX + 1] = { 39 [TCA_PEDIT_KEY_EX_HTYPE] = 40 NLA_POLICY_MAX(NLA_U16, TCA_PEDIT_HDR_TYPE_MAX), 41 [TCA_PEDIT_KEY_EX_CMD] = NLA_POLICY_MAX(NLA_U16, TCA_PEDIT_CMD_MAX), 42 }; 43 44 static struct tcf_pedit_key_ex *tcf_pedit_keys_ex_parse(struct nlattr *nla, 45 u8 n, struct netlink_ext_ack *extack) 46 { 47 struct tcf_pedit_key_ex *keys_ex; 48 struct tcf_pedit_key_ex *k; 49 const struct nlattr *ka; 50 int err = -EINVAL; 51 int rem; 52 53 if (!nla) 54 return NULL; 55 56 keys_ex = kzalloc_objs(*k, n); 57 if (!keys_ex) 58 return ERR_PTR(-ENOMEM); 59 60 k = keys_ex; 61 62 nla_for_each_nested(ka, nla, rem) { 63 struct nlattr *tb[TCA_PEDIT_KEY_EX_MAX + 1]; 64 65 if (!n) { 66 NL_SET_ERR_MSG_MOD(extack, "Can't parse more extended keys than requested"); 67 err = -EINVAL; 68 goto err_out; 69 } 70 n--; 71 72 if (nla_type(ka) != TCA_PEDIT_KEY_EX) { 73 NL_SET_ERR_MSG_ATTR(extack, ka, "Unknown attribute, expected extended key"); 74 err = -EINVAL; 75 goto err_out; 76 } 77 78 err = nla_parse_nested_deprecated(tb, TCA_PEDIT_KEY_EX_MAX, 79 ka, pedit_key_ex_policy, 80 NULL); 81 if (err) 82 goto err_out; 83 84 if (NL_REQ_ATTR_CHECK(extack, nla, tb, TCA_PEDIT_KEY_EX_HTYPE)) { 85 NL_SET_ERR_MSG(extack, "Missing required attribute"); 86 err = -EINVAL; 87 goto err_out; 88 } 89 90 if (NL_REQ_ATTR_CHECK(extack, nla, tb, TCA_PEDIT_KEY_EX_CMD)) { 91 NL_SET_ERR_MSG(extack, "Missing required attribute"); 92 err = -EINVAL; 93 goto err_out; 94 } 95 96 k->htype = nla_get_u16(tb[TCA_PEDIT_KEY_EX_HTYPE]); 97 k->cmd = nla_get_u16(tb[TCA_PEDIT_KEY_EX_CMD]); 98 99 k++; 100 } 101 102 if (n) { 103 NL_SET_ERR_MSG_MOD(extack, "Not enough extended keys to parse"); 104 err = -EINVAL; 105 goto err_out; 106 } 107 108 return keys_ex; 109 110 err_out: 111 kfree(keys_ex); 112 return ERR_PTR(err); 113 } 114 115 static int tcf_pedit_key_ex_dump(struct sk_buff *skb, 116 struct tcf_pedit_key_ex *keys_ex, int n) 117 { 118 struct nlattr *keys_start = nla_nest_start_noflag(skb, 119 TCA_PEDIT_KEYS_EX); 120 121 if (!keys_start) 122 goto nla_failure; 123 for (; n > 0; n--) { 124 struct nlattr *key_start; 125 126 key_start = nla_nest_start_noflag(skb, TCA_PEDIT_KEY_EX); 127 if (!key_start) 128 goto nla_failure; 129 130 if (nla_put_u16(skb, TCA_PEDIT_KEY_EX_HTYPE, keys_ex->htype) || 131 nla_put_u16(skb, TCA_PEDIT_KEY_EX_CMD, keys_ex->cmd)) 132 goto nla_failure; 133 134 nla_nest_end(skb, key_start); 135 136 keys_ex++; 137 } 138 139 nla_nest_end(skb, keys_start); 140 141 return 0; 142 nla_failure: 143 nla_nest_cancel(skb, keys_start); 144 return -EINVAL; 145 } 146 147 static void tcf_pedit_cleanup_rcu(struct rcu_head *head) 148 { 149 struct tcf_pedit_parms *parms = 150 container_of(head, struct tcf_pedit_parms, rcu); 151 152 kfree(parms->tcfp_keys_ex); 153 kfree(parms->tcfp_keys); 154 155 kfree(parms); 156 } 157 158 static int tcf_pedit_init(struct net *net, struct nlattr *nla, 159 struct nlattr *est, struct tc_action **a, 160 struct tcf_proto *tp, u32 flags, 161 struct netlink_ext_ack *extack) 162 { 163 struct tc_action_net *tn = net_generic(net, act_pedit_ops.net_id); 164 bool bind = flags & TCA_ACT_FLAGS_BIND; 165 struct tcf_chain *goto_ch = NULL; 166 struct tcf_pedit_parms *oparms, *nparms; 167 struct nlattr *tb[TCA_PEDIT_MAX + 1]; 168 struct tc_pedit *parm; 169 struct nlattr *pattr; 170 struct tcf_pedit *p; 171 int ret = 0, err; 172 int i, ksize; 173 u32 index; 174 175 if (!nla) { 176 NL_SET_ERR_MSG_MOD(extack, "Pedit requires attributes to be passed"); 177 return -EINVAL; 178 } 179 180 err = nla_parse_nested_deprecated(tb, TCA_PEDIT_MAX, nla, 181 pedit_policy, NULL); 182 if (err < 0) 183 return err; 184 185 pattr = tb[TCA_PEDIT_PARMS]; 186 if (!pattr) 187 pattr = tb[TCA_PEDIT_PARMS_EX]; 188 if (!pattr) { 189 NL_SET_ERR_MSG_MOD(extack, "Missing required TCA_PEDIT_PARMS or TCA_PEDIT_PARMS_EX pedit attribute"); 190 return -EINVAL; 191 } 192 193 parm = nla_data(pattr); 194 195 index = parm->index; 196 err = tcf_idr_check_alloc(tn, &index, a, bind); 197 if (!err) { 198 ret = tcf_idr_create_from_flags(tn, index, est, a, 199 &act_pedit_ops, bind, flags); 200 if (ret) { 201 tcf_idr_cleanup(tn, index); 202 return ret; 203 } 204 ret = ACT_P_CREATED; 205 } else if (err > 0) { 206 if (bind) 207 return ACT_P_BOUND; 208 if (!(flags & TCA_ACT_FLAGS_REPLACE)) { 209 ret = -EEXIST; 210 goto out_release; 211 } 212 } else { 213 return err; 214 } 215 216 if (!parm->nkeys) { 217 NL_SET_ERR_MSG_MOD(extack, "Pedit requires keys to be passed"); 218 ret = -EINVAL; 219 goto out_release; 220 } 221 ksize = parm->nkeys * sizeof(struct tc_pedit_key); 222 if (nla_len(pattr) < sizeof(*parm) + ksize) { 223 NL_SET_ERR_MSG_ATTR(extack, pattr, "Length of TCA_PEDIT_PARMS or TCA_PEDIT_PARMS_EX pedit attribute is invalid"); 224 ret = -EINVAL; 225 goto out_release; 226 } 227 228 nparms = kzalloc_obj(*nparms); 229 if (!nparms) { 230 ret = -ENOMEM; 231 goto out_release; 232 } 233 234 nparms->tcfp_keys_ex = 235 tcf_pedit_keys_ex_parse(tb[TCA_PEDIT_KEYS_EX], parm->nkeys, extack); 236 if (IS_ERR(nparms->tcfp_keys_ex)) { 237 ret = PTR_ERR(nparms->tcfp_keys_ex); 238 goto out_free; 239 } 240 241 err = tcf_action_check_ctrlact(parm->action, tp, &goto_ch, extack); 242 if (err < 0) { 243 ret = err; 244 goto out_free_ex; 245 } 246 247 nparms->tcfp_flags = parm->flags; 248 nparms->tcfp_nkeys = parm->nkeys; 249 250 nparms->tcfp_keys = kmemdup(parm->keys, ksize, GFP_KERNEL); 251 if (!nparms->tcfp_keys) { 252 ret = -ENOMEM; 253 goto put_chain; 254 } 255 256 for (i = 0; i < nparms->tcfp_nkeys; ++i) { 257 u32 offmask = nparms->tcfp_keys[i].offmask; 258 u32 cur = nparms->tcfp_keys[i].off; 259 260 /* The AT option can be added to static offsets in the datapath */ 261 if (!offmask && cur % 4) { 262 NL_SET_ERR_MSG_MOD(extack, "Offsets must be on 32bit boundaries"); 263 ret = -EINVAL; 264 goto out_free_keys; 265 } 266 267 /* sanitize the shift value for any later use */ 268 nparms->tcfp_keys[i].shift = min_t(size_t, 269 BITS_PER_TYPE(int) - 1, 270 nparms->tcfp_keys[i].shift); 271 272 } 273 274 p = to_pedit(*a); 275 nparms->action = parm->action; 276 spin_lock_bh(&p->tcf_lock); 277 goto_ch = tcf_action_set_ctrlact(*a, parm->action, goto_ch); 278 oparms = rcu_replace_pointer(p->parms, nparms, 1); 279 spin_unlock_bh(&p->tcf_lock); 280 281 if (oparms) 282 call_rcu(&oparms->rcu, tcf_pedit_cleanup_rcu); 283 284 if (goto_ch) 285 tcf_chain_put_by_act(goto_ch); 286 287 return ret; 288 289 out_free_keys: 290 kfree(nparms->tcfp_keys); 291 put_chain: 292 if (goto_ch) 293 tcf_chain_put_by_act(goto_ch); 294 out_free_ex: 295 kfree(nparms->tcfp_keys_ex); 296 out_free: 297 kfree(nparms); 298 out_release: 299 tcf_idr_release(*a, bind); 300 return ret; 301 } 302 303 static void tcf_pedit_cleanup(struct tc_action *a) 304 { 305 struct tcf_pedit *p = to_pedit(a); 306 struct tcf_pedit_parms *parms; 307 308 parms = rcu_dereference_protected(p->parms, 1); 309 310 if (parms) 311 call_rcu(&parms->rcu, tcf_pedit_cleanup_rcu); 312 } 313 314 static bool offset_valid(struct sk_buff *skb, int offset, int len) 315 { 316 if (offset < -(int)skb_headroom(skb)) 317 return false; 318 319 return offset <= (int)skb->len - len; 320 } 321 322 static int pedit_l4_skb_offset(struct sk_buff *skb, int *hoffset, const int header_type) 323 { 324 const int noff = skb_network_offset(skb); 325 int ret = -EINVAL; 326 struct iphdr _iph; 327 328 switch (skb->protocol) { 329 case htons(ETH_P_IP): { 330 const struct iphdr *iph = skb_header_pointer(skb, noff, sizeof(_iph), &_iph); 331 332 if (!iph) 333 goto out; 334 *hoffset = noff + iph->ihl * 4; 335 ret = 0; 336 break; 337 } 338 case htons(ETH_P_IPV6): 339 ret = ipv6_find_hdr(skb, hoffset, header_type, NULL, NULL) == header_type ? 0 : -EINVAL; 340 break; 341 } 342 out: 343 return ret; 344 } 345 346 static int pedit_skb_hdr_offset(struct sk_buff *skb, 347 enum pedit_header_type htype, int *hoffset) 348 { 349 int ret = -EINVAL; 350 /* 'htype' is validated in the netlink parsing */ 351 switch (htype) { 352 case TCA_PEDIT_KEY_EX_HDR_TYPE_ETH: 353 if (skb_mac_header_was_set(skb)) { 354 *hoffset = skb_mac_offset(skb); 355 ret = 0; 356 } 357 break; 358 case TCA_PEDIT_KEY_EX_HDR_TYPE_NETWORK: 359 case TCA_PEDIT_KEY_EX_HDR_TYPE_IP4: 360 case TCA_PEDIT_KEY_EX_HDR_TYPE_IP6: 361 *hoffset = skb_network_offset(skb); 362 ret = 0; 363 break; 364 case TCA_PEDIT_KEY_EX_HDR_TYPE_TCP: 365 ret = pedit_l4_skb_offset(skb, hoffset, IPPROTO_TCP); 366 break; 367 case TCA_PEDIT_KEY_EX_HDR_TYPE_UDP: 368 ret = pedit_l4_skb_offset(skb, hoffset, IPPROTO_UDP); 369 break; 370 default: 371 break; 372 } 373 return ret; 374 } 375 376 TC_INDIRECT_SCOPE int tcf_pedit_act(struct sk_buff *skb, 377 const struct tc_action *a, 378 struct tcf_result *res) 379 { 380 enum pedit_header_type htype = TCA_PEDIT_KEY_EX_HDR_TYPE_NETWORK; 381 enum pedit_cmd cmd = TCA_PEDIT_KEY_EX_CMD_SET; 382 struct tcf_pedit *p = to_pedit(a); 383 struct tcf_pedit_key_ex *tkey_ex; 384 struct tcf_pedit_parms *parms; 385 struct tc_pedit_key *tkey; 386 int i; 387 388 parms = rcu_dereference_bh(p->parms); 389 390 tcf_lastuse_update(&p->tcf_tm); 391 tcf_action_update_bstats(&p->common, skb); 392 393 tkey = parms->tcfp_keys; 394 tkey_ex = parms->tcfp_keys_ex; 395 396 for (i = parms->tcfp_nkeys; i > 0; i--, tkey++) { 397 int write_offset, write_len; 398 int offset = tkey->off; 399 int hoffset = 0; 400 u32 cur_val, val; 401 u32 *ptr; 402 int rc; 403 404 if (tkey_ex) { 405 htype = tkey_ex->htype; 406 cmd = tkey_ex->cmd; 407 408 tkey_ex++; 409 } 410 411 rc = pedit_skb_hdr_offset(skb, htype, &hoffset); 412 if (rc) { 413 pr_info_ratelimited("tc action pedit unable to extract header offset for header type (0x%x)\n", htype); 414 goto bad; 415 } 416 417 if (tkey->offmask) { 418 u8 *d, _d; 419 int at_offset; 420 421 if (check_add_overflow(hoffset, (int)tkey->at, &at_offset) || 422 !offset_valid(skb, at_offset, sizeof(_d))) { 423 pr_info_ratelimited("tc action pedit 'at' offset %d out of bounds\n", 424 hoffset + tkey->at); 425 goto bad; 426 } 427 d = skb_header_pointer(skb, at_offset, 428 sizeof(_d), &_d); 429 if (!d) 430 goto bad; 431 432 offset += (*d & tkey->offmask) >> tkey->shift; 433 if (offset % 4) { 434 pr_info_ratelimited("tc action pedit offset must be on 32 bit boundaries\n"); 435 goto bad; 436 } 437 } 438 439 if (check_add_overflow(hoffset, offset, &write_offset)) { 440 pr_info_ratelimited("tc action pedit offset overflow\n"); 441 goto bad; 442 } 443 444 if (!offset_valid(skb, write_offset, sizeof(*ptr))) { 445 pr_info_ratelimited("tc action pedit offset %d out of bounds\n", 446 write_offset); 447 goto bad; 448 } 449 450 if (write_offset < 0) { 451 if (skb_cow(skb, -write_offset)) 452 goto bad; 453 if (write_offset + (int)sizeof(*ptr) > 0) { 454 if (skb_ensure_writable(skb, 455 min_t(int, skb->len, 456 write_offset + (int)sizeof(*ptr)))) 457 goto bad; 458 } 459 } else { 460 if (check_add_overflow(write_offset, (int)sizeof(*ptr), 461 &write_len)) 462 goto bad; 463 if (skb_ensure_writable(skb, min_t(int, skb->len, 464 write_len))) 465 goto bad; 466 } 467 468 ptr = (u32 *)(skb->data + write_offset); 469 cur_val = get_unaligned(ptr); 470 /* just do it, baby */ 471 switch (cmd) { 472 case TCA_PEDIT_KEY_EX_CMD_SET: 473 val = tkey->val; 474 break; 475 case TCA_PEDIT_KEY_EX_CMD_ADD: 476 val = (cur_val + tkey->val) & ~tkey->mask; 477 break; 478 default: 479 pr_info_ratelimited("tc action pedit bad command (%d)\n", cmd); 480 goto bad; 481 } 482 483 put_unaligned((cur_val & tkey->mask) ^ val, ptr); 484 } 485 486 goto done; 487 488 bad: 489 tcf_action_inc_overlimit_qstats(&p->common); 490 done: 491 return parms->action; 492 } 493 494 static void tcf_pedit_stats_update(struct tc_action *a, u64 bytes, u64 packets, 495 u64 drops, u64 lastuse, bool hw) 496 { 497 struct tcf_pedit *d = to_pedit(a); 498 struct tcf_t *tm = &d->tcf_tm; 499 500 tcf_action_update_stats(a, bytes, packets, drops, hw); 501 tm->lastuse = max_t(u64, tm->lastuse, lastuse); 502 } 503 504 static int tcf_pedit_dump(struct sk_buff *skb, struct tc_action *a, 505 int bind, int ref) 506 { 507 unsigned char *b = skb_tail_pointer(skb); 508 const struct tcf_pedit *p = to_pedit(a); 509 const struct tcf_pedit_parms *parms; 510 struct tc_pedit *opt; 511 struct tcf_t t; 512 int s; 513 514 rcu_read_lock(); 515 parms = rcu_dereference(p->parms); 516 s = struct_size(opt, keys, parms->tcfp_nkeys); 517 518 opt = kzalloc(s, GFP_ATOMIC); 519 if (unlikely(!opt)) { 520 rcu_read_unlock(); 521 return -ENOBUFS; 522 } 523 opt->nkeys = parms->tcfp_nkeys; 524 525 memcpy(opt->keys, parms->tcfp_keys, 526 flex_array_size(opt, keys, parms->tcfp_nkeys)); 527 opt->index = p->tcf_index; 528 opt->flags = parms->tcfp_flags; 529 opt->action = parms->action; 530 opt->refcnt = refcount_read(&p->tcf_refcnt) - ref; 531 opt->bindcnt = atomic_read(&p->tcf_bindcnt) - bind; 532 533 if (parms->tcfp_keys_ex) { 534 if (tcf_pedit_key_ex_dump(skb, parms->tcfp_keys_ex, 535 parms->tcfp_nkeys)) 536 goto nla_put_failure; 537 538 if (nla_put(skb, TCA_PEDIT_PARMS_EX, s, opt)) 539 goto nla_put_failure; 540 } else { 541 if (nla_put(skb, TCA_PEDIT_PARMS, s, opt)) 542 goto nla_put_failure; 543 } 544 545 tcf_tm_dump(&t, &p->tcf_tm); 546 if (nla_put_64bit(skb, TCA_PEDIT_TM, sizeof(t), &t, TCA_PEDIT_PAD)) 547 goto nla_put_failure; 548 rcu_read_unlock(); 549 550 kfree(opt); 551 return skb->len; 552 553 nla_put_failure: 554 rcu_read_unlock(); 555 nlmsg_trim(skb, b); 556 kfree(opt); 557 return -1; 558 } 559 560 static int tcf_pedit_offload_act_setup(struct tc_action *act, void *entry_data, 561 u32 *index_inc, bool bind, 562 struct netlink_ext_ack *extack) 563 { 564 if (bind) { 565 struct flow_action_entry *entry = entry_data; 566 int k; 567 568 for (k = 0; k < tcf_pedit_nkeys(act); k++) { 569 switch (tcf_pedit_cmd(act, k)) { 570 case TCA_PEDIT_KEY_EX_CMD_SET: 571 entry->id = FLOW_ACTION_MANGLE; 572 break; 573 case TCA_PEDIT_KEY_EX_CMD_ADD: 574 entry->id = FLOW_ACTION_ADD; 575 break; 576 default: 577 NL_SET_ERR_MSG_MOD(extack, "Unsupported pedit command offload"); 578 return -EOPNOTSUPP; 579 } 580 entry->mangle.htype = tcf_pedit_htype(act, k); 581 entry->mangle.mask = tcf_pedit_mask(act, k); 582 entry->mangle.val = tcf_pedit_val(act, k); 583 entry->mangle.offset = tcf_pedit_offset(act, k); 584 entry->hw_stats = tc_act_hw_stats(act->hw_stats); 585 entry++; 586 } 587 *index_inc = k; 588 } else { 589 struct flow_offload_action *fl_action = entry_data; 590 u32 cmd = tcf_pedit_cmd(act, 0); 591 int k; 592 593 switch (cmd) { 594 case TCA_PEDIT_KEY_EX_CMD_SET: 595 fl_action->id = FLOW_ACTION_MANGLE; 596 break; 597 case TCA_PEDIT_KEY_EX_CMD_ADD: 598 fl_action->id = FLOW_ACTION_ADD; 599 break; 600 default: 601 NL_SET_ERR_MSG_MOD(extack, "Unsupported pedit command offload"); 602 return -EOPNOTSUPP; 603 } 604 605 for (k = 1; k < tcf_pedit_nkeys(act); k++) { 606 if (cmd != tcf_pedit_cmd(act, k)) { 607 NL_SET_ERR_MSG_MOD(extack, "Unsupported pedit command offload"); 608 return -EOPNOTSUPP; 609 } 610 } 611 } 612 613 return 0; 614 } 615 616 static struct tc_action_ops act_pedit_ops = { 617 .kind = "pedit", 618 .id = TCA_ID_PEDIT, 619 .owner = THIS_MODULE, 620 .act = tcf_pedit_act, 621 .stats_update = tcf_pedit_stats_update, 622 .dump = tcf_pedit_dump, 623 .cleanup = tcf_pedit_cleanup, 624 .init = tcf_pedit_init, 625 .offload_act_setup = tcf_pedit_offload_act_setup, 626 .size = sizeof(struct tcf_pedit), 627 }; 628 MODULE_ALIAS_NET_ACT("pedit"); 629 630 static __net_init int pedit_init_net(struct net *net) 631 { 632 struct tc_action_net *tn = net_generic(net, act_pedit_ops.net_id); 633 634 return tc_action_net_init(net, tn, &act_pedit_ops); 635 } 636 637 static void __net_exit pedit_exit_net(struct list_head *net_list) 638 { 639 tc_action_net_exit(net_list, act_pedit_ops.net_id); 640 } 641 642 static struct pernet_operations pedit_net_ops = { 643 .init = pedit_init_net, 644 .exit_batch = pedit_exit_net, 645 .id = &act_pedit_ops.net_id, 646 .size = sizeof(struct tc_action_net), 647 }; 648 649 MODULE_AUTHOR("Jamal Hadi Salim(2002-4)"); 650 MODULE_DESCRIPTION("Generic Packet Editor actions"); 651 MODULE_LICENSE("GPL"); 652 653 static int __init pedit_init_module(void) 654 { 655 return tcf_register_action(&act_pedit_ops, &pedit_net_ops); 656 } 657 658 static void __exit pedit_cleanup_module(void) 659 { 660 tcf_unregister_action(&act_pedit_ops, &pedit_net_ops); 661 } 662 663 module_init(pedit_init_module); 664 module_exit(pedit_cleanup_module); 665