1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * net/sched/act_pedit.c Generic packet editor 4 * 5 * Authors: Jamal Hadi Salim (2002-4) 6 */ 7 8 #include <linux/types.h> 9 #include <linux/kernel.h> 10 #include <linux/string.h> 11 #include <linux/errno.h> 12 #include <linux/skbuff.h> 13 #include <linux/rtnetlink.h> 14 #include <linux/module.h> 15 #include <linux/init.h> 16 #include <linux/ip.h> 17 #include <linux/ipv6.h> 18 #include <linux/slab.h> 19 #include <net/ipv6.h> 20 #include <net/netlink.h> 21 #include <net/pkt_sched.h> 22 #include <linux/tc_act/tc_pedit.h> 23 #include <net/tc_act/tc_pedit.h> 24 #include <uapi/linux/tc_act/tc_pedit.h> 25 #include <net/pkt_cls.h> 26 #include <net/tc_wrapper.h> 27 28 static struct tc_action_ops act_pedit_ops; 29 30 static const struct nla_policy pedit_policy[TCA_PEDIT_MAX + 1] = { 31 [TCA_PEDIT_PARMS] = { .len = sizeof(struct tc_pedit) }, 32 [TCA_PEDIT_PARMS_EX] = { .len = sizeof(struct tc_pedit) }, 33 [TCA_PEDIT_KEYS_EX] = { .type = NLA_NESTED }, 34 }; 35 36 static const struct nla_policy pedit_key_ex_policy[TCA_PEDIT_KEY_EX_MAX + 1] = { 37 [TCA_PEDIT_KEY_EX_HTYPE] = 38 NLA_POLICY_MAX(NLA_U16, TCA_PEDIT_HDR_TYPE_MAX), 39 [TCA_PEDIT_KEY_EX_CMD] = NLA_POLICY_MAX(NLA_U16, TCA_PEDIT_CMD_MAX), 40 }; 41 42 static struct tcf_pedit_key_ex *tcf_pedit_keys_ex_parse(struct nlattr *nla, 43 u8 n, struct netlink_ext_ack *extack) 44 { 45 struct tcf_pedit_key_ex *keys_ex; 46 struct tcf_pedit_key_ex *k; 47 const struct nlattr *ka; 48 int err = -EINVAL; 49 int rem; 50 51 if (!nla) 52 return NULL; 53 54 keys_ex = kcalloc(n, sizeof(*k), GFP_KERNEL); 55 if (!keys_ex) 56 return ERR_PTR(-ENOMEM); 57 58 k = keys_ex; 59 60 nla_for_each_nested(ka, nla, rem) { 61 struct nlattr *tb[TCA_PEDIT_KEY_EX_MAX + 1]; 62 63 if (!n) { 64 NL_SET_ERR_MSG_MOD(extack, "Can't parse more extended keys than requested"); 65 err = -EINVAL; 66 goto err_out; 67 } 68 n--; 69 70 if (nla_type(ka) != TCA_PEDIT_KEY_EX) { 71 NL_SET_ERR_MSG_ATTR(extack, ka, "Unknown attribute, expected extended key"); 72 err = -EINVAL; 73 goto err_out; 74 } 75 76 err = nla_parse_nested_deprecated(tb, TCA_PEDIT_KEY_EX_MAX, 77 ka, pedit_key_ex_policy, 78 NULL); 79 if (err) 80 goto err_out; 81 82 if (NL_REQ_ATTR_CHECK(extack, nla, tb, TCA_PEDIT_KEY_EX_HTYPE)) { 83 NL_SET_ERR_MSG(extack, "Missing required attribute"); 84 err = -EINVAL; 85 goto err_out; 86 } 87 88 if (NL_REQ_ATTR_CHECK(extack, nla, tb, TCA_PEDIT_KEY_EX_CMD)) { 89 NL_SET_ERR_MSG(extack, "Missing required attribute"); 90 err = -EINVAL; 91 goto err_out; 92 } 93 94 k->htype = nla_get_u16(tb[TCA_PEDIT_KEY_EX_HTYPE]); 95 k->cmd = nla_get_u16(tb[TCA_PEDIT_KEY_EX_CMD]); 96 97 k++; 98 } 99 100 if (n) { 101 NL_SET_ERR_MSG_MOD(extack, "Not enough extended keys to parse"); 102 err = -EINVAL; 103 goto err_out; 104 } 105 106 return keys_ex; 107 108 err_out: 109 kfree(keys_ex); 110 return ERR_PTR(err); 111 } 112 113 static int tcf_pedit_key_ex_dump(struct sk_buff *skb, 114 struct tcf_pedit_key_ex *keys_ex, int n) 115 { 116 struct nlattr *keys_start = nla_nest_start_noflag(skb, 117 TCA_PEDIT_KEYS_EX); 118 119 if (!keys_start) 120 goto nla_failure; 121 for (; n > 0; n--) { 122 struct nlattr *key_start; 123 124 key_start = nla_nest_start_noflag(skb, TCA_PEDIT_KEY_EX); 125 if (!key_start) 126 goto nla_failure; 127 128 if (nla_put_u16(skb, TCA_PEDIT_KEY_EX_HTYPE, keys_ex->htype) || 129 nla_put_u16(skb, TCA_PEDIT_KEY_EX_CMD, keys_ex->cmd)) 130 goto nla_failure; 131 132 nla_nest_end(skb, key_start); 133 134 keys_ex++; 135 } 136 137 nla_nest_end(skb, keys_start); 138 139 return 0; 140 nla_failure: 141 nla_nest_cancel(skb, keys_start); 142 return -EINVAL; 143 } 144 145 static void tcf_pedit_cleanup_rcu(struct rcu_head *head) 146 { 147 struct tcf_pedit_parms *parms = 148 container_of(head, struct tcf_pedit_parms, rcu); 149 150 kfree(parms->tcfp_keys_ex); 151 kfree(parms->tcfp_keys); 152 153 kfree(parms); 154 } 155 156 static int tcf_pedit_init(struct net *net, struct nlattr *nla, 157 struct nlattr *est, struct tc_action **a, 158 struct tcf_proto *tp, u32 flags, 159 struct netlink_ext_ack *extack) 160 { 161 struct tc_action_net *tn = net_generic(net, act_pedit_ops.net_id); 162 bool bind = flags & TCA_ACT_FLAGS_BIND; 163 struct tcf_chain *goto_ch = NULL; 164 struct tcf_pedit_parms *oparms, *nparms; 165 struct nlattr *tb[TCA_PEDIT_MAX + 1]; 166 struct tc_pedit *parm; 167 struct nlattr *pattr; 168 struct tcf_pedit *p; 169 int ret = 0, err; 170 int i, ksize; 171 u32 index; 172 173 if (!nla) { 174 NL_SET_ERR_MSG_MOD(extack, "Pedit requires attributes to be passed"); 175 return -EINVAL; 176 } 177 178 err = nla_parse_nested_deprecated(tb, TCA_PEDIT_MAX, nla, 179 pedit_policy, NULL); 180 if (err < 0) 181 return err; 182 183 pattr = tb[TCA_PEDIT_PARMS]; 184 if (!pattr) 185 pattr = tb[TCA_PEDIT_PARMS_EX]; 186 if (!pattr) { 187 NL_SET_ERR_MSG_MOD(extack, "Missing required TCA_PEDIT_PARMS or TCA_PEDIT_PARMS_EX pedit attribute"); 188 return -EINVAL; 189 } 190 191 parm = nla_data(pattr); 192 193 index = parm->index; 194 err = tcf_idr_check_alloc(tn, &index, a, bind); 195 if (!err) { 196 ret = tcf_idr_create_from_flags(tn, index, est, a, 197 &act_pedit_ops, bind, flags); 198 if (ret) { 199 tcf_idr_cleanup(tn, index); 200 return ret; 201 } 202 ret = ACT_P_CREATED; 203 } else if (err > 0) { 204 if (bind) 205 return ACT_P_BOUND; 206 if (!(flags & TCA_ACT_FLAGS_REPLACE)) { 207 ret = -EEXIST; 208 goto out_release; 209 } 210 } else { 211 return err; 212 } 213 214 if (!parm->nkeys) { 215 NL_SET_ERR_MSG_MOD(extack, "Pedit requires keys to be passed"); 216 ret = -EINVAL; 217 goto out_release; 218 } 219 ksize = parm->nkeys * sizeof(struct tc_pedit_key); 220 if (nla_len(pattr) < sizeof(*parm) + ksize) { 221 NL_SET_ERR_MSG_ATTR(extack, pattr, "Length of TCA_PEDIT_PARMS or TCA_PEDIT_PARMS_EX pedit attribute is invalid"); 222 ret = -EINVAL; 223 goto out_release; 224 } 225 226 nparms = kzalloc(sizeof(*nparms), GFP_KERNEL); 227 if (!nparms) { 228 ret = -ENOMEM; 229 goto out_release; 230 } 231 232 nparms->tcfp_keys_ex = 233 tcf_pedit_keys_ex_parse(tb[TCA_PEDIT_KEYS_EX], parm->nkeys, extack); 234 if (IS_ERR(nparms->tcfp_keys_ex)) { 235 ret = PTR_ERR(nparms->tcfp_keys_ex); 236 goto out_free; 237 } 238 239 err = tcf_action_check_ctrlact(parm->action, tp, &goto_ch, extack); 240 if (err < 0) { 241 ret = err; 242 goto out_free_ex; 243 } 244 245 nparms->tcfp_off_max_hint = 0; 246 nparms->tcfp_flags = parm->flags; 247 nparms->tcfp_nkeys = parm->nkeys; 248 249 nparms->tcfp_keys = kmemdup(parm->keys, ksize, GFP_KERNEL); 250 if (!nparms->tcfp_keys) { 251 ret = -ENOMEM; 252 goto put_chain; 253 } 254 255 for (i = 0; i < nparms->tcfp_nkeys; ++i) { 256 u32 offmask = nparms->tcfp_keys[i].offmask; 257 u32 cur = nparms->tcfp_keys[i].off; 258 259 /* The AT option can be added to static offsets in the datapath */ 260 if (!offmask && cur % 4) { 261 NL_SET_ERR_MSG_MOD(extack, "Offsets must be on 32bit boundaries"); 262 ret = -EINVAL; 263 goto out_free_keys; 264 } 265 266 /* sanitize the shift value for any later use */ 267 nparms->tcfp_keys[i].shift = min_t(size_t, 268 BITS_PER_TYPE(int) - 1, 269 nparms->tcfp_keys[i].shift); 270 271 /* The AT option can read a single byte, we can bound the actual 272 * value with uchar max. 273 */ 274 cur += (0xff & offmask) >> nparms->tcfp_keys[i].shift; 275 276 /* Each key touches 4 bytes starting from the computed offset */ 277 nparms->tcfp_off_max_hint = 278 max(nparms->tcfp_off_max_hint, cur + 4); 279 } 280 281 p = to_pedit(*a); 282 283 spin_lock_bh(&p->tcf_lock); 284 goto_ch = tcf_action_set_ctrlact(*a, parm->action, goto_ch); 285 oparms = rcu_replace_pointer(p->parms, nparms, 1); 286 spin_unlock_bh(&p->tcf_lock); 287 288 if (oparms) 289 call_rcu(&oparms->rcu, tcf_pedit_cleanup_rcu); 290 291 if (goto_ch) 292 tcf_chain_put_by_act(goto_ch); 293 294 return ret; 295 296 out_free_keys: 297 kfree(nparms->tcfp_keys); 298 put_chain: 299 if (goto_ch) 300 tcf_chain_put_by_act(goto_ch); 301 out_free_ex: 302 kfree(nparms->tcfp_keys_ex); 303 out_free: 304 kfree(nparms); 305 out_release: 306 tcf_idr_release(*a, bind); 307 return ret; 308 } 309 310 static void tcf_pedit_cleanup(struct tc_action *a) 311 { 312 struct tcf_pedit *p = to_pedit(a); 313 struct tcf_pedit_parms *parms; 314 315 parms = rcu_dereference_protected(p->parms, 1); 316 317 if (parms) 318 call_rcu(&parms->rcu, tcf_pedit_cleanup_rcu); 319 } 320 321 static bool offset_valid(struct sk_buff *skb, int offset) 322 { 323 if (offset > 0 && offset > skb->len) 324 return false; 325 326 if (offset < 0 && -offset > skb_headroom(skb)) 327 return false; 328 329 return true; 330 } 331 332 static int pedit_l4_skb_offset(struct sk_buff *skb, int *hoffset, const int header_type) 333 { 334 const int noff = skb_network_offset(skb); 335 int ret = -EINVAL; 336 struct iphdr _iph; 337 338 switch (skb->protocol) { 339 case htons(ETH_P_IP): { 340 const struct iphdr *iph = skb_header_pointer(skb, noff, sizeof(_iph), &_iph); 341 342 if (!iph) 343 goto out; 344 *hoffset = noff + iph->ihl * 4; 345 ret = 0; 346 break; 347 } 348 case htons(ETH_P_IPV6): 349 ret = ipv6_find_hdr(skb, hoffset, header_type, NULL, NULL) == header_type ? 0 : -EINVAL; 350 break; 351 } 352 out: 353 return ret; 354 } 355 356 static int pedit_skb_hdr_offset(struct sk_buff *skb, 357 enum pedit_header_type htype, int *hoffset) 358 { 359 int ret = -EINVAL; 360 /* 'htype' is validated in the netlink parsing */ 361 switch (htype) { 362 case TCA_PEDIT_KEY_EX_HDR_TYPE_ETH: 363 if (skb_mac_header_was_set(skb)) { 364 *hoffset = skb_mac_offset(skb); 365 ret = 0; 366 } 367 break; 368 case TCA_PEDIT_KEY_EX_HDR_TYPE_NETWORK: 369 case TCA_PEDIT_KEY_EX_HDR_TYPE_IP4: 370 case TCA_PEDIT_KEY_EX_HDR_TYPE_IP6: 371 *hoffset = skb_network_offset(skb); 372 ret = 0; 373 break; 374 case TCA_PEDIT_KEY_EX_HDR_TYPE_TCP: 375 ret = pedit_l4_skb_offset(skb, hoffset, IPPROTO_TCP); 376 break; 377 case TCA_PEDIT_KEY_EX_HDR_TYPE_UDP: 378 ret = pedit_l4_skb_offset(skb, hoffset, IPPROTO_UDP); 379 break; 380 default: 381 break; 382 } 383 return ret; 384 } 385 386 TC_INDIRECT_SCOPE int tcf_pedit_act(struct sk_buff *skb, 387 const struct tc_action *a, 388 struct tcf_result *res) 389 { 390 enum pedit_header_type htype = TCA_PEDIT_KEY_EX_HDR_TYPE_NETWORK; 391 enum pedit_cmd cmd = TCA_PEDIT_KEY_EX_CMD_SET; 392 struct tcf_pedit *p = to_pedit(a); 393 struct tcf_pedit_key_ex *tkey_ex; 394 struct tcf_pedit_parms *parms; 395 struct tc_pedit_key *tkey; 396 u32 max_offset; 397 int i; 398 399 parms = rcu_dereference_bh(p->parms); 400 401 max_offset = (skb_transport_header_was_set(skb) ? 402 skb_transport_offset(skb) : 403 skb_network_offset(skb)) + 404 parms->tcfp_off_max_hint; 405 if (skb_ensure_writable(skb, min(skb->len, max_offset))) 406 goto done; 407 408 tcf_lastuse_update(&p->tcf_tm); 409 tcf_action_update_bstats(&p->common, skb); 410 411 tkey = parms->tcfp_keys; 412 tkey_ex = parms->tcfp_keys_ex; 413 414 for (i = parms->tcfp_nkeys; i > 0; i--, tkey++) { 415 int offset = tkey->off; 416 int hoffset = 0; 417 u32 *ptr, hdata; 418 u32 val; 419 int rc; 420 421 if (tkey_ex) { 422 htype = tkey_ex->htype; 423 cmd = tkey_ex->cmd; 424 425 tkey_ex++; 426 } 427 428 rc = pedit_skb_hdr_offset(skb, htype, &hoffset); 429 if (rc) { 430 pr_info_ratelimited("tc action pedit unable to extract header offset for header type (0x%x)\n", htype); 431 goto bad; 432 } 433 434 if (tkey->offmask) { 435 u8 *d, _d; 436 437 if (!offset_valid(skb, hoffset + tkey->at)) { 438 pr_info_ratelimited("tc action pedit 'at' offset %d out of bounds\n", 439 hoffset + tkey->at); 440 goto bad; 441 } 442 d = skb_header_pointer(skb, hoffset + tkey->at, 443 sizeof(_d), &_d); 444 if (!d) 445 goto bad; 446 447 offset += (*d & tkey->offmask) >> tkey->shift; 448 if (offset % 4) { 449 pr_info_ratelimited("tc action pedit offset must be on 32 bit boundaries\n"); 450 goto bad; 451 } 452 } 453 454 if (!offset_valid(skb, hoffset + offset)) { 455 pr_info_ratelimited("tc action pedit offset %d out of bounds\n", hoffset + offset); 456 goto bad; 457 } 458 459 ptr = skb_header_pointer(skb, hoffset + offset, 460 sizeof(hdata), &hdata); 461 if (!ptr) 462 goto bad; 463 /* just do it, baby */ 464 switch (cmd) { 465 case TCA_PEDIT_KEY_EX_CMD_SET: 466 val = tkey->val; 467 break; 468 case TCA_PEDIT_KEY_EX_CMD_ADD: 469 val = (*ptr + tkey->val) & ~tkey->mask; 470 break; 471 default: 472 pr_info_ratelimited("tc action pedit bad command (%d)\n", cmd); 473 goto bad; 474 } 475 476 *ptr = ((*ptr & tkey->mask) ^ val); 477 if (ptr == &hdata) 478 skb_store_bits(skb, hoffset + offset, ptr, 4); 479 } 480 481 goto done; 482 483 bad: 484 tcf_action_inc_overlimit_qstats(&p->common); 485 done: 486 return p->tcf_action; 487 } 488 489 static void tcf_pedit_stats_update(struct tc_action *a, u64 bytes, u64 packets, 490 u64 drops, u64 lastuse, bool hw) 491 { 492 struct tcf_pedit *d = to_pedit(a); 493 struct tcf_t *tm = &d->tcf_tm; 494 495 tcf_action_update_stats(a, bytes, packets, drops, hw); 496 tm->lastuse = max_t(u64, tm->lastuse, lastuse); 497 } 498 499 static int tcf_pedit_dump(struct sk_buff *skb, struct tc_action *a, 500 int bind, int ref) 501 { 502 unsigned char *b = skb_tail_pointer(skb); 503 struct tcf_pedit *p = to_pedit(a); 504 struct tcf_pedit_parms *parms; 505 struct tc_pedit *opt; 506 struct tcf_t t; 507 int s; 508 509 spin_lock_bh(&p->tcf_lock); 510 parms = rcu_dereference_protected(p->parms, 1); 511 s = struct_size(opt, keys, parms->tcfp_nkeys); 512 513 opt = kzalloc(s, GFP_ATOMIC); 514 if (unlikely(!opt)) { 515 spin_unlock_bh(&p->tcf_lock); 516 return -ENOBUFS; 517 } 518 opt->nkeys = parms->tcfp_nkeys; 519 520 memcpy(opt->keys, parms->tcfp_keys, 521 flex_array_size(opt, keys, parms->tcfp_nkeys)); 522 opt->index = p->tcf_index; 523 opt->flags = parms->tcfp_flags; 524 opt->action = p->tcf_action; 525 opt->refcnt = refcount_read(&p->tcf_refcnt) - ref; 526 opt->bindcnt = atomic_read(&p->tcf_bindcnt) - bind; 527 528 if (parms->tcfp_keys_ex) { 529 if (tcf_pedit_key_ex_dump(skb, parms->tcfp_keys_ex, 530 parms->tcfp_nkeys)) 531 goto nla_put_failure; 532 533 if (nla_put(skb, TCA_PEDIT_PARMS_EX, s, opt)) 534 goto nla_put_failure; 535 } else { 536 if (nla_put(skb, TCA_PEDIT_PARMS, s, opt)) 537 goto nla_put_failure; 538 } 539 540 tcf_tm_dump(&t, &p->tcf_tm); 541 if (nla_put_64bit(skb, TCA_PEDIT_TM, sizeof(t), &t, TCA_PEDIT_PAD)) 542 goto nla_put_failure; 543 spin_unlock_bh(&p->tcf_lock); 544 545 kfree(opt); 546 return skb->len; 547 548 nla_put_failure: 549 spin_unlock_bh(&p->tcf_lock); 550 nlmsg_trim(skb, b); 551 kfree(opt); 552 return -1; 553 } 554 555 static int tcf_pedit_offload_act_setup(struct tc_action *act, void *entry_data, 556 u32 *index_inc, bool bind, 557 struct netlink_ext_ack *extack) 558 { 559 if (bind) { 560 struct flow_action_entry *entry = entry_data; 561 int k; 562 563 for (k = 0; k < tcf_pedit_nkeys(act); k++) { 564 switch (tcf_pedit_cmd(act, k)) { 565 case TCA_PEDIT_KEY_EX_CMD_SET: 566 entry->id = FLOW_ACTION_MANGLE; 567 break; 568 case TCA_PEDIT_KEY_EX_CMD_ADD: 569 entry->id = FLOW_ACTION_ADD; 570 break; 571 default: 572 NL_SET_ERR_MSG_MOD(extack, "Unsupported pedit command offload"); 573 return -EOPNOTSUPP; 574 } 575 entry->mangle.htype = tcf_pedit_htype(act, k); 576 entry->mangle.mask = tcf_pedit_mask(act, k); 577 entry->mangle.val = tcf_pedit_val(act, k); 578 entry->mangle.offset = tcf_pedit_offset(act, k); 579 entry->hw_stats = tc_act_hw_stats(act->hw_stats); 580 entry++; 581 } 582 *index_inc = k; 583 } else { 584 struct flow_offload_action *fl_action = entry_data; 585 u32 cmd = tcf_pedit_cmd(act, 0); 586 int k; 587 588 switch (cmd) { 589 case TCA_PEDIT_KEY_EX_CMD_SET: 590 fl_action->id = FLOW_ACTION_MANGLE; 591 break; 592 case TCA_PEDIT_KEY_EX_CMD_ADD: 593 fl_action->id = FLOW_ACTION_ADD; 594 break; 595 default: 596 NL_SET_ERR_MSG_MOD(extack, "Unsupported pedit command offload"); 597 return -EOPNOTSUPP; 598 } 599 600 for (k = 1; k < tcf_pedit_nkeys(act); k++) { 601 if (cmd != tcf_pedit_cmd(act, k)) { 602 NL_SET_ERR_MSG_MOD(extack, "Unsupported pedit command offload"); 603 return -EOPNOTSUPP; 604 } 605 } 606 } 607 608 return 0; 609 } 610 611 static struct tc_action_ops act_pedit_ops = { 612 .kind = "pedit", 613 .id = TCA_ID_PEDIT, 614 .owner = THIS_MODULE, 615 .act = tcf_pedit_act, 616 .stats_update = tcf_pedit_stats_update, 617 .dump = tcf_pedit_dump, 618 .cleanup = tcf_pedit_cleanup, 619 .init = tcf_pedit_init, 620 .offload_act_setup = tcf_pedit_offload_act_setup, 621 .size = sizeof(struct tcf_pedit), 622 }; 623 MODULE_ALIAS_NET_ACT("pedit"); 624 625 static __net_init int pedit_init_net(struct net *net) 626 { 627 struct tc_action_net *tn = net_generic(net, act_pedit_ops.net_id); 628 629 return tc_action_net_init(net, tn, &act_pedit_ops); 630 } 631 632 static void __net_exit pedit_exit_net(struct list_head *net_list) 633 { 634 tc_action_net_exit(net_list, act_pedit_ops.net_id); 635 } 636 637 static struct pernet_operations pedit_net_ops = { 638 .init = pedit_init_net, 639 .exit_batch = pedit_exit_net, 640 .id = &act_pedit_ops.net_id, 641 .size = sizeof(struct tc_action_net), 642 }; 643 644 MODULE_AUTHOR("Jamal Hadi Salim(2002-4)"); 645 MODULE_DESCRIPTION("Generic Packet Editor actions"); 646 MODULE_LICENSE("GPL"); 647 648 static int __init pedit_init_module(void) 649 { 650 return tcf_register_action(&act_pedit_ops, &pedit_net_ops); 651 } 652 653 static void __exit pedit_cleanup_module(void) 654 { 655 tcf_unregister_action(&act_pedit_ops, &pedit_net_ops); 656 } 657 658 module_init(pedit_init_module); 659 module_exit(pedit_cleanup_module); 660