1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * net/sched/act_api.c Packet action API. 4 * 5 * Author: Jamal Hadi Salim 6 */ 7 8 #include <linux/types.h> 9 #include <linux/kernel.h> 10 #include <linux/string.h> 11 #include <linux/errno.h> 12 #include <linux/slab.h> 13 #include <linux/skbuff.h> 14 #include <linux/init.h> 15 #include <linux/kmod.h> 16 #include <linux/err.h> 17 #include <linux/module.h> 18 #include <net/net_namespace.h> 19 #include <net/sock.h> 20 #include <net/sch_generic.h> 21 #include <net/pkt_cls.h> 22 #include <net/tc_act/tc_pedit.h> 23 #include <net/act_api.h> 24 #include <net/netlink.h> 25 #include <net/flow_offload.h> 26 #include <net/tc_wrapper.h> 27 28 #ifdef CONFIG_INET 29 DEFINE_STATIC_KEY_FALSE(tcf_frag_xmit_count); 30 EXPORT_SYMBOL_GPL(tcf_frag_xmit_count); 31 #endif 32 33 int tcf_dev_queue_xmit(struct sk_buff *skb, int (*xmit)(struct sk_buff *skb)) 34 { 35 #ifdef CONFIG_INET 36 if (static_branch_unlikely(&tcf_frag_xmit_count)) 37 return sch_frag_xmit_hook(skb, xmit); 38 #endif 39 40 return xmit(skb); 41 } 42 EXPORT_SYMBOL_GPL(tcf_dev_queue_xmit); 43 44 static void tcf_action_goto_chain_exec(const struct tcf_chain *chain, 45 struct tcf_result *res) 46 { 47 res->goto_tp = rcu_dereference_bh(chain->filter_chain); 48 } 49 50 static void tcf_free_cookie_rcu(struct rcu_head *p) 51 { 52 struct tc_cookie *cookie = container_of(p, struct tc_cookie, rcu); 53 54 kfree(cookie->data); 55 kfree(cookie); 56 } 57 58 static void tcf_set_action_cookie(struct tc_cookie __rcu **old_cookie, 59 struct tc_cookie *new_cookie) 60 { 61 struct tc_cookie *old; 62 63 old = unrcu_pointer(xchg(old_cookie, RCU_INITIALIZER(new_cookie))); 64 if (old) 65 call_rcu(&old->rcu, tcf_free_cookie_rcu); 66 } 67 68 int tcf_action_check_ctrlact(int action, struct tcf_proto *tp, 69 struct tcf_chain **newchain, 70 struct netlink_ext_ack *extack) 71 { 72 int opcode = TC_ACT_EXT_OPCODE(action), ret = -EINVAL; 73 u32 chain_index; 74 75 if (!opcode) 76 ret = action > TC_ACT_VALUE_MAX ? -EINVAL : 0; 77 else if (opcode <= TC_ACT_EXT_OPCODE_MAX || action == TC_ACT_UNSPEC) 78 ret = 0; 79 if (ret) { 80 NL_SET_ERR_MSG(extack, "invalid control action"); 81 goto end; 82 } 83 84 if (TC_ACT_EXT_CMP(action, TC_ACT_GOTO_CHAIN)) { 85 chain_index = action & TC_ACT_EXT_VAL_MASK; 86 if (!tp || !newchain) { 87 ret = -EINVAL; 88 NL_SET_ERR_MSG(extack, 89 "can't goto NULL proto/chain"); 90 goto end; 91 } 92 *newchain = tcf_chain_get_by_act(tp->chain->block, chain_index); 93 if (!*newchain) { 94 ret = -ENOMEM; 95 NL_SET_ERR_MSG(extack, 96 "can't allocate goto_chain"); 97 } 98 } 99 end: 100 return ret; 101 } 102 EXPORT_SYMBOL(tcf_action_check_ctrlact); 103 104 struct tcf_chain *tcf_action_set_ctrlact(struct tc_action *a, int action, 105 struct tcf_chain *goto_chain) 106 { 107 a->tcfa_action = action; 108 goto_chain = rcu_replace_pointer(a->goto_chain, goto_chain, 1); 109 return goto_chain; 110 } 111 EXPORT_SYMBOL(tcf_action_set_ctrlact); 112 113 static void free_tcf(struct tc_action *p) 114 { 115 struct tcf_chain *chain = rcu_dereference_protected(p->goto_chain, 1); 116 117 free_percpu(p->cpu_bstats); 118 free_percpu(p->cpu_bstats_hw); 119 free_percpu(p->cpu_qstats); 120 121 tcf_set_action_cookie(&p->user_cookie, NULL); 122 if (chain) 123 tcf_chain_put_by_act(chain); 124 125 kfree_rcu(p, tcfa_rcu); 126 } 127 128 static void offload_action_hw_count_set(struct tc_action *act, 129 u32 hw_count) 130 { 131 act->in_hw_count = hw_count; 132 } 133 134 static void offload_action_hw_count_inc(struct tc_action *act, 135 u32 hw_count) 136 { 137 act->in_hw_count += hw_count; 138 } 139 140 static void offload_action_hw_count_dec(struct tc_action *act, 141 u32 hw_count) 142 { 143 act->in_hw_count = act->in_hw_count > hw_count ? 144 act->in_hw_count - hw_count : 0; 145 } 146 147 static unsigned int tcf_offload_act_num_actions_single(struct tc_action *act) 148 { 149 unsigned int count; 150 151 if (is_tcf_pedit(act)) { 152 spin_lock_bh(&act->tcfa_lock); 153 count = tcf_pedit_nkeys_locked(act); 154 spin_unlock_bh(&act->tcfa_lock); 155 return count; 156 } 157 return 1; 158 } 159 160 static bool tc_act_skip_hw(u32 flags) 161 { 162 return (flags & TCA_ACT_FLAGS_SKIP_HW) ? true : false; 163 } 164 165 static bool tc_act_skip_sw(u32 flags) 166 { 167 return (flags & TCA_ACT_FLAGS_SKIP_SW) ? true : false; 168 } 169 170 /* SKIP_HW and SKIP_SW are mutually exclusive flags. */ 171 static bool tc_act_flags_valid(u32 flags) 172 { 173 flags &= TCA_ACT_FLAGS_SKIP_HW | TCA_ACT_FLAGS_SKIP_SW; 174 175 return flags ^ (TCA_ACT_FLAGS_SKIP_HW | TCA_ACT_FLAGS_SKIP_SW); 176 } 177 178 static int offload_action_init(struct flow_offload_action *fl_action, 179 struct tc_action *act, 180 enum offload_act_command cmd, 181 struct netlink_ext_ack *extack) 182 { 183 int err; 184 185 fl_action->extack = extack; 186 fl_action->command = cmd; 187 fl_action->index = act->tcfa_index; 188 fl_action->cookie = (unsigned long)act; 189 190 if (act->ops->offload_act_setup) { 191 spin_lock_bh(&act->tcfa_lock); 192 err = act->ops->offload_act_setup(act, fl_action, NULL, 193 false, extack); 194 spin_unlock_bh(&act->tcfa_lock); 195 return err; 196 } 197 198 return -EOPNOTSUPP; 199 } 200 201 static int tcf_action_offload_cmd_ex(struct flow_offload_action *fl_act, 202 u32 *hw_count) 203 { 204 int err; 205 206 err = flow_indr_dev_setup_offload(NULL, NULL, TC_SETUP_ACT, 207 fl_act, NULL, NULL); 208 if (err < 0) 209 return err; 210 211 if (hw_count) 212 *hw_count = err; 213 214 return 0; 215 } 216 217 static int tcf_action_offload_cmd_cb_ex(struct flow_offload_action *fl_act, 218 u32 *hw_count, 219 flow_indr_block_bind_cb_t *cb, 220 void *cb_priv) 221 { 222 int err; 223 224 err = cb(NULL, NULL, cb_priv, TC_SETUP_ACT, NULL, fl_act, NULL); 225 if (err < 0) 226 return err; 227 228 if (hw_count) 229 *hw_count = 1; 230 231 return 0; 232 } 233 234 static int tcf_action_offload_cmd(struct flow_offload_action *fl_act, 235 u32 *hw_count, 236 flow_indr_block_bind_cb_t *cb, 237 void *cb_priv) 238 { 239 return cb ? tcf_action_offload_cmd_cb_ex(fl_act, hw_count, 240 cb, cb_priv) : 241 tcf_action_offload_cmd_ex(fl_act, hw_count); 242 } 243 244 static int tcf_action_offload_add_ex(struct tc_action *action, 245 struct netlink_ext_ack *extack, 246 flow_indr_block_bind_cb_t *cb, 247 void *cb_priv) 248 { 249 bool skip_sw = tc_act_skip_sw(action->tcfa_flags); 250 struct tc_action *actions[TCA_ACT_MAX_PRIO] = { 251 [0] = action, 252 }; 253 struct flow_offload_action *fl_action; 254 u32 in_hw_count = 0; 255 int num, err = 0; 256 257 if (tc_act_skip_hw(action->tcfa_flags)) 258 return 0; 259 260 num = tcf_offload_act_num_actions_single(action); 261 fl_action = offload_action_alloc(num); 262 if (!fl_action) 263 return -ENOMEM; 264 265 err = offload_action_init(fl_action, action, FLOW_ACT_REPLACE, extack); 266 if (err) 267 goto fl_err; 268 269 err = tc_setup_action(&fl_action->action, actions, 0, extack); 270 if (err) { 271 NL_SET_ERR_MSG_MOD(extack, 272 "Failed to setup tc actions for offload"); 273 goto fl_err; 274 } 275 276 err = tcf_action_offload_cmd(fl_action, &in_hw_count, cb, cb_priv); 277 if (!err) 278 cb ? offload_action_hw_count_inc(action, in_hw_count) : 279 offload_action_hw_count_set(action, in_hw_count); 280 281 if (skip_sw && !tc_act_in_hw(action)) 282 err = -EINVAL; 283 284 tc_cleanup_offload_action(&fl_action->action); 285 286 fl_err: 287 kfree(fl_action); 288 289 return err; 290 } 291 292 /* offload the tc action after it is inserted */ 293 static int tcf_action_offload_add(struct tc_action *action, 294 struct netlink_ext_ack *extack) 295 { 296 return tcf_action_offload_add_ex(action, extack, NULL, NULL); 297 } 298 299 int tcf_action_update_hw_stats(struct tc_action *action) 300 { 301 struct flow_offload_action fl_act = {}; 302 int err; 303 304 err = offload_action_init(&fl_act, action, FLOW_ACT_STATS, NULL); 305 if (err) 306 return err; 307 308 err = tcf_action_offload_cmd(&fl_act, NULL, NULL, NULL); 309 if (!err) { 310 preempt_disable(); 311 tcf_action_stats_update(action, fl_act.stats.bytes, 312 fl_act.stats.pkts, 313 fl_act.stats.drops, 314 fl_act.stats.lastused, 315 true); 316 preempt_enable(); 317 action->used_hw_stats = fl_act.stats.used_hw_stats; 318 action->used_hw_stats_valid = true; 319 } else { 320 return -EOPNOTSUPP; 321 } 322 323 return 0; 324 } 325 EXPORT_SYMBOL(tcf_action_update_hw_stats); 326 327 static int tcf_action_offload_del_ex(struct tc_action *action, 328 flow_indr_block_bind_cb_t *cb, 329 void *cb_priv) 330 { 331 struct flow_offload_action fl_act = {}; 332 u32 in_hw_count = 0; 333 int err = 0; 334 335 if (!tc_act_in_hw(action)) 336 return 0; 337 338 err = offload_action_init(&fl_act, action, FLOW_ACT_DESTROY, NULL); 339 if (err) 340 return err; 341 342 err = tcf_action_offload_cmd(&fl_act, &in_hw_count, cb, cb_priv); 343 if (err < 0) 344 return err; 345 346 if (!cb && action->in_hw_count != in_hw_count) 347 return -EINVAL; 348 349 /* do not need to update hw state when deleting action */ 350 if (cb && in_hw_count) 351 offload_action_hw_count_dec(action, in_hw_count); 352 353 return 0; 354 } 355 356 static int tcf_action_offload_del(struct tc_action *action) 357 { 358 return tcf_action_offload_del_ex(action, NULL, NULL); 359 } 360 361 static void tcf_action_cleanup(struct tc_action *p) 362 { 363 tcf_action_offload_del(p); 364 if (p->ops->cleanup) 365 p->ops->cleanup(p); 366 367 gen_kill_estimator(&p->tcfa_rate_est); 368 free_tcf(p); 369 } 370 371 static int __tcf_action_put(struct tc_action *p, bool bind) 372 { 373 struct tcf_idrinfo *idrinfo = p->idrinfo; 374 375 if (refcount_dec_and_mutex_lock(&p->tcfa_refcnt, &idrinfo->lock)) { 376 if (bind) 377 atomic_dec(&p->tcfa_bindcnt); 378 idr_remove(&idrinfo->action_idr, p->tcfa_index); 379 mutex_unlock(&idrinfo->lock); 380 381 tcf_action_cleanup(p); 382 return 1; 383 } 384 385 if (bind) 386 atomic_dec(&p->tcfa_bindcnt); 387 388 return 0; 389 } 390 391 static int __tcf_idr_release(struct tc_action *p, bool bind, bool strict) 392 { 393 int ret = 0; 394 395 /* Release with strict==1 and bind==0 is only called through act API 396 * interface (classifiers always bind). Only case when action with 397 * positive reference count and zero bind count can exist is when it was 398 * also created with act API (unbinding last classifier will destroy the 399 * action if it was created by classifier). So only case when bind count 400 * can be changed after initial check is when unbound action is 401 * destroyed by act API while classifier binds to action with same id 402 * concurrently. This result either creation of new action(same behavior 403 * as before), or reusing existing action if concurrent process 404 * increments reference count before action is deleted. Both scenarios 405 * are acceptable. 406 */ 407 if (p) { 408 if (!bind && strict && atomic_read(&p->tcfa_bindcnt) > 0) 409 return -EPERM; 410 411 if (__tcf_action_put(p, bind)) 412 ret = ACT_P_DELETED; 413 } 414 415 return ret; 416 } 417 418 int tcf_idr_release(struct tc_action *a, bool bind) 419 { 420 const struct tc_action_ops *ops = a->ops; 421 int ret; 422 423 ret = __tcf_idr_release(a, bind, false); 424 if (ret == ACT_P_DELETED) 425 module_put(ops->owner); 426 return ret; 427 } 428 EXPORT_SYMBOL(tcf_idr_release); 429 430 static size_t tcf_action_shared_attrs_size(const struct tc_action *act) 431 { 432 struct tc_cookie *user_cookie; 433 u32 cookie_len = 0; 434 435 rcu_read_lock(); 436 user_cookie = rcu_dereference(act->user_cookie); 437 438 if (user_cookie) 439 cookie_len = nla_total_size(user_cookie->len); 440 rcu_read_unlock(); 441 442 return nla_total_size(0) /* action number nested */ 443 + nla_total_size(IFNAMSIZ) /* TCA_ACT_KIND */ 444 + cookie_len /* TCA_ACT_COOKIE */ 445 + nla_total_size(sizeof(struct nla_bitfield32)) /* TCA_ACT_HW_STATS */ 446 + nla_total_size(0) /* TCA_ACT_STATS nested */ 447 + nla_total_size(sizeof(struct nla_bitfield32)) /* TCA_ACT_FLAGS */ 448 /* TCA_STATS_BASIC */ 449 + nla_total_size_64bit(sizeof(struct gnet_stats_basic)) 450 /* TCA_STATS_PKT64 */ 451 + nla_total_size_64bit(sizeof(u64)) 452 /* TCA_STATS_QUEUE */ 453 + nla_total_size_64bit(sizeof(struct gnet_stats_queue)) 454 + nla_total_size(0) /* TCA_ACT_OPTIONS nested */ 455 + nla_total_size(sizeof(struct tcf_t)); /* TCA_GACT_TM */ 456 } 457 458 static size_t tcf_action_full_attrs_size(size_t sz) 459 { 460 return NLMSG_HDRLEN /* struct nlmsghdr */ 461 + sizeof(struct tcamsg) 462 + nla_total_size(0) /* TCA_ACT_TAB nested */ 463 + sz; 464 } 465 466 static size_t tcf_action_fill_size(const struct tc_action *act) 467 { 468 size_t sz = tcf_action_shared_attrs_size(act); 469 470 if (act->ops->get_fill_size) 471 return act->ops->get_fill_size(act) + sz; 472 return sz; 473 } 474 475 static int 476 tcf_action_dump_terse(struct sk_buff *skb, struct tc_action *a, bool from_act) 477 { 478 unsigned char *b = skb_tail_pointer(skb); 479 struct tc_cookie *cookie; 480 481 if (nla_put_string(skb, TCA_ACT_KIND, a->ops->kind)) 482 goto nla_put_failure; 483 if (tcf_action_copy_stats(skb, a, 0)) 484 goto nla_put_failure; 485 if (from_act && nla_put_u32(skb, TCA_ACT_INDEX, a->tcfa_index)) 486 goto nla_put_failure; 487 488 rcu_read_lock(); 489 cookie = rcu_dereference(a->user_cookie); 490 if (cookie) { 491 if (nla_put(skb, TCA_ACT_COOKIE, cookie->len, cookie->data)) { 492 rcu_read_unlock(); 493 goto nla_put_failure; 494 } 495 } 496 rcu_read_unlock(); 497 498 return 0; 499 500 nla_put_failure: 501 nlmsg_trim(skb, b); 502 return -1; 503 } 504 505 static int 506 tcf_action_dump_1(struct sk_buff *skb, struct tc_action *a, int bind, int ref) 507 { 508 unsigned char *b = skb_tail_pointer(skb); 509 struct nlattr *nest; 510 int err = -EINVAL; 511 u32 flags; 512 513 if (tcf_action_dump_terse(skb, a, false)) 514 goto nla_put_failure; 515 516 if (a->hw_stats != TCA_ACT_HW_STATS_ANY && 517 nla_put_bitfield32(skb, TCA_ACT_HW_STATS, 518 a->hw_stats, TCA_ACT_HW_STATS_ANY)) 519 goto nla_put_failure; 520 521 if (a->used_hw_stats_valid && 522 nla_put_bitfield32(skb, TCA_ACT_USED_HW_STATS, 523 a->used_hw_stats, TCA_ACT_HW_STATS_ANY)) 524 goto nla_put_failure; 525 526 flags = a->tcfa_flags & TCA_ACT_FLAGS_USER_MASK; 527 if (flags && 528 nla_put_bitfield32(skb, TCA_ACT_FLAGS, 529 flags, flags)) 530 goto nla_put_failure; 531 532 if (nla_put_u32(skb, TCA_ACT_IN_HW_COUNT, a->in_hw_count)) 533 goto nla_put_failure; 534 535 nest = nla_nest_start_noflag(skb, TCA_ACT_OPTIONS); 536 if (nest == NULL) 537 goto nla_put_failure; 538 err = tcf_action_dump_old(skb, a, bind, ref); 539 if (err > 0) { 540 nla_nest_end(skb, nest); 541 return err; 542 } 543 544 nla_put_failure: 545 nlmsg_trim(skb, b); 546 return -1; 547 } 548 549 static int tcf_dump_walker(struct tcf_idrinfo *idrinfo, struct sk_buff *skb, 550 struct netlink_callback *cb) 551 { 552 int err = 0, index = -1, s_i = 0, n_i = 0; 553 u32 act_flags = cb->args[2]; 554 unsigned long jiffy_since = cb->args[3]; 555 struct nlattr *nest; 556 struct idr *idr = &idrinfo->action_idr; 557 struct tc_action *p; 558 unsigned long id = 1; 559 unsigned long tmp; 560 561 mutex_lock(&idrinfo->lock); 562 563 s_i = cb->args[0]; 564 565 idr_for_each_entry_ul(idr, p, tmp, id) { 566 index++; 567 if (index < s_i) 568 continue; 569 if (IS_ERR(p)) 570 continue; 571 572 if (jiffy_since && 573 time_after(jiffy_since, 574 (unsigned long)p->tcfa_tm.lastuse)) 575 continue; 576 577 tcf_action_update_hw_stats(p); 578 579 nest = nla_nest_start_noflag(skb, n_i); 580 if (!nest) { 581 index--; 582 goto nla_put_failure; 583 } 584 err = (act_flags & TCA_ACT_FLAG_TERSE_DUMP) ? 585 tcf_action_dump_terse(skb, p, true) : 586 tcf_action_dump_1(skb, p, 0, 0); 587 if (err < 0) { 588 index--; 589 nlmsg_trim(skb, nest); 590 goto done; 591 } 592 nla_nest_end(skb, nest); 593 n_i++; 594 if (!(act_flags & TCA_ACT_FLAG_LARGE_DUMP_ON) && 595 n_i >= TCA_ACT_MAX_PRIO) 596 goto done; 597 } 598 done: 599 if (index >= 0) 600 cb->args[0] = index + 1; 601 602 mutex_unlock(&idrinfo->lock); 603 if (n_i) { 604 if (act_flags & TCA_ACT_FLAG_LARGE_DUMP_ON) 605 cb->args[1] = n_i; 606 } 607 return n_i; 608 609 nla_put_failure: 610 nla_nest_cancel(skb, nest); 611 goto done; 612 } 613 614 static int tcf_idr_release_unsafe(struct tc_action *p) 615 { 616 if (atomic_read(&p->tcfa_bindcnt) > 0) 617 return -EPERM; 618 619 if (refcount_dec_and_test(&p->tcfa_refcnt)) { 620 idr_remove(&p->idrinfo->action_idr, p->tcfa_index); 621 tcf_action_cleanup(p); 622 return ACT_P_DELETED; 623 } 624 625 return 0; 626 } 627 628 static int tcf_del_walker(struct tcf_idrinfo *idrinfo, struct sk_buff *skb, 629 const struct tc_action_ops *ops, 630 struct netlink_ext_ack *extack) 631 { 632 struct nlattr *nest; 633 int n_i = 0; 634 int ret = -EINVAL; 635 struct idr *idr = &idrinfo->action_idr; 636 struct tc_action *p; 637 unsigned long id = 1; 638 unsigned long tmp; 639 640 nest = nla_nest_start_noflag(skb, 0); 641 if (nest == NULL) 642 goto nla_put_failure; 643 if (nla_put_string(skb, TCA_ACT_KIND, ops->kind)) 644 goto nla_put_failure; 645 646 ret = 0; 647 mutex_lock(&idrinfo->lock); 648 idr_for_each_entry_ul(idr, p, tmp, id) { 649 if (IS_ERR(p)) 650 continue; 651 ret = tcf_idr_release_unsafe(p); 652 if (ret == ACT_P_DELETED) 653 module_put(ops->owner); 654 else if (ret < 0) 655 break; 656 n_i++; 657 } 658 mutex_unlock(&idrinfo->lock); 659 if (ret < 0) { 660 if (n_i) 661 NL_SET_ERR_MSG(extack, "Unable to flush all TC actions"); 662 else 663 goto nla_put_failure; 664 } 665 666 ret = nla_put_u32(skb, TCA_FCNT, n_i); 667 if (ret) 668 goto nla_put_failure; 669 nla_nest_end(skb, nest); 670 671 return n_i; 672 nla_put_failure: 673 nla_nest_cancel(skb, nest); 674 return ret; 675 } 676 677 int tcf_generic_walker(struct tc_action_net *tn, struct sk_buff *skb, 678 struct netlink_callback *cb, int type, 679 const struct tc_action_ops *ops, 680 struct netlink_ext_ack *extack) 681 { 682 struct tcf_idrinfo *idrinfo = tn->idrinfo; 683 684 if (type == RTM_DELACTION) { 685 return tcf_del_walker(idrinfo, skb, ops, extack); 686 } else if (type == RTM_GETACTION) { 687 return tcf_dump_walker(idrinfo, skb, cb); 688 } else { 689 WARN(1, "tcf_generic_walker: unknown command %d\n", type); 690 NL_SET_ERR_MSG(extack, "tcf_generic_walker: unknown command"); 691 return -EINVAL; 692 } 693 } 694 EXPORT_SYMBOL(tcf_generic_walker); 695 696 int tcf_idr_search(struct tc_action_net *tn, struct tc_action **a, u32 index) 697 { 698 struct tcf_idrinfo *idrinfo = tn->idrinfo; 699 struct tc_action *p; 700 701 mutex_lock(&idrinfo->lock); 702 p = idr_find(&idrinfo->action_idr, index); 703 if (IS_ERR(p)) 704 p = NULL; 705 else if (p) 706 refcount_inc(&p->tcfa_refcnt); 707 mutex_unlock(&idrinfo->lock); 708 709 if (p) { 710 *a = p; 711 return true; 712 } 713 return false; 714 } 715 EXPORT_SYMBOL(tcf_idr_search); 716 717 static int __tcf_generic_walker(struct net *net, struct sk_buff *skb, 718 struct netlink_callback *cb, int type, 719 const struct tc_action_ops *ops, 720 struct netlink_ext_ack *extack) 721 { 722 struct tc_action_net *tn = net_generic(net, ops->net_id); 723 724 if (unlikely(ops->walk)) 725 return ops->walk(net, skb, cb, type, ops, extack); 726 727 return tcf_generic_walker(tn, skb, cb, type, ops, extack); 728 } 729 730 static int __tcf_idr_search(struct net *net, 731 const struct tc_action_ops *ops, 732 struct tc_action **a, u32 index) 733 { 734 struct tc_action_net *tn = net_generic(net, ops->net_id); 735 736 if (unlikely(ops->lookup)) 737 return ops->lookup(net, a, index); 738 739 return tcf_idr_search(tn, a, index); 740 } 741 742 static int tcf_idr_delete_index(struct tcf_idrinfo *idrinfo, u32 index) 743 { 744 struct tc_action *p; 745 int ret = 0; 746 747 mutex_lock(&idrinfo->lock); 748 p = idr_find(&idrinfo->action_idr, index); 749 if (!p) { 750 mutex_unlock(&idrinfo->lock); 751 return -ENOENT; 752 } 753 754 if (!atomic_read(&p->tcfa_bindcnt)) { 755 if (refcount_dec_and_test(&p->tcfa_refcnt)) { 756 struct module *owner = p->ops->owner; 757 758 WARN_ON(p != idr_remove(&idrinfo->action_idr, 759 p->tcfa_index)); 760 mutex_unlock(&idrinfo->lock); 761 762 tcf_action_cleanup(p); 763 module_put(owner); 764 return 0; 765 } 766 ret = 0; 767 } else { 768 ret = -EPERM; 769 } 770 771 mutex_unlock(&idrinfo->lock); 772 return ret; 773 } 774 775 int tcf_idr_create(struct tc_action_net *tn, u32 index, struct nlattr *est, 776 struct tc_action **a, const struct tc_action_ops *ops, 777 int bind, bool cpustats, u32 flags) 778 { 779 struct tc_action *p = kzalloc(ops->size, GFP_KERNEL); 780 struct tcf_idrinfo *idrinfo = tn->idrinfo; 781 int err = -ENOMEM; 782 783 if (unlikely(!p)) 784 return -ENOMEM; 785 refcount_set(&p->tcfa_refcnt, 1); 786 if (bind) 787 atomic_set(&p->tcfa_bindcnt, 1); 788 789 if (cpustats) { 790 p->cpu_bstats = netdev_alloc_pcpu_stats(struct gnet_stats_basic_sync); 791 if (!p->cpu_bstats) 792 goto err1; 793 p->cpu_bstats_hw = netdev_alloc_pcpu_stats(struct gnet_stats_basic_sync); 794 if (!p->cpu_bstats_hw) 795 goto err2; 796 p->cpu_qstats = alloc_percpu(struct gnet_stats_queue); 797 if (!p->cpu_qstats) 798 goto err3; 799 } 800 gnet_stats_basic_sync_init(&p->tcfa_bstats); 801 gnet_stats_basic_sync_init(&p->tcfa_bstats_hw); 802 spin_lock_init(&p->tcfa_lock); 803 p->tcfa_index = index; 804 p->tcfa_tm.install = jiffies; 805 p->tcfa_tm.lastuse = jiffies; 806 p->tcfa_tm.firstuse = 0; 807 p->tcfa_flags = flags; 808 if (est) { 809 err = gen_new_estimator(&p->tcfa_bstats, p->cpu_bstats, 810 &p->tcfa_rate_est, 811 &p->tcfa_lock, false, est); 812 if (err) 813 goto err4; 814 } 815 816 p->idrinfo = idrinfo; 817 __module_get(ops->owner); 818 p->ops = ops; 819 *a = p; 820 return 0; 821 err4: 822 free_percpu(p->cpu_qstats); 823 err3: 824 free_percpu(p->cpu_bstats_hw); 825 err2: 826 free_percpu(p->cpu_bstats); 827 err1: 828 kfree(p); 829 return err; 830 } 831 EXPORT_SYMBOL(tcf_idr_create); 832 833 int tcf_idr_create_from_flags(struct tc_action_net *tn, u32 index, 834 struct nlattr *est, struct tc_action **a, 835 const struct tc_action_ops *ops, int bind, 836 u32 flags) 837 { 838 /* Set cpustats according to actions flags. */ 839 return tcf_idr_create(tn, index, est, a, ops, bind, 840 !(flags & TCA_ACT_FLAGS_NO_PERCPU_STATS), flags); 841 } 842 EXPORT_SYMBOL(tcf_idr_create_from_flags); 843 844 /* Cleanup idr index that was allocated but not initialized. */ 845 846 void tcf_idr_cleanup(struct tc_action_net *tn, u32 index) 847 { 848 struct tcf_idrinfo *idrinfo = tn->idrinfo; 849 850 mutex_lock(&idrinfo->lock); 851 /* Remove ERR_PTR(-EBUSY) allocated by tcf_idr_check_alloc */ 852 WARN_ON(!IS_ERR(idr_remove(&idrinfo->action_idr, index))); 853 mutex_unlock(&idrinfo->lock); 854 } 855 EXPORT_SYMBOL(tcf_idr_cleanup); 856 857 /* Check if action with specified index exists. If actions is found, increments 858 * its reference and bind counters, and return 1. Otherwise insert temporary 859 * error pointer (to prevent concurrent users from inserting actions with same 860 * index) and return 0. 861 * 862 * May return -EAGAIN for binding actions in case of a parallel add/delete on 863 * the requested index. 864 */ 865 866 int tcf_idr_check_alloc(struct tc_action_net *tn, u32 *index, 867 struct tc_action **a, int bind) 868 { 869 struct tcf_idrinfo *idrinfo = tn->idrinfo; 870 struct tc_action *p; 871 int ret; 872 u32 max; 873 874 if (*index) { 875 rcu_read_lock(); 876 p = idr_find(&idrinfo->action_idr, *index); 877 878 if (IS_ERR(p)) { 879 /* This means that another process allocated 880 * index but did not assign the pointer yet. 881 */ 882 rcu_read_unlock(); 883 return -EAGAIN; 884 } 885 886 if (!p) { 887 /* Empty slot, try to allocate it */ 888 max = *index; 889 rcu_read_unlock(); 890 goto new; 891 } 892 893 if (!refcount_inc_not_zero(&p->tcfa_refcnt)) { 894 /* Action was deleted in parallel */ 895 rcu_read_unlock(); 896 return -EAGAIN; 897 } 898 899 if (bind) 900 atomic_inc(&p->tcfa_bindcnt); 901 *a = p; 902 903 rcu_read_unlock(); 904 905 return 1; 906 } else { 907 /* Find a slot */ 908 *index = 1; 909 max = UINT_MAX; 910 } 911 912 new: 913 *a = NULL; 914 915 mutex_lock(&idrinfo->lock); 916 ret = idr_alloc_u32(&idrinfo->action_idr, ERR_PTR(-EBUSY), index, max, 917 GFP_KERNEL); 918 mutex_unlock(&idrinfo->lock); 919 920 /* N binds raced for action allocation, 921 * retry for all the ones that failed. 922 */ 923 if (ret == -ENOSPC && *index == max) 924 ret = -EAGAIN; 925 926 return ret; 927 } 928 EXPORT_SYMBOL(tcf_idr_check_alloc); 929 930 void tcf_idrinfo_destroy(const struct tc_action_ops *ops, 931 struct tcf_idrinfo *idrinfo) 932 { 933 struct idr *idr = &idrinfo->action_idr; 934 bool mutex_taken = false; 935 struct tc_action *p; 936 unsigned long id = 1; 937 unsigned long tmp; 938 int ret; 939 940 idr_for_each_entry_ul(idr, p, tmp, id) { 941 if (IS_ERR(p)) 942 continue; 943 if (tc_act_in_hw(p) && !mutex_taken) { 944 rtnl_lock(); 945 mutex_taken = true; 946 } 947 ret = __tcf_idr_release(p, false, true); 948 if (ret == ACT_P_DELETED) 949 module_put(ops->owner); 950 else if (ret < 0) 951 return; 952 } 953 if (mutex_taken) 954 rtnl_unlock(); 955 idr_destroy(&idrinfo->action_idr); 956 } 957 EXPORT_SYMBOL(tcf_idrinfo_destroy); 958 959 static LIST_HEAD(act_base); 960 static DEFINE_RWLOCK(act_mod_lock); 961 /* since act ops id is stored in pernet subsystem list, 962 * then there is no way to walk through only all the action 963 * subsystem, so we keep tc action pernet ops id for 964 * reoffload to walk through. 965 */ 966 static LIST_HEAD(act_pernet_id_list); 967 static DEFINE_MUTEX(act_id_mutex); 968 struct tc_act_pernet_id { 969 struct list_head list; 970 unsigned int id; 971 }; 972 973 static int tcf_pernet_add_id_list(unsigned int id) 974 { 975 struct tc_act_pernet_id *id_ptr; 976 int ret = 0; 977 978 mutex_lock(&act_id_mutex); 979 list_for_each_entry(id_ptr, &act_pernet_id_list, list) { 980 if (id_ptr->id == id) { 981 ret = -EEXIST; 982 goto err_out; 983 } 984 } 985 986 id_ptr = kzalloc_obj(*id_ptr); 987 if (!id_ptr) { 988 ret = -ENOMEM; 989 goto err_out; 990 } 991 id_ptr->id = id; 992 993 list_add_tail(&id_ptr->list, &act_pernet_id_list); 994 995 err_out: 996 mutex_unlock(&act_id_mutex); 997 return ret; 998 } 999 1000 static void tcf_pernet_del_id_list(unsigned int id) 1001 { 1002 struct tc_act_pernet_id *id_ptr; 1003 1004 mutex_lock(&act_id_mutex); 1005 list_for_each_entry(id_ptr, &act_pernet_id_list, list) { 1006 if (id_ptr->id == id) { 1007 list_del(&id_ptr->list); 1008 kfree(id_ptr); 1009 break; 1010 } 1011 } 1012 mutex_unlock(&act_id_mutex); 1013 } 1014 1015 int tcf_register_action(struct tc_action_ops *act, 1016 struct pernet_operations *ops) 1017 { 1018 struct tc_action_ops *a; 1019 int ret; 1020 1021 if (!act->act || !act->dump || !act->init) 1022 return -EINVAL; 1023 1024 /* We have to register pernet ops before making the action ops visible, 1025 * otherwise tcf_action_init_1() could get a partially initialized 1026 * netns. 1027 */ 1028 ret = register_pernet_subsys(ops); 1029 if (ret) 1030 return ret; 1031 1032 if (ops->id) { 1033 ret = tcf_pernet_add_id_list(*ops->id); 1034 if (ret) 1035 goto err_id; 1036 } 1037 1038 write_lock(&act_mod_lock); 1039 list_for_each_entry(a, &act_base, head) { 1040 if (act->id == a->id || (strcmp(act->kind, a->kind) == 0)) { 1041 ret = -EEXIST; 1042 goto err_out; 1043 } 1044 } 1045 list_add_tail(&act->head, &act_base); 1046 write_unlock(&act_mod_lock); 1047 1048 return 0; 1049 1050 err_out: 1051 write_unlock(&act_mod_lock); 1052 if (ops->id) 1053 tcf_pernet_del_id_list(*ops->id); 1054 err_id: 1055 unregister_pernet_subsys(ops); 1056 return ret; 1057 } 1058 EXPORT_SYMBOL(tcf_register_action); 1059 1060 int tcf_unregister_action(struct tc_action_ops *act, 1061 struct pernet_operations *ops) 1062 { 1063 struct tc_action_ops *a; 1064 int err = -ENOENT; 1065 1066 write_lock(&act_mod_lock); 1067 list_for_each_entry(a, &act_base, head) { 1068 if (a == act) { 1069 list_del(&act->head); 1070 err = 0; 1071 break; 1072 } 1073 } 1074 write_unlock(&act_mod_lock); 1075 if (!err) { 1076 unregister_pernet_subsys(ops); 1077 if (ops->id) 1078 tcf_pernet_del_id_list(*ops->id); 1079 } 1080 return err; 1081 } 1082 EXPORT_SYMBOL(tcf_unregister_action); 1083 1084 /* lookup by name */ 1085 static struct tc_action_ops *tc_lookup_action_n(char *kind) 1086 { 1087 struct tc_action_ops *a, *res = NULL; 1088 1089 if (kind) { 1090 read_lock(&act_mod_lock); 1091 list_for_each_entry(a, &act_base, head) { 1092 if (strcmp(kind, a->kind) == 0) { 1093 if (try_module_get(a->owner)) 1094 res = a; 1095 break; 1096 } 1097 } 1098 read_unlock(&act_mod_lock); 1099 } 1100 return res; 1101 } 1102 1103 /* lookup by nlattr */ 1104 static struct tc_action_ops *tc_lookup_action(struct nlattr *kind) 1105 { 1106 struct tc_action_ops *a, *res = NULL; 1107 1108 if (kind) { 1109 read_lock(&act_mod_lock); 1110 list_for_each_entry(a, &act_base, head) { 1111 if (nla_strcmp(kind, a->kind) == 0) { 1112 if (try_module_get(a->owner)) 1113 res = a; 1114 break; 1115 } 1116 } 1117 read_unlock(&act_mod_lock); 1118 } 1119 return res; 1120 } 1121 1122 /*TCA_ACT_MAX_PRIO is 32, there count up to 32 */ 1123 #define TCA_ACT_MAX_PRIO_MASK 0x1FF 1124 int tcf_action_exec(struct sk_buff *skb, struct tc_action **actions, 1125 int nr_actions, struct tcf_result *res) 1126 { 1127 u32 jmp_prgcnt = 0; 1128 u32 jmp_ttl = TCA_ACT_MAX_PRIO; /*matches actions per filter */ 1129 int i; 1130 int ret = TC_ACT_OK; 1131 1132 if (skb_skip_tc_classify(skb)) 1133 return TC_ACT_OK; 1134 1135 restart_act_graph: 1136 for (i = 0; i < nr_actions; i++) { 1137 const struct tc_action *a = actions[i]; 1138 int repeat_ttl; 1139 1140 if (jmp_prgcnt > 0) { 1141 jmp_prgcnt -= 1; 1142 continue; 1143 } 1144 1145 if (tc_act_skip_sw(a->tcfa_flags)) 1146 continue; 1147 1148 repeat_ttl = 32; 1149 repeat: 1150 ret = tc_act(skb, a, res); 1151 if (unlikely(ret == TC_ACT_REPEAT)) { 1152 if (--repeat_ttl != 0) 1153 goto repeat; 1154 /* suspicious opcode, stop pipeline */ 1155 net_warn_ratelimited("TC_ACT_REPEAT abuse ?\n"); 1156 return TC_ACT_OK; 1157 } 1158 if (TC_ACT_EXT_CMP(ret, TC_ACT_JUMP)) { 1159 jmp_prgcnt = ret & TCA_ACT_MAX_PRIO_MASK; 1160 if (!jmp_prgcnt || (jmp_prgcnt > nr_actions)) { 1161 /* faulty opcode, stop pipeline */ 1162 return TC_ACT_OK; 1163 } else { 1164 jmp_ttl -= 1; 1165 if (jmp_ttl > 0) 1166 goto restart_act_graph; 1167 else /* faulty graph, stop pipeline */ 1168 return TC_ACT_OK; 1169 } 1170 } else if (TC_ACT_EXT_CMP(ret, TC_ACT_GOTO_CHAIN)) { 1171 struct tcf_chain *chain = rcu_dereference_bh(a->goto_chain); 1172 1173 if (unlikely(!chain)) { 1174 tcf_set_drop_reason(skb, 1175 SKB_DROP_REASON_TC_CHAIN_NOTFOUND); 1176 return TC_ACT_SHOT; 1177 } 1178 tcf_action_goto_chain_exec(chain, res); 1179 } 1180 1181 if (ret != TC_ACT_PIPE) 1182 break; 1183 } 1184 1185 return ret; 1186 } 1187 EXPORT_SYMBOL(tcf_action_exec); 1188 1189 int tcf_action_destroy(struct tc_action *actions[], int bind) 1190 { 1191 const struct tc_action_ops *ops; 1192 struct tc_action *a; 1193 int ret = 0, i; 1194 1195 tcf_act_for_each_action(i, a, actions) { 1196 actions[i] = NULL; 1197 ops = a->ops; 1198 ret = __tcf_idr_release(a, bind, true); 1199 if (ret == ACT_P_DELETED) 1200 module_put(ops->owner); 1201 else if (ret < 0) 1202 return ret; 1203 } 1204 return ret; 1205 } 1206 1207 static int tcf_action_put(struct tc_action *p) 1208 { 1209 return __tcf_action_put(p, false); 1210 } 1211 1212 static void tcf_action_put_many(struct tc_action *actions[]) 1213 { 1214 struct tc_action *a; 1215 int i; 1216 1217 tcf_act_for_each_action(i, a, actions) { 1218 const struct tc_action_ops *ops = a->ops; 1219 if (tcf_action_put(a)) 1220 module_put(ops->owner); 1221 } 1222 } 1223 1224 static void tca_put_bound_many(struct tc_action *actions[], int init_res[]) 1225 { 1226 struct tc_action *a; 1227 int i; 1228 1229 tcf_act_for_each_action(i, a, actions) { 1230 const struct tc_action_ops *ops = a->ops; 1231 1232 if (init_res[i] == ACT_P_CREATED) 1233 continue; 1234 1235 if (tcf_action_put(a)) 1236 module_put(ops->owner); 1237 } 1238 } 1239 1240 int 1241 tcf_action_dump_old(struct sk_buff *skb, struct tc_action *a, int bind, int ref) 1242 { 1243 return a->ops->dump(skb, a, bind, ref); 1244 } 1245 1246 int tcf_action_dump(struct sk_buff *skb, struct tc_action *actions[], 1247 int bind, int ref, bool terse) 1248 { 1249 struct tc_action *a; 1250 int err = -EINVAL, i; 1251 struct nlattr *nest; 1252 1253 tcf_act_for_each_action(i, a, actions) { 1254 nest = nla_nest_start_noflag(skb, i + 1); 1255 if (nest == NULL) 1256 goto nla_put_failure; 1257 err = terse ? tcf_action_dump_terse(skb, a, false) : 1258 tcf_action_dump_1(skb, a, bind, ref); 1259 if (err < 0) 1260 goto errout; 1261 nla_nest_end(skb, nest); 1262 } 1263 1264 return 0; 1265 1266 nla_put_failure: 1267 err = -EINVAL; 1268 errout: 1269 nla_nest_cancel(skb, nest); 1270 return err; 1271 } 1272 1273 static struct tc_cookie *nla_memdup_cookie(struct nlattr **tb) 1274 { 1275 struct tc_cookie *c = kzalloc_obj(*c); 1276 if (!c) 1277 return NULL; 1278 1279 c->data = nla_memdup(tb[TCA_ACT_COOKIE], GFP_KERNEL); 1280 if (!c->data) { 1281 kfree(c); 1282 return NULL; 1283 } 1284 c->len = nla_len(tb[TCA_ACT_COOKIE]); 1285 1286 return c; 1287 } 1288 1289 static u8 tcf_action_hw_stats_get(struct nlattr *hw_stats_attr) 1290 { 1291 struct nla_bitfield32 hw_stats_bf; 1292 1293 /* If the user did not pass the attr, that means he does 1294 * not care about the type. Return "any" in that case 1295 * which is setting on all supported types. 1296 */ 1297 if (!hw_stats_attr) 1298 return TCA_ACT_HW_STATS_ANY; 1299 hw_stats_bf = nla_get_bitfield32(hw_stats_attr); 1300 return hw_stats_bf.value; 1301 } 1302 1303 static const struct nla_policy tcf_action_policy[TCA_ACT_MAX + 1] = { 1304 [TCA_ACT_KIND] = { .type = NLA_STRING }, 1305 [TCA_ACT_INDEX] = { .type = NLA_U32 }, 1306 [TCA_ACT_COOKIE] = { .type = NLA_BINARY, 1307 .len = TC_COOKIE_MAX_SIZE }, 1308 [TCA_ACT_OPTIONS] = { .type = NLA_NESTED }, 1309 [TCA_ACT_FLAGS] = NLA_POLICY_BITFIELD32(TCA_ACT_FLAGS_NO_PERCPU_STATS | 1310 TCA_ACT_FLAGS_SKIP_HW | 1311 TCA_ACT_FLAGS_SKIP_SW), 1312 [TCA_ACT_HW_STATS] = NLA_POLICY_BITFIELD32(TCA_ACT_HW_STATS_ANY), 1313 }; 1314 1315 void tcf_idr_insert_many(struct tc_action *actions[], int init_res[]) 1316 { 1317 struct tc_action *a; 1318 int i; 1319 1320 tcf_act_for_each_action(i, a, actions) { 1321 struct tcf_idrinfo *idrinfo; 1322 1323 if (init_res[i] == ACT_P_BOUND) 1324 continue; 1325 1326 idrinfo = a->idrinfo; 1327 mutex_lock(&idrinfo->lock); 1328 /* Replace ERR_PTR(-EBUSY) allocated by tcf_idr_check_alloc */ 1329 idr_replace(&idrinfo->action_idr, a, a->tcfa_index); 1330 mutex_unlock(&idrinfo->lock); 1331 } 1332 } 1333 1334 struct tc_action_ops *tc_action_load_ops(struct nlattr *nla, u32 flags, 1335 struct netlink_ext_ack *extack) 1336 { 1337 bool police = flags & TCA_ACT_FLAGS_POLICE; 1338 struct nlattr *tb[TCA_ACT_MAX + 1]; 1339 struct tc_action_ops *a_o; 1340 char act_name[IFNAMSIZ]; 1341 struct nlattr *kind; 1342 int err; 1343 1344 if (!police) { 1345 err = nla_parse_nested_deprecated(tb, TCA_ACT_MAX, nla, 1346 tcf_action_policy, extack); 1347 if (err < 0) 1348 return ERR_PTR(err); 1349 err = -EINVAL; 1350 kind = tb[TCA_ACT_KIND]; 1351 if (!kind) { 1352 NL_SET_ERR_MSG(extack, "TC action kind must be specified"); 1353 return ERR_PTR(err); 1354 } 1355 if (nla_strscpy(act_name, kind, IFNAMSIZ) < 0) { 1356 NL_SET_ERR_MSG(extack, "TC action name too long"); 1357 return ERR_PTR(err); 1358 } 1359 } else { 1360 if (strscpy(act_name, "police", IFNAMSIZ) < 0) { 1361 NL_SET_ERR_MSG(extack, "TC action name too long"); 1362 return ERR_PTR(-EINVAL); 1363 } 1364 } 1365 1366 a_o = tc_lookup_action_n(act_name); 1367 if (a_o == NULL) { 1368 #ifdef CONFIG_MODULES 1369 bool rtnl_held = !(flags & TCA_ACT_FLAGS_NO_RTNL); 1370 1371 if (rtnl_held) 1372 rtnl_unlock(); 1373 request_module(NET_ACT_ALIAS_PREFIX "%s", act_name); 1374 if (rtnl_held) 1375 rtnl_lock(); 1376 1377 a_o = tc_lookup_action_n(act_name); 1378 1379 /* We dropped the RTNL semaphore in order to 1380 * perform the module load. So, even if we 1381 * succeeded in loading the module we have to 1382 * tell the caller to replay the request. We 1383 * indicate this using -EAGAIN. 1384 */ 1385 if (a_o != NULL) { 1386 module_put(a_o->owner); 1387 return ERR_PTR(-EAGAIN); 1388 } 1389 #endif 1390 NL_SET_ERR_MSG(extack, "Failed to load TC action module"); 1391 return ERR_PTR(-ENOENT); 1392 } 1393 1394 return a_o; 1395 } 1396 1397 struct tc_action *tcf_action_init_1(struct net *net, struct tcf_proto *tp, 1398 struct nlattr *nla, struct nlattr *est, 1399 struct tc_action_ops *a_o, int *init_res, 1400 u32 flags, struct netlink_ext_ack *extack) 1401 { 1402 bool police = flags & TCA_ACT_FLAGS_POLICE; 1403 struct nla_bitfield32 userflags = { 0, 0 }; 1404 struct tc_cookie *user_cookie = NULL; 1405 u8 hw_stats = TCA_ACT_HW_STATS_ANY; 1406 struct nlattr *tb[TCA_ACT_MAX + 1]; 1407 struct tc_action *a; 1408 int err; 1409 1410 /* backward compatibility for policer */ 1411 if (!police) { 1412 err = nla_parse_nested_deprecated(tb, TCA_ACT_MAX, nla, 1413 tcf_action_policy, extack); 1414 if (err < 0) 1415 return ERR_PTR(err); 1416 if (tb[TCA_ACT_COOKIE]) { 1417 user_cookie = nla_memdup_cookie(tb); 1418 if (!user_cookie) { 1419 NL_SET_ERR_MSG(extack, "No memory to generate TC cookie"); 1420 err = -ENOMEM; 1421 goto err_out; 1422 } 1423 } 1424 hw_stats = tcf_action_hw_stats_get(tb[TCA_ACT_HW_STATS]); 1425 if (tb[TCA_ACT_FLAGS]) { 1426 userflags = nla_get_bitfield32(tb[TCA_ACT_FLAGS]); 1427 if (!tc_act_flags_valid(userflags.value)) { 1428 err = -EINVAL; 1429 goto err_out; 1430 } 1431 } 1432 1433 err = a_o->init(net, tb[TCA_ACT_OPTIONS], est, &a, tp, 1434 userflags.value | flags, extack); 1435 } else { 1436 err = a_o->init(net, nla, est, &a, tp, userflags.value | flags, 1437 extack); 1438 } 1439 if (err < 0) 1440 goto err_out; 1441 *init_res = err; 1442 1443 if (!police && tb[TCA_ACT_COOKIE]) 1444 tcf_set_action_cookie(&a->user_cookie, user_cookie); 1445 1446 if (!police) 1447 a->hw_stats = hw_stats; 1448 1449 return a; 1450 1451 err_out: 1452 if (user_cookie) { 1453 kfree(user_cookie->data); 1454 kfree(user_cookie); 1455 } 1456 return ERR_PTR(err); 1457 } 1458 1459 static bool tc_act_bind(u32 flags) 1460 { 1461 return !!(flags & TCA_ACT_FLAGS_BIND); 1462 } 1463 1464 /* Returns numbers of initialized actions or negative error. */ 1465 1466 int tcf_action_init(struct net *net, struct tcf_proto *tp, struct nlattr *nla, 1467 struct nlattr *est, struct tc_action *actions[], 1468 int init_res[], size_t *attr_size, 1469 u32 flags, u32 fl_flags, 1470 struct netlink_ext_ack *extack) 1471 { 1472 struct tc_action_ops *ops[TCA_ACT_MAX_PRIO] = {}; 1473 struct nlattr *tb[TCA_ACT_MAX_PRIO + 2]; 1474 struct tc_action *act; 1475 size_t sz = 0; 1476 int err; 1477 int i; 1478 1479 err = nla_parse_nested_deprecated(tb, TCA_ACT_MAX_PRIO + 1, nla, NULL, 1480 extack); 1481 if (err < 0) 1482 return err; 1483 1484 /* The nested attributes are parsed as types, but they are really an 1485 * array of actions. So we parse one more than we can handle, and return 1486 * an error if the last one is set (as that indicates that the request 1487 * contained more than the maximum number of actions). 1488 */ 1489 if (tb[TCA_ACT_MAX_PRIO + 1]) { 1490 NL_SET_ERR_MSG_FMT(extack, 1491 "Only %d actions supported per filter", 1492 TCA_ACT_MAX_PRIO); 1493 return -EINVAL; 1494 } 1495 1496 for (i = 1; i <= TCA_ACT_MAX_PRIO && tb[i]; i++) { 1497 struct tc_action_ops *a_o; 1498 1499 a_o = tc_action_load_ops(tb[i], flags, extack); 1500 if (IS_ERR(a_o)) { 1501 err = PTR_ERR(a_o); 1502 goto err_mod; 1503 } 1504 ops[i - 1] = a_o; 1505 } 1506 1507 for (i = 1; i <= TCA_ACT_MAX_PRIO && tb[i]; i++) { 1508 act = tcf_action_init_1(net, tp, tb[i], est, ops[i - 1], 1509 &init_res[i - 1], flags, extack); 1510 if (IS_ERR(act)) { 1511 err = PTR_ERR(act); 1512 goto err; 1513 } 1514 sz += tcf_action_fill_size(act); 1515 /* Start from index 0 */ 1516 actions[i - 1] = act; 1517 if (tc_act_bind(flags)) { 1518 bool skip_sw = tc_skip_sw(fl_flags); 1519 bool skip_hw = tc_skip_hw(fl_flags); 1520 1521 if (tc_act_bind(act->tcfa_flags)) { 1522 /* Action is created by classifier and is not 1523 * standalone. Check that the user did not set 1524 * any action flags different than the 1525 * classifier flags, and inherit the flags from 1526 * the classifier for the compatibility case 1527 * where no flags were specified at all. 1528 */ 1529 if ((tc_act_skip_sw(act->tcfa_flags) && !skip_sw) || 1530 (tc_act_skip_hw(act->tcfa_flags) && !skip_hw)) { 1531 NL_SET_ERR_MSG(extack, 1532 "Mismatch between action and filter offload flags"); 1533 err = -EINVAL; 1534 goto err; 1535 } 1536 if (skip_sw) 1537 act->tcfa_flags |= TCA_ACT_FLAGS_SKIP_SW; 1538 if (skip_hw) 1539 act->tcfa_flags |= TCA_ACT_FLAGS_SKIP_HW; 1540 continue; 1541 } 1542 1543 /* Action is standalone */ 1544 if (skip_sw != tc_act_skip_sw(act->tcfa_flags) || 1545 skip_hw != tc_act_skip_hw(act->tcfa_flags)) { 1546 NL_SET_ERR_MSG(extack, 1547 "Mismatch between action and filter offload flags"); 1548 err = -EINVAL; 1549 goto err; 1550 } 1551 } else { 1552 err = tcf_action_offload_add(act, extack); 1553 if (tc_act_skip_sw(act->tcfa_flags) && err) 1554 goto err; 1555 } 1556 } 1557 1558 /* We have to commit them all together, because if any error happened in 1559 * between, we could not handle the failure gracefully. 1560 */ 1561 tcf_idr_insert_many(actions, init_res); 1562 1563 *attr_size = tcf_action_full_attrs_size(sz); 1564 err = i - 1; 1565 goto err_mod; 1566 1567 err: 1568 tcf_action_destroy(actions, flags & TCA_ACT_FLAGS_BIND); 1569 err_mod: 1570 for (i = 0; i < TCA_ACT_MAX_PRIO && ops[i]; i++) 1571 module_put(ops[i]->owner); 1572 return err; 1573 } 1574 1575 void tcf_action_update_stats(struct tc_action *a, u64 bytes, u64 packets, 1576 u64 drops, bool hw) 1577 { 1578 if (a->cpu_bstats) { 1579 _bstats_update(this_cpu_ptr(a->cpu_bstats), bytes, packets); 1580 1581 this_cpu_add(a->cpu_qstats->drops, drops); 1582 1583 if (hw) 1584 _bstats_update(this_cpu_ptr(a->cpu_bstats_hw), 1585 bytes, packets); 1586 return; 1587 } 1588 1589 _bstats_update(&a->tcfa_bstats, bytes, packets); 1590 atomic_add(drops, &a->tcfa_drops); 1591 if (hw) 1592 _bstats_update(&a->tcfa_bstats_hw, bytes, packets); 1593 } 1594 EXPORT_SYMBOL(tcf_action_update_stats); 1595 1596 int tcf_action_copy_stats(struct sk_buff *skb, struct tc_action *p, 1597 int compat_mode) 1598 { 1599 struct gnet_stats_queue qstats = {0}; 1600 struct gnet_dump d; 1601 int err = 0; 1602 1603 if (p == NULL) 1604 goto errout; 1605 1606 /* compat_mode being true specifies a call that is supposed 1607 * to add additional backward compatibility statistic TLVs. 1608 */ 1609 if (compat_mode) { 1610 if (p->type == TCA_OLD_COMPAT) 1611 err = gnet_stats_start_copy_compat(skb, 0, 1612 TCA_STATS, 1613 TCA_XSTATS, 1614 &p->tcfa_lock, &d, 1615 TCA_PAD); 1616 else 1617 return 0; 1618 } else 1619 err = gnet_stats_start_copy(skb, TCA_ACT_STATS, 1620 &p->tcfa_lock, &d, TCA_ACT_PAD); 1621 1622 if (err < 0) 1623 goto errout; 1624 1625 qstats.drops = atomic_read(&p->tcfa_drops); 1626 qstats.overlimits = atomic_read(&p->tcfa_overlimits); 1627 1628 if (gnet_stats_copy_basic(&d, p->cpu_bstats, 1629 &p->tcfa_bstats, false) < 0 || 1630 gnet_stats_copy_basic_hw(&d, p->cpu_bstats_hw, 1631 &p->tcfa_bstats_hw, false) < 0 || 1632 gnet_stats_copy_rate_est(&d, &p->tcfa_rate_est) < 0 || 1633 gnet_stats_copy_queue(&d, p->cpu_qstats, 1634 &qstats, 1635 qstats.qlen) < 0) 1636 goto errout; 1637 1638 if (gnet_stats_finish_copy(&d) < 0) 1639 goto errout; 1640 1641 return 0; 1642 1643 errout: 1644 return -1; 1645 } 1646 1647 static int tca_get_fill(struct sk_buff *skb, struct tc_action *actions[], 1648 u32 portid, u32 seq, u16 flags, int event, int bind, 1649 int ref, struct netlink_ext_ack *extack) 1650 { 1651 struct tcamsg *t; 1652 struct nlmsghdr *nlh; 1653 unsigned char *b = skb_tail_pointer(skb); 1654 struct nlattr *nest; 1655 1656 nlh = nlmsg_put(skb, portid, seq, event, sizeof(*t), flags); 1657 if (!nlh) 1658 goto out_nlmsg_trim; 1659 t = nlmsg_data(nlh); 1660 t->tca_family = AF_UNSPEC; 1661 t->tca__pad1 = 0; 1662 t->tca__pad2 = 0; 1663 1664 if (extack && extack->_msg && 1665 nla_put_string(skb, TCA_ROOT_EXT_WARN_MSG, extack->_msg)) 1666 goto out_nlmsg_trim; 1667 1668 nest = nla_nest_start_noflag(skb, TCA_ACT_TAB); 1669 if (!nest) 1670 goto out_nlmsg_trim; 1671 1672 if (tcf_action_dump(skb, actions, bind, ref, false) < 0) 1673 goto out_nlmsg_trim; 1674 1675 nla_nest_end(skb, nest); 1676 1677 nlh->nlmsg_len = skb_tail_pointer(skb) - b; 1678 1679 return skb->len; 1680 1681 out_nlmsg_trim: 1682 nlmsg_trim(skb, b); 1683 return -1; 1684 } 1685 1686 static int 1687 tcf_get_notify(struct net *net, u32 portid, struct nlmsghdr *n, 1688 struct tc_action *actions[], int event, 1689 struct netlink_ext_ack *extack) 1690 { 1691 struct sk_buff *skb; 1692 1693 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); 1694 if (!skb) 1695 return -ENOBUFS; 1696 if (tca_get_fill(skb, actions, portid, n->nlmsg_seq, 0, event, 1697 0, 1, NULL) <= 0) { 1698 NL_SET_ERR_MSG(extack, "Failed to fill netlink attributes while adding TC action"); 1699 kfree_skb(skb); 1700 return -EINVAL; 1701 } 1702 1703 return rtnl_unicast(skb, net, portid); 1704 } 1705 1706 static struct tc_action *tcf_action_get_1(struct net *net, struct nlattr *nla, 1707 struct nlmsghdr *n, u32 portid, 1708 struct netlink_ext_ack *extack) 1709 { 1710 struct nlattr *tb[TCA_ACT_MAX + 1]; 1711 const struct tc_action_ops *ops; 1712 struct tc_action *a; 1713 int index; 1714 int err; 1715 1716 err = nla_parse_nested_deprecated(tb, TCA_ACT_MAX, nla, 1717 tcf_action_policy, extack); 1718 if (err < 0) 1719 goto err_out; 1720 1721 err = -EINVAL; 1722 if (tb[TCA_ACT_INDEX] == NULL || 1723 nla_len(tb[TCA_ACT_INDEX]) < sizeof(index)) { 1724 NL_SET_ERR_MSG(extack, "Invalid TC action index value"); 1725 goto err_out; 1726 } 1727 index = nla_get_u32(tb[TCA_ACT_INDEX]); 1728 1729 err = -EINVAL; 1730 ops = tc_lookup_action(tb[TCA_ACT_KIND]); 1731 if (!ops) { /* could happen in batch of actions */ 1732 NL_SET_ERR_MSG(extack, "Specified TC action kind not found"); 1733 goto err_out; 1734 } 1735 err = -ENOENT; 1736 if (__tcf_idr_search(net, ops, &a, index) == 0) { 1737 NL_SET_ERR_MSG(extack, "TC action with specified index not found"); 1738 goto err_mod; 1739 } 1740 1741 module_put(ops->owner); 1742 return a; 1743 1744 err_mod: 1745 module_put(ops->owner); 1746 err_out: 1747 return ERR_PTR(err); 1748 } 1749 1750 static int tca_action_flush(struct net *net, struct nlattr *nla, 1751 struct nlmsghdr *n, u32 portid, 1752 struct netlink_ext_ack *extack) 1753 { 1754 struct sk_buff *skb; 1755 unsigned char *b; 1756 struct nlmsghdr *nlh; 1757 struct tcamsg *t; 1758 struct netlink_callback dcb; 1759 struct nlattr *nest; 1760 struct nlattr *tb[TCA_ACT_MAX + 1]; 1761 const struct tc_action_ops *ops; 1762 struct nlattr *kind; 1763 int err = -ENOMEM; 1764 1765 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); 1766 if (!skb) 1767 return err; 1768 1769 b = skb_tail_pointer(skb); 1770 1771 err = nla_parse_nested_deprecated(tb, TCA_ACT_MAX, nla, 1772 tcf_action_policy, extack); 1773 if (err < 0) 1774 goto err_out; 1775 1776 err = -EINVAL; 1777 kind = tb[TCA_ACT_KIND]; 1778 ops = tc_lookup_action(kind); 1779 if (!ops) { /*some idjot trying to flush unknown action */ 1780 NL_SET_ERR_MSG(extack, "Cannot flush unknown TC action"); 1781 goto err_out; 1782 } 1783 1784 nlh = nlmsg_put(skb, portid, n->nlmsg_seq, RTM_DELACTION, 1785 sizeof(*t), 0); 1786 if (!nlh) { 1787 NL_SET_ERR_MSG(extack, "Failed to create TC action flush notification"); 1788 goto out_module_put; 1789 } 1790 t = nlmsg_data(nlh); 1791 t->tca_family = AF_UNSPEC; 1792 t->tca__pad1 = 0; 1793 t->tca__pad2 = 0; 1794 1795 nest = nla_nest_start_noflag(skb, TCA_ACT_TAB); 1796 if (!nest) { 1797 NL_SET_ERR_MSG(extack, "Failed to add new netlink message"); 1798 goto out_module_put; 1799 } 1800 1801 err = __tcf_generic_walker(net, skb, &dcb, RTM_DELACTION, ops, extack); 1802 if (err <= 0) { 1803 nla_nest_cancel(skb, nest); 1804 goto out_module_put; 1805 } 1806 1807 nla_nest_end(skb, nest); 1808 1809 nlh->nlmsg_len = skb_tail_pointer(skb) - b; 1810 nlh->nlmsg_flags |= NLM_F_ROOT; 1811 module_put(ops->owner); 1812 err = rtnetlink_send(skb, net, portid, RTNLGRP_TC, 1813 n->nlmsg_flags & NLM_F_ECHO); 1814 if (err < 0) 1815 NL_SET_ERR_MSG(extack, "Failed to send TC action flush notification"); 1816 1817 return err; 1818 1819 out_module_put: 1820 module_put(ops->owner); 1821 err_out: 1822 kfree_skb(skb); 1823 return err; 1824 } 1825 1826 static int tcf_action_delete(struct net *net, struct tc_action *actions[]) 1827 { 1828 struct tc_action *a; 1829 int i; 1830 1831 tcf_act_for_each_action(i, a, actions) { 1832 const struct tc_action_ops *ops = a->ops; 1833 /* Actions can be deleted concurrently so we must save their 1834 * type and id to search again after reference is released. 1835 */ 1836 struct tcf_idrinfo *idrinfo = a->idrinfo; 1837 u32 act_index = a->tcfa_index; 1838 1839 actions[i] = NULL; 1840 if (tcf_action_put(a)) { 1841 /* last reference, action was deleted concurrently */ 1842 module_put(ops->owner); 1843 } else { 1844 int ret; 1845 1846 /* now do the delete */ 1847 ret = tcf_idr_delete_index(idrinfo, act_index); 1848 if (ret < 0) 1849 return ret; 1850 } 1851 } 1852 return 0; 1853 } 1854 1855 static struct sk_buff *tcf_reoffload_del_notify_msg(struct net *net, 1856 struct tc_action *action) 1857 { 1858 size_t attr_size = tcf_action_fill_size(action); 1859 struct tc_action *actions[TCA_ACT_MAX_PRIO] = { 1860 [0] = action, 1861 }; 1862 struct sk_buff *skb; 1863 1864 skb = alloc_skb(max(attr_size, NLMSG_GOODSIZE), GFP_KERNEL); 1865 if (!skb) 1866 return ERR_PTR(-ENOBUFS); 1867 1868 if (tca_get_fill(skb, actions, 0, 0, 0, RTM_DELACTION, 0, 1, NULL) <= 0) { 1869 kfree_skb(skb); 1870 return ERR_PTR(-EINVAL); 1871 } 1872 1873 return skb; 1874 } 1875 1876 static int tcf_reoffload_del_notify(struct net *net, struct tc_action *action) 1877 { 1878 const struct tc_action_ops *ops = action->ops; 1879 struct sk_buff *skb; 1880 int ret; 1881 1882 if (!rtnl_notify_needed(net, 0, RTNLGRP_TC)) { 1883 skb = NULL; 1884 } else { 1885 skb = tcf_reoffload_del_notify_msg(net, action); 1886 if (IS_ERR(skb)) 1887 return PTR_ERR(skb); 1888 } 1889 1890 ret = tcf_idr_release_unsafe(action); 1891 if (ret == ACT_P_DELETED) { 1892 module_put(ops->owner); 1893 ret = rtnetlink_maybe_send(skb, net, 0, RTNLGRP_TC, 0); 1894 } else { 1895 kfree_skb(skb); 1896 } 1897 1898 return ret; 1899 } 1900 1901 int tcf_action_reoffload_cb(flow_indr_block_bind_cb_t *cb, 1902 void *cb_priv, bool add) 1903 { 1904 struct tc_act_pernet_id *id_ptr; 1905 struct tcf_idrinfo *idrinfo; 1906 struct tc_action_net *tn; 1907 struct tc_action *p; 1908 unsigned int act_id; 1909 unsigned long tmp; 1910 unsigned long id; 1911 struct idr *idr; 1912 struct net *net; 1913 int ret; 1914 1915 if (!cb) 1916 return -EINVAL; 1917 1918 down_read(&net_rwsem); 1919 mutex_lock(&act_id_mutex); 1920 1921 for_each_net(net) { 1922 list_for_each_entry(id_ptr, &act_pernet_id_list, list) { 1923 act_id = id_ptr->id; 1924 tn = net_generic(net, act_id); 1925 if (!tn) 1926 continue; 1927 idrinfo = tn->idrinfo; 1928 if (!idrinfo) 1929 continue; 1930 1931 mutex_lock(&idrinfo->lock); 1932 idr = &idrinfo->action_idr; 1933 idr_for_each_entry_ul(idr, p, tmp, id) { 1934 if (IS_ERR(p) || tc_act_bind(p->tcfa_flags)) 1935 continue; 1936 if (add) { 1937 tcf_action_offload_add_ex(p, NULL, cb, 1938 cb_priv); 1939 continue; 1940 } 1941 1942 /* cb unregister to update hw count */ 1943 ret = tcf_action_offload_del_ex(p, cb, cb_priv); 1944 if (ret < 0) 1945 continue; 1946 if (tc_act_skip_sw(p->tcfa_flags) && 1947 !tc_act_in_hw(p)) 1948 tcf_reoffload_del_notify(net, p); 1949 } 1950 mutex_unlock(&idrinfo->lock); 1951 } 1952 } 1953 mutex_unlock(&act_id_mutex); 1954 up_read(&net_rwsem); 1955 1956 return 0; 1957 } 1958 1959 static struct sk_buff *tcf_del_notify_msg(struct net *net, struct nlmsghdr *n, 1960 struct tc_action *actions[], 1961 u32 portid, size_t attr_size, 1962 struct netlink_ext_ack *extack) 1963 { 1964 struct sk_buff *skb; 1965 1966 skb = alloc_skb(max(attr_size, NLMSG_GOODSIZE), GFP_KERNEL); 1967 if (!skb) 1968 return ERR_PTR(-ENOBUFS); 1969 1970 if (tca_get_fill(skb, actions, portid, n->nlmsg_seq, 0, RTM_DELACTION, 1971 0, 2, extack) <= 0) { 1972 NL_SET_ERR_MSG(extack, "Failed to fill netlink TC action attributes"); 1973 kfree_skb(skb); 1974 return ERR_PTR(-EINVAL); 1975 } 1976 1977 return skb; 1978 } 1979 1980 static int tcf_del_notify(struct net *net, struct nlmsghdr *n, 1981 struct tc_action *actions[], u32 portid, 1982 size_t attr_size, struct netlink_ext_ack *extack) 1983 { 1984 struct sk_buff *skb; 1985 int ret; 1986 1987 if (!rtnl_notify_needed(net, n->nlmsg_flags, RTNLGRP_TC)) { 1988 skb = NULL; 1989 } else { 1990 skb = tcf_del_notify_msg(net, n, actions, portid, attr_size, 1991 extack); 1992 if (IS_ERR(skb)) 1993 return PTR_ERR(skb); 1994 } 1995 1996 /* now do the delete */ 1997 ret = tcf_action_delete(net, actions); 1998 if (ret < 0) { 1999 NL_SET_ERR_MSG(extack, "Failed to delete TC action"); 2000 kfree_skb(skb); 2001 return ret; 2002 } 2003 2004 return rtnetlink_maybe_send(skb, net, portid, RTNLGRP_TC, 2005 n->nlmsg_flags & NLM_F_ECHO); 2006 } 2007 2008 static int 2009 tca_action_gd(struct net *net, struct nlattr *nla, struct nlmsghdr *n, 2010 u32 portid, int event, struct netlink_ext_ack *extack) 2011 { 2012 int i, ret; 2013 struct nlattr *tb[TCA_ACT_MAX_PRIO + 1]; 2014 struct tc_action *act; 2015 size_t attr_size = 0; 2016 struct tc_action *actions[TCA_ACT_MAX_PRIO] = {}; 2017 2018 ret = nla_parse_nested_deprecated(tb, TCA_ACT_MAX_PRIO, nla, NULL, 2019 extack); 2020 if (ret < 0) 2021 return ret; 2022 2023 if (event == RTM_DELACTION && n->nlmsg_flags & NLM_F_ROOT) { 2024 if (tb[1]) 2025 return tca_action_flush(net, tb[1], n, portid, extack); 2026 2027 NL_SET_ERR_MSG(extack, "Invalid netlink attributes while flushing TC action"); 2028 return -EINVAL; 2029 } 2030 2031 for (i = 1; i <= TCA_ACT_MAX_PRIO && tb[i]; i++) { 2032 act = tcf_action_get_1(net, tb[i], n, portid, extack); 2033 if (IS_ERR(act)) { 2034 ret = PTR_ERR(act); 2035 goto err; 2036 } 2037 attr_size += tcf_action_fill_size(act); 2038 actions[i - 1] = act; 2039 } 2040 2041 attr_size = tcf_action_full_attrs_size(attr_size); 2042 2043 if (event == RTM_GETACTION) 2044 ret = tcf_get_notify(net, portid, n, actions, event, extack); 2045 else { /* delete */ 2046 ret = tcf_del_notify(net, n, actions, portid, attr_size, extack); 2047 if (ret) 2048 goto err; 2049 return 0; 2050 } 2051 err: 2052 tcf_action_put_many(actions); 2053 return ret; 2054 } 2055 2056 static struct sk_buff *tcf_add_notify_msg(struct net *net, struct nlmsghdr *n, 2057 struct tc_action *actions[], 2058 u32 portid, size_t attr_size, 2059 struct netlink_ext_ack *extack) 2060 { 2061 struct sk_buff *skb; 2062 2063 skb = alloc_skb(max(attr_size, NLMSG_GOODSIZE), GFP_KERNEL); 2064 if (!skb) 2065 return ERR_PTR(-ENOBUFS); 2066 2067 if (tca_get_fill(skb, actions, portid, n->nlmsg_seq, n->nlmsg_flags, 2068 RTM_NEWACTION, 0, 0, extack) <= 0) { 2069 NL_SET_ERR_MSG(extack, "Failed to fill netlink attributes while adding TC action"); 2070 kfree_skb(skb); 2071 return ERR_PTR(-EINVAL); 2072 } 2073 2074 return skb; 2075 } 2076 2077 static int tcf_add_notify(struct net *net, struct nlmsghdr *n, 2078 struct tc_action *actions[], u32 portid, 2079 size_t attr_size, struct netlink_ext_ack *extack) 2080 { 2081 struct sk_buff *skb; 2082 2083 if (!rtnl_notify_needed(net, n->nlmsg_flags, RTNLGRP_TC)) { 2084 skb = NULL; 2085 } else { 2086 skb = tcf_add_notify_msg(net, n, actions, portid, attr_size, 2087 extack); 2088 if (IS_ERR(skb)) 2089 return PTR_ERR(skb); 2090 } 2091 2092 return rtnetlink_maybe_send(skb, net, portid, RTNLGRP_TC, 2093 n->nlmsg_flags & NLM_F_ECHO); 2094 } 2095 2096 static int tcf_action_add(struct net *net, struct nlattr *nla, 2097 struct nlmsghdr *n, u32 portid, u32 flags, 2098 struct netlink_ext_ack *extack) 2099 { 2100 size_t attr_size = 0; 2101 int loop, ret; 2102 struct tc_action *actions[TCA_ACT_MAX_PRIO] = {}; 2103 int init_res[TCA_ACT_MAX_PRIO] = {}; 2104 2105 for (loop = 0; loop < 10; loop++) { 2106 ret = tcf_action_init(net, NULL, nla, NULL, actions, init_res, 2107 &attr_size, flags, 0, extack); 2108 if (ret != -EAGAIN) 2109 break; 2110 } 2111 2112 if (ret < 0) 2113 return ret; 2114 2115 ret = tcf_add_notify(net, n, actions, portid, attr_size, extack); 2116 2117 /* only put bound actions */ 2118 tca_put_bound_many(actions, init_res); 2119 2120 return ret; 2121 } 2122 2123 static const struct nla_policy tcaa_policy[TCA_ROOT_MAX + 1] = { 2124 [TCA_ROOT_FLAGS] = NLA_POLICY_BITFIELD32(TCA_ACT_FLAG_LARGE_DUMP_ON | 2125 TCA_ACT_FLAG_TERSE_DUMP), 2126 [TCA_ROOT_TIME_DELTA] = { .type = NLA_U32 }, 2127 }; 2128 2129 static int tc_ctl_action(struct sk_buff *skb, struct nlmsghdr *n, 2130 struct netlink_ext_ack *extack) 2131 { 2132 struct net *net = sock_net(skb->sk); 2133 struct nlattr *tca[TCA_ROOT_MAX + 1]; 2134 u32 portid = NETLINK_CB(skb).portid; 2135 u32 flags = 0; 2136 int ret = 0; 2137 2138 if ((n->nlmsg_type != RTM_GETACTION) && 2139 !netlink_capable(skb, CAP_NET_ADMIN)) 2140 return -EPERM; 2141 2142 ret = nlmsg_parse_deprecated(n, sizeof(struct tcamsg), tca, 2143 TCA_ROOT_MAX, NULL, extack); 2144 if (ret < 0) 2145 return ret; 2146 2147 if (tca[TCA_ACT_TAB] == NULL) { 2148 NL_SET_ERR_MSG(extack, "Netlink action attributes missing"); 2149 return -EINVAL; 2150 } 2151 2152 /* n->nlmsg_flags & NLM_F_CREATE */ 2153 switch (n->nlmsg_type) { 2154 case RTM_NEWACTION: 2155 /* we are going to assume all other flags 2156 * imply create only if it doesn't exist 2157 * Note that CREATE | EXCL implies that 2158 * but since we want avoid ambiguity (eg when flags 2159 * is zero) then just set this 2160 */ 2161 if (n->nlmsg_flags & NLM_F_REPLACE) 2162 flags = TCA_ACT_FLAGS_REPLACE; 2163 ret = tcf_action_add(net, tca[TCA_ACT_TAB], n, portid, flags, 2164 extack); 2165 break; 2166 case RTM_DELACTION: 2167 ret = tca_action_gd(net, tca[TCA_ACT_TAB], n, 2168 portid, RTM_DELACTION, extack); 2169 break; 2170 case RTM_GETACTION: 2171 ret = tca_action_gd(net, tca[TCA_ACT_TAB], n, 2172 portid, RTM_GETACTION, extack); 2173 break; 2174 default: 2175 BUG(); 2176 } 2177 2178 return ret; 2179 } 2180 2181 static struct nlattr *find_dump_kind(struct nlattr **nla) 2182 { 2183 struct nlattr *tb1, *tb2[TCA_ACT_MAX + 1]; 2184 struct nlattr *tb[TCA_ACT_MAX_PRIO + 1]; 2185 struct nlattr *kind; 2186 2187 tb1 = nla[TCA_ACT_TAB]; 2188 if (tb1 == NULL) 2189 return NULL; 2190 2191 if (nla_parse_deprecated(tb, TCA_ACT_MAX_PRIO, nla_data(tb1), NLMSG_ALIGN(nla_len(tb1)), NULL, NULL) < 0) 2192 return NULL; 2193 2194 if (tb[1] == NULL) 2195 return NULL; 2196 if (nla_parse_nested_deprecated(tb2, TCA_ACT_MAX, tb[1], tcf_action_policy, NULL) < 0) 2197 return NULL; 2198 kind = tb2[TCA_ACT_KIND]; 2199 2200 return kind; 2201 } 2202 2203 static int tc_dump_action(struct sk_buff *skb, struct netlink_callback *cb) 2204 { 2205 struct net *net = sock_net(skb->sk); 2206 struct nlmsghdr *nlh; 2207 unsigned char *b = skb_tail_pointer(skb); 2208 struct nlattr *nest; 2209 struct tc_action_ops *a_o; 2210 int ret = 0; 2211 struct tcamsg *t = (struct tcamsg *) nlmsg_data(cb->nlh); 2212 struct nlattr *tb[TCA_ROOT_MAX + 1]; 2213 struct nlattr *count_attr = NULL; 2214 unsigned long jiffy_since = 0; 2215 struct nlattr *kind = NULL; 2216 struct nla_bitfield32 bf; 2217 u32 msecs_since = 0; 2218 u32 act_count = 0; 2219 2220 ret = nlmsg_parse_deprecated(cb->nlh, sizeof(struct tcamsg), tb, 2221 TCA_ROOT_MAX, tcaa_policy, cb->extack); 2222 if (ret < 0) 2223 return ret; 2224 2225 kind = find_dump_kind(tb); 2226 if (kind == NULL) { 2227 pr_info("tc_dump_action: action bad kind\n"); 2228 return 0; 2229 } 2230 2231 a_o = tc_lookup_action(kind); 2232 if (a_o == NULL) 2233 return 0; 2234 2235 cb->args[2] = 0; 2236 if (tb[TCA_ROOT_FLAGS]) { 2237 bf = nla_get_bitfield32(tb[TCA_ROOT_FLAGS]); 2238 cb->args[2] = bf.value; 2239 } 2240 2241 if (tb[TCA_ROOT_TIME_DELTA]) { 2242 msecs_since = nla_get_u32(tb[TCA_ROOT_TIME_DELTA]); 2243 } 2244 2245 nlh = nlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq, 2246 cb->nlh->nlmsg_type, sizeof(*t), 0); 2247 if (!nlh) 2248 goto out_module_put; 2249 2250 if (msecs_since) 2251 jiffy_since = jiffies - msecs_to_jiffies(msecs_since); 2252 2253 t = nlmsg_data(nlh); 2254 t->tca_family = AF_UNSPEC; 2255 t->tca__pad1 = 0; 2256 t->tca__pad2 = 0; 2257 cb->args[3] = jiffy_since; 2258 count_attr = nla_reserve(skb, TCA_ROOT_COUNT, sizeof(u32)); 2259 if (!count_attr) 2260 goto out_module_put; 2261 2262 nest = nla_nest_start_noflag(skb, TCA_ACT_TAB); 2263 if (nest == NULL) 2264 goto out_module_put; 2265 2266 ret = __tcf_generic_walker(net, skb, cb, RTM_GETACTION, a_o, NULL); 2267 if (ret < 0) 2268 goto out_module_put; 2269 2270 if (ret > 0) { 2271 nla_nest_end(skb, nest); 2272 ret = skb->len; 2273 act_count = cb->args[1]; 2274 memcpy(nla_data(count_attr), &act_count, sizeof(u32)); 2275 cb->args[1] = 0; 2276 } else 2277 nlmsg_trim(skb, b); 2278 2279 nlh->nlmsg_len = skb_tail_pointer(skb) - b; 2280 if (NETLINK_CB(cb->skb).portid && ret) 2281 nlh->nlmsg_flags |= NLM_F_MULTI; 2282 module_put(a_o->owner); 2283 return skb->len; 2284 2285 out_module_put: 2286 module_put(a_o->owner); 2287 nlmsg_trim(skb, b); 2288 return skb->len; 2289 } 2290 2291 static const struct rtnl_msg_handler tc_action_rtnl_msg_handlers[] __initconst = { 2292 {.msgtype = RTM_NEWACTION, .doit = tc_ctl_action}, 2293 {.msgtype = RTM_DELACTION, .doit = tc_ctl_action}, 2294 {.msgtype = RTM_GETACTION, .doit = tc_ctl_action, 2295 .dumpit = tc_dump_action}, 2296 }; 2297 2298 static int __init tc_action_init(void) 2299 { 2300 rtnl_register_many(tc_action_rtnl_msg_handlers); 2301 return 0; 2302 } 2303 2304 subsys_initcall(tc_action_init); 2305