1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * net/sched/act_api.c Packet action API. 4 * 5 * Author: Jamal Hadi Salim 6 */ 7 8 #include <linux/types.h> 9 #include <linux/kernel.h> 10 #include <linux/string.h> 11 #include <linux/errno.h> 12 #include <linux/slab.h> 13 #include <linux/skbuff.h> 14 #include <linux/init.h> 15 #include <linux/kmod.h> 16 #include <linux/err.h> 17 #include <linux/module.h> 18 #include <net/net_namespace.h> 19 #include <net/sock.h> 20 #include <net/sch_generic.h> 21 #include <net/pkt_cls.h> 22 #include <net/tc_act/tc_pedit.h> 23 #include <net/act_api.h> 24 #include <net/netlink.h> 25 #include <net/flow_offload.h> 26 #include <net/tc_wrapper.h> 27 28 #ifdef CONFIG_INET 29 DEFINE_STATIC_KEY_FALSE(tcf_frag_xmit_count); 30 EXPORT_SYMBOL_GPL(tcf_frag_xmit_count); 31 #endif 32 33 int tcf_dev_queue_xmit(struct sk_buff *skb, int (*xmit)(struct sk_buff *skb)) 34 { 35 #ifdef CONFIG_INET 36 if (static_branch_unlikely(&tcf_frag_xmit_count)) 37 return sch_frag_xmit_hook(skb, xmit); 38 #endif 39 40 return xmit(skb); 41 } 42 EXPORT_SYMBOL_GPL(tcf_dev_queue_xmit); 43 44 static void tcf_action_goto_chain_exec(const struct tcf_chain *chain, 45 struct tcf_result *res) 46 { 47 res->goto_tp = rcu_dereference_bh(chain->filter_chain); 48 } 49 50 static void tcf_free_cookie_rcu(struct rcu_head *p) 51 { 52 struct tc_cookie *cookie = container_of(p, struct tc_cookie, rcu); 53 54 kfree(cookie->data); 55 kfree(cookie); 56 } 57 58 static void tcf_set_action_cookie(struct tc_cookie __rcu **old_cookie, 59 struct tc_cookie *new_cookie) 60 { 61 struct tc_cookie *old; 62 63 old = unrcu_pointer(xchg(old_cookie, RCU_INITIALIZER(new_cookie))); 64 if (old) 65 call_rcu(&old->rcu, tcf_free_cookie_rcu); 66 } 67 68 int tcf_action_check_ctrlact(int action, struct tcf_proto *tp, 69 struct tcf_chain **newchain, 70 struct netlink_ext_ack *extack) 71 { 72 int opcode = TC_ACT_EXT_OPCODE(action), ret = -EINVAL; 73 u32 chain_index; 74 75 if (!opcode) 76 ret = action > TC_ACT_VALUE_MAX ? -EINVAL : 0; 77 else if (opcode <= TC_ACT_EXT_OPCODE_MAX || action == TC_ACT_UNSPEC) 78 ret = 0; 79 if (ret) { 80 NL_SET_ERR_MSG(extack, "invalid control action"); 81 goto end; 82 } 83 84 if (TC_ACT_EXT_CMP(action, TC_ACT_GOTO_CHAIN)) { 85 chain_index = action & TC_ACT_EXT_VAL_MASK; 86 if (!tp || !newchain) { 87 ret = -EINVAL; 88 NL_SET_ERR_MSG(extack, 89 "can't goto NULL proto/chain"); 90 goto end; 91 } 92 *newchain = tcf_chain_get_by_act(tp->chain->block, chain_index); 93 if (!*newchain) { 94 ret = -ENOMEM; 95 NL_SET_ERR_MSG(extack, 96 "can't allocate goto_chain"); 97 } 98 } 99 end: 100 return ret; 101 } 102 EXPORT_SYMBOL(tcf_action_check_ctrlact); 103 104 struct tcf_chain *tcf_action_set_ctrlact(struct tc_action *a, int action, 105 struct tcf_chain *goto_chain) 106 { 107 a->tcfa_action = action; 108 goto_chain = rcu_replace_pointer(a->goto_chain, goto_chain, 1); 109 return goto_chain; 110 } 111 EXPORT_SYMBOL(tcf_action_set_ctrlact); 112 113 static void free_tcf(struct tc_action *p) 114 { 115 struct tcf_chain *chain = rcu_dereference_protected(p->goto_chain, 1); 116 117 free_percpu(p->cpu_bstats); 118 free_percpu(p->cpu_bstats_hw); 119 free_percpu(p->cpu_qstats); 120 121 tcf_set_action_cookie(&p->user_cookie, NULL); 122 if (chain) 123 tcf_chain_put_by_act(chain); 124 125 kfree_rcu(p, tcfa_rcu); 126 } 127 128 static void offload_action_hw_count_set(struct tc_action *act, 129 u32 hw_count) 130 { 131 act->in_hw_count = hw_count; 132 } 133 134 static void offload_action_hw_count_inc(struct tc_action *act, 135 u32 hw_count) 136 { 137 act->in_hw_count += hw_count; 138 } 139 140 static void offload_action_hw_count_dec(struct tc_action *act, 141 u32 hw_count) 142 { 143 act->in_hw_count = act->in_hw_count > hw_count ? 144 act->in_hw_count - hw_count : 0; 145 } 146 147 static unsigned int tcf_offload_act_num_actions_single(struct tc_action *act) 148 { 149 unsigned int count; 150 151 if (is_tcf_pedit(act)) { 152 spin_lock_bh(&act->tcfa_lock); 153 count = tcf_pedit_nkeys_locked(act); 154 spin_unlock_bh(&act->tcfa_lock); 155 return count; 156 } 157 return 1; 158 } 159 160 static bool tc_act_skip_hw(u32 flags) 161 { 162 return (flags & TCA_ACT_FLAGS_SKIP_HW) ? true : false; 163 } 164 165 static bool tc_act_skip_sw(u32 flags) 166 { 167 return (flags & TCA_ACT_FLAGS_SKIP_SW) ? true : false; 168 } 169 170 /* SKIP_HW and SKIP_SW are mutually exclusive flags. */ 171 static bool tc_act_flags_valid(u32 flags) 172 { 173 flags &= TCA_ACT_FLAGS_SKIP_HW | TCA_ACT_FLAGS_SKIP_SW; 174 175 return flags ^ (TCA_ACT_FLAGS_SKIP_HW | TCA_ACT_FLAGS_SKIP_SW); 176 } 177 178 static int offload_action_init(struct flow_offload_action *fl_action, 179 struct tc_action *act, 180 enum offload_act_command cmd, 181 struct netlink_ext_ack *extack) 182 { 183 int err; 184 185 fl_action->extack = extack; 186 fl_action->command = cmd; 187 fl_action->index = act->tcfa_index; 188 fl_action->cookie = (unsigned long)act; 189 190 if (act->ops->offload_act_setup) { 191 spin_lock_bh(&act->tcfa_lock); 192 err = act->ops->offload_act_setup(act, fl_action, NULL, 193 false, extack); 194 spin_unlock_bh(&act->tcfa_lock); 195 return err; 196 } 197 198 return -EOPNOTSUPP; 199 } 200 201 static int tcf_action_offload_cmd_ex(struct flow_offload_action *fl_act, 202 u32 *hw_count) 203 { 204 int err; 205 206 err = flow_indr_dev_setup_offload(NULL, NULL, TC_SETUP_ACT, 207 fl_act, NULL, NULL); 208 if (err < 0) 209 return err; 210 211 if (hw_count) 212 *hw_count = err; 213 214 return 0; 215 } 216 217 static int tcf_action_offload_cmd_cb_ex(struct flow_offload_action *fl_act, 218 u32 *hw_count, 219 flow_indr_block_bind_cb_t *cb, 220 void *cb_priv) 221 { 222 int err; 223 224 err = cb(NULL, NULL, cb_priv, TC_SETUP_ACT, NULL, fl_act, NULL); 225 if (err < 0) 226 return err; 227 228 if (hw_count) 229 *hw_count = 1; 230 231 return 0; 232 } 233 234 static int tcf_action_offload_cmd(struct flow_offload_action *fl_act, 235 u32 *hw_count, 236 flow_indr_block_bind_cb_t *cb, 237 void *cb_priv) 238 { 239 return cb ? tcf_action_offload_cmd_cb_ex(fl_act, hw_count, 240 cb, cb_priv) : 241 tcf_action_offload_cmd_ex(fl_act, hw_count); 242 } 243 244 static int tcf_action_offload_add_ex(struct tc_action *action, 245 struct netlink_ext_ack *extack, 246 flow_indr_block_bind_cb_t *cb, 247 void *cb_priv) 248 { 249 bool skip_sw = tc_act_skip_sw(action->tcfa_flags); 250 struct tc_action *actions[TCA_ACT_MAX_PRIO] = { 251 [0] = action, 252 }; 253 struct flow_offload_action *fl_action; 254 u32 in_hw_count = 0; 255 int num, err = 0; 256 257 if (tc_act_skip_hw(action->tcfa_flags)) 258 return 0; 259 260 num = tcf_offload_act_num_actions_single(action); 261 fl_action = offload_action_alloc(num); 262 if (!fl_action) 263 return -ENOMEM; 264 265 err = offload_action_init(fl_action, action, FLOW_ACT_REPLACE, extack); 266 if (err) 267 goto fl_err; 268 269 err = tc_setup_action(&fl_action->action, actions, 0, extack); 270 if (err) { 271 NL_SET_ERR_MSG_MOD(extack, 272 "Failed to setup tc actions for offload"); 273 goto fl_err; 274 } 275 276 err = tcf_action_offload_cmd(fl_action, &in_hw_count, cb, cb_priv); 277 if (!err) 278 cb ? offload_action_hw_count_inc(action, in_hw_count) : 279 offload_action_hw_count_set(action, in_hw_count); 280 281 if (skip_sw && !tc_act_in_hw(action)) 282 err = -EINVAL; 283 284 tc_cleanup_offload_action(&fl_action->action); 285 286 fl_err: 287 kfree(fl_action); 288 289 return err; 290 } 291 292 /* offload the tc action after it is inserted */ 293 static int tcf_action_offload_add(struct tc_action *action, 294 struct netlink_ext_ack *extack) 295 { 296 return tcf_action_offload_add_ex(action, extack, NULL, NULL); 297 } 298 299 int tcf_action_update_hw_stats(struct tc_action *action) 300 { 301 struct flow_offload_action fl_act = {}; 302 int err; 303 304 err = offload_action_init(&fl_act, action, FLOW_ACT_STATS, NULL); 305 if (err) 306 return err; 307 308 err = tcf_action_offload_cmd(&fl_act, NULL, NULL, NULL); 309 if (!err) { 310 preempt_disable(); 311 tcf_action_stats_update(action, fl_act.stats.bytes, 312 fl_act.stats.pkts, 313 fl_act.stats.drops, 314 fl_act.stats.lastused, 315 true); 316 preempt_enable(); 317 action->used_hw_stats = fl_act.stats.used_hw_stats; 318 action->used_hw_stats_valid = true; 319 } else { 320 return -EOPNOTSUPP; 321 } 322 323 return 0; 324 } 325 EXPORT_SYMBOL(tcf_action_update_hw_stats); 326 327 static int tcf_action_offload_del_ex(struct tc_action *action, 328 flow_indr_block_bind_cb_t *cb, 329 void *cb_priv) 330 { 331 struct flow_offload_action fl_act = {}; 332 u32 in_hw_count = 0; 333 int err = 0; 334 335 if (!tc_act_in_hw(action)) 336 return 0; 337 338 err = offload_action_init(&fl_act, action, FLOW_ACT_DESTROY, NULL); 339 if (err) 340 return err; 341 342 err = tcf_action_offload_cmd(&fl_act, &in_hw_count, cb, cb_priv); 343 if (err < 0) 344 return err; 345 346 if (!cb && action->in_hw_count != in_hw_count) 347 return -EINVAL; 348 349 /* do not need to update hw state when deleting action */ 350 if (cb && in_hw_count) 351 offload_action_hw_count_dec(action, in_hw_count); 352 353 return 0; 354 } 355 356 static int tcf_action_offload_del(struct tc_action *action) 357 { 358 return tcf_action_offload_del_ex(action, NULL, NULL); 359 } 360 361 static void tcf_action_cleanup(struct tc_action *p) 362 { 363 tcf_action_offload_del(p); 364 if (p->ops->cleanup) 365 p->ops->cleanup(p); 366 367 gen_kill_estimator(&p->tcfa_rate_est); 368 free_tcf(p); 369 } 370 371 static int __tcf_action_put(struct tc_action *p, bool bind) 372 { 373 struct tcf_idrinfo *idrinfo = p->idrinfo; 374 375 if (refcount_dec_and_mutex_lock(&p->tcfa_refcnt, &idrinfo->lock)) { 376 if (bind) 377 atomic_dec(&p->tcfa_bindcnt); 378 idr_remove(&idrinfo->action_idr, p->tcfa_index); 379 mutex_unlock(&idrinfo->lock); 380 381 tcf_action_cleanup(p); 382 return 1; 383 } 384 385 if (bind) 386 atomic_dec(&p->tcfa_bindcnt); 387 388 return 0; 389 } 390 391 static int __tcf_idr_release(struct tc_action *p, bool bind, bool strict) 392 { 393 int ret = 0; 394 395 /* Release with strict==1 and bind==0 is only called through act API 396 * interface (classifiers always bind). Only case when action with 397 * positive reference count and zero bind count can exist is when it was 398 * also created with act API (unbinding last classifier will destroy the 399 * action if it was created by classifier). So only case when bind count 400 * can be changed after initial check is when unbound action is 401 * destroyed by act API while classifier binds to action with same id 402 * concurrently. This result either creation of new action(same behavior 403 * as before), or reusing existing action if concurrent process 404 * increments reference count before action is deleted. Both scenarios 405 * are acceptable. 406 */ 407 if (p) { 408 if (!bind && strict && atomic_read(&p->tcfa_bindcnt) > 0) 409 return -EPERM; 410 411 if (__tcf_action_put(p, bind)) 412 ret = ACT_P_DELETED; 413 } 414 415 return ret; 416 } 417 418 int tcf_idr_release(struct tc_action *a, bool bind) 419 { 420 const struct tc_action_ops *ops = a->ops; 421 int ret; 422 423 ret = __tcf_idr_release(a, bind, false); 424 if (ret == ACT_P_DELETED) 425 module_put(ops->owner); 426 return ret; 427 } 428 EXPORT_SYMBOL(tcf_idr_release); 429 430 static size_t tcf_action_shared_attrs_size(const struct tc_action *act) 431 { 432 struct tc_cookie *user_cookie; 433 u32 cookie_len = 0; 434 435 rcu_read_lock(); 436 user_cookie = rcu_dereference(act->user_cookie); 437 438 if (user_cookie) 439 cookie_len = nla_total_size(user_cookie->len); 440 rcu_read_unlock(); 441 442 return nla_total_size(0) /* action number nested */ 443 + nla_total_size(IFNAMSIZ) /* TCA_ACT_KIND */ 444 + cookie_len /* TCA_ACT_COOKIE */ 445 + nla_total_size(sizeof(struct nla_bitfield32)) /* TCA_ACT_HW_STATS */ 446 /* TCA_ACT_USED_HW_STATS */ 447 + nla_total_size(sizeof(struct nla_bitfield32)) 448 + nla_total_size(sizeof(u32)) /* TCA_ACT_IN_HW_COUNT */ 449 + nla_total_size(0) /* TCA_ACT_STATS nested */ 450 + nla_total_size(sizeof(struct nla_bitfield32)) /* TCA_ACT_FLAGS */ 451 /* TCA_STATS_BASIC */ 452 + nla_total_size_64bit(sizeof(struct gnet_stats_basic)) 453 /* TCA_STATS_BASIC_HW */ 454 + nla_total_size_64bit(sizeof(struct gnet_stats_basic)) 455 /* TCA_STATS_PKT64, emitted by both of the basic copies above */ 456 + 2 * nla_total_size_64bit(sizeof(u64)) 457 /* TCA_STATS_RATE_EST */ 458 + nla_total_size_64bit(sizeof(struct gnet_stats_rate_est)) 459 /* TCA_STATS_RATE_EST64 */ 460 + nla_total_size_64bit(sizeof(struct gnet_stats_rate_est64)) 461 /* TCA_STATS_QUEUE */ 462 + nla_total_size_64bit(sizeof(struct gnet_stats_queue)) 463 + nla_total_size(0) /* TCA_ACT_OPTIONS nested */ 464 /* TCA_GACT_TM; actions dump their tcf_t with nla_put_64bit(), 465 * which may emit an extra NLA_PAD attribute. 466 */ 467 + nla_total_size_64bit(sizeof(struct tcf_t)); 468 } 469 470 static size_t tcf_action_full_attrs_size(size_t sz) 471 { 472 return NLMSG_HDRLEN /* struct nlmsghdr */ 473 + sizeof(struct tcamsg) 474 + nla_total_size(0) /* TCA_ACT_TAB nested */ 475 + sz; 476 } 477 478 /* tca_get_fill() may append TCA_ROOT_EXT_WARN_MSG from extack->_msg */ 479 static size_t tcf_action_warn_attr_size(const struct netlink_ext_ack *extack) 480 { 481 if (unlikely(extack && extack->_msg)) 482 return nla_total_size(strlen(extack->_msg) + 1); 483 484 return 0; 485 } 486 487 static size_t tcf_action_fill_size(const struct tc_action *act) 488 { 489 size_t sz = tcf_action_shared_attrs_size(act); 490 491 if (act->ops->get_fill_size) 492 return act->ops->get_fill_size(act) + sz; 493 return sz; 494 } 495 496 static int 497 tcf_action_dump_terse(struct sk_buff *skb, struct tc_action *a, bool from_act) 498 { 499 unsigned char *b = skb_tail_pointer(skb); 500 struct tc_cookie *cookie; 501 502 if (nla_put_string(skb, TCA_ACT_KIND, a->ops->kind)) 503 goto nla_put_failure; 504 if (tcf_action_copy_stats(skb, a, 0)) 505 goto nla_put_failure; 506 if (from_act && nla_put_u32(skb, TCA_ACT_INDEX, a->tcfa_index)) 507 goto nla_put_failure; 508 509 rcu_read_lock(); 510 cookie = rcu_dereference(a->user_cookie); 511 if (cookie) { 512 if (nla_put(skb, TCA_ACT_COOKIE, cookie->len, cookie->data)) { 513 rcu_read_unlock(); 514 goto nla_put_failure; 515 } 516 } 517 rcu_read_unlock(); 518 519 return 0; 520 521 nla_put_failure: 522 nlmsg_trim(skb, b); 523 return -1; 524 } 525 526 static int 527 tcf_action_dump_1(struct sk_buff *skb, struct tc_action *a, int bind, int ref) 528 { 529 unsigned char *b = skb_tail_pointer(skb); 530 struct nlattr *nest; 531 int err = -EINVAL; 532 u32 flags; 533 534 if (tcf_action_dump_terse(skb, a, false)) 535 goto nla_put_failure; 536 537 if (a->hw_stats != TCA_ACT_HW_STATS_ANY && 538 nla_put_bitfield32(skb, TCA_ACT_HW_STATS, 539 a->hw_stats, TCA_ACT_HW_STATS_ANY)) 540 goto nla_put_failure; 541 542 if (a->used_hw_stats_valid && 543 nla_put_bitfield32(skb, TCA_ACT_USED_HW_STATS, 544 a->used_hw_stats, TCA_ACT_HW_STATS_ANY)) 545 goto nla_put_failure; 546 547 flags = a->tcfa_flags & TCA_ACT_FLAGS_USER_MASK; 548 if (flags && 549 nla_put_bitfield32(skb, TCA_ACT_FLAGS, 550 flags, flags)) 551 goto nla_put_failure; 552 553 if (nla_put_u32(skb, TCA_ACT_IN_HW_COUNT, a->in_hw_count)) 554 goto nla_put_failure; 555 556 nest = nla_nest_start_noflag(skb, TCA_ACT_OPTIONS); 557 if (nest == NULL) 558 goto nla_put_failure; 559 err = tcf_action_dump_old(skb, a, bind, ref); 560 if (err > 0) { 561 nla_nest_end(skb, nest); 562 return err; 563 } 564 565 nla_put_failure: 566 nlmsg_trim(skb, b); 567 return -1; 568 } 569 570 static int tcf_dump_walker(struct tcf_idrinfo *idrinfo, struct sk_buff *skb, 571 struct netlink_callback *cb) 572 { 573 int err = 0, index = -1, s_i = 0, n_i = 0; 574 u32 act_flags = cb->args[2]; 575 unsigned long jiffy_since = cb->args[3]; 576 struct nlattr *nest; 577 struct idr *idr = &idrinfo->action_idr; 578 struct tc_action *p; 579 unsigned long id = 1; 580 unsigned long tmp; 581 582 mutex_lock(&idrinfo->lock); 583 584 s_i = cb->args[0]; 585 586 idr_for_each_entry_ul(idr, p, tmp, id) { 587 index++; 588 if (index < s_i) 589 continue; 590 if (IS_ERR(p)) 591 continue; 592 593 if (jiffy_since && 594 time_after(jiffy_since, 595 (unsigned long)p->tcfa_tm.lastuse)) 596 continue; 597 598 tcf_action_update_hw_stats(p); 599 600 nest = nla_nest_start_noflag(skb, n_i); 601 if (!nest) { 602 index--; 603 goto nla_put_failure; 604 } 605 err = (act_flags & TCA_ACT_FLAG_TERSE_DUMP) ? 606 tcf_action_dump_terse(skb, p, true) : 607 tcf_action_dump_1(skb, p, 0, 0); 608 if (err < 0) { 609 index--; 610 nlmsg_trim(skb, nest); 611 goto done; 612 } 613 nla_nest_end(skb, nest); 614 n_i++; 615 if (!(act_flags & TCA_ACT_FLAG_LARGE_DUMP_ON) && 616 n_i >= TCA_ACT_MAX_PRIO) 617 goto done; 618 } 619 done: 620 if (index >= 0) 621 cb->args[0] = index + 1; 622 623 mutex_unlock(&idrinfo->lock); 624 if (n_i) { 625 if (act_flags & TCA_ACT_FLAG_LARGE_DUMP_ON) 626 cb->args[1] = n_i; 627 } 628 return n_i; 629 630 nla_put_failure: 631 nla_nest_cancel(skb, nest); 632 goto done; 633 } 634 635 static int tcf_idr_release_unsafe(struct tc_action *p) 636 { 637 if (atomic_read(&p->tcfa_bindcnt) > 0) 638 return -EPERM; 639 640 if (refcount_dec_and_test(&p->tcfa_refcnt)) { 641 idr_remove(&p->idrinfo->action_idr, p->tcfa_index); 642 tcf_action_cleanup(p); 643 return ACT_P_DELETED; 644 } 645 646 return 0; 647 } 648 649 static int tcf_del_walker(struct tcf_idrinfo *idrinfo, struct sk_buff *skb, 650 const struct tc_action_ops *ops, 651 struct netlink_ext_ack *extack) 652 { 653 struct nlattr *nest; 654 int n_i = 0; 655 int ret = -EINVAL; 656 struct idr *idr = &idrinfo->action_idr; 657 struct tc_action *p; 658 unsigned long id = 1; 659 unsigned long tmp; 660 661 nest = nla_nest_start_noflag(skb, 0); 662 if (nest == NULL) 663 goto nla_put_failure; 664 if (nla_put_string(skb, TCA_ACT_KIND, ops->kind)) 665 goto nla_put_failure; 666 667 ret = 0; 668 mutex_lock(&idrinfo->lock); 669 idr_for_each_entry_ul(idr, p, tmp, id) { 670 if (IS_ERR(p)) 671 continue; 672 ret = tcf_idr_release_unsafe(p); 673 if (ret == ACT_P_DELETED) 674 module_put(ops->owner); 675 else if (ret < 0) 676 break; 677 n_i++; 678 } 679 mutex_unlock(&idrinfo->lock); 680 if (ret < 0) { 681 if (n_i) 682 NL_SET_ERR_MSG(extack, "Unable to flush all TC actions"); 683 else 684 goto nla_put_failure; 685 } 686 687 ret = nla_put_u32(skb, TCA_FCNT, n_i); 688 if (ret) 689 goto nla_put_failure; 690 nla_nest_end(skb, nest); 691 692 return n_i; 693 nla_put_failure: 694 nla_nest_cancel(skb, nest); 695 return ret; 696 } 697 698 int tcf_generic_walker(struct tc_action_net *tn, struct sk_buff *skb, 699 struct netlink_callback *cb, int type, 700 const struct tc_action_ops *ops, 701 struct netlink_ext_ack *extack) 702 { 703 struct tcf_idrinfo *idrinfo = tn->idrinfo; 704 705 if (type == RTM_DELACTION) { 706 return tcf_del_walker(idrinfo, skb, ops, extack); 707 } else if (type == RTM_GETACTION) { 708 return tcf_dump_walker(idrinfo, skb, cb); 709 } else { 710 WARN(1, "tcf_generic_walker: unknown command %d\n", type); 711 NL_SET_ERR_MSG(extack, "tcf_generic_walker: unknown command"); 712 return -EINVAL; 713 } 714 } 715 EXPORT_SYMBOL(tcf_generic_walker); 716 717 int tcf_idr_search(struct tc_action_net *tn, struct tc_action **a, u32 index) 718 { 719 struct tcf_idrinfo *idrinfo = tn->idrinfo; 720 struct tc_action *p; 721 722 mutex_lock(&idrinfo->lock); 723 p = idr_find(&idrinfo->action_idr, index); 724 if (IS_ERR(p)) 725 p = NULL; 726 else if (p) 727 refcount_inc(&p->tcfa_refcnt); 728 mutex_unlock(&idrinfo->lock); 729 730 if (p) { 731 *a = p; 732 return true; 733 } 734 return false; 735 } 736 EXPORT_SYMBOL(tcf_idr_search); 737 738 static int __tcf_generic_walker(struct net *net, struct sk_buff *skb, 739 struct netlink_callback *cb, int type, 740 const struct tc_action_ops *ops, 741 struct netlink_ext_ack *extack) 742 { 743 struct tc_action_net *tn = net_generic(net, ops->net_id); 744 745 if (unlikely(ops->walk)) 746 return ops->walk(net, skb, cb, type, ops, extack); 747 748 return tcf_generic_walker(tn, skb, cb, type, ops, extack); 749 } 750 751 static int __tcf_idr_search(struct net *net, 752 const struct tc_action_ops *ops, 753 struct tc_action **a, u32 index) 754 { 755 struct tc_action_net *tn = net_generic(net, ops->net_id); 756 757 if (unlikely(ops->lookup)) 758 return ops->lookup(net, a, index); 759 760 return tcf_idr_search(tn, a, index); 761 } 762 763 static int tcf_idr_delete_index(struct tcf_idrinfo *idrinfo, u32 index) 764 { 765 struct tc_action *p; 766 int ret = 0; 767 768 mutex_lock(&idrinfo->lock); 769 p = idr_find(&idrinfo->action_idr, index); 770 if (IS_ERR_OR_NULL(p)) { 771 mutex_unlock(&idrinfo->lock); 772 return -ENOENT; 773 } 774 775 if (!atomic_read(&p->tcfa_bindcnt)) { 776 if (refcount_dec_and_test(&p->tcfa_refcnt)) { 777 struct module *owner = p->ops->owner; 778 779 WARN_ON(p != idr_remove(&idrinfo->action_idr, 780 p->tcfa_index)); 781 mutex_unlock(&idrinfo->lock); 782 783 tcf_action_cleanup(p); 784 module_put(owner); 785 return 0; 786 } 787 ret = 0; 788 } else { 789 ret = -EPERM; 790 } 791 792 mutex_unlock(&idrinfo->lock); 793 return ret; 794 } 795 796 int tcf_idr_create(struct tc_action_net *tn, u32 index, struct nlattr *est, 797 struct tc_action **a, const struct tc_action_ops *ops, 798 int bind, bool cpustats, u32 flags) 799 { 800 struct tc_action *p = kzalloc(ops->size, GFP_KERNEL); 801 struct tcf_idrinfo *idrinfo = tn->idrinfo; 802 int err = -ENOMEM; 803 804 if (unlikely(!p)) 805 return -ENOMEM; 806 refcount_set(&p->tcfa_refcnt, 1); 807 if (bind) 808 atomic_set(&p->tcfa_bindcnt, 1); 809 810 if (cpustats) { 811 p->cpu_bstats = netdev_alloc_pcpu_stats(struct gnet_stats_basic_sync); 812 if (!p->cpu_bstats) 813 goto err1; 814 p->cpu_bstats_hw = netdev_alloc_pcpu_stats(struct gnet_stats_basic_sync); 815 if (!p->cpu_bstats_hw) 816 goto err2; 817 p->cpu_qstats = alloc_percpu(struct gnet_stats_queue); 818 if (!p->cpu_qstats) 819 goto err3; 820 } 821 gnet_stats_basic_sync_init(&p->tcfa_bstats); 822 gnet_stats_basic_sync_init(&p->tcfa_bstats_hw); 823 spin_lock_init(&p->tcfa_lock); 824 p->tcfa_index = index; 825 p->tcfa_tm.install = jiffies; 826 p->tcfa_tm.lastuse = jiffies; 827 p->tcfa_tm.firstuse = 0; 828 p->tcfa_flags = flags; 829 if (est) { 830 err = gen_new_estimator(&p->tcfa_bstats, p->cpu_bstats, 831 &p->tcfa_rate_est, 832 &p->tcfa_lock, false, est); 833 if (err) 834 goto err4; 835 } 836 837 p->idrinfo = idrinfo; 838 __module_get(ops->owner); 839 p->ops = ops; 840 *a = p; 841 return 0; 842 err4: 843 free_percpu(p->cpu_qstats); 844 err3: 845 free_percpu(p->cpu_bstats_hw); 846 err2: 847 free_percpu(p->cpu_bstats); 848 err1: 849 kfree(p); 850 return err; 851 } 852 EXPORT_SYMBOL(tcf_idr_create); 853 854 int tcf_idr_create_from_flags(struct tc_action_net *tn, u32 index, 855 struct nlattr *est, struct tc_action **a, 856 const struct tc_action_ops *ops, int bind, 857 u32 flags) 858 { 859 /* Set cpustats according to actions flags. */ 860 return tcf_idr_create(tn, index, est, a, ops, bind, 861 !(flags & TCA_ACT_FLAGS_NO_PERCPU_STATS), flags); 862 } 863 EXPORT_SYMBOL(tcf_idr_create_from_flags); 864 865 /* Cleanup idr index that was allocated but not initialized. */ 866 867 void tcf_idr_cleanup(struct tc_action_net *tn, u32 index) 868 { 869 struct tcf_idrinfo *idrinfo = tn->idrinfo; 870 871 mutex_lock(&idrinfo->lock); 872 /* Remove ERR_PTR(-EBUSY) allocated by tcf_idr_check_alloc */ 873 WARN_ON(!IS_ERR(idr_remove(&idrinfo->action_idr, index))); 874 mutex_unlock(&idrinfo->lock); 875 } 876 EXPORT_SYMBOL(tcf_idr_cleanup); 877 878 /* Check if action with specified index exists. If actions is found, increments 879 * its reference and bind counters, and return 1. Otherwise insert temporary 880 * error pointer (to prevent concurrent users from inserting actions with same 881 * index) and return 0. 882 * 883 * May return -EAGAIN for binding actions in case of a parallel add/delete on 884 * the requested index. 885 */ 886 887 int tcf_idr_check_alloc(struct tc_action_net *tn, u32 *index, 888 struct tc_action **a, int bind) 889 { 890 struct tcf_idrinfo *idrinfo = tn->idrinfo; 891 struct tc_action *p; 892 int ret; 893 u32 max; 894 895 if (*index) { 896 rcu_read_lock(); 897 p = idr_find(&idrinfo->action_idr, *index); 898 899 if (IS_ERR(p)) { 900 /* This means that another process allocated 901 * index but did not assign the pointer yet. 902 */ 903 rcu_read_unlock(); 904 return -EAGAIN; 905 } 906 907 if (!p) { 908 /* Empty slot, try to allocate it */ 909 max = *index; 910 rcu_read_unlock(); 911 goto new; 912 } 913 914 if (!refcount_inc_not_zero(&p->tcfa_refcnt)) { 915 /* Action was deleted in parallel */ 916 rcu_read_unlock(); 917 return -EAGAIN; 918 } 919 920 if (bind) 921 atomic_inc(&p->tcfa_bindcnt); 922 *a = p; 923 924 rcu_read_unlock(); 925 926 return 1; 927 } else { 928 /* Find a slot */ 929 *index = 1; 930 max = UINT_MAX; 931 } 932 933 new: 934 *a = NULL; 935 936 mutex_lock(&idrinfo->lock); 937 ret = idr_alloc_u32(&idrinfo->action_idr, ERR_PTR(-EBUSY), index, max, 938 GFP_KERNEL); 939 mutex_unlock(&idrinfo->lock); 940 941 /* N binds raced for action allocation, 942 * retry for all the ones that failed. 943 */ 944 if (ret == -ENOSPC && *index == max) 945 ret = -EAGAIN; 946 947 return ret; 948 } 949 EXPORT_SYMBOL(tcf_idr_check_alloc); 950 951 void tcf_idrinfo_destroy(const struct tc_action_ops *ops, 952 struct tcf_idrinfo *idrinfo) 953 { 954 struct idr *idr = &idrinfo->action_idr; 955 bool mutex_taken = false; 956 struct tc_action *p; 957 unsigned long id = 1; 958 unsigned long tmp; 959 int ret; 960 961 idr_for_each_entry_ul(idr, p, tmp, id) { 962 if (IS_ERR(p)) 963 continue; 964 if (tc_act_in_hw(p) && !mutex_taken) { 965 rtnl_lock(); 966 mutex_taken = true; 967 } 968 ret = __tcf_idr_release(p, false, true); 969 if (ret == ACT_P_DELETED) 970 module_put(ops->owner); 971 else if (ret < 0) 972 return; 973 } 974 if (mutex_taken) 975 rtnl_unlock(); 976 idr_destroy(&idrinfo->action_idr); 977 } 978 EXPORT_SYMBOL(tcf_idrinfo_destroy); 979 980 static LIST_HEAD(act_base); 981 static DEFINE_RWLOCK(act_mod_lock); 982 /* since act ops id is stored in pernet subsystem list, 983 * then there is no way to walk through only all the action 984 * subsystem, so we keep tc action pernet ops id for 985 * reoffload to walk through. 986 */ 987 static LIST_HEAD(act_pernet_id_list); 988 static DEFINE_MUTEX(act_id_mutex); 989 struct tc_act_pernet_id { 990 struct list_head list; 991 unsigned int id; 992 }; 993 994 static int tcf_pernet_add_id_list(unsigned int id) 995 { 996 struct tc_act_pernet_id *id_ptr; 997 int ret = 0; 998 999 mutex_lock(&act_id_mutex); 1000 list_for_each_entry(id_ptr, &act_pernet_id_list, list) { 1001 if (id_ptr->id == id) { 1002 ret = -EEXIST; 1003 goto err_out; 1004 } 1005 } 1006 1007 id_ptr = kzalloc_obj(*id_ptr); 1008 if (!id_ptr) { 1009 ret = -ENOMEM; 1010 goto err_out; 1011 } 1012 id_ptr->id = id; 1013 1014 list_add_tail(&id_ptr->list, &act_pernet_id_list); 1015 1016 err_out: 1017 mutex_unlock(&act_id_mutex); 1018 return ret; 1019 } 1020 1021 static void tcf_pernet_del_id_list(unsigned int id) 1022 { 1023 struct tc_act_pernet_id *id_ptr; 1024 1025 mutex_lock(&act_id_mutex); 1026 list_for_each_entry(id_ptr, &act_pernet_id_list, list) { 1027 if (id_ptr->id == id) { 1028 list_del(&id_ptr->list); 1029 kfree(id_ptr); 1030 break; 1031 } 1032 } 1033 mutex_unlock(&act_id_mutex); 1034 } 1035 1036 int tcf_register_action(struct tc_action_ops *act, 1037 struct pernet_operations *ops) 1038 { 1039 struct tc_action_ops *a; 1040 int ret; 1041 1042 if (!act->act || !act->dump || !act->init) 1043 return -EINVAL; 1044 1045 /* We have to register pernet ops before making the action ops visible, 1046 * otherwise tcf_action_init_1() could get a partially initialized 1047 * netns. 1048 */ 1049 ret = register_pernet_subsys(ops); 1050 if (ret) 1051 return ret; 1052 1053 if (ops->id) { 1054 ret = tcf_pernet_add_id_list(*ops->id); 1055 if (ret) 1056 goto err_id; 1057 } 1058 1059 write_lock(&act_mod_lock); 1060 list_for_each_entry(a, &act_base, head) { 1061 if (act->id == a->id || (strcmp(act->kind, a->kind) == 0)) { 1062 ret = -EEXIST; 1063 goto err_out; 1064 } 1065 } 1066 list_add_tail(&act->head, &act_base); 1067 write_unlock(&act_mod_lock); 1068 1069 return 0; 1070 1071 err_out: 1072 write_unlock(&act_mod_lock); 1073 if (ops->id) 1074 tcf_pernet_del_id_list(*ops->id); 1075 err_id: 1076 unregister_pernet_subsys(ops); 1077 return ret; 1078 } 1079 EXPORT_SYMBOL(tcf_register_action); 1080 1081 int tcf_unregister_action(struct tc_action_ops *act, 1082 struct pernet_operations *ops) 1083 { 1084 struct tc_action_ops *a; 1085 int err = -ENOENT; 1086 1087 write_lock(&act_mod_lock); 1088 list_for_each_entry(a, &act_base, head) { 1089 if (a == act) { 1090 list_del(&act->head); 1091 err = 0; 1092 break; 1093 } 1094 } 1095 write_unlock(&act_mod_lock); 1096 if (!err) { 1097 unregister_pernet_subsys(ops); 1098 if (ops->id) 1099 tcf_pernet_del_id_list(*ops->id); 1100 } 1101 return err; 1102 } 1103 EXPORT_SYMBOL(tcf_unregister_action); 1104 1105 /* lookup by name */ 1106 static struct tc_action_ops *tc_lookup_action_n(char *kind) 1107 { 1108 struct tc_action_ops *a, *res = NULL; 1109 1110 if (kind) { 1111 read_lock(&act_mod_lock); 1112 list_for_each_entry(a, &act_base, head) { 1113 if (strcmp(kind, a->kind) == 0) { 1114 if (try_module_get(a->owner)) 1115 res = a; 1116 break; 1117 } 1118 } 1119 read_unlock(&act_mod_lock); 1120 } 1121 return res; 1122 } 1123 1124 /* lookup by nlattr */ 1125 static struct tc_action_ops *tc_lookup_action(struct nlattr *kind) 1126 { 1127 struct tc_action_ops *a, *res = NULL; 1128 1129 if (kind) { 1130 read_lock(&act_mod_lock); 1131 list_for_each_entry(a, &act_base, head) { 1132 if (nla_strcmp(kind, a->kind) == 0) { 1133 if (try_module_get(a->owner)) 1134 res = a; 1135 break; 1136 } 1137 } 1138 read_unlock(&act_mod_lock); 1139 } 1140 return res; 1141 } 1142 1143 /*TCA_ACT_MAX_PRIO is 32, there count up to 32 */ 1144 #define TCA_ACT_MAX_PRIO_MASK 0x1FF 1145 int tcf_action_exec(struct sk_buff *skb, struct tc_action **actions, 1146 int nr_actions, struct tcf_result *res) 1147 { 1148 u32 jmp_prgcnt = 0; 1149 u32 jmp_ttl = TCA_ACT_MAX_PRIO; /*matches actions per filter */ 1150 int i; 1151 int ret = TC_ACT_OK; 1152 1153 if (skb_skip_tc_classify(skb)) 1154 return TC_ACT_OK; 1155 1156 restart_act_graph: 1157 for (i = 0; i < nr_actions; i++) { 1158 const struct tc_action *a = actions[i]; 1159 int repeat_ttl; 1160 1161 if (jmp_prgcnt > 0) { 1162 jmp_prgcnt -= 1; 1163 continue; 1164 } 1165 1166 if (tc_act_skip_sw(a->tcfa_flags)) 1167 continue; 1168 1169 repeat_ttl = 32; 1170 repeat: 1171 ret = tc_act(skb, a, res); 1172 if (unlikely(ret == TC_ACT_REPEAT)) { 1173 if (--repeat_ttl != 0) 1174 goto repeat; 1175 /* suspicious opcode, stop pipeline */ 1176 net_warn_ratelimited("TC_ACT_REPEAT abuse ?\n"); 1177 return TC_ACT_OK; 1178 } 1179 if (TC_ACT_EXT_CMP(ret, TC_ACT_JUMP)) { 1180 jmp_prgcnt = ret & TCA_ACT_MAX_PRIO_MASK; 1181 if (!jmp_prgcnt || (jmp_prgcnt > nr_actions)) { 1182 /* faulty opcode, stop pipeline */ 1183 return TC_ACT_OK; 1184 } else { 1185 jmp_ttl -= 1; 1186 if (jmp_ttl > 0) 1187 goto restart_act_graph; 1188 else /* faulty graph, stop pipeline */ 1189 return TC_ACT_OK; 1190 } 1191 } else if (TC_ACT_EXT_CMP(ret, TC_ACT_GOTO_CHAIN)) { 1192 struct tcf_chain *chain = rcu_dereference_bh(a->goto_chain); 1193 1194 if (unlikely(!chain)) { 1195 tcf_set_drop_reason(skb, 1196 SKB_DROP_REASON_TC_CHAIN_NOTFOUND); 1197 return TC_ACT_SHOT; 1198 } 1199 tcf_action_goto_chain_exec(chain, res); 1200 } 1201 1202 if (ret != TC_ACT_PIPE) 1203 break; 1204 } 1205 1206 return ret; 1207 } 1208 EXPORT_SYMBOL(tcf_action_exec); 1209 1210 int tcf_action_destroy(struct tc_action *actions[], int bind) 1211 { 1212 struct tc_action *a; 1213 int ret = 0, i; 1214 1215 tcf_act_for_each_action(i, a, actions) { 1216 actions[i] = NULL; 1217 /* Drop our reference even if the action is still bound to a filter. */ 1218 ret = tcf_idr_release(a, bind); 1219 } 1220 return ret; 1221 } 1222 1223 static int tcf_action_put(struct tc_action *p) 1224 { 1225 return __tcf_action_put(p, false); 1226 } 1227 1228 static void tcf_action_put_many(struct tc_action *actions[]) 1229 { 1230 int i; 1231 1232 /* Deletion may have cleared entries before failing. */ 1233 for (i = 0; i < TCA_ACT_MAX_PRIO; i++) { 1234 struct tc_action *a = actions[i]; 1235 const struct tc_action_ops *ops; 1236 1237 if (!a) 1238 continue; 1239 ops = a->ops; 1240 if (tcf_action_put(a)) 1241 module_put(ops->owner); 1242 } 1243 } 1244 1245 static void tca_put_bound_many(struct tc_action *actions[], int init_res[]) 1246 { 1247 struct tc_action *a; 1248 int i; 1249 1250 tcf_act_for_each_action(i, a, actions) { 1251 const struct tc_action_ops *ops = a->ops; 1252 1253 if (init_res[i] == ACT_P_CREATED) 1254 continue; 1255 1256 if (tcf_action_put(a)) 1257 module_put(ops->owner); 1258 } 1259 } 1260 1261 int 1262 tcf_action_dump_old(struct sk_buff *skb, struct tc_action *a, int bind, int ref) 1263 { 1264 return a->ops->dump(skb, a, bind, ref); 1265 } 1266 1267 int tcf_action_dump(struct sk_buff *skb, struct tc_action *actions[], 1268 int bind, int ref, bool terse) 1269 { 1270 struct tc_action *a; 1271 int err = -EINVAL, i; 1272 struct nlattr *nest; 1273 1274 tcf_act_for_each_action(i, a, actions) { 1275 nest = nla_nest_start_noflag(skb, i + 1); 1276 if (nest == NULL) 1277 goto nla_put_failure; 1278 err = terse ? tcf_action_dump_terse(skb, a, false) : 1279 tcf_action_dump_1(skb, a, bind, ref); 1280 if (err < 0) 1281 goto errout; 1282 nla_nest_end(skb, nest); 1283 } 1284 1285 return 0; 1286 1287 nla_put_failure: 1288 err = -EINVAL; 1289 errout: 1290 nla_nest_cancel(skb, nest); 1291 return err; 1292 } 1293 1294 static struct tc_cookie *nla_memdup_cookie(struct nlattr **tb) 1295 { 1296 struct tc_cookie *c = kzalloc_obj(*c); 1297 if (!c) 1298 return NULL; 1299 1300 c->data = nla_memdup(tb[TCA_ACT_COOKIE], GFP_KERNEL); 1301 if (!c->data) { 1302 kfree(c); 1303 return NULL; 1304 } 1305 c->len = nla_len(tb[TCA_ACT_COOKIE]); 1306 1307 return c; 1308 } 1309 1310 static u8 tcf_action_hw_stats_get(struct nlattr *hw_stats_attr) 1311 { 1312 struct nla_bitfield32 hw_stats_bf; 1313 1314 /* If the user did not pass the attr, that means he does 1315 * not care about the type. Return "any" in that case 1316 * which is setting on all supported types. 1317 */ 1318 if (!hw_stats_attr) 1319 return TCA_ACT_HW_STATS_ANY; 1320 hw_stats_bf = nla_get_bitfield32(hw_stats_attr); 1321 return hw_stats_bf.value; 1322 } 1323 1324 static const struct nla_policy tcf_action_policy[TCA_ACT_MAX + 1] = { 1325 [TCA_ACT_KIND] = { .type = NLA_STRING }, 1326 [TCA_ACT_INDEX] = { .type = NLA_U32 }, 1327 [TCA_ACT_COOKIE] = { .type = NLA_BINARY, 1328 .len = TC_COOKIE_MAX_SIZE }, 1329 [TCA_ACT_OPTIONS] = { .type = NLA_NESTED }, 1330 [TCA_ACT_FLAGS] = NLA_POLICY_BITFIELD32(TCA_ACT_FLAGS_NO_PERCPU_STATS | 1331 TCA_ACT_FLAGS_SKIP_HW | 1332 TCA_ACT_FLAGS_SKIP_SW), 1333 [TCA_ACT_HW_STATS] = NLA_POLICY_BITFIELD32(TCA_ACT_HW_STATS_ANY), 1334 }; 1335 1336 void tcf_idr_insert_many(struct tc_action *actions[], int init_res[]) 1337 { 1338 struct tc_action *a; 1339 int i; 1340 1341 tcf_act_for_each_action(i, a, actions) { 1342 struct tcf_idrinfo *idrinfo; 1343 1344 if (init_res[i] == ACT_P_BOUND) 1345 continue; 1346 1347 idrinfo = a->idrinfo; 1348 mutex_lock(&idrinfo->lock); 1349 /* Replace ERR_PTR(-EBUSY) allocated by tcf_idr_check_alloc */ 1350 idr_replace(&idrinfo->action_idr, a, a->tcfa_index); 1351 mutex_unlock(&idrinfo->lock); 1352 } 1353 } 1354 1355 struct tc_action_ops *tc_action_load_ops(struct nlattr *nla, u32 flags, 1356 struct netlink_ext_ack *extack) 1357 { 1358 bool police = flags & TCA_ACT_FLAGS_POLICE; 1359 struct nlattr *tb[TCA_ACT_MAX + 1]; 1360 struct tc_action_ops *a_o; 1361 char act_name[IFNAMSIZ]; 1362 struct nlattr *kind; 1363 int err; 1364 1365 if (!police) { 1366 err = nla_parse_nested_deprecated(tb, TCA_ACT_MAX, nla, 1367 tcf_action_policy, extack); 1368 if (err < 0) 1369 return ERR_PTR(err); 1370 err = -EINVAL; 1371 kind = tb[TCA_ACT_KIND]; 1372 if (!kind) { 1373 NL_SET_ERR_MSG(extack, "TC action kind must be specified"); 1374 return ERR_PTR(err); 1375 } 1376 if (nla_strscpy(act_name, kind, IFNAMSIZ) < 0) { 1377 NL_SET_ERR_MSG(extack, "TC action name too long"); 1378 return ERR_PTR(err); 1379 } 1380 } else { 1381 if (strscpy(act_name, "police", IFNAMSIZ) < 0) { 1382 NL_SET_ERR_MSG(extack, "TC action name too long"); 1383 return ERR_PTR(-EINVAL); 1384 } 1385 } 1386 1387 a_o = tc_lookup_action_n(act_name); 1388 if (a_o == NULL) { 1389 #ifdef CONFIG_MODULES 1390 bool rtnl_held = !(flags & TCA_ACT_FLAGS_NO_RTNL); 1391 1392 if (rtnl_held) 1393 rtnl_unlock(); 1394 request_module(NET_ACT_ALIAS_PREFIX "%s", act_name); 1395 if (rtnl_held) 1396 rtnl_lock(); 1397 1398 a_o = tc_lookup_action_n(act_name); 1399 1400 /* We dropped the RTNL semaphore in order to 1401 * perform the module load. So, even if we 1402 * succeeded in loading the module we have to 1403 * tell the caller to replay the request. We 1404 * indicate this using -EAGAIN. 1405 */ 1406 if (a_o != NULL) { 1407 module_put(a_o->owner); 1408 return ERR_PTR(-EAGAIN); 1409 } 1410 #endif 1411 NL_SET_ERR_MSG(extack, "Failed to load TC action module"); 1412 return ERR_PTR(-ENOENT); 1413 } 1414 1415 return a_o; 1416 } 1417 1418 struct tc_action *tcf_action_init_1(struct net *net, struct tcf_proto *tp, 1419 struct nlattr *nla, struct nlattr *est, 1420 struct tc_action_ops *a_o, int *init_res, 1421 u32 flags, struct netlink_ext_ack *extack) 1422 { 1423 bool police = flags & TCA_ACT_FLAGS_POLICE; 1424 struct nla_bitfield32 userflags = { 0, 0 }; 1425 struct tc_cookie *user_cookie = NULL; 1426 u8 hw_stats = TCA_ACT_HW_STATS_ANY; 1427 struct nlattr *tb[TCA_ACT_MAX + 1]; 1428 struct tc_action *a; 1429 int err; 1430 1431 /* backward compatibility for policer */ 1432 if (!police) { 1433 err = nla_parse_nested_deprecated(tb, TCA_ACT_MAX, nla, 1434 tcf_action_policy, extack); 1435 if (err < 0) 1436 return ERR_PTR(err); 1437 if (tb[TCA_ACT_COOKIE]) { 1438 user_cookie = nla_memdup_cookie(tb); 1439 if (!user_cookie) { 1440 NL_SET_ERR_MSG(extack, "No memory to generate TC cookie"); 1441 err = -ENOMEM; 1442 goto err_out; 1443 } 1444 } 1445 hw_stats = tcf_action_hw_stats_get(tb[TCA_ACT_HW_STATS]); 1446 if (tb[TCA_ACT_FLAGS]) { 1447 userflags = nla_get_bitfield32(tb[TCA_ACT_FLAGS]); 1448 if (!tc_act_flags_valid(userflags.value)) { 1449 err = -EINVAL; 1450 goto err_out; 1451 } 1452 } 1453 1454 err = a_o->init(net, tb[TCA_ACT_OPTIONS], est, &a, tp, 1455 userflags.value | flags, extack); 1456 } else { 1457 err = a_o->init(net, nla, est, &a, tp, userflags.value | flags, 1458 extack); 1459 } 1460 if (err < 0) 1461 goto err_out; 1462 *init_res = err; 1463 1464 if (!police && tb[TCA_ACT_COOKIE]) 1465 tcf_set_action_cookie(&a->user_cookie, user_cookie); 1466 1467 if (!police) 1468 a->hw_stats = hw_stats; 1469 1470 return a; 1471 1472 err_out: 1473 if (user_cookie) { 1474 kfree(user_cookie->data); 1475 kfree(user_cookie); 1476 } 1477 return ERR_PTR(err); 1478 } 1479 1480 static bool tc_act_bind(u32 flags) 1481 { 1482 return !!(flags & TCA_ACT_FLAGS_BIND); 1483 } 1484 1485 /* Returns numbers of initialized actions or negative error. */ 1486 1487 int tcf_action_init(struct net *net, struct tcf_proto *tp, struct nlattr *nla, 1488 struct nlattr *est, struct tc_action *actions[], 1489 int init_res[], size_t *attr_size, 1490 u32 flags, u32 fl_flags, 1491 struct netlink_ext_ack *extack) 1492 { 1493 struct tc_action_ops *ops[TCA_ACT_MAX_PRIO] = {}; 1494 struct nlattr *tb[TCA_ACT_MAX_PRIO + 2]; 1495 struct tc_action *act; 1496 size_t sz = 0; 1497 int err; 1498 int i; 1499 1500 err = nla_parse_nested_deprecated(tb, TCA_ACT_MAX_PRIO + 1, nla, NULL, 1501 extack); 1502 if (err < 0) 1503 return err; 1504 1505 /* The nested attributes are parsed as types, but they are really an 1506 * array of actions. So we parse one more than we can handle, and return 1507 * an error if the last one is set (as that indicates that the request 1508 * contained more than the maximum number of actions). 1509 */ 1510 if (tb[TCA_ACT_MAX_PRIO + 1]) { 1511 NL_SET_ERR_MSG_FMT(extack, 1512 "Only %d actions supported per filter", 1513 TCA_ACT_MAX_PRIO); 1514 return -EINVAL; 1515 } 1516 1517 for (i = 1; i <= TCA_ACT_MAX_PRIO && tb[i]; i++) { 1518 struct tc_action_ops *a_o; 1519 1520 a_o = tc_action_load_ops(tb[i], flags, extack); 1521 if (IS_ERR(a_o)) { 1522 err = PTR_ERR(a_o); 1523 goto err_mod; 1524 } 1525 ops[i - 1] = a_o; 1526 } 1527 1528 for (i = 1; i <= TCA_ACT_MAX_PRIO && tb[i]; i++) { 1529 act = tcf_action_init_1(net, tp, tb[i], est, ops[i - 1], 1530 &init_res[i - 1], flags, extack); 1531 if (IS_ERR(act)) { 1532 err = PTR_ERR(act); 1533 goto err; 1534 } 1535 sz += tcf_action_fill_size(act); 1536 /* Start from index 0 */ 1537 actions[i - 1] = act; 1538 if (tc_act_bind(flags)) { 1539 bool skip_sw = tc_skip_sw(fl_flags); 1540 bool skip_hw = tc_skip_hw(fl_flags); 1541 1542 if (tc_act_bind(act->tcfa_flags)) { 1543 /* Action is created by classifier and is not 1544 * standalone. Check that the user did not set 1545 * any action flags different than the 1546 * classifier flags, and inherit the flags from 1547 * the classifier for the compatibility case 1548 * where no flags were specified at all. 1549 */ 1550 if ((tc_act_skip_sw(act->tcfa_flags) && !skip_sw) || 1551 (tc_act_skip_hw(act->tcfa_flags) && !skip_hw)) { 1552 NL_SET_ERR_MSG(extack, 1553 "Mismatch between action and filter offload flags"); 1554 err = -EINVAL; 1555 goto err; 1556 } 1557 if (skip_sw) 1558 act->tcfa_flags |= TCA_ACT_FLAGS_SKIP_SW; 1559 if (skip_hw) 1560 act->tcfa_flags |= TCA_ACT_FLAGS_SKIP_HW; 1561 continue; 1562 } 1563 1564 /* Action is standalone */ 1565 if (skip_sw != tc_act_skip_sw(act->tcfa_flags) || 1566 skip_hw != tc_act_skip_hw(act->tcfa_flags)) { 1567 NL_SET_ERR_MSG(extack, 1568 "Mismatch between action and filter offload flags"); 1569 err = -EINVAL; 1570 goto err; 1571 } 1572 } else { 1573 err = tcf_action_offload_add(act, extack); 1574 if (tc_act_skip_sw(act->tcfa_flags) && err) 1575 goto err; 1576 } 1577 } 1578 1579 /* We have to commit them all together, because if any error happened in 1580 * between, we could not handle the failure gracefully. 1581 */ 1582 tcf_idr_insert_many(actions, init_res); 1583 1584 *attr_size = tcf_action_full_attrs_size(sz); 1585 err = i - 1; 1586 goto err_mod; 1587 1588 err: 1589 tcf_action_destroy(actions, flags & TCA_ACT_FLAGS_BIND); 1590 err_mod: 1591 for (i = 0; i < TCA_ACT_MAX_PRIO && ops[i]; i++) 1592 module_put(ops[i]->owner); 1593 return err; 1594 } 1595 1596 void tcf_action_update_stats(struct tc_action *a, u64 bytes, u64 packets, 1597 u64 drops, bool hw) 1598 { 1599 if (a->cpu_bstats) { 1600 _bstats_update(this_cpu_ptr(a->cpu_bstats), bytes, packets); 1601 1602 this_cpu_add(a->cpu_qstats->drops, drops); 1603 1604 if (hw) 1605 _bstats_update(this_cpu_ptr(a->cpu_bstats_hw), 1606 bytes, packets); 1607 return; 1608 } 1609 1610 _bstats_update(&a->tcfa_bstats, bytes, packets); 1611 atomic_add(drops, &a->tcfa_drops); 1612 if (hw) 1613 _bstats_update(&a->tcfa_bstats_hw, bytes, packets); 1614 } 1615 EXPORT_SYMBOL(tcf_action_update_stats); 1616 1617 int tcf_action_copy_stats(struct sk_buff *skb, struct tc_action *p, 1618 int compat_mode) 1619 { 1620 struct gnet_stats_queue qstats = {0}; 1621 struct gnet_dump d; 1622 int err = 0; 1623 1624 if (p == NULL) 1625 goto errout; 1626 1627 /* compat_mode being true specifies a call that is supposed 1628 * to add additional backward compatibility statistic TLVs. 1629 */ 1630 if (compat_mode) { 1631 if (p->type == TCA_OLD_COMPAT) 1632 err = gnet_stats_start_copy_compat(skb, 0, 1633 TCA_STATS, 1634 TCA_XSTATS, 1635 &p->tcfa_lock, &d, 1636 TCA_PAD); 1637 else 1638 return 0; 1639 } else 1640 err = gnet_stats_start_copy(skb, TCA_ACT_STATS, 1641 &p->tcfa_lock, &d, TCA_ACT_PAD); 1642 1643 if (err < 0) 1644 goto errout; 1645 1646 qstats.drops = atomic_read(&p->tcfa_drops); 1647 qstats.overlimits = atomic_read(&p->tcfa_overlimits); 1648 1649 if (gnet_stats_copy_basic(&d, p->cpu_bstats, 1650 &p->tcfa_bstats, false) < 0 || 1651 gnet_stats_copy_basic_hw(&d, p->cpu_bstats_hw, 1652 &p->tcfa_bstats_hw, false) < 0 || 1653 gnet_stats_copy_rate_est(&d, &p->tcfa_rate_est) < 0 || 1654 gnet_stats_copy_queue(&d, p->cpu_qstats, 1655 &qstats, 1656 qstats.qlen) < 0) 1657 goto errout; 1658 1659 if (gnet_stats_finish_copy(&d) < 0) 1660 goto errout; 1661 1662 return 0; 1663 1664 errout: 1665 return -1; 1666 } 1667 1668 static int tca_get_fill(struct sk_buff *skb, struct tc_action *actions[], 1669 u32 portid, u32 seq, u16 flags, int event, int bind, 1670 int ref, struct netlink_ext_ack *extack) 1671 { 1672 struct tcamsg *t; 1673 struct nlmsghdr *nlh; 1674 unsigned char *b = skb_tail_pointer(skb); 1675 struct nlattr *nest; 1676 1677 nlh = nlmsg_put(skb, portid, seq, event, sizeof(*t), flags); 1678 if (!nlh) 1679 goto out_nlmsg_trim; 1680 t = nlmsg_data(nlh); 1681 t->tca_family = AF_UNSPEC; 1682 t->tca__pad1 = 0; 1683 t->tca__pad2 = 0; 1684 1685 if (extack && extack->_msg && 1686 nla_put_string(skb, TCA_ROOT_EXT_WARN_MSG, extack->_msg)) 1687 goto out_nlmsg_trim; 1688 1689 nest = nla_nest_start_noflag(skb, TCA_ACT_TAB); 1690 if (!nest) 1691 goto out_nlmsg_trim; 1692 1693 if (tcf_action_dump(skb, actions, bind, ref, false) < 0) 1694 goto out_nlmsg_trim; 1695 1696 nla_nest_end(skb, nest); 1697 1698 nlh->nlmsg_len = skb_tail_pointer(skb) - b; 1699 1700 return skb->len; 1701 1702 out_nlmsg_trim: 1703 nlmsg_trim(skb, b); 1704 return -1; 1705 } 1706 1707 static int 1708 tcf_get_notify(struct net *net, u32 portid, struct nlmsghdr *n, 1709 struct tc_action *actions[], size_t attr_size, int event, 1710 struct netlink_ext_ack *extack) 1711 { 1712 struct sk_buff *skb; 1713 1714 skb = alloc_skb(max(attr_size, NLMSG_GOODSIZE), GFP_KERNEL); 1715 if (!skb) 1716 return -ENOBUFS; 1717 if (tca_get_fill(skb, actions, portid, n->nlmsg_seq, 0, event, 1718 0, 1, NULL) <= 0) { 1719 NL_SET_ERR_MSG(extack, "Failed to fill netlink attributes while adding TC action"); 1720 kfree_skb(skb); 1721 return -EINVAL; 1722 } 1723 1724 return rtnl_unicast(skb, net, portid); 1725 } 1726 1727 static struct tc_action *tcf_action_get_1(struct net *net, struct nlattr *nla, 1728 struct nlmsghdr *n, u32 portid, 1729 struct netlink_ext_ack *extack) 1730 { 1731 struct nlattr *tb[TCA_ACT_MAX + 1]; 1732 const struct tc_action_ops *ops; 1733 struct tc_action *a; 1734 int index; 1735 int err; 1736 1737 err = nla_parse_nested_deprecated(tb, TCA_ACT_MAX, nla, 1738 tcf_action_policy, extack); 1739 if (err < 0) 1740 goto err_out; 1741 1742 err = -EINVAL; 1743 if (tb[TCA_ACT_INDEX] == NULL || 1744 nla_len(tb[TCA_ACT_INDEX]) < sizeof(index)) { 1745 NL_SET_ERR_MSG(extack, "Invalid TC action index value"); 1746 goto err_out; 1747 } 1748 index = nla_get_u32(tb[TCA_ACT_INDEX]); 1749 1750 err = -EINVAL; 1751 ops = tc_lookup_action(tb[TCA_ACT_KIND]); 1752 if (!ops) { /* could happen in batch of actions */ 1753 NL_SET_ERR_MSG(extack, "Specified TC action kind not found"); 1754 goto err_out; 1755 } 1756 err = -ENOENT; 1757 if (__tcf_idr_search(net, ops, &a, index) == 0) { 1758 NL_SET_ERR_MSG(extack, "TC action with specified index not found"); 1759 goto err_mod; 1760 } 1761 1762 module_put(ops->owner); 1763 return a; 1764 1765 err_mod: 1766 module_put(ops->owner); 1767 err_out: 1768 return ERR_PTR(err); 1769 } 1770 1771 static int tca_action_flush(struct net *net, struct nlattr *nla, 1772 struct nlmsghdr *n, u32 portid, 1773 struct netlink_ext_ack *extack) 1774 { 1775 struct sk_buff *skb; 1776 unsigned char *b; 1777 struct nlmsghdr *nlh; 1778 struct tcamsg *t; 1779 struct netlink_callback dcb; 1780 struct nlattr *nest; 1781 struct nlattr *tb[TCA_ACT_MAX + 1]; 1782 const struct tc_action_ops *ops; 1783 struct nlattr *kind; 1784 int err = -ENOMEM; 1785 1786 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); 1787 if (!skb) 1788 return err; 1789 1790 b = skb_tail_pointer(skb); 1791 1792 err = nla_parse_nested_deprecated(tb, TCA_ACT_MAX, nla, 1793 tcf_action_policy, extack); 1794 if (err < 0) 1795 goto err_out; 1796 1797 err = -EINVAL; 1798 kind = tb[TCA_ACT_KIND]; 1799 ops = tc_lookup_action(kind); 1800 if (!ops) { /*some idjot trying to flush unknown action */ 1801 NL_SET_ERR_MSG(extack, "Cannot flush unknown TC action"); 1802 goto err_out; 1803 } 1804 1805 nlh = nlmsg_put(skb, portid, n->nlmsg_seq, RTM_DELACTION, 1806 sizeof(*t), 0); 1807 if (!nlh) { 1808 NL_SET_ERR_MSG(extack, "Failed to create TC action flush notification"); 1809 goto out_module_put; 1810 } 1811 t = nlmsg_data(nlh); 1812 t->tca_family = AF_UNSPEC; 1813 t->tca__pad1 = 0; 1814 t->tca__pad2 = 0; 1815 1816 nest = nla_nest_start_noflag(skb, TCA_ACT_TAB); 1817 if (!nest) { 1818 NL_SET_ERR_MSG(extack, "Failed to add new netlink message"); 1819 goto out_module_put; 1820 } 1821 1822 err = __tcf_generic_walker(net, skb, &dcb, RTM_DELACTION, ops, extack); 1823 if (err <= 0) { 1824 nla_nest_cancel(skb, nest); 1825 goto out_module_put; 1826 } 1827 1828 nla_nest_end(skb, nest); 1829 1830 nlh->nlmsg_len = skb_tail_pointer(skb) - b; 1831 nlh->nlmsg_flags |= NLM_F_ROOT; 1832 module_put(ops->owner); 1833 err = rtnetlink_send(skb, net, portid, RTNLGRP_TC, 1834 n->nlmsg_flags & NLM_F_ECHO); 1835 if (err < 0) 1836 NL_SET_ERR_MSG(extack, "Failed to send TC action flush notification"); 1837 1838 return err; 1839 1840 out_module_put: 1841 module_put(ops->owner); 1842 err_out: 1843 kfree_skb(skb); 1844 return err; 1845 } 1846 1847 static int tcf_action_delete(struct net *net, struct tc_action *actions[]) 1848 { 1849 struct tc_action *a; 1850 int i; 1851 1852 tcf_act_for_each_action(i, a, actions) { 1853 const struct tc_action_ops *ops = a->ops; 1854 /* Actions can be deleted concurrently so we must save their 1855 * type and id to search again after reference is released. 1856 */ 1857 struct tcf_idrinfo *idrinfo = a->idrinfo; 1858 u32 act_index = a->tcfa_index; 1859 1860 actions[i] = NULL; 1861 if (tcf_action_put(a)) { 1862 /* last reference, action was deleted concurrently */ 1863 module_put(ops->owner); 1864 } else { 1865 int ret; 1866 1867 /* now do the delete */ 1868 ret = tcf_idr_delete_index(idrinfo, act_index); 1869 if (ret < 0) 1870 return ret; 1871 } 1872 } 1873 return 0; 1874 } 1875 1876 static struct sk_buff *tcf_reoffload_del_notify_msg(struct net *net, 1877 struct tc_action *action) 1878 { 1879 struct tc_action *actions[TCA_ACT_MAX_PRIO] = { 1880 [0] = action, 1881 }; 1882 struct sk_buff *skb; 1883 size_t attr_size; 1884 1885 attr_size = tcf_action_full_attrs_size(tcf_action_fill_size(action)); 1886 1887 skb = alloc_skb(max(attr_size, NLMSG_GOODSIZE), GFP_KERNEL); 1888 if (!skb) 1889 return ERR_PTR(-ENOBUFS); 1890 1891 if (tca_get_fill(skb, actions, 0, 0, 0, RTM_DELACTION, 0, 1, NULL) <= 0) { 1892 kfree_skb(skb); 1893 return ERR_PTR(-EINVAL); 1894 } 1895 1896 return skb; 1897 } 1898 1899 static int tcf_reoffload_del_notify(struct net *net, struct tc_action *action) 1900 { 1901 const struct tc_action_ops *ops = action->ops; 1902 struct sk_buff *skb = NULL; 1903 int ret; 1904 1905 if (rtnl_notify_needed(net, 0, RTNLGRP_TC)) { 1906 skb = tcf_reoffload_del_notify_msg(net, action); 1907 /* The action has already lost its hardware instance and is 1908 * skip_sw, so it must be released whether or not the 1909 * notification can be built. Drop the notification rather 1910 * than leave an action behind that processes no packets. 1911 */ 1912 if (IS_ERR(skb)) 1913 skb = NULL; 1914 } 1915 1916 ret = tcf_idr_release_unsafe(action); 1917 if (ret == ACT_P_DELETED) { 1918 module_put(ops->owner); 1919 ret = rtnetlink_maybe_send(skb, net, 0, RTNLGRP_TC, 0); 1920 } else { 1921 kfree_skb(skb); 1922 } 1923 1924 return ret; 1925 } 1926 1927 int tcf_action_reoffload_cb(flow_indr_block_bind_cb_t *cb, 1928 void *cb_priv, bool add) 1929 { 1930 struct tc_act_pernet_id *id_ptr; 1931 struct tcf_idrinfo *idrinfo; 1932 struct tc_action_net *tn; 1933 struct tc_action *p; 1934 unsigned int act_id; 1935 unsigned long tmp; 1936 unsigned long id; 1937 struct idr *idr; 1938 struct net *net; 1939 int ret; 1940 1941 if (!cb) 1942 return -EINVAL; 1943 1944 down_read(&net_rwsem); 1945 mutex_lock(&act_id_mutex); 1946 1947 for_each_net(net) { 1948 list_for_each_entry(id_ptr, &act_pernet_id_list, list) { 1949 act_id = id_ptr->id; 1950 tn = net_generic(net, act_id); 1951 if (!tn) 1952 continue; 1953 idrinfo = tn->idrinfo; 1954 if (!idrinfo) 1955 continue; 1956 1957 mutex_lock(&idrinfo->lock); 1958 idr = &idrinfo->action_idr; 1959 idr_for_each_entry_ul(idr, p, tmp, id) { 1960 if (IS_ERR(p) || tc_act_bind(p->tcfa_flags)) 1961 continue; 1962 if (add) { 1963 tcf_action_offload_add_ex(p, NULL, cb, 1964 cb_priv); 1965 continue; 1966 } 1967 1968 /* cb unregister to update hw count */ 1969 ret = tcf_action_offload_del_ex(p, cb, cb_priv); 1970 if (ret < 0) 1971 continue; 1972 if (tc_act_skip_sw(p->tcfa_flags) && 1973 !tc_act_in_hw(p)) 1974 tcf_reoffload_del_notify(net, p); 1975 } 1976 mutex_unlock(&idrinfo->lock); 1977 } 1978 } 1979 mutex_unlock(&act_id_mutex); 1980 up_read(&net_rwsem); 1981 1982 return 0; 1983 } 1984 1985 static struct sk_buff *tcf_del_notify_msg(struct net *net, struct nlmsghdr *n, 1986 struct tc_action *actions[], 1987 u32 portid, size_t attr_size, 1988 struct netlink_ext_ack *extack) 1989 { 1990 struct sk_buff *skb; 1991 1992 skb = alloc_skb(max(attr_size + tcf_action_warn_attr_size(extack), 1993 NLMSG_GOODSIZE), GFP_KERNEL); 1994 if (!skb) 1995 return ERR_PTR(-ENOBUFS); 1996 1997 if (tca_get_fill(skb, actions, portid, n->nlmsg_seq, 0, RTM_DELACTION, 1998 0, 2, extack) <= 0) { 1999 NL_SET_ERR_MSG(extack, "Failed to fill netlink TC action attributes"); 2000 kfree_skb(skb); 2001 return ERR_PTR(-EINVAL); 2002 } 2003 2004 return skb; 2005 } 2006 2007 static int tcf_del_notify(struct net *net, struct nlmsghdr *n, 2008 struct tc_action *actions[], u32 portid, 2009 size_t attr_size, struct netlink_ext_ack *extack) 2010 { 2011 struct sk_buff *skb; 2012 int ret; 2013 2014 if (!rtnl_notify_needed(net, n->nlmsg_flags, RTNLGRP_TC)) { 2015 skb = NULL; 2016 } else { 2017 skb = tcf_del_notify_msg(net, n, actions, portid, attr_size, 2018 extack); 2019 if (IS_ERR(skb)) 2020 return PTR_ERR(skb); 2021 } 2022 2023 /* now do the delete */ 2024 ret = tcf_action_delete(net, actions); 2025 if (ret < 0) { 2026 NL_SET_ERR_MSG(extack, "Failed to delete TC action"); 2027 kfree_skb(skb); 2028 return ret; 2029 } 2030 2031 return rtnetlink_maybe_send(skb, net, portid, RTNLGRP_TC, 2032 n->nlmsg_flags & NLM_F_ECHO); 2033 } 2034 2035 static int 2036 tca_action_gd(struct net *net, struct nlattr *nla, struct nlmsghdr *n, 2037 u32 portid, int event, struct netlink_ext_ack *extack) 2038 { 2039 int i, ret; 2040 struct nlattr *tb[TCA_ACT_MAX_PRIO + 1]; 2041 struct tc_action *act; 2042 size_t attr_size = 0; 2043 struct tc_action *actions[TCA_ACT_MAX_PRIO] = {}; 2044 2045 ret = nla_parse_nested_deprecated(tb, TCA_ACT_MAX_PRIO, nla, NULL, 2046 extack); 2047 if (ret < 0) 2048 return ret; 2049 2050 if (event == RTM_DELACTION && n->nlmsg_flags & NLM_F_ROOT) { 2051 if (tb[1]) 2052 return tca_action_flush(net, tb[1], n, portid, extack); 2053 2054 NL_SET_ERR_MSG(extack, "Invalid netlink attributes while flushing TC action"); 2055 return -EINVAL; 2056 } 2057 2058 for (i = 1; i <= TCA_ACT_MAX_PRIO && tb[i]; i++) { 2059 act = tcf_action_get_1(net, tb[i], n, portid, extack); 2060 if (IS_ERR(act)) { 2061 ret = PTR_ERR(act); 2062 goto err; 2063 } 2064 attr_size += tcf_action_fill_size(act); 2065 actions[i - 1] = act; 2066 } 2067 2068 attr_size = tcf_action_full_attrs_size(attr_size); 2069 2070 if (event == RTM_GETACTION) 2071 ret = tcf_get_notify(net, portid, n, actions, attr_size, event, 2072 extack); 2073 else { /* delete */ 2074 ret = tcf_del_notify(net, n, actions, portid, attr_size, extack); 2075 if (ret) 2076 goto err; 2077 return 0; 2078 } 2079 err: 2080 tcf_action_put_many(actions); 2081 return ret; 2082 } 2083 2084 static struct sk_buff *tcf_add_notify_msg(struct net *net, struct nlmsghdr *n, 2085 struct tc_action *actions[], 2086 u32 portid, size_t attr_size, 2087 struct netlink_ext_ack *extack) 2088 { 2089 struct sk_buff *skb; 2090 2091 skb = alloc_skb(max(attr_size + tcf_action_warn_attr_size(extack), 2092 NLMSG_GOODSIZE), GFP_KERNEL); 2093 if (!skb) 2094 return ERR_PTR(-ENOBUFS); 2095 2096 if (tca_get_fill(skb, actions, portid, n->nlmsg_seq, n->nlmsg_flags, 2097 RTM_NEWACTION, 0, 0, extack) <= 0) { 2098 NL_SET_ERR_MSG(extack, "Failed to fill netlink attributes while adding TC action"); 2099 kfree_skb(skb); 2100 return ERR_PTR(-EINVAL); 2101 } 2102 2103 return skb; 2104 } 2105 2106 static int tcf_add_notify(struct net *net, struct nlmsghdr *n, 2107 struct tc_action *actions[], u32 portid, 2108 size_t attr_size, struct netlink_ext_ack *extack) 2109 { 2110 struct sk_buff *skb; 2111 2112 if (!rtnl_notify_needed(net, n->nlmsg_flags, RTNLGRP_TC)) { 2113 skb = NULL; 2114 } else { 2115 skb = tcf_add_notify_msg(net, n, actions, portid, attr_size, 2116 extack); 2117 if (IS_ERR(skb)) 2118 return PTR_ERR(skb); 2119 } 2120 2121 return rtnetlink_maybe_send(skb, net, portid, RTNLGRP_TC, 2122 n->nlmsg_flags & NLM_F_ECHO); 2123 } 2124 2125 static int tcf_action_add(struct net *net, struct nlattr *nla, 2126 struct nlmsghdr *n, u32 portid, u32 flags, 2127 struct netlink_ext_ack *extack) 2128 { 2129 size_t attr_size = 0; 2130 int loop, ret; 2131 struct tc_action *actions[TCA_ACT_MAX_PRIO] = {}; 2132 int init_res[TCA_ACT_MAX_PRIO] = {}; 2133 2134 for (loop = 0; loop < 10; loop++) { 2135 ret = tcf_action_init(net, NULL, nla, NULL, actions, init_res, 2136 &attr_size, flags, 0, extack); 2137 if (ret != -EAGAIN) 2138 break; 2139 } 2140 2141 if (ret < 0) 2142 return ret; 2143 2144 ret = tcf_add_notify(net, n, actions, portid, attr_size, extack); 2145 2146 /* only put bound actions */ 2147 tca_put_bound_many(actions, init_res); 2148 2149 return ret; 2150 } 2151 2152 static const struct nla_policy tcaa_policy[TCA_ROOT_MAX + 1] = { 2153 [TCA_ROOT_FLAGS] = NLA_POLICY_BITFIELD32(TCA_ACT_FLAG_LARGE_DUMP_ON | 2154 TCA_ACT_FLAG_TERSE_DUMP), 2155 [TCA_ROOT_TIME_DELTA] = { .type = NLA_U32 }, 2156 }; 2157 2158 static int tc_ctl_action(struct sk_buff *skb, struct nlmsghdr *n, 2159 struct netlink_ext_ack *extack) 2160 { 2161 struct net *net = sock_net(skb->sk); 2162 struct nlattr *tca[TCA_ROOT_MAX + 1]; 2163 u32 portid = NETLINK_CB(skb).portid; 2164 u32 flags = 0; 2165 int ret = 0; 2166 2167 if ((n->nlmsg_type != RTM_GETACTION) && 2168 !netlink_capable(skb, CAP_NET_ADMIN)) 2169 return -EPERM; 2170 2171 ret = nlmsg_parse_deprecated(n, sizeof(struct tcamsg), tca, 2172 TCA_ROOT_MAX, NULL, extack); 2173 if (ret < 0) 2174 return ret; 2175 2176 if (tca[TCA_ACT_TAB] == NULL) { 2177 NL_SET_ERR_MSG(extack, "Netlink action attributes missing"); 2178 return -EINVAL; 2179 } 2180 2181 /* n->nlmsg_flags & NLM_F_CREATE */ 2182 switch (n->nlmsg_type) { 2183 case RTM_NEWACTION: 2184 /* we are going to assume all other flags 2185 * imply create only if it doesn't exist 2186 * Note that CREATE | EXCL implies that 2187 * but since we want avoid ambiguity (eg when flags 2188 * is zero) then just set this 2189 */ 2190 if (n->nlmsg_flags & NLM_F_REPLACE) 2191 flags = TCA_ACT_FLAGS_REPLACE; 2192 ret = tcf_action_add(net, tca[TCA_ACT_TAB], n, portid, flags, 2193 extack); 2194 break; 2195 case RTM_DELACTION: 2196 ret = tca_action_gd(net, tca[TCA_ACT_TAB], n, 2197 portid, RTM_DELACTION, extack); 2198 break; 2199 case RTM_GETACTION: 2200 ret = tca_action_gd(net, tca[TCA_ACT_TAB], n, 2201 portid, RTM_GETACTION, extack); 2202 break; 2203 default: 2204 BUG(); 2205 } 2206 2207 return ret; 2208 } 2209 2210 static struct nlattr *find_dump_kind(struct nlattr **nla) 2211 { 2212 struct nlattr *tb1, *tb2[TCA_ACT_MAX + 1]; 2213 struct nlattr *tb[TCA_ACT_MAX_PRIO + 1]; 2214 struct nlattr *kind; 2215 2216 tb1 = nla[TCA_ACT_TAB]; 2217 if (tb1 == NULL) 2218 return NULL; 2219 2220 if (nla_parse_deprecated(tb, TCA_ACT_MAX_PRIO, nla_data(tb1), NLMSG_ALIGN(nla_len(tb1)), NULL, NULL) < 0) 2221 return NULL; 2222 2223 if (tb[1] == NULL) 2224 return NULL; 2225 if (nla_parse_nested_deprecated(tb2, TCA_ACT_MAX, tb[1], tcf_action_policy, NULL) < 0) 2226 return NULL; 2227 kind = tb2[TCA_ACT_KIND]; 2228 2229 return kind; 2230 } 2231 2232 static int tc_dump_action(struct sk_buff *skb, struct netlink_callback *cb) 2233 { 2234 struct net *net = sock_net(skb->sk); 2235 struct nlmsghdr *nlh; 2236 unsigned char *b = skb_tail_pointer(skb); 2237 struct nlattr *nest; 2238 struct tc_action_ops *a_o; 2239 int ret = 0; 2240 struct tcamsg *t = (struct tcamsg *) nlmsg_data(cb->nlh); 2241 struct nlattr *tb[TCA_ROOT_MAX + 1]; 2242 struct nlattr *count_attr = NULL; 2243 unsigned long jiffy_since = 0; 2244 struct nlattr *kind = NULL; 2245 struct nla_bitfield32 bf; 2246 u32 msecs_since = 0; 2247 u32 act_count = 0; 2248 2249 ret = nlmsg_parse_deprecated(cb->nlh, sizeof(struct tcamsg), tb, 2250 TCA_ROOT_MAX, tcaa_policy, cb->extack); 2251 if (ret < 0) 2252 return ret; 2253 2254 kind = find_dump_kind(tb); 2255 if (kind == NULL) { 2256 pr_info("tc_dump_action: action bad kind\n"); 2257 return 0; 2258 } 2259 2260 a_o = tc_lookup_action(kind); 2261 if (a_o == NULL) 2262 return 0; 2263 2264 cb->args[2] = 0; 2265 if (tb[TCA_ROOT_FLAGS]) { 2266 bf = nla_get_bitfield32(tb[TCA_ROOT_FLAGS]); 2267 cb->args[2] = bf.value; 2268 } 2269 2270 if (tb[TCA_ROOT_TIME_DELTA]) { 2271 msecs_since = nla_get_u32(tb[TCA_ROOT_TIME_DELTA]); 2272 } 2273 2274 nlh = nlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq, 2275 cb->nlh->nlmsg_type, sizeof(*t), 0); 2276 if (!nlh) 2277 goto out_module_put; 2278 2279 if (msecs_since) 2280 jiffy_since = jiffies - msecs_to_jiffies(msecs_since); 2281 2282 t = nlmsg_data(nlh); 2283 t->tca_family = AF_UNSPEC; 2284 t->tca__pad1 = 0; 2285 t->tca__pad2 = 0; 2286 cb->args[3] = jiffy_since; 2287 count_attr = nla_reserve(skb, TCA_ROOT_COUNT, sizeof(u32)); 2288 if (!count_attr) 2289 goto out_module_put; 2290 2291 nest = nla_nest_start_noflag(skb, TCA_ACT_TAB); 2292 if (nest == NULL) 2293 goto out_module_put; 2294 2295 ret = __tcf_generic_walker(net, skb, cb, RTM_GETACTION, a_o, NULL); 2296 if (ret < 0) 2297 goto out_module_put; 2298 2299 if (ret > 0) { 2300 nla_nest_end(skb, nest); 2301 ret = skb->len; 2302 act_count = cb->args[1]; 2303 memcpy(nla_data(count_attr), &act_count, sizeof(u32)); 2304 cb->args[1] = 0; 2305 } else 2306 nlmsg_trim(skb, b); 2307 2308 nlh->nlmsg_len = skb_tail_pointer(skb) - b; 2309 if (NETLINK_CB(cb->skb).portid && ret) 2310 nlh->nlmsg_flags |= NLM_F_MULTI; 2311 module_put(a_o->owner); 2312 return skb->len; 2313 2314 out_module_put: 2315 module_put(a_o->owner); 2316 nlmsg_trim(skb, b); 2317 return skb->len; 2318 } 2319 2320 static const struct rtnl_msg_handler tc_action_rtnl_msg_handlers[] __initconst = { 2321 {.msgtype = RTM_NEWACTION, .doit = tc_ctl_action}, 2322 {.msgtype = RTM_DELACTION, .doit = tc_ctl_action}, 2323 {.msgtype = RTM_GETACTION, .doit = tc_ctl_action, 2324 .dumpit = tc_dump_action}, 2325 }; 2326 2327 static int __init tc_action_init(void) 2328 { 2329 rtnl_register_many(tc_action_rtnl_msg_handlers); 2330 return 0; 2331 } 2332 2333 subsys_initcall(tc_action_init); 2334