1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * net/sched/act_api.c Packet action API. 4 * 5 * Author: Jamal Hadi Salim 6 */ 7 8 #include <linux/types.h> 9 #include <linux/kernel.h> 10 #include <linux/string.h> 11 #include <linux/errno.h> 12 #include <linux/slab.h> 13 #include <linux/skbuff.h> 14 #include <linux/init.h> 15 #include <linux/kmod.h> 16 #include <linux/err.h> 17 #include <linux/module.h> 18 #include <net/net_namespace.h> 19 #include <net/sock.h> 20 #include <net/sch_generic.h> 21 #include <net/pkt_cls.h> 22 #include <net/tc_act/tc_pedit.h> 23 #include <net/act_api.h> 24 #include <net/netlink.h> 25 #include <net/flow_offload.h> 26 #include <net/tc_wrapper.h> 27 28 #ifdef CONFIG_INET 29 DEFINE_STATIC_KEY_FALSE(tcf_frag_xmit_count); 30 EXPORT_SYMBOL_GPL(tcf_frag_xmit_count); 31 #endif 32 33 int tcf_dev_queue_xmit(struct sk_buff *skb, int (*xmit)(struct sk_buff *skb)) 34 { 35 #ifdef CONFIG_INET 36 if (static_branch_unlikely(&tcf_frag_xmit_count)) 37 return sch_frag_xmit_hook(skb, xmit); 38 #endif 39 40 return xmit(skb); 41 } 42 EXPORT_SYMBOL_GPL(tcf_dev_queue_xmit); 43 44 static void tcf_action_goto_chain_exec(const struct tcf_chain *chain, 45 struct tcf_result *res) 46 { 47 res->goto_tp = rcu_dereference_bh(chain->filter_chain); 48 } 49 50 static void tcf_free_cookie_rcu(struct rcu_head *p) 51 { 52 struct tc_cookie *cookie = container_of(p, struct tc_cookie, rcu); 53 54 kfree(cookie->data); 55 kfree(cookie); 56 } 57 58 static void tcf_set_action_cookie(struct tc_cookie __rcu **old_cookie, 59 struct tc_cookie *new_cookie) 60 { 61 struct tc_cookie *old; 62 63 old = unrcu_pointer(xchg(old_cookie, RCU_INITIALIZER(new_cookie))); 64 if (old) 65 call_rcu(&old->rcu, tcf_free_cookie_rcu); 66 } 67 68 int tcf_action_check_ctrlact(int action, struct tcf_proto *tp, 69 struct tcf_chain **newchain, 70 struct netlink_ext_ack *extack) 71 { 72 int opcode = TC_ACT_EXT_OPCODE(action), ret = -EINVAL; 73 u32 chain_index; 74 75 if (!opcode) 76 ret = action > TC_ACT_VALUE_MAX ? -EINVAL : 0; 77 else if (opcode <= TC_ACT_EXT_OPCODE_MAX || action == TC_ACT_UNSPEC) 78 ret = 0; 79 if (ret) { 80 NL_SET_ERR_MSG(extack, "invalid control action"); 81 goto end; 82 } 83 84 if (TC_ACT_EXT_CMP(action, TC_ACT_GOTO_CHAIN)) { 85 chain_index = action & TC_ACT_EXT_VAL_MASK; 86 if (!tp || !newchain) { 87 ret = -EINVAL; 88 NL_SET_ERR_MSG(extack, 89 "can't goto NULL proto/chain"); 90 goto end; 91 } 92 *newchain = tcf_chain_get_by_act(tp->chain->block, chain_index); 93 if (!*newchain) { 94 ret = -ENOMEM; 95 NL_SET_ERR_MSG(extack, 96 "can't allocate goto_chain"); 97 } 98 } 99 end: 100 return ret; 101 } 102 EXPORT_SYMBOL(tcf_action_check_ctrlact); 103 104 struct tcf_chain *tcf_action_set_ctrlact(struct tc_action *a, int action, 105 struct tcf_chain *goto_chain) 106 { 107 a->tcfa_action = action; 108 goto_chain = rcu_replace_pointer(a->goto_chain, goto_chain, 1); 109 return goto_chain; 110 } 111 EXPORT_SYMBOL(tcf_action_set_ctrlact); 112 113 static void free_tcf(struct tc_action *p) 114 { 115 struct tcf_chain *chain = rcu_dereference_protected(p->goto_chain, 1); 116 117 free_percpu(p->cpu_bstats); 118 free_percpu(p->cpu_bstats_hw); 119 free_percpu(p->cpu_qstats); 120 121 tcf_set_action_cookie(&p->user_cookie, NULL); 122 if (chain) 123 tcf_chain_put_by_act(chain); 124 125 kfree_rcu(p, tcfa_rcu); 126 } 127 128 static void offload_action_hw_count_set(struct tc_action *act, 129 u32 hw_count) 130 { 131 act->in_hw_count = hw_count; 132 } 133 134 static void offload_action_hw_count_inc(struct tc_action *act, 135 u32 hw_count) 136 { 137 act->in_hw_count += hw_count; 138 } 139 140 static void offload_action_hw_count_dec(struct tc_action *act, 141 u32 hw_count) 142 { 143 act->in_hw_count = act->in_hw_count > hw_count ? 144 act->in_hw_count - hw_count : 0; 145 } 146 147 static unsigned int tcf_offload_act_num_actions_single(struct tc_action *act) 148 { 149 unsigned int count; 150 151 if (is_tcf_pedit(act)) { 152 spin_lock_bh(&act->tcfa_lock); 153 count = tcf_pedit_nkeys_locked(act); 154 spin_unlock_bh(&act->tcfa_lock); 155 return count; 156 } 157 return 1; 158 } 159 160 static bool tc_act_skip_hw(u32 flags) 161 { 162 return (flags & TCA_ACT_FLAGS_SKIP_HW) ? true : false; 163 } 164 165 static bool tc_act_skip_sw(u32 flags) 166 { 167 return (flags & TCA_ACT_FLAGS_SKIP_SW) ? true : false; 168 } 169 170 /* SKIP_HW and SKIP_SW are mutually exclusive flags. */ 171 static bool tc_act_flags_valid(u32 flags) 172 { 173 flags &= TCA_ACT_FLAGS_SKIP_HW | TCA_ACT_FLAGS_SKIP_SW; 174 175 return flags ^ (TCA_ACT_FLAGS_SKIP_HW | TCA_ACT_FLAGS_SKIP_SW); 176 } 177 178 static int offload_action_init(struct flow_offload_action *fl_action, 179 struct tc_action *act, 180 enum offload_act_command cmd, 181 struct netlink_ext_ack *extack) 182 { 183 int err; 184 185 fl_action->extack = extack; 186 fl_action->command = cmd; 187 fl_action->index = act->tcfa_index; 188 fl_action->cookie = (unsigned long)act; 189 190 if (act->ops->offload_act_setup) { 191 spin_lock_bh(&act->tcfa_lock); 192 err = act->ops->offload_act_setup(act, fl_action, NULL, 193 false, extack); 194 spin_unlock_bh(&act->tcfa_lock); 195 return err; 196 } 197 198 return -EOPNOTSUPP; 199 } 200 201 static int tcf_action_offload_cmd_ex(struct flow_offload_action *fl_act, 202 u32 *hw_count) 203 { 204 int err; 205 206 err = flow_indr_dev_setup_offload(NULL, NULL, TC_SETUP_ACT, 207 fl_act, NULL, NULL); 208 if (err < 0) 209 return err; 210 211 if (hw_count) 212 *hw_count = err; 213 214 return 0; 215 } 216 217 static int tcf_action_offload_cmd_cb_ex(struct flow_offload_action *fl_act, 218 u32 *hw_count, 219 flow_indr_block_bind_cb_t *cb, 220 void *cb_priv) 221 { 222 int err; 223 224 err = cb(NULL, NULL, cb_priv, TC_SETUP_ACT, NULL, fl_act, NULL); 225 if (err < 0) 226 return err; 227 228 if (hw_count) 229 *hw_count = 1; 230 231 return 0; 232 } 233 234 static int tcf_action_offload_cmd(struct flow_offload_action *fl_act, 235 u32 *hw_count, 236 flow_indr_block_bind_cb_t *cb, 237 void *cb_priv) 238 { 239 return cb ? tcf_action_offload_cmd_cb_ex(fl_act, hw_count, 240 cb, cb_priv) : 241 tcf_action_offload_cmd_ex(fl_act, hw_count); 242 } 243 244 static int tcf_action_offload_add_ex(struct tc_action *action, 245 struct netlink_ext_ack *extack, 246 flow_indr_block_bind_cb_t *cb, 247 void *cb_priv) 248 { 249 bool skip_sw = tc_act_skip_sw(action->tcfa_flags); 250 struct tc_action *actions[TCA_ACT_MAX_PRIO] = { 251 [0] = action, 252 }; 253 struct flow_offload_action *fl_action; 254 u32 in_hw_count = 0; 255 int num, err = 0; 256 257 if (tc_act_skip_hw(action->tcfa_flags)) 258 return 0; 259 260 num = tcf_offload_act_num_actions_single(action); 261 fl_action = offload_action_alloc(num); 262 if (!fl_action) 263 return -ENOMEM; 264 265 err = offload_action_init(fl_action, action, FLOW_ACT_REPLACE, extack); 266 if (err) 267 goto fl_err; 268 269 err = tc_setup_action(&fl_action->action, actions, 0, extack); 270 if (err) { 271 NL_SET_ERR_MSG_MOD(extack, 272 "Failed to setup tc actions for offload"); 273 goto fl_err; 274 } 275 276 err = tcf_action_offload_cmd(fl_action, &in_hw_count, cb, cb_priv); 277 if (!err) 278 cb ? offload_action_hw_count_inc(action, in_hw_count) : 279 offload_action_hw_count_set(action, in_hw_count); 280 281 if (skip_sw && !tc_act_in_hw(action)) 282 err = -EINVAL; 283 284 tc_cleanup_offload_action(&fl_action->action); 285 286 fl_err: 287 kfree(fl_action); 288 289 return err; 290 } 291 292 /* offload the tc action after it is inserted */ 293 static int tcf_action_offload_add(struct tc_action *action, 294 struct netlink_ext_ack *extack) 295 { 296 return tcf_action_offload_add_ex(action, extack, NULL, NULL); 297 } 298 299 int tcf_action_update_hw_stats(struct tc_action *action) 300 { 301 struct flow_offload_action fl_act = {}; 302 int err; 303 304 err = offload_action_init(&fl_act, action, FLOW_ACT_STATS, NULL); 305 if (err) 306 return err; 307 308 err = tcf_action_offload_cmd(&fl_act, NULL, NULL, NULL); 309 if (!err) { 310 preempt_disable(); 311 tcf_action_stats_update(action, fl_act.stats.bytes, 312 fl_act.stats.pkts, 313 fl_act.stats.drops, 314 fl_act.stats.lastused, 315 true); 316 preempt_enable(); 317 action->used_hw_stats = fl_act.stats.used_hw_stats; 318 action->used_hw_stats_valid = true; 319 } else { 320 return -EOPNOTSUPP; 321 } 322 323 return 0; 324 } 325 EXPORT_SYMBOL(tcf_action_update_hw_stats); 326 327 static int tcf_action_offload_del_ex(struct tc_action *action, 328 flow_indr_block_bind_cb_t *cb, 329 void *cb_priv) 330 { 331 struct flow_offload_action fl_act = {}; 332 u32 in_hw_count = 0; 333 int err = 0; 334 335 if (!tc_act_in_hw(action)) 336 return 0; 337 338 err = offload_action_init(&fl_act, action, FLOW_ACT_DESTROY, NULL); 339 if (err) 340 return err; 341 342 err = tcf_action_offload_cmd(&fl_act, &in_hw_count, cb, cb_priv); 343 if (err < 0) 344 return err; 345 346 if (!cb && action->in_hw_count != in_hw_count) 347 return -EINVAL; 348 349 /* do not need to update hw state when deleting action */ 350 if (cb && in_hw_count) 351 offload_action_hw_count_dec(action, in_hw_count); 352 353 return 0; 354 } 355 356 static int tcf_action_offload_del(struct tc_action *action) 357 { 358 return tcf_action_offload_del_ex(action, NULL, NULL); 359 } 360 361 static void tcf_action_cleanup(struct tc_action *p) 362 { 363 tcf_action_offload_del(p); 364 if (p->ops->cleanup) 365 p->ops->cleanup(p); 366 367 gen_kill_estimator(&p->tcfa_rate_est); 368 free_tcf(p); 369 } 370 371 static int __tcf_action_put(struct tc_action *p, bool bind) 372 { 373 struct tcf_idrinfo *idrinfo = p->idrinfo; 374 375 mutex_lock(&idrinfo->lock); 376 if (bind) 377 atomic_dec(&p->tcfa_bindcnt); 378 if (refcount_dec_and_test(&p->tcfa_refcnt)) { 379 idr_remove(&idrinfo->action_idr, p->tcfa_index); 380 mutex_unlock(&idrinfo->lock); 381 382 tcf_action_cleanup(p); 383 return 1; 384 } 385 386 mutex_unlock(&idrinfo->lock); 387 return 0; 388 } 389 390 static int __tcf_idr_release(struct tc_action *p, bool bind, bool strict) 391 { 392 int ret = 0; 393 394 /* Release with strict==1 and bind==0 is only called through act API 395 * interface (classifiers always bind). Only case when action with 396 * positive reference count and zero bind count can exist is when it was 397 * also created with act API (unbinding last classifier will destroy the 398 * action if it was created by classifier). So only case when bind count 399 * can be changed after initial check is when unbound action is 400 * destroyed by act API while classifier binds to action with same id 401 * concurrently. This result either creation of new action(same behavior 402 * as before), or reusing existing action if concurrent process 403 * increments reference count before action is deleted. Both scenarios 404 * are acceptable. 405 */ 406 if (p) { 407 if (!bind && strict && atomic_read(&p->tcfa_bindcnt) > 0) 408 return -EPERM; 409 410 if (__tcf_action_put(p, bind)) 411 ret = ACT_P_DELETED; 412 } 413 414 return ret; 415 } 416 417 int tcf_idr_release(struct tc_action *a, bool bind) 418 { 419 const struct tc_action_ops *ops = a->ops; 420 int ret; 421 422 ret = __tcf_idr_release(a, bind, false); 423 if (ret == ACT_P_DELETED) 424 module_put(ops->owner); 425 return ret; 426 } 427 EXPORT_SYMBOL(tcf_idr_release); 428 429 static size_t tcf_action_shared_attrs_size(const struct tc_action *act) 430 { 431 struct tc_cookie *user_cookie; 432 u32 cookie_len = 0; 433 434 rcu_read_lock(); 435 user_cookie = rcu_dereference(act->user_cookie); 436 437 if (user_cookie) 438 cookie_len = nla_total_size(user_cookie->len); 439 rcu_read_unlock(); 440 441 return nla_total_size(0) /* action number nested */ 442 + nla_total_size(IFNAMSIZ) /* TCA_ACT_KIND */ 443 + cookie_len /* TCA_ACT_COOKIE */ 444 + nla_total_size(sizeof(struct nla_bitfield32)) /* TCA_ACT_HW_STATS */ 445 /* TCA_ACT_USED_HW_STATS */ 446 + nla_total_size(sizeof(struct nla_bitfield32)) 447 + nla_total_size(sizeof(u32)) /* TCA_ACT_IN_HW_COUNT */ 448 + nla_total_size(0) /* TCA_ACT_STATS nested */ 449 + nla_total_size(sizeof(struct nla_bitfield32)) /* TCA_ACT_FLAGS */ 450 /* TCA_STATS_BASIC */ 451 + nla_total_size_64bit(sizeof(struct gnet_stats_basic)) 452 /* TCA_STATS_BASIC_HW */ 453 + nla_total_size_64bit(sizeof(struct gnet_stats_basic)) 454 /* TCA_STATS_PKT64, emitted by both of the basic copies above */ 455 + 2 * nla_total_size_64bit(sizeof(u64)) 456 /* TCA_STATS_RATE_EST */ 457 + nla_total_size_64bit(sizeof(struct gnet_stats_rate_est)) 458 /* TCA_STATS_RATE_EST64 */ 459 + nla_total_size_64bit(sizeof(struct gnet_stats_rate_est64)) 460 /* TCA_STATS_QUEUE */ 461 + nla_total_size_64bit(sizeof(struct gnet_stats_queue)) 462 + nla_total_size(0) /* TCA_ACT_OPTIONS nested */ 463 /* TCA_GACT_TM; actions dump their tcf_t with nla_put_64bit(), 464 * which may emit an extra NLA_PAD attribute. 465 */ 466 + nla_total_size_64bit(sizeof(struct tcf_t)); 467 } 468 469 static size_t tcf_action_full_attrs_size(size_t sz) 470 { 471 return NLMSG_HDRLEN /* struct nlmsghdr */ 472 + sizeof(struct tcamsg) 473 + nla_total_size(0) /* TCA_ACT_TAB nested */ 474 + sz; 475 } 476 477 /* tca_get_fill() may append TCA_ROOT_EXT_WARN_MSG from extack->_msg */ 478 static size_t tcf_action_warn_attr_size(const struct netlink_ext_ack *extack) 479 { 480 if (unlikely(extack && extack->_msg)) 481 return nla_total_size(strlen(extack->_msg) + 1); 482 483 return 0; 484 } 485 486 static size_t tcf_action_fill_size(const struct tc_action *act) 487 { 488 size_t sz = tcf_action_shared_attrs_size(act); 489 490 if (act->ops->get_fill_size) 491 return act->ops->get_fill_size(act) + sz; 492 return sz; 493 } 494 495 static int 496 tcf_action_dump_terse(struct sk_buff *skb, struct tc_action *a, bool from_act) 497 { 498 unsigned char *b = skb_tail_pointer(skb); 499 struct tc_cookie *cookie; 500 501 if (nla_put_string(skb, TCA_ACT_KIND, a->ops->kind)) 502 goto nla_put_failure; 503 if (tcf_action_copy_stats(skb, a, 0)) 504 goto nla_put_failure; 505 if (from_act && nla_put_u32(skb, TCA_ACT_INDEX, a->tcfa_index)) 506 goto nla_put_failure; 507 508 rcu_read_lock(); 509 cookie = rcu_dereference(a->user_cookie); 510 if (cookie) { 511 if (nla_put(skb, TCA_ACT_COOKIE, cookie->len, cookie->data)) { 512 rcu_read_unlock(); 513 goto nla_put_failure; 514 } 515 } 516 rcu_read_unlock(); 517 518 return 0; 519 520 nla_put_failure: 521 nlmsg_trim(skb, b); 522 return -1; 523 } 524 525 static int 526 tcf_action_dump_1(struct sk_buff *skb, struct tc_action *a, int bind, int ref) 527 { 528 unsigned char *b = skb_tail_pointer(skb); 529 struct nlattr *nest; 530 int err = -EINVAL; 531 u32 flags; 532 533 if (tcf_action_dump_terse(skb, a, false)) 534 goto nla_put_failure; 535 536 if (a->hw_stats != TCA_ACT_HW_STATS_ANY && 537 nla_put_bitfield32(skb, TCA_ACT_HW_STATS, 538 a->hw_stats, TCA_ACT_HW_STATS_ANY)) 539 goto nla_put_failure; 540 541 if (a->used_hw_stats_valid && 542 nla_put_bitfield32(skb, TCA_ACT_USED_HW_STATS, 543 a->used_hw_stats, TCA_ACT_HW_STATS_ANY)) 544 goto nla_put_failure; 545 546 flags = a->tcfa_flags & TCA_ACT_FLAGS_USER_MASK; 547 if (flags && 548 nla_put_bitfield32(skb, TCA_ACT_FLAGS, 549 flags, flags)) 550 goto nla_put_failure; 551 552 if (nla_put_u32(skb, TCA_ACT_IN_HW_COUNT, a->in_hw_count)) 553 goto nla_put_failure; 554 555 nest = nla_nest_start_noflag(skb, TCA_ACT_OPTIONS); 556 if (nest == NULL) 557 goto nla_put_failure; 558 err = tcf_action_dump_old(skb, a, bind, ref); 559 if (err > 0) { 560 nla_nest_end(skb, nest); 561 return err; 562 } 563 564 nla_put_failure: 565 nlmsg_trim(skb, b); 566 return -1; 567 } 568 569 static int tcf_dump_walker(struct tcf_idrinfo *idrinfo, struct sk_buff *skb, 570 struct netlink_callback *cb) 571 { 572 int err = 0, index = -1, s_i = 0, n_i = 0; 573 u32 act_flags = cb->args[2]; 574 unsigned long jiffy_since = cb->args[3]; 575 struct nlattr *nest; 576 struct idr *idr = &idrinfo->action_idr; 577 struct tc_action *p; 578 unsigned long id = 1; 579 unsigned long tmp; 580 581 mutex_lock(&idrinfo->lock); 582 583 s_i = cb->args[0]; 584 585 idr_for_each_entry_ul(idr, p, tmp, id) { 586 index++; 587 if (index < s_i) 588 continue; 589 if (IS_ERR(p)) 590 continue; 591 592 if (jiffy_since && 593 time_after(jiffy_since, 594 (unsigned long)p->tcfa_tm.lastuse)) 595 continue; 596 597 tcf_action_update_hw_stats(p); 598 599 nest = nla_nest_start_noflag(skb, n_i); 600 if (!nest) { 601 index--; 602 goto nla_put_failure; 603 } 604 err = (act_flags & TCA_ACT_FLAG_TERSE_DUMP) ? 605 tcf_action_dump_terse(skb, p, true) : 606 tcf_action_dump_1(skb, p, 0, 0); 607 if (err < 0) { 608 index--; 609 nlmsg_trim(skb, nest); 610 goto done; 611 } 612 nla_nest_end(skb, nest); 613 n_i++; 614 if (!(act_flags & TCA_ACT_FLAG_LARGE_DUMP_ON) && 615 n_i >= TCA_ACT_MAX_PRIO) 616 goto done; 617 } 618 done: 619 if (index >= 0) 620 cb->args[0] = index + 1; 621 622 mutex_unlock(&idrinfo->lock); 623 if (n_i) { 624 if (act_flags & TCA_ACT_FLAG_LARGE_DUMP_ON) 625 cb->args[1] = n_i; 626 } 627 return n_i; 628 629 nla_put_failure: 630 nla_nest_cancel(skb, nest); 631 goto done; 632 } 633 634 static int tcf_idr_release_unsafe(struct tc_action *p) 635 { 636 if (atomic_read(&p->tcfa_bindcnt) > 0) 637 return -EPERM; 638 639 if (refcount_dec_and_test(&p->tcfa_refcnt)) { 640 idr_remove(&p->idrinfo->action_idr, p->tcfa_index); 641 tcf_action_cleanup(p); 642 return ACT_P_DELETED; 643 } 644 645 return 0; 646 } 647 648 static int tcf_del_walker(struct tcf_idrinfo *idrinfo, struct sk_buff *skb, 649 const struct tc_action_ops *ops, 650 struct netlink_ext_ack *extack) 651 { 652 struct nlattr *nest; 653 int n_i = 0; 654 int ret = -EINVAL; 655 struct idr *idr = &idrinfo->action_idr; 656 struct tc_action *p; 657 unsigned long id = 1; 658 unsigned long tmp; 659 660 nest = nla_nest_start_noflag(skb, 0); 661 if (nest == NULL) 662 goto nla_put_failure; 663 if (nla_put_string(skb, TCA_ACT_KIND, ops->kind)) 664 goto nla_put_failure; 665 666 ret = 0; 667 mutex_lock(&idrinfo->lock); 668 idr_for_each_entry_ul(idr, p, tmp, id) { 669 if (IS_ERR(p)) 670 continue; 671 ret = tcf_idr_release_unsafe(p); 672 if (ret == ACT_P_DELETED) 673 module_put(ops->owner); 674 else if (ret < 0) 675 break; 676 n_i++; 677 } 678 mutex_unlock(&idrinfo->lock); 679 if (ret < 0) { 680 if (n_i) 681 NL_SET_ERR_MSG(extack, "Unable to flush all TC actions"); 682 else 683 goto nla_put_failure; 684 } 685 686 ret = nla_put_u32(skb, TCA_FCNT, n_i); 687 if (ret) 688 goto nla_put_failure; 689 nla_nest_end(skb, nest); 690 691 return n_i; 692 nla_put_failure: 693 nla_nest_cancel(skb, nest); 694 return ret; 695 } 696 697 int tcf_generic_walker(struct tc_action_net *tn, struct sk_buff *skb, 698 struct netlink_callback *cb, int type, 699 const struct tc_action_ops *ops, 700 struct netlink_ext_ack *extack) 701 { 702 struct tcf_idrinfo *idrinfo = tn->idrinfo; 703 704 if (type == RTM_DELACTION) { 705 return tcf_del_walker(idrinfo, skb, ops, extack); 706 } else if (type == RTM_GETACTION) { 707 return tcf_dump_walker(idrinfo, skb, cb); 708 } else { 709 WARN(1, "tcf_generic_walker: unknown command %d\n", type); 710 NL_SET_ERR_MSG(extack, "tcf_generic_walker: unknown command"); 711 return -EINVAL; 712 } 713 } 714 EXPORT_SYMBOL(tcf_generic_walker); 715 716 int tcf_idr_search(struct tc_action_net *tn, struct tc_action **a, u32 index) 717 { 718 struct tcf_idrinfo *idrinfo = tn->idrinfo; 719 struct tc_action *p; 720 721 mutex_lock(&idrinfo->lock); 722 p = idr_find(&idrinfo->action_idr, index); 723 if (IS_ERR(p)) 724 p = NULL; 725 else if (p) 726 refcount_inc(&p->tcfa_refcnt); 727 mutex_unlock(&idrinfo->lock); 728 729 if (p) { 730 *a = p; 731 return true; 732 } 733 return false; 734 } 735 EXPORT_SYMBOL(tcf_idr_search); 736 737 static int __tcf_generic_walker(struct net *net, struct sk_buff *skb, 738 struct netlink_callback *cb, int type, 739 const struct tc_action_ops *ops, 740 struct netlink_ext_ack *extack) 741 { 742 struct tc_action_net *tn = net_generic(net, ops->net_id); 743 744 if (unlikely(ops->walk)) 745 return ops->walk(net, skb, cb, type, ops, extack); 746 747 return tcf_generic_walker(tn, skb, cb, type, ops, extack); 748 } 749 750 static int __tcf_idr_search(struct net *net, 751 const struct tc_action_ops *ops, 752 struct tc_action **a, u32 index) 753 { 754 struct tc_action_net *tn = net_generic(net, ops->net_id); 755 756 if (unlikely(ops->lookup)) 757 return ops->lookup(net, a, index); 758 759 return tcf_idr_search(tn, a, index); 760 } 761 762 static int tcf_idr_delete_index(struct tcf_idrinfo *idrinfo, u32 index) 763 { 764 struct tc_action *p; 765 int ret = 0; 766 767 mutex_lock(&idrinfo->lock); 768 p = idr_find(&idrinfo->action_idr, index); 769 if (IS_ERR_OR_NULL(p)) { 770 mutex_unlock(&idrinfo->lock); 771 return -ENOENT; 772 } 773 774 if (!atomic_read(&p->tcfa_bindcnt)) { 775 if (refcount_dec_and_test(&p->tcfa_refcnt)) { 776 struct module *owner = p->ops->owner; 777 778 WARN_ON(p != idr_remove(&idrinfo->action_idr, 779 p->tcfa_index)); 780 mutex_unlock(&idrinfo->lock); 781 782 tcf_action_cleanup(p); 783 module_put(owner); 784 return 0; 785 } 786 ret = 0; 787 } else { 788 ret = -EPERM; 789 } 790 791 mutex_unlock(&idrinfo->lock); 792 return ret; 793 } 794 795 int tcf_idr_create(struct tc_action_net *tn, u32 index, struct nlattr *est, 796 struct tc_action **a, const struct tc_action_ops *ops, 797 int bind, bool cpustats, u32 flags) 798 { 799 struct tc_action *p = kzalloc(ops->size, GFP_KERNEL); 800 struct tcf_idrinfo *idrinfo = tn->idrinfo; 801 int err = -ENOMEM; 802 803 if (unlikely(!p)) 804 return -ENOMEM; 805 refcount_set(&p->tcfa_refcnt, 1); 806 if (bind) 807 atomic_set(&p->tcfa_bindcnt, 1); 808 809 if (cpustats) { 810 p->cpu_bstats = netdev_alloc_pcpu_stats(struct gnet_stats_basic_sync); 811 if (!p->cpu_bstats) 812 goto err1; 813 p->cpu_bstats_hw = netdev_alloc_pcpu_stats(struct gnet_stats_basic_sync); 814 if (!p->cpu_bstats_hw) 815 goto err2; 816 p->cpu_qstats = alloc_percpu(struct gnet_stats_queue); 817 if (!p->cpu_qstats) 818 goto err3; 819 } 820 gnet_stats_basic_sync_init(&p->tcfa_bstats); 821 gnet_stats_basic_sync_init(&p->tcfa_bstats_hw); 822 spin_lock_init(&p->tcfa_lock); 823 p->tcfa_index = index; 824 p->tcfa_tm.install = jiffies; 825 p->tcfa_tm.lastuse = jiffies; 826 p->tcfa_tm.firstuse = 0; 827 p->tcfa_flags = flags; 828 if (est) { 829 err = gen_new_estimator(&p->tcfa_bstats, p->cpu_bstats, 830 &p->tcfa_rate_est, 831 &p->tcfa_lock, false, est); 832 if (err) 833 goto err4; 834 } 835 836 p->idrinfo = idrinfo; 837 __module_get(ops->owner); 838 p->ops = ops; 839 *a = p; 840 return 0; 841 err4: 842 free_percpu(p->cpu_qstats); 843 err3: 844 free_percpu(p->cpu_bstats_hw); 845 err2: 846 free_percpu(p->cpu_bstats); 847 err1: 848 kfree(p); 849 return err; 850 } 851 EXPORT_SYMBOL(tcf_idr_create); 852 853 int tcf_idr_create_from_flags(struct tc_action_net *tn, u32 index, 854 struct nlattr *est, struct tc_action **a, 855 const struct tc_action_ops *ops, int bind, 856 u32 flags) 857 { 858 /* Set cpustats according to actions flags. */ 859 return tcf_idr_create(tn, index, est, a, ops, bind, 860 !(flags & TCA_ACT_FLAGS_NO_PERCPU_STATS), flags); 861 } 862 EXPORT_SYMBOL(tcf_idr_create_from_flags); 863 864 /* Cleanup idr index that was allocated but not initialized. */ 865 866 void tcf_idr_cleanup(struct tc_action_net *tn, u32 index) 867 { 868 struct tcf_idrinfo *idrinfo = tn->idrinfo; 869 870 mutex_lock(&idrinfo->lock); 871 /* Remove ERR_PTR(-EBUSY) allocated by tcf_idr_check_alloc */ 872 WARN_ON(!IS_ERR(idr_remove(&idrinfo->action_idr, index))); 873 mutex_unlock(&idrinfo->lock); 874 } 875 EXPORT_SYMBOL(tcf_idr_cleanup); 876 877 /* Check if action with specified index exists. If actions is found, increments 878 * its reference and bind counters, and return 1. Otherwise insert temporary 879 * error pointer (to prevent concurrent users from inserting actions with same 880 * index) and return 0. 881 * 882 * May return -EAGAIN for binding actions in case of a parallel add/delete on 883 * the requested index. 884 */ 885 886 int tcf_idr_check_alloc(struct tc_action_net *tn, u32 *index, 887 struct tc_action **a, int bind) 888 { 889 struct tcf_idrinfo *idrinfo = tn->idrinfo; 890 struct tc_action *p; 891 int ret; 892 u32 max; 893 894 if (*index) { 895 rcu_read_lock(); 896 p = idr_find(&idrinfo->action_idr, *index); 897 898 if (IS_ERR(p)) { 899 /* This means that another process allocated 900 * index but did not assign the pointer yet. 901 */ 902 rcu_read_unlock(); 903 return -EAGAIN; 904 } 905 906 if (!p) { 907 /* Empty slot, try to allocate it */ 908 max = *index; 909 rcu_read_unlock(); 910 goto new; 911 } 912 913 if (!refcount_inc_not_zero(&p->tcfa_refcnt)) { 914 /* Action was deleted in parallel */ 915 rcu_read_unlock(); 916 return -EAGAIN; 917 } 918 919 if (bind) 920 atomic_inc(&p->tcfa_bindcnt); 921 *a = p; 922 923 rcu_read_unlock(); 924 925 return 1; 926 } else { 927 /* Find a slot */ 928 *index = 1; 929 max = UINT_MAX; 930 } 931 932 new: 933 *a = NULL; 934 935 mutex_lock(&idrinfo->lock); 936 ret = idr_alloc_u32(&idrinfo->action_idr, ERR_PTR(-EBUSY), index, max, 937 GFP_KERNEL); 938 mutex_unlock(&idrinfo->lock); 939 940 /* N binds raced for action allocation, 941 * retry for all the ones that failed. 942 */ 943 if (ret == -ENOSPC && *index == max) 944 ret = -EAGAIN; 945 946 return ret; 947 } 948 EXPORT_SYMBOL(tcf_idr_check_alloc); 949 950 void tcf_idrinfo_destroy(const struct tc_action_ops *ops, 951 struct tcf_idrinfo *idrinfo) 952 { 953 struct idr *idr = &idrinfo->action_idr; 954 bool mutex_taken = false; 955 struct tc_action *p; 956 unsigned long id = 1; 957 unsigned long tmp; 958 int ret; 959 960 idr_for_each_entry_ul(idr, p, tmp, id) { 961 if (IS_ERR(p)) 962 continue; 963 if (tc_act_in_hw(p) && !mutex_taken) { 964 rtnl_lock(); 965 mutex_taken = true; 966 } 967 ret = __tcf_idr_release(p, false, true); 968 if (ret == ACT_P_DELETED) 969 module_put(ops->owner); 970 else if (ret < 0) 971 return; 972 } 973 if (mutex_taken) 974 rtnl_unlock(); 975 idr_destroy(&idrinfo->action_idr); 976 } 977 EXPORT_SYMBOL(tcf_idrinfo_destroy); 978 979 static LIST_HEAD(act_base); 980 static DEFINE_RWLOCK(act_mod_lock); 981 /* since act ops id is stored in pernet subsystem list, 982 * then there is no way to walk through only all the action 983 * subsystem, so we keep tc action pernet ops id for 984 * reoffload to walk through. 985 */ 986 static LIST_HEAD(act_pernet_id_list); 987 static DEFINE_MUTEX(act_id_mutex); 988 struct tc_act_pernet_id { 989 struct list_head list; 990 unsigned int id; 991 }; 992 993 static int tcf_pernet_add_id_list(unsigned int id) 994 { 995 struct tc_act_pernet_id *id_ptr; 996 int ret = 0; 997 998 mutex_lock(&act_id_mutex); 999 list_for_each_entry(id_ptr, &act_pernet_id_list, list) { 1000 if (id_ptr->id == id) { 1001 ret = -EEXIST; 1002 goto err_out; 1003 } 1004 } 1005 1006 id_ptr = kzalloc_obj(*id_ptr); 1007 if (!id_ptr) { 1008 ret = -ENOMEM; 1009 goto err_out; 1010 } 1011 id_ptr->id = id; 1012 1013 list_add_tail(&id_ptr->list, &act_pernet_id_list); 1014 1015 err_out: 1016 mutex_unlock(&act_id_mutex); 1017 return ret; 1018 } 1019 1020 static void tcf_pernet_del_id_list(unsigned int id) 1021 { 1022 struct tc_act_pernet_id *id_ptr; 1023 1024 mutex_lock(&act_id_mutex); 1025 list_for_each_entry(id_ptr, &act_pernet_id_list, list) { 1026 if (id_ptr->id == id) { 1027 list_del(&id_ptr->list); 1028 kfree(id_ptr); 1029 break; 1030 } 1031 } 1032 mutex_unlock(&act_id_mutex); 1033 } 1034 1035 int tcf_register_action(struct tc_action_ops *act, 1036 struct pernet_operations *ops) 1037 { 1038 struct tc_action_ops *a; 1039 int ret; 1040 1041 if (!act->act || !act->dump || !act->init) 1042 return -EINVAL; 1043 1044 /* We have to register pernet ops before making the action ops visible, 1045 * otherwise tcf_action_init_1() could get a partially initialized 1046 * netns. 1047 */ 1048 ret = register_pernet_subsys(ops); 1049 if (ret) 1050 return ret; 1051 1052 if (ops->id) { 1053 ret = tcf_pernet_add_id_list(*ops->id); 1054 if (ret) 1055 goto err_id; 1056 } 1057 1058 write_lock(&act_mod_lock); 1059 list_for_each_entry(a, &act_base, head) { 1060 if (act->id == a->id || (strcmp(act->kind, a->kind) == 0)) { 1061 ret = -EEXIST; 1062 goto err_out; 1063 } 1064 } 1065 list_add_tail(&act->head, &act_base); 1066 write_unlock(&act_mod_lock); 1067 1068 return 0; 1069 1070 err_out: 1071 write_unlock(&act_mod_lock); 1072 if (ops->id) 1073 tcf_pernet_del_id_list(*ops->id); 1074 err_id: 1075 unregister_pernet_subsys(ops); 1076 return ret; 1077 } 1078 EXPORT_SYMBOL(tcf_register_action); 1079 1080 int tcf_unregister_action(struct tc_action_ops *act, 1081 struct pernet_operations *ops) 1082 { 1083 struct tc_action_ops *a; 1084 int err = -ENOENT; 1085 1086 write_lock(&act_mod_lock); 1087 list_for_each_entry(a, &act_base, head) { 1088 if (a == act) { 1089 list_del(&act->head); 1090 err = 0; 1091 break; 1092 } 1093 } 1094 write_unlock(&act_mod_lock); 1095 if (!err) { 1096 unregister_pernet_subsys(ops); 1097 if (ops->id) 1098 tcf_pernet_del_id_list(*ops->id); 1099 } 1100 return err; 1101 } 1102 EXPORT_SYMBOL(tcf_unregister_action); 1103 1104 /* lookup by name */ 1105 static struct tc_action_ops *tc_lookup_action_n(char *kind) 1106 { 1107 struct tc_action_ops *a, *res = NULL; 1108 1109 if (kind) { 1110 read_lock(&act_mod_lock); 1111 list_for_each_entry(a, &act_base, head) { 1112 if (strcmp(kind, a->kind) == 0) { 1113 if (try_module_get(a->owner)) 1114 res = a; 1115 break; 1116 } 1117 } 1118 read_unlock(&act_mod_lock); 1119 } 1120 return res; 1121 } 1122 1123 /* lookup by nlattr */ 1124 static struct tc_action_ops *tc_lookup_action(struct nlattr *kind) 1125 { 1126 struct tc_action_ops *a, *res = NULL; 1127 1128 if (kind) { 1129 read_lock(&act_mod_lock); 1130 list_for_each_entry(a, &act_base, head) { 1131 if (nla_strcmp(kind, a->kind) == 0) { 1132 if (try_module_get(a->owner)) 1133 res = a; 1134 break; 1135 } 1136 } 1137 read_unlock(&act_mod_lock); 1138 } 1139 return res; 1140 } 1141 1142 /*TCA_ACT_MAX_PRIO is 32, there count up to 32 */ 1143 #define TCA_ACT_MAX_PRIO_MASK 0x1FF 1144 int tcf_action_exec(struct sk_buff *skb, struct tc_action **actions, 1145 int nr_actions, struct tcf_result *res) 1146 { 1147 u32 jmp_prgcnt = 0; 1148 u32 jmp_ttl = TCA_ACT_MAX_PRIO; /*matches actions per filter */ 1149 int i; 1150 int ret = TC_ACT_OK; 1151 1152 if (skb_skip_tc_classify(skb)) 1153 return TC_ACT_OK; 1154 1155 restart_act_graph: 1156 for (i = 0; i < nr_actions; i++) { 1157 const struct tc_action *a = actions[i]; 1158 int repeat_ttl; 1159 1160 if (jmp_prgcnt > 0) { 1161 jmp_prgcnt -= 1; 1162 continue; 1163 } 1164 1165 if (tc_act_skip_sw(a->tcfa_flags)) 1166 continue; 1167 1168 repeat_ttl = 32; 1169 repeat: 1170 ret = tc_act(skb, a, res); 1171 if (unlikely(ret == TC_ACT_REPEAT)) { 1172 if (--repeat_ttl != 0) 1173 goto repeat; 1174 /* suspicious opcode, stop pipeline */ 1175 net_warn_ratelimited("TC_ACT_REPEAT abuse ?\n"); 1176 return TC_ACT_OK; 1177 } 1178 if (TC_ACT_EXT_CMP(ret, TC_ACT_JUMP)) { 1179 jmp_prgcnt = ret & TCA_ACT_MAX_PRIO_MASK; 1180 if (!jmp_prgcnt || (jmp_prgcnt > nr_actions)) { 1181 /* faulty opcode, stop pipeline */ 1182 return TC_ACT_OK; 1183 } else { 1184 jmp_ttl -= 1; 1185 if (jmp_ttl > 0) 1186 goto restart_act_graph; 1187 else /* faulty graph, stop pipeline */ 1188 return TC_ACT_OK; 1189 } 1190 } else if (TC_ACT_EXT_CMP(ret, TC_ACT_GOTO_CHAIN)) { 1191 struct tcf_chain *chain = rcu_dereference_bh(a->goto_chain); 1192 1193 if (unlikely(!chain)) { 1194 tcf_set_drop_reason(skb, 1195 SKB_DROP_REASON_TC_CHAIN_NOTFOUND); 1196 return TC_ACT_SHOT; 1197 } 1198 tcf_action_goto_chain_exec(chain, res); 1199 } 1200 1201 if (ret != TC_ACT_PIPE) 1202 break; 1203 } 1204 1205 return ret; 1206 } 1207 EXPORT_SYMBOL(tcf_action_exec); 1208 1209 int tcf_action_destroy(struct tc_action *actions[], int bind) 1210 { 1211 struct tc_action *a; 1212 int ret = 0, i; 1213 1214 tcf_act_for_each_action(i, a, actions) { 1215 actions[i] = NULL; 1216 /* Drop our reference even if the action is still bound to a filter. */ 1217 ret = tcf_idr_release(a, bind); 1218 } 1219 return ret; 1220 } 1221 1222 static int tcf_action_put(struct tc_action *p) 1223 { 1224 return __tcf_action_put(p, false); 1225 } 1226 1227 static void tcf_action_put_many(struct tc_action *actions[]) 1228 { 1229 int i; 1230 1231 /* Deletion may have cleared entries before failing. */ 1232 for (i = 0; i < TCA_ACT_MAX_PRIO; i++) { 1233 struct tc_action *a = actions[i]; 1234 const struct tc_action_ops *ops; 1235 1236 if (!a) 1237 continue; 1238 ops = a->ops; 1239 if (tcf_action_put(a)) 1240 module_put(ops->owner); 1241 } 1242 } 1243 1244 static void tca_put_bound_many(struct tc_action *actions[], int init_res[]) 1245 { 1246 struct tc_action *a; 1247 int i; 1248 1249 tcf_act_for_each_action(i, a, actions) { 1250 const struct tc_action_ops *ops = a->ops; 1251 1252 if (init_res[i] == ACT_P_CREATED) 1253 continue; 1254 1255 if (tcf_action_put(a)) 1256 module_put(ops->owner); 1257 } 1258 } 1259 1260 int 1261 tcf_action_dump_old(struct sk_buff *skb, struct tc_action *a, int bind, int ref) 1262 { 1263 return a->ops->dump(skb, a, bind, ref); 1264 } 1265 1266 int tcf_action_dump(struct sk_buff *skb, struct tc_action *actions[], 1267 int bind, int ref, bool terse) 1268 { 1269 struct tc_action *a; 1270 int err = -EINVAL, i; 1271 struct nlattr *nest; 1272 1273 tcf_act_for_each_action(i, a, actions) { 1274 nest = nla_nest_start_noflag(skb, i + 1); 1275 if (nest == NULL) 1276 goto nla_put_failure; 1277 err = terse ? tcf_action_dump_terse(skb, a, false) : 1278 tcf_action_dump_1(skb, a, bind, ref); 1279 if (err < 0) 1280 goto errout; 1281 nla_nest_end(skb, nest); 1282 } 1283 1284 return 0; 1285 1286 nla_put_failure: 1287 err = -EINVAL; 1288 errout: 1289 nla_nest_cancel(skb, nest); 1290 return err; 1291 } 1292 1293 static struct tc_cookie *nla_memdup_cookie(struct nlattr **tb) 1294 { 1295 struct tc_cookie *c = kzalloc_obj(*c); 1296 if (!c) 1297 return NULL; 1298 1299 c->data = nla_memdup(tb[TCA_ACT_COOKIE], GFP_KERNEL); 1300 if (!c->data) { 1301 kfree(c); 1302 return NULL; 1303 } 1304 c->len = nla_len(tb[TCA_ACT_COOKIE]); 1305 1306 return c; 1307 } 1308 1309 static u8 tcf_action_hw_stats_get(struct nlattr *hw_stats_attr) 1310 { 1311 struct nla_bitfield32 hw_stats_bf; 1312 1313 /* If the user did not pass the attr, that means he does 1314 * not care about the type. Return "any" in that case 1315 * which is setting on all supported types. 1316 */ 1317 if (!hw_stats_attr) 1318 return TCA_ACT_HW_STATS_ANY; 1319 hw_stats_bf = nla_get_bitfield32(hw_stats_attr); 1320 return hw_stats_bf.value; 1321 } 1322 1323 static const struct nla_policy tcf_action_policy[TCA_ACT_MAX + 1] = { 1324 [TCA_ACT_KIND] = { .type = NLA_STRING }, 1325 [TCA_ACT_INDEX] = { .type = NLA_U32 }, 1326 [TCA_ACT_COOKIE] = { .type = NLA_BINARY, 1327 .len = TC_COOKIE_MAX_SIZE }, 1328 [TCA_ACT_OPTIONS] = { .type = NLA_NESTED }, 1329 [TCA_ACT_FLAGS] = NLA_POLICY_BITFIELD32(TCA_ACT_FLAGS_NO_PERCPU_STATS | 1330 TCA_ACT_FLAGS_SKIP_HW | 1331 TCA_ACT_FLAGS_SKIP_SW), 1332 [TCA_ACT_HW_STATS] = NLA_POLICY_BITFIELD32(TCA_ACT_HW_STATS_ANY), 1333 }; 1334 1335 void tcf_idr_insert_many(struct tc_action *actions[], int init_res[]) 1336 { 1337 struct tc_action *a; 1338 int i; 1339 1340 tcf_act_for_each_action(i, a, actions) { 1341 struct tcf_idrinfo *idrinfo; 1342 1343 if (init_res[i] == ACT_P_BOUND) 1344 continue; 1345 1346 idrinfo = a->idrinfo; 1347 mutex_lock(&idrinfo->lock); 1348 /* Replace ERR_PTR(-EBUSY) allocated by tcf_idr_check_alloc */ 1349 idr_replace(&idrinfo->action_idr, a, a->tcfa_index); 1350 mutex_unlock(&idrinfo->lock); 1351 } 1352 } 1353 1354 struct tc_action_ops *tc_action_load_ops(struct nlattr *nla, u32 flags, 1355 struct netlink_ext_ack *extack) 1356 { 1357 bool police = flags & TCA_ACT_FLAGS_POLICE; 1358 struct nlattr *tb[TCA_ACT_MAX + 1]; 1359 struct tc_action_ops *a_o; 1360 char act_name[IFNAMSIZ]; 1361 struct nlattr *kind; 1362 int err; 1363 1364 if (!police) { 1365 err = nla_parse_nested_deprecated(tb, TCA_ACT_MAX, nla, 1366 tcf_action_policy, extack); 1367 if (err < 0) 1368 return ERR_PTR(err); 1369 err = -EINVAL; 1370 kind = tb[TCA_ACT_KIND]; 1371 if (!kind) { 1372 NL_SET_ERR_MSG(extack, "TC action kind must be specified"); 1373 return ERR_PTR(err); 1374 } 1375 if (nla_strscpy(act_name, kind, IFNAMSIZ) < 0) { 1376 NL_SET_ERR_MSG(extack, "TC action name too long"); 1377 return ERR_PTR(err); 1378 } 1379 } else { 1380 if (strscpy(act_name, "police", IFNAMSIZ) < 0) { 1381 NL_SET_ERR_MSG(extack, "TC action name too long"); 1382 return ERR_PTR(-EINVAL); 1383 } 1384 } 1385 1386 a_o = tc_lookup_action_n(act_name); 1387 if (a_o == NULL) { 1388 #ifdef CONFIG_MODULES 1389 bool rtnl_held = !(flags & TCA_ACT_FLAGS_NO_RTNL); 1390 1391 if (rtnl_held) 1392 rtnl_unlock(); 1393 request_module(NET_ACT_ALIAS_PREFIX "%s", act_name); 1394 if (rtnl_held) 1395 rtnl_lock(); 1396 1397 a_o = tc_lookup_action_n(act_name); 1398 1399 /* We dropped the RTNL semaphore in order to 1400 * perform the module load. So, even if we 1401 * succeeded in loading the module we have to 1402 * tell the caller to replay the request. We 1403 * indicate this using -EAGAIN. 1404 */ 1405 if (a_o != NULL) { 1406 module_put(a_o->owner); 1407 return ERR_PTR(-EAGAIN); 1408 } 1409 #endif 1410 NL_SET_ERR_MSG(extack, "Failed to load TC action module"); 1411 return ERR_PTR(-ENOENT); 1412 } 1413 1414 return a_o; 1415 } 1416 1417 struct tc_action *tcf_action_init_1(struct net *net, struct tcf_proto *tp, 1418 struct nlattr *nla, struct nlattr *est, 1419 struct tc_action_ops *a_o, int *init_res, 1420 u32 flags, struct netlink_ext_ack *extack) 1421 { 1422 bool police = flags & TCA_ACT_FLAGS_POLICE; 1423 struct nla_bitfield32 userflags = { 0, 0 }; 1424 struct tc_cookie *user_cookie = NULL; 1425 u8 hw_stats = TCA_ACT_HW_STATS_ANY; 1426 struct nlattr *tb[TCA_ACT_MAX + 1]; 1427 struct tc_action *a; 1428 int err; 1429 1430 /* backward compatibility for policer */ 1431 if (!police) { 1432 err = nla_parse_nested_deprecated(tb, TCA_ACT_MAX, nla, 1433 tcf_action_policy, extack); 1434 if (err < 0) 1435 return ERR_PTR(err); 1436 if (tb[TCA_ACT_COOKIE]) { 1437 user_cookie = nla_memdup_cookie(tb); 1438 if (!user_cookie) { 1439 NL_SET_ERR_MSG(extack, "No memory to generate TC cookie"); 1440 err = -ENOMEM; 1441 goto err_out; 1442 } 1443 } 1444 hw_stats = tcf_action_hw_stats_get(tb[TCA_ACT_HW_STATS]); 1445 if (tb[TCA_ACT_FLAGS]) { 1446 userflags = nla_get_bitfield32(tb[TCA_ACT_FLAGS]); 1447 if (!tc_act_flags_valid(userflags.value)) { 1448 err = -EINVAL; 1449 goto err_out; 1450 } 1451 } 1452 1453 err = a_o->init(net, tb[TCA_ACT_OPTIONS], est, &a, tp, 1454 userflags.value | flags, extack); 1455 } else { 1456 err = a_o->init(net, nla, est, &a, tp, userflags.value | flags, 1457 extack); 1458 } 1459 if (err < 0) 1460 goto err_out; 1461 *init_res = err; 1462 1463 if (!police && tb[TCA_ACT_COOKIE]) 1464 tcf_set_action_cookie(&a->user_cookie, user_cookie); 1465 1466 if (!police) 1467 a->hw_stats = hw_stats; 1468 1469 return a; 1470 1471 err_out: 1472 if (user_cookie) { 1473 kfree(user_cookie->data); 1474 kfree(user_cookie); 1475 } 1476 return ERR_PTR(err); 1477 } 1478 1479 static bool tc_act_bind(u32 flags) 1480 { 1481 return !!(flags & TCA_ACT_FLAGS_BIND); 1482 } 1483 1484 /* Returns numbers of initialized actions or negative error. */ 1485 1486 int tcf_action_init(struct net *net, struct tcf_proto *tp, struct nlattr *nla, 1487 struct nlattr *est, struct tc_action *actions[], 1488 int init_res[], size_t *attr_size, 1489 u32 flags, u32 fl_flags, 1490 struct netlink_ext_ack *extack) 1491 { 1492 struct tc_action_ops *ops[TCA_ACT_MAX_PRIO] = {}; 1493 struct nlattr *tb[TCA_ACT_MAX_PRIO + 2]; 1494 struct tc_action *act; 1495 size_t sz = 0; 1496 int err; 1497 int i; 1498 1499 err = nla_parse_nested_deprecated(tb, TCA_ACT_MAX_PRIO + 1, nla, NULL, 1500 extack); 1501 if (err < 0) 1502 return err; 1503 1504 /* The nested attributes are parsed as types, but they are really an 1505 * array of actions. So we parse one more than we can handle, and return 1506 * an error if the last one is set (as that indicates that the request 1507 * contained more than the maximum number of actions). 1508 */ 1509 if (tb[TCA_ACT_MAX_PRIO + 1]) { 1510 NL_SET_ERR_MSG_FMT(extack, 1511 "Only %d actions supported per filter", 1512 TCA_ACT_MAX_PRIO); 1513 return -EINVAL; 1514 } 1515 1516 for (i = 1; i <= TCA_ACT_MAX_PRIO && tb[i]; i++) { 1517 struct tc_action_ops *a_o; 1518 1519 a_o = tc_action_load_ops(tb[i], flags, extack); 1520 if (IS_ERR(a_o)) { 1521 err = PTR_ERR(a_o); 1522 goto err_mod; 1523 } 1524 ops[i - 1] = a_o; 1525 } 1526 1527 for (i = 1; i <= TCA_ACT_MAX_PRIO && tb[i]; i++) { 1528 act = tcf_action_init_1(net, tp, tb[i], est, ops[i - 1], 1529 &init_res[i - 1], flags, extack); 1530 if (IS_ERR(act)) { 1531 err = PTR_ERR(act); 1532 goto err; 1533 } 1534 sz += tcf_action_fill_size(act); 1535 /* Start from index 0 */ 1536 actions[i - 1] = act; 1537 if (tc_act_bind(flags)) { 1538 bool skip_sw = tc_skip_sw(fl_flags); 1539 bool skip_hw = tc_skip_hw(fl_flags); 1540 1541 if (tc_act_bind(act->tcfa_flags)) { 1542 /* Action is created by classifier and is not 1543 * standalone. Check that the user did not set 1544 * any action flags different than the 1545 * classifier flags, and inherit the flags from 1546 * the classifier for the compatibility case 1547 * where no flags were specified at all. 1548 */ 1549 if ((tc_act_skip_sw(act->tcfa_flags) && !skip_sw) || 1550 (tc_act_skip_hw(act->tcfa_flags) && !skip_hw)) { 1551 NL_SET_ERR_MSG(extack, 1552 "Mismatch between action and filter offload flags"); 1553 err = -EINVAL; 1554 goto err; 1555 } 1556 if (skip_sw) 1557 act->tcfa_flags |= TCA_ACT_FLAGS_SKIP_SW; 1558 if (skip_hw) 1559 act->tcfa_flags |= TCA_ACT_FLAGS_SKIP_HW; 1560 continue; 1561 } 1562 1563 /* Action is standalone */ 1564 if (skip_sw != tc_act_skip_sw(act->tcfa_flags) || 1565 skip_hw != tc_act_skip_hw(act->tcfa_flags)) { 1566 NL_SET_ERR_MSG(extack, 1567 "Mismatch between action and filter offload flags"); 1568 err = -EINVAL; 1569 goto err; 1570 } 1571 } else { 1572 err = tcf_action_offload_add(act, extack); 1573 if (tc_act_skip_sw(act->tcfa_flags) && err) 1574 goto err; 1575 } 1576 } 1577 1578 /* We have to commit them all together, because if any error happened in 1579 * between, we could not handle the failure gracefully. 1580 */ 1581 tcf_idr_insert_many(actions, init_res); 1582 1583 *attr_size = tcf_action_full_attrs_size(sz); 1584 err = i - 1; 1585 goto err_mod; 1586 1587 err: 1588 tcf_action_destroy(actions, flags & TCA_ACT_FLAGS_BIND); 1589 err_mod: 1590 for (i = 0; i < TCA_ACT_MAX_PRIO && ops[i]; i++) 1591 module_put(ops[i]->owner); 1592 return err; 1593 } 1594 1595 void tcf_action_update_stats(struct tc_action *a, u64 bytes, u64 packets, 1596 u64 drops, bool hw) 1597 { 1598 if (a->cpu_bstats) { 1599 _bstats_update(this_cpu_ptr(a->cpu_bstats), bytes, packets); 1600 1601 this_cpu_add(a->cpu_qstats->drops, drops); 1602 1603 if (hw) 1604 _bstats_update(this_cpu_ptr(a->cpu_bstats_hw), 1605 bytes, packets); 1606 return; 1607 } 1608 1609 _bstats_update(&a->tcfa_bstats, bytes, packets); 1610 atomic_add(drops, &a->tcfa_drops); 1611 if (hw) 1612 _bstats_update(&a->tcfa_bstats_hw, bytes, packets); 1613 } 1614 EXPORT_SYMBOL(tcf_action_update_stats); 1615 1616 int tcf_action_copy_stats(struct sk_buff *skb, struct tc_action *p, 1617 int compat_mode) 1618 { 1619 struct gnet_stats_queue qstats = {0}; 1620 struct gnet_dump d; 1621 int err = 0; 1622 1623 if (p == NULL) 1624 goto errout; 1625 1626 /* compat_mode being true specifies a call that is supposed 1627 * to add additional backward compatibility statistic TLVs. 1628 */ 1629 if (compat_mode) { 1630 if (p->type == TCA_OLD_COMPAT) 1631 err = gnet_stats_start_copy_compat(skb, 0, 1632 TCA_STATS, 1633 TCA_XSTATS, 1634 &p->tcfa_lock, &d, 1635 TCA_PAD); 1636 else 1637 return 0; 1638 } else 1639 err = gnet_stats_start_copy(skb, TCA_ACT_STATS, 1640 &p->tcfa_lock, &d, TCA_ACT_PAD); 1641 1642 if (err < 0) 1643 goto errout; 1644 1645 qstats.drops = atomic_read(&p->tcfa_drops); 1646 qstats.overlimits = atomic_read(&p->tcfa_overlimits); 1647 1648 if (gnet_stats_copy_basic(&d, p->cpu_bstats, 1649 &p->tcfa_bstats, false) < 0 || 1650 gnet_stats_copy_basic_hw(&d, p->cpu_bstats_hw, 1651 &p->tcfa_bstats_hw, false) < 0 || 1652 gnet_stats_copy_rate_est(&d, &p->tcfa_rate_est) < 0 || 1653 gnet_stats_copy_queue(&d, p->cpu_qstats, 1654 &qstats, 1655 qstats.qlen) < 0) 1656 goto errout; 1657 1658 if (gnet_stats_finish_copy(&d) < 0) 1659 goto errout; 1660 1661 return 0; 1662 1663 errout: 1664 return -1; 1665 } 1666 1667 static int tca_get_fill(struct sk_buff *skb, struct tc_action *actions[], 1668 u32 portid, u32 seq, u16 flags, int event, int bind, 1669 int ref, struct netlink_ext_ack *extack) 1670 { 1671 struct tcamsg *t; 1672 struct nlmsghdr *nlh; 1673 unsigned char *b = skb_tail_pointer(skb); 1674 struct nlattr *nest; 1675 1676 nlh = nlmsg_put(skb, portid, seq, event, sizeof(*t), flags); 1677 if (!nlh) 1678 goto out_nlmsg_trim; 1679 t = nlmsg_data(nlh); 1680 t->tca_family = AF_UNSPEC; 1681 t->tca__pad1 = 0; 1682 t->tca__pad2 = 0; 1683 1684 if (extack && extack->_msg && 1685 nla_put_string(skb, TCA_ROOT_EXT_WARN_MSG, extack->_msg)) 1686 goto out_nlmsg_trim; 1687 1688 nest = nla_nest_start_noflag(skb, TCA_ACT_TAB); 1689 if (!nest) 1690 goto out_nlmsg_trim; 1691 1692 if (tcf_action_dump(skb, actions, bind, ref, false) < 0) 1693 goto out_nlmsg_trim; 1694 1695 nla_nest_end(skb, nest); 1696 1697 nlh->nlmsg_len = skb_tail_pointer(skb) - b; 1698 1699 return skb->len; 1700 1701 out_nlmsg_trim: 1702 nlmsg_trim(skb, b); 1703 return -1; 1704 } 1705 1706 static int 1707 tcf_get_notify(struct net *net, u32 portid, struct nlmsghdr *n, 1708 struct tc_action *actions[], size_t attr_size, int event, 1709 struct netlink_ext_ack *extack) 1710 { 1711 struct sk_buff *skb; 1712 1713 skb = alloc_skb(max(attr_size, NLMSG_GOODSIZE), GFP_KERNEL); 1714 if (!skb) 1715 return -ENOBUFS; 1716 if (tca_get_fill(skb, actions, portid, n->nlmsg_seq, 0, event, 1717 0, 1, NULL) <= 0) { 1718 NL_SET_ERR_MSG(extack, "Failed to fill netlink attributes while adding TC action"); 1719 kfree_skb(skb); 1720 return -EINVAL; 1721 } 1722 1723 return rtnl_unicast(skb, net, portid); 1724 } 1725 1726 static struct tc_action *tcf_action_get_1(struct net *net, struct nlattr *nla, 1727 struct nlmsghdr *n, u32 portid, 1728 struct netlink_ext_ack *extack) 1729 { 1730 struct nlattr *tb[TCA_ACT_MAX + 1]; 1731 const struct tc_action_ops *ops; 1732 struct tc_action *a; 1733 int index; 1734 int err; 1735 1736 err = nla_parse_nested_deprecated(tb, TCA_ACT_MAX, nla, 1737 tcf_action_policy, extack); 1738 if (err < 0) 1739 goto err_out; 1740 1741 err = -EINVAL; 1742 if (tb[TCA_ACT_INDEX] == NULL || 1743 nla_len(tb[TCA_ACT_INDEX]) < sizeof(index)) { 1744 NL_SET_ERR_MSG(extack, "Invalid TC action index value"); 1745 goto err_out; 1746 } 1747 index = nla_get_u32(tb[TCA_ACT_INDEX]); 1748 1749 err = -EINVAL; 1750 ops = tc_lookup_action(tb[TCA_ACT_KIND]); 1751 if (!ops) { /* could happen in batch of actions */ 1752 NL_SET_ERR_MSG(extack, "Specified TC action kind not found"); 1753 goto err_out; 1754 } 1755 err = -ENOENT; 1756 if (__tcf_idr_search(net, ops, &a, index) == 0) { 1757 NL_SET_ERR_MSG(extack, "TC action with specified index not found"); 1758 goto err_mod; 1759 } 1760 1761 module_put(ops->owner); 1762 return a; 1763 1764 err_mod: 1765 module_put(ops->owner); 1766 err_out: 1767 return ERR_PTR(err); 1768 } 1769 1770 static int tca_action_flush(struct net *net, struct nlattr *nla, 1771 struct nlmsghdr *n, u32 portid, 1772 struct netlink_ext_ack *extack) 1773 { 1774 struct sk_buff *skb; 1775 unsigned char *b; 1776 struct nlmsghdr *nlh; 1777 struct tcamsg *t; 1778 struct netlink_callback dcb; 1779 struct nlattr *nest; 1780 struct nlattr *tb[TCA_ACT_MAX + 1]; 1781 const struct tc_action_ops *ops; 1782 struct nlattr *kind; 1783 int err = -ENOMEM; 1784 1785 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); 1786 if (!skb) 1787 return err; 1788 1789 b = skb_tail_pointer(skb); 1790 1791 err = nla_parse_nested_deprecated(tb, TCA_ACT_MAX, nla, 1792 tcf_action_policy, extack); 1793 if (err < 0) 1794 goto err_out; 1795 1796 err = -EINVAL; 1797 kind = tb[TCA_ACT_KIND]; 1798 ops = tc_lookup_action(kind); 1799 if (!ops) { /*some idjot trying to flush unknown action */ 1800 NL_SET_ERR_MSG(extack, "Cannot flush unknown TC action"); 1801 goto err_out; 1802 } 1803 1804 nlh = nlmsg_put(skb, portid, n->nlmsg_seq, RTM_DELACTION, 1805 sizeof(*t), 0); 1806 if (!nlh) { 1807 NL_SET_ERR_MSG(extack, "Failed to create TC action flush notification"); 1808 goto out_module_put; 1809 } 1810 t = nlmsg_data(nlh); 1811 t->tca_family = AF_UNSPEC; 1812 t->tca__pad1 = 0; 1813 t->tca__pad2 = 0; 1814 1815 nest = nla_nest_start_noflag(skb, TCA_ACT_TAB); 1816 if (!nest) { 1817 NL_SET_ERR_MSG(extack, "Failed to add new netlink message"); 1818 goto out_module_put; 1819 } 1820 1821 err = __tcf_generic_walker(net, skb, &dcb, RTM_DELACTION, ops, extack); 1822 if (err <= 0) { 1823 nla_nest_cancel(skb, nest); 1824 goto out_module_put; 1825 } 1826 1827 nla_nest_end(skb, nest); 1828 1829 nlh->nlmsg_len = skb_tail_pointer(skb) - b; 1830 nlh->nlmsg_flags |= NLM_F_ROOT; 1831 module_put(ops->owner); 1832 err = rtnetlink_send(skb, net, portid, RTNLGRP_TC, 1833 n->nlmsg_flags & NLM_F_ECHO); 1834 if (err < 0) 1835 NL_SET_ERR_MSG(extack, "Failed to send TC action flush notification"); 1836 1837 return err; 1838 1839 out_module_put: 1840 module_put(ops->owner); 1841 err_out: 1842 kfree_skb(skb); 1843 return err; 1844 } 1845 1846 static int tcf_action_delete(struct net *net, struct tc_action *actions[]) 1847 { 1848 struct tc_action *a; 1849 int i; 1850 1851 tcf_act_for_each_action(i, a, actions) { 1852 const struct tc_action_ops *ops = a->ops; 1853 /* Actions can be deleted concurrently so we must save their 1854 * type and id to search again after reference is released. 1855 */ 1856 struct tcf_idrinfo *idrinfo = a->idrinfo; 1857 u32 act_index = a->tcfa_index; 1858 1859 actions[i] = NULL; 1860 if (tcf_action_put(a)) { 1861 /* last reference, action was deleted concurrently */ 1862 module_put(ops->owner); 1863 } else { 1864 int ret; 1865 1866 /* now do the delete */ 1867 ret = tcf_idr_delete_index(idrinfo, act_index); 1868 if (ret < 0) 1869 return ret; 1870 } 1871 } 1872 return 0; 1873 } 1874 1875 static struct sk_buff *tcf_reoffload_del_notify_msg(struct net *net, 1876 struct tc_action *action) 1877 { 1878 struct tc_action *actions[TCA_ACT_MAX_PRIO] = { 1879 [0] = action, 1880 }; 1881 struct sk_buff *skb; 1882 size_t attr_size; 1883 1884 attr_size = tcf_action_full_attrs_size(tcf_action_fill_size(action)); 1885 1886 skb = alloc_skb(max(attr_size, NLMSG_GOODSIZE), GFP_KERNEL); 1887 if (!skb) 1888 return ERR_PTR(-ENOBUFS); 1889 1890 if (tca_get_fill(skb, actions, 0, 0, 0, RTM_DELACTION, 0, 1, NULL) <= 0) { 1891 kfree_skb(skb); 1892 return ERR_PTR(-EINVAL); 1893 } 1894 1895 return skb; 1896 } 1897 1898 static int tcf_reoffload_del_notify(struct net *net, struct tc_action *action) 1899 { 1900 const struct tc_action_ops *ops = action->ops; 1901 struct sk_buff *skb = NULL; 1902 int ret; 1903 1904 if (rtnl_notify_needed(net, 0, RTNLGRP_TC)) { 1905 skb = tcf_reoffload_del_notify_msg(net, action); 1906 /* The action has already lost its hardware instance and is 1907 * skip_sw, so it must be released whether or not the 1908 * notification can be built. Drop the notification rather 1909 * than leave an action behind that processes no packets. 1910 */ 1911 if (IS_ERR(skb)) 1912 skb = NULL; 1913 } 1914 1915 ret = tcf_idr_release_unsafe(action); 1916 if (ret == ACT_P_DELETED) { 1917 module_put(ops->owner); 1918 ret = rtnetlink_maybe_send(skb, net, 0, RTNLGRP_TC, 0); 1919 } else { 1920 kfree_skb(skb); 1921 } 1922 1923 return ret; 1924 } 1925 1926 int tcf_action_reoffload_cb(flow_indr_block_bind_cb_t *cb, 1927 void *cb_priv, bool add) 1928 { 1929 struct tc_act_pernet_id *id_ptr; 1930 struct tcf_idrinfo *idrinfo; 1931 struct tc_action_net *tn; 1932 struct tc_action *p; 1933 unsigned int act_id; 1934 unsigned long tmp; 1935 unsigned long id; 1936 struct idr *idr; 1937 struct net *net; 1938 int ret; 1939 1940 if (!cb) 1941 return -EINVAL; 1942 1943 down_read(&net_rwsem); 1944 mutex_lock(&act_id_mutex); 1945 1946 for_each_net(net) { 1947 list_for_each_entry(id_ptr, &act_pernet_id_list, list) { 1948 act_id = id_ptr->id; 1949 tn = net_generic(net, act_id); 1950 if (!tn) 1951 continue; 1952 idrinfo = tn->idrinfo; 1953 if (!idrinfo) 1954 continue; 1955 1956 mutex_lock(&idrinfo->lock); 1957 idr = &idrinfo->action_idr; 1958 idr_for_each_entry_ul(idr, p, tmp, id) { 1959 if (IS_ERR(p) || tc_act_bind(p->tcfa_flags)) 1960 continue; 1961 if (add) { 1962 tcf_action_offload_add_ex(p, NULL, cb, 1963 cb_priv); 1964 continue; 1965 } 1966 1967 /* cb unregister to update hw count */ 1968 ret = tcf_action_offload_del_ex(p, cb, cb_priv); 1969 if (ret < 0) 1970 continue; 1971 if (tc_act_skip_sw(p->tcfa_flags) && 1972 !tc_act_in_hw(p)) 1973 tcf_reoffload_del_notify(net, p); 1974 } 1975 mutex_unlock(&idrinfo->lock); 1976 } 1977 } 1978 mutex_unlock(&act_id_mutex); 1979 up_read(&net_rwsem); 1980 1981 return 0; 1982 } 1983 1984 static struct sk_buff *tcf_del_notify_msg(struct net *net, struct nlmsghdr *n, 1985 struct tc_action *actions[], 1986 u32 portid, size_t attr_size, 1987 struct netlink_ext_ack *extack) 1988 { 1989 struct sk_buff *skb; 1990 1991 skb = alloc_skb(max(attr_size + tcf_action_warn_attr_size(extack), 1992 NLMSG_GOODSIZE), GFP_KERNEL); 1993 if (!skb) 1994 return ERR_PTR(-ENOBUFS); 1995 1996 if (tca_get_fill(skb, actions, portid, n->nlmsg_seq, 0, RTM_DELACTION, 1997 0, 2, extack) <= 0) { 1998 NL_SET_ERR_MSG(extack, "Failed to fill netlink TC action attributes"); 1999 kfree_skb(skb); 2000 return ERR_PTR(-EINVAL); 2001 } 2002 2003 return skb; 2004 } 2005 2006 static int tcf_del_notify(struct net *net, struct nlmsghdr *n, 2007 struct tc_action *actions[], u32 portid, 2008 size_t attr_size, struct netlink_ext_ack *extack) 2009 { 2010 struct sk_buff *skb; 2011 int ret; 2012 2013 if (!rtnl_notify_needed(net, n->nlmsg_flags, RTNLGRP_TC)) { 2014 skb = NULL; 2015 } else { 2016 skb = tcf_del_notify_msg(net, n, actions, portid, attr_size, 2017 extack); 2018 if (IS_ERR(skb)) 2019 return PTR_ERR(skb); 2020 } 2021 2022 /* now do the delete */ 2023 ret = tcf_action_delete(net, actions); 2024 if (ret < 0) { 2025 NL_SET_ERR_MSG(extack, "Failed to delete TC action"); 2026 kfree_skb(skb); 2027 return ret; 2028 } 2029 2030 return rtnetlink_maybe_send(skb, net, portid, RTNLGRP_TC, 2031 n->nlmsg_flags & NLM_F_ECHO); 2032 } 2033 2034 static int 2035 tca_action_gd(struct net *net, struct nlattr *nla, struct nlmsghdr *n, 2036 u32 portid, int event, struct netlink_ext_ack *extack) 2037 { 2038 int i, ret; 2039 struct nlattr *tb[TCA_ACT_MAX_PRIO + 1]; 2040 struct tc_action *act; 2041 size_t attr_size = 0; 2042 struct tc_action *actions[TCA_ACT_MAX_PRIO] = {}; 2043 2044 ret = nla_parse_nested_deprecated(tb, TCA_ACT_MAX_PRIO, nla, NULL, 2045 extack); 2046 if (ret < 0) 2047 return ret; 2048 2049 if (event == RTM_DELACTION && n->nlmsg_flags & NLM_F_ROOT) { 2050 if (tb[1]) 2051 return tca_action_flush(net, tb[1], n, portid, extack); 2052 2053 NL_SET_ERR_MSG(extack, "Invalid netlink attributes while flushing TC action"); 2054 return -EINVAL; 2055 } 2056 2057 for (i = 1; i <= TCA_ACT_MAX_PRIO && tb[i]; i++) { 2058 act = tcf_action_get_1(net, tb[i], n, portid, extack); 2059 if (IS_ERR(act)) { 2060 ret = PTR_ERR(act); 2061 goto err; 2062 } 2063 attr_size += tcf_action_fill_size(act); 2064 actions[i - 1] = act; 2065 } 2066 2067 attr_size = tcf_action_full_attrs_size(attr_size); 2068 2069 if (event == RTM_GETACTION) 2070 ret = tcf_get_notify(net, portid, n, actions, attr_size, event, 2071 extack); 2072 else { /* delete */ 2073 ret = tcf_del_notify(net, n, actions, portid, attr_size, extack); 2074 if (ret) 2075 goto err; 2076 return 0; 2077 } 2078 err: 2079 tcf_action_put_many(actions); 2080 return ret; 2081 } 2082 2083 static struct sk_buff *tcf_add_notify_msg(struct net *net, struct nlmsghdr *n, 2084 struct tc_action *actions[], 2085 u32 portid, size_t attr_size, 2086 struct netlink_ext_ack *extack) 2087 { 2088 struct sk_buff *skb; 2089 2090 skb = alloc_skb(max(attr_size + tcf_action_warn_attr_size(extack), 2091 NLMSG_GOODSIZE), GFP_KERNEL); 2092 if (!skb) 2093 return ERR_PTR(-ENOBUFS); 2094 2095 if (tca_get_fill(skb, actions, portid, n->nlmsg_seq, n->nlmsg_flags, 2096 RTM_NEWACTION, 0, 0, extack) <= 0) { 2097 NL_SET_ERR_MSG(extack, "Failed to fill netlink attributes while adding TC action"); 2098 kfree_skb(skb); 2099 return ERR_PTR(-EINVAL); 2100 } 2101 2102 return skb; 2103 } 2104 2105 static int tcf_add_notify(struct net *net, struct nlmsghdr *n, 2106 struct tc_action *actions[], u32 portid, 2107 size_t attr_size, struct netlink_ext_ack *extack) 2108 { 2109 struct sk_buff *skb; 2110 2111 if (!rtnl_notify_needed(net, n->nlmsg_flags, RTNLGRP_TC)) { 2112 skb = NULL; 2113 } else { 2114 skb = tcf_add_notify_msg(net, n, actions, portid, attr_size, 2115 extack); 2116 if (IS_ERR(skb)) 2117 return PTR_ERR(skb); 2118 } 2119 2120 return rtnetlink_maybe_send(skb, net, portid, RTNLGRP_TC, 2121 n->nlmsg_flags & NLM_F_ECHO); 2122 } 2123 2124 static int tcf_action_add(struct net *net, struct nlattr *nla, 2125 struct nlmsghdr *n, u32 portid, u32 flags, 2126 struct netlink_ext_ack *extack) 2127 { 2128 size_t attr_size = 0; 2129 int loop, ret; 2130 struct tc_action *actions[TCA_ACT_MAX_PRIO] = {}; 2131 int init_res[TCA_ACT_MAX_PRIO] = {}; 2132 2133 for (loop = 0; loop < 10; loop++) { 2134 ret = tcf_action_init(net, NULL, nla, NULL, actions, init_res, 2135 &attr_size, flags, 0, extack); 2136 if (ret != -EAGAIN) 2137 break; 2138 } 2139 2140 if (ret < 0) 2141 return ret; 2142 2143 ret = tcf_add_notify(net, n, actions, portid, attr_size, extack); 2144 2145 /* only put bound actions */ 2146 tca_put_bound_many(actions, init_res); 2147 2148 return ret; 2149 } 2150 2151 static const struct nla_policy tcaa_policy[TCA_ROOT_MAX + 1] = { 2152 [TCA_ROOT_FLAGS] = NLA_POLICY_BITFIELD32(TCA_ACT_FLAG_LARGE_DUMP_ON | 2153 TCA_ACT_FLAG_TERSE_DUMP), 2154 [TCA_ROOT_TIME_DELTA] = { .type = NLA_U32 }, 2155 }; 2156 2157 static int tc_ctl_action(struct sk_buff *skb, struct nlmsghdr *n, 2158 struct netlink_ext_ack *extack) 2159 { 2160 struct net *net = sock_net(skb->sk); 2161 struct nlattr *tca[TCA_ROOT_MAX + 1]; 2162 u32 portid = NETLINK_CB(skb).portid; 2163 u32 flags = 0; 2164 int ret = 0; 2165 2166 if ((n->nlmsg_type != RTM_GETACTION) && 2167 !netlink_capable(skb, CAP_NET_ADMIN)) 2168 return -EPERM; 2169 2170 ret = nlmsg_parse_deprecated(n, sizeof(struct tcamsg), tca, 2171 TCA_ROOT_MAX, NULL, extack); 2172 if (ret < 0) 2173 return ret; 2174 2175 if (tca[TCA_ACT_TAB] == NULL) { 2176 NL_SET_ERR_MSG(extack, "Netlink action attributes missing"); 2177 return -EINVAL; 2178 } 2179 2180 /* n->nlmsg_flags & NLM_F_CREATE */ 2181 switch (n->nlmsg_type) { 2182 case RTM_NEWACTION: 2183 /* we are going to assume all other flags 2184 * imply create only if it doesn't exist 2185 * Note that CREATE | EXCL implies that 2186 * but since we want avoid ambiguity (eg when flags 2187 * is zero) then just set this 2188 */ 2189 if (n->nlmsg_flags & NLM_F_REPLACE) 2190 flags = TCA_ACT_FLAGS_REPLACE; 2191 ret = tcf_action_add(net, tca[TCA_ACT_TAB], n, portid, flags, 2192 extack); 2193 break; 2194 case RTM_DELACTION: 2195 ret = tca_action_gd(net, tca[TCA_ACT_TAB], n, 2196 portid, RTM_DELACTION, extack); 2197 break; 2198 case RTM_GETACTION: 2199 ret = tca_action_gd(net, tca[TCA_ACT_TAB], n, 2200 portid, RTM_GETACTION, extack); 2201 break; 2202 default: 2203 BUG(); 2204 } 2205 2206 return ret; 2207 } 2208 2209 static struct nlattr *find_dump_kind(struct nlattr **nla) 2210 { 2211 struct nlattr *tb1, *tb2[TCA_ACT_MAX + 1]; 2212 struct nlattr *tb[TCA_ACT_MAX_PRIO + 1]; 2213 struct nlattr *kind; 2214 2215 tb1 = nla[TCA_ACT_TAB]; 2216 if (tb1 == NULL) 2217 return NULL; 2218 2219 if (nla_parse_deprecated(tb, TCA_ACT_MAX_PRIO, nla_data(tb1), NLMSG_ALIGN(nla_len(tb1)), NULL, NULL) < 0) 2220 return NULL; 2221 2222 if (tb[1] == NULL) 2223 return NULL; 2224 if (nla_parse_nested_deprecated(tb2, TCA_ACT_MAX, tb[1], tcf_action_policy, NULL) < 0) 2225 return NULL; 2226 kind = tb2[TCA_ACT_KIND]; 2227 2228 return kind; 2229 } 2230 2231 static int tc_dump_action(struct sk_buff *skb, struct netlink_callback *cb) 2232 { 2233 struct net *net = sock_net(skb->sk); 2234 struct nlmsghdr *nlh; 2235 unsigned char *b = skb_tail_pointer(skb); 2236 struct nlattr *nest; 2237 struct tc_action_ops *a_o; 2238 int ret = 0; 2239 struct tcamsg *t = (struct tcamsg *) nlmsg_data(cb->nlh); 2240 struct nlattr *tb[TCA_ROOT_MAX + 1]; 2241 struct nlattr *count_attr = NULL; 2242 unsigned long jiffy_since = 0; 2243 struct nlattr *kind = NULL; 2244 struct nla_bitfield32 bf; 2245 u32 msecs_since = 0; 2246 u32 act_count = 0; 2247 2248 ret = nlmsg_parse_deprecated(cb->nlh, sizeof(struct tcamsg), tb, 2249 TCA_ROOT_MAX, tcaa_policy, cb->extack); 2250 if (ret < 0) 2251 return ret; 2252 2253 kind = find_dump_kind(tb); 2254 if (kind == NULL) { 2255 pr_info("tc_dump_action: action bad kind\n"); 2256 return 0; 2257 } 2258 2259 a_o = tc_lookup_action(kind); 2260 if (a_o == NULL) 2261 return 0; 2262 2263 cb->args[2] = 0; 2264 if (tb[TCA_ROOT_FLAGS]) { 2265 bf = nla_get_bitfield32(tb[TCA_ROOT_FLAGS]); 2266 cb->args[2] = bf.value; 2267 } 2268 2269 if (tb[TCA_ROOT_TIME_DELTA]) { 2270 msecs_since = nla_get_u32(tb[TCA_ROOT_TIME_DELTA]); 2271 } 2272 2273 nlh = nlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq, 2274 cb->nlh->nlmsg_type, sizeof(*t), 0); 2275 if (!nlh) 2276 goto out_module_put; 2277 2278 if (msecs_since) 2279 jiffy_since = jiffies - msecs_to_jiffies(msecs_since); 2280 2281 t = nlmsg_data(nlh); 2282 t->tca_family = AF_UNSPEC; 2283 t->tca__pad1 = 0; 2284 t->tca__pad2 = 0; 2285 cb->args[3] = jiffy_since; 2286 count_attr = nla_reserve(skb, TCA_ROOT_COUNT, sizeof(u32)); 2287 if (!count_attr) 2288 goto out_module_put; 2289 2290 nest = nla_nest_start_noflag(skb, TCA_ACT_TAB); 2291 if (nest == NULL) 2292 goto out_module_put; 2293 2294 ret = __tcf_generic_walker(net, skb, cb, RTM_GETACTION, a_o, NULL); 2295 if (ret < 0) 2296 goto out_module_put; 2297 2298 if (ret > 0) { 2299 nla_nest_end(skb, nest); 2300 ret = skb->len; 2301 act_count = cb->args[1]; 2302 memcpy(nla_data(count_attr), &act_count, sizeof(u32)); 2303 cb->args[1] = 0; 2304 } else 2305 nlmsg_trim(skb, b); 2306 2307 nlh->nlmsg_len = skb_tail_pointer(skb) - b; 2308 if (NETLINK_CB(cb->skb).portid && ret) 2309 nlh->nlmsg_flags |= NLM_F_MULTI; 2310 module_put(a_o->owner); 2311 return skb->len; 2312 2313 out_module_put: 2314 module_put(a_o->owner); 2315 nlmsg_trim(skb, b); 2316 return skb->len; 2317 } 2318 2319 static const struct rtnl_msg_handler tc_action_rtnl_msg_handlers[] __initconst = { 2320 {.msgtype = RTM_NEWACTION, .doit = tc_ctl_action}, 2321 {.msgtype = RTM_DELACTION, .doit = tc_ctl_action}, 2322 {.msgtype = RTM_GETACTION, .doit = tc_ctl_action, 2323 .dumpit = tc_dump_action}, 2324 }; 2325 2326 static int __init tc_action_init(void) 2327 { 2328 rtnl_register_many(tc_action_rtnl_msg_handlers); 2329 return 0; 2330 } 2331 2332 subsys_initcall(tc_action_init); 2333