1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * net/sched/cls_api.c Packet classifier API. 4 * 5 * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru> 6 * 7 * Changes: 8 * 9 * Eduardo J. Blanco <ejbs@netlabs.com.uy> :990222: kmod support 10 */ 11 12 #include <linux/module.h> 13 #include <linux/types.h> 14 #include <linux/kernel.h> 15 #include <linux/string.h> 16 #include <linux/errno.h> 17 #include <linux/err.h> 18 #include <linux/skbuff.h> 19 #include <linux/init.h> 20 #include <linux/kmod.h> 21 #include <linux/slab.h> 22 #include <linux/idr.h> 23 #include <linux/jhash.h> 24 #include <linux/rculist.h> 25 #include <linux/rhashtable.h> 26 #include <net/net_namespace.h> 27 #include <net/sock.h> 28 #include <net/netlink.h> 29 #include <net/pkt_sched.h> 30 #include <net/pkt_cls.h> 31 #include <net/tc_act/tc_pedit.h> 32 #include <net/tc_act/tc_mirred.h> 33 #include <net/tc_act/tc_vlan.h> 34 #include <net/tc_act/tc_tunnel_key.h> 35 #include <net/tc_act/tc_csum.h> 36 #include <net/tc_act/tc_gact.h> 37 #include <net/tc_act/tc_police.h> 38 #include <net/tc_act/tc_sample.h> 39 #include <net/tc_act/tc_skbedit.h> 40 #include <net/tc_act/tc_ct.h> 41 #include <net/tc_act/tc_mpls.h> 42 #include <net/tc_act/tc_gate.h> 43 #include <net/flow_offload.h> 44 #include <net/tc_wrapper.h> 45 46 /* The list of all installed classifier types */ 47 static LIST_HEAD(tcf_proto_base); 48 49 /* Protects list of registered TC modules. It is pure SMP lock. */ 50 static DEFINE_RWLOCK(cls_mod_lock); 51 52 static struct xarray tcf_exts_miss_cookies_xa; 53 struct tcf_exts_miss_cookie_node { 54 const struct tcf_chain *chain; 55 const struct tcf_proto *tp; 56 const struct tcf_exts *exts; 57 u32 chain_index; 58 u32 tp_prio; 59 u32 handle; 60 u32 miss_cookie_base; 61 struct rcu_head rcu; 62 }; 63 64 /* Each tc action entry cookie will be comprised of 32bit miss_cookie_base + 65 * action index in the exts tc actions array. 66 */ 67 union tcf_exts_miss_cookie { 68 struct { 69 u32 miss_cookie_base; 70 u32 act_index; 71 }; 72 u64 miss_cookie; 73 }; 74 75 #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT) 76 static int 77 tcf_exts_miss_cookie_base_alloc(struct tcf_exts *exts, struct tcf_proto *tp, 78 u32 handle) 79 { 80 struct tcf_exts_miss_cookie_node *n; 81 static u32 next; 82 int err; 83 84 if (WARN_ON(!handle || !tp->ops->get_exts)) 85 return -EINVAL; 86 87 n = kzalloc(sizeof(*n), GFP_KERNEL); 88 if (!n) 89 return -ENOMEM; 90 91 n->chain_index = tp->chain->index; 92 n->chain = tp->chain; 93 n->tp_prio = tp->prio; 94 n->tp = tp; 95 n->exts = exts; 96 n->handle = handle; 97 98 err = xa_alloc_cyclic(&tcf_exts_miss_cookies_xa, &n->miss_cookie_base, 99 n, xa_limit_32b, &next, GFP_KERNEL); 100 if (err) 101 goto err_xa_alloc; 102 103 exts->miss_cookie_node = n; 104 return 0; 105 106 err_xa_alloc: 107 kfree(n); 108 return err; 109 } 110 111 static void tcf_exts_miss_cookie_base_destroy(struct tcf_exts *exts) 112 { 113 struct tcf_exts_miss_cookie_node *n; 114 115 if (!exts->miss_cookie_node) 116 return; 117 118 n = exts->miss_cookie_node; 119 xa_erase(&tcf_exts_miss_cookies_xa, n->miss_cookie_base); 120 kfree_rcu(n, rcu); 121 } 122 123 static struct tcf_exts_miss_cookie_node * 124 tcf_exts_miss_cookie_lookup(u64 miss_cookie, int *act_index) 125 { 126 union tcf_exts_miss_cookie mc = { .miss_cookie = miss_cookie, }; 127 128 *act_index = mc.act_index; 129 return xa_load(&tcf_exts_miss_cookies_xa, mc.miss_cookie_base); 130 } 131 #else /* IS_ENABLED(CONFIG_NET_TC_SKB_EXT) */ 132 static int 133 tcf_exts_miss_cookie_base_alloc(struct tcf_exts *exts, struct tcf_proto *tp, 134 u32 handle) 135 { 136 return 0; 137 } 138 139 static void tcf_exts_miss_cookie_base_destroy(struct tcf_exts *exts) 140 { 141 } 142 #endif /* IS_ENABLED(CONFIG_NET_TC_SKB_EXT) */ 143 144 static u64 tcf_exts_miss_cookie_get(u32 miss_cookie_base, int act_index) 145 { 146 union tcf_exts_miss_cookie mc = { .act_index = act_index, }; 147 148 if (!miss_cookie_base) 149 return 0; 150 151 mc.miss_cookie_base = miss_cookie_base; 152 return mc.miss_cookie; 153 } 154 155 #ifdef CONFIG_NET_CLS_ACT 156 DEFINE_STATIC_KEY_FALSE(tc_skb_ext_tc); 157 EXPORT_SYMBOL(tc_skb_ext_tc); 158 159 void tc_skb_ext_tc_enable(void) 160 { 161 static_branch_inc(&tc_skb_ext_tc); 162 } 163 EXPORT_SYMBOL(tc_skb_ext_tc_enable); 164 165 void tc_skb_ext_tc_disable(void) 166 { 167 static_branch_dec(&tc_skb_ext_tc); 168 } 169 EXPORT_SYMBOL(tc_skb_ext_tc_disable); 170 #endif 171 172 static u32 destroy_obj_hashfn(const struct tcf_proto *tp) 173 { 174 return jhash_3words(tp->chain->index, tp->prio, 175 (__force __u32)tp->protocol, 0); 176 } 177 178 static void tcf_proto_signal_destroying(struct tcf_chain *chain, 179 struct tcf_proto *tp) 180 { 181 struct tcf_block *block = chain->block; 182 183 mutex_lock(&block->proto_destroy_lock); 184 hash_add_rcu(block->proto_destroy_ht, &tp->destroy_ht_node, 185 destroy_obj_hashfn(tp)); 186 mutex_unlock(&block->proto_destroy_lock); 187 } 188 189 static bool tcf_proto_cmp(const struct tcf_proto *tp1, 190 const struct tcf_proto *tp2) 191 { 192 return tp1->chain->index == tp2->chain->index && 193 tp1->prio == tp2->prio && 194 tp1->protocol == tp2->protocol; 195 } 196 197 static bool tcf_proto_exists_destroying(struct tcf_chain *chain, 198 struct tcf_proto *tp) 199 { 200 u32 hash = destroy_obj_hashfn(tp); 201 struct tcf_proto *iter; 202 bool found = false; 203 204 rcu_read_lock(); 205 hash_for_each_possible_rcu(chain->block->proto_destroy_ht, iter, 206 destroy_ht_node, hash) { 207 if (tcf_proto_cmp(tp, iter)) { 208 found = true; 209 break; 210 } 211 } 212 rcu_read_unlock(); 213 214 return found; 215 } 216 217 static void 218 tcf_proto_signal_destroyed(struct tcf_chain *chain, struct tcf_proto *tp) 219 { 220 struct tcf_block *block = chain->block; 221 222 mutex_lock(&block->proto_destroy_lock); 223 if (hash_hashed(&tp->destroy_ht_node)) 224 hash_del_rcu(&tp->destroy_ht_node); 225 mutex_unlock(&block->proto_destroy_lock); 226 } 227 228 /* Find classifier type by string name */ 229 230 static const struct tcf_proto_ops *__tcf_proto_lookup_ops(const char *kind) 231 { 232 const struct tcf_proto_ops *t, *res = NULL; 233 234 if (kind) { 235 read_lock(&cls_mod_lock); 236 list_for_each_entry(t, &tcf_proto_base, head) { 237 if (strcmp(kind, t->kind) == 0) { 238 if (try_module_get(t->owner)) 239 res = t; 240 break; 241 } 242 } 243 read_unlock(&cls_mod_lock); 244 } 245 return res; 246 } 247 248 static const struct tcf_proto_ops * 249 tcf_proto_lookup_ops(const char *kind, bool rtnl_held, 250 struct netlink_ext_ack *extack) 251 { 252 const struct tcf_proto_ops *ops; 253 254 ops = __tcf_proto_lookup_ops(kind); 255 if (ops) 256 return ops; 257 #ifdef CONFIG_MODULES 258 if (rtnl_held) 259 rtnl_unlock(); 260 request_module(NET_CLS_ALIAS_PREFIX "%s", kind); 261 if (rtnl_held) 262 rtnl_lock(); 263 ops = __tcf_proto_lookup_ops(kind); 264 /* We dropped the RTNL semaphore in order to perform 265 * the module load. So, even if we succeeded in loading 266 * the module we have to replay the request. We indicate 267 * this using -EAGAIN. 268 */ 269 if (ops) { 270 module_put(ops->owner); 271 return ERR_PTR(-EAGAIN); 272 } 273 #endif 274 NL_SET_ERR_MSG(extack, "TC classifier not found"); 275 return ERR_PTR(-ENOENT); 276 } 277 278 /* Register(unregister) new classifier type */ 279 280 int register_tcf_proto_ops(struct tcf_proto_ops *ops) 281 { 282 struct tcf_proto_ops *t; 283 int rc = -EEXIST; 284 285 write_lock(&cls_mod_lock); 286 list_for_each_entry(t, &tcf_proto_base, head) 287 if (!strcmp(ops->kind, t->kind)) 288 goto out; 289 290 list_add_tail(&ops->head, &tcf_proto_base); 291 rc = 0; 292 out: 293 write_unlock(&cls_mod_lock); 294 return rc; 295 } 296 EXPORT_SYMBOL(register_tcf_proto_ops); 297 298 static struct workqueue_struct *tc_filter_wq; 299 300 void unregister_tcf_proto_ops(struct tcf_proto_ops *ops) 301 { 302 struct tcf_proto_ops *t; 303 int rc = -ENOENT; 304 305 /* Wait for outstanding call_rcu()s, if any, from a 306 * tcf_proto_ops's destroy() handler. 307 */ 308 rcu_barrier(); 309 flush_workqueue(tc_filter_wq); 310 311 write_lock(&cls_mod_lock); 312 list_for_each_entry(t, &tcf_proto_base, head) { 313 if (t == ops) { 314 list_del(&t->head); 315 rc = 0; 316 break; 317 } 318 } 319 write_unlock(&cls_mod_lock); 320 321 WARN(rc, "unregister tc filter kind(%s) failed %d\n", ops->kind, rc); 322 } 323 EXPORT_SYMBOL(unregister_tcf_proto_ops); 324 325 bool tcf_queue_work(struct rcu_work *rwork, work_func_t func) 326 { 327 INIT_RCU_WORK(rwork, func); 328 return queue_rcu_work(tc_filter_wq, rwork); 329 } 330 EXPORT_SYMBOL(tcf_queue_work); 331 332 /* Select new prio value from the range, managed by kernel. */ 333 334 static inline u32 tcf_auto_prio(struct tcf_proto *tp) 335 { 336 u32 first = TC_H_MAKE(0xC0000000U, 0U); 337 338 if (tp) 339 first = tp->prio - 1; 340 341 return TC_H_MAJ(first); 342 } 343 344 static bool tcf_proto_check_kind(struct nlattr *kind, char *name) 345 { 346 if (kind) 347 return nla_strscpy(name, kind, IFNAMSIZ) < 0; 348 memset(name, 0, IFNAMSIZ); 349 return false; 350 } 351 352 static bool tcf_proto_is_unlocked(const char *kind) 353 { 354 const struct tcf_proto_ops *ops; 355 bool ret; 356 357 if (strlen(kind) == 0) 358 return false; 359 360 ops = tcf_proto_lookup_ops(kind, false, NULL); 361 /* On error return false to take rtnl lock. Proto lookup/create 362 * functions will perform lookup again and properly handle errors. 363 */ 364 if (IS_ERR(ops)) 365 return false; 366 367 ret = !!(ops->flags & TCF_PROTO_OPS_DOIT_UNLOCKED); 368 module_put(ops->owner); 369 return ret; 370 } 371 372 static struct tcf_proto *tcf_proto_create(const char *kind, u32 protocol, 373 u32 prio, struct tcf_chain *chain, 374 bool rtnl_held, 375 struct netlink_ext_ack *extack) 376 { 377 struct tcf_proto *tp; 378 int err; 379 380 tp = kzalloc(sizeof(*tp), GFP_KERNEL); 381 if (!tp) 382 return ERR_PTR(-ENOBUFS); 383 384 tp->ops = tcf_proto_lookup_ops(kind, rtnl_held, extack); 385 if (IS_ERR(tp->ops)) { 386 err = PTR_ERR(tp->ops); 387 goto errout; 388 } 389 tp->classify = tp->ops->classify; 390 tp->protocol = protocol; 391 tp->prio = prio; 392 tp->chain = chain; 393 spin_lock_init(&tp->lock); 394 refcount_set(&tp->refcnt, 1); 395 396 err = tp->ops->init(tp); 397 if (err) { 398 module_put(tp->ops->owner); 399 goto errout; 400 } 401 return tp; 402 403 errout: 404 kfree(tp); 405 return ERR_PTR(err); 406 } 407 408 static void tcf_proto_get(struct tcf_proto *tp) 409 { 410 refcount_inc(&tp->refcnt); 411 } 412 413 static void tcf_maintain_bypass(struct tcf_block *block) 414 { 415 int filtercnt = atomic_read(&block->filtercnt); 416 int skipswcnt = atomic_read(&block->skipswcnt); 417 bool bypass_wanted = filtercnt > 0 && filtercnt == skipswcnt; 418 419 if (bypass_wanted != block->bypass_wanted) { 420 #ifdef CONFIG_NET_CLS_ACT 421 if (bypass_wanted) 422 static_branch_inc(&tcf_bypass_check_needed_key); 423 else 424 static_branch_dec(&tcf_bypass_check_needed_key); 425 #endif 426 block->bypass_wanted = bypass_wanted; 427 } 428 } 429 430 static void tcf_block_filter_cnt_update(struct tcf_block *block, bool *counted, bool add) 431 { 432 lockdep_assert_not_held(&block->cb_lock); 433 434 down_write(&block->cb_lock); 435 if (*counted != add) { 436 if (add) { 437 atomic_inc(&block->filtercnt); 438 *counted = true; 439 } else { 440 atomic_dec(&block->filtercnt); 441 *counted = false; 442 } 443 } 444 tcf_maintain_bypass(block); 445 up_write(&block->cb_lock); 446 } 447 448 static void tcf_chain_put(struct tcf_chain *chain); 449 450 static void tcf_proto_destroy(struct tcf_proto *tp, bool rtnl_held, 451 bool sig_destroy, struct netlink_ext_ack *extack) 452 { 453 tp->ops->destroy(tp, rtnl_held, extack); 454 tcf_block_filter_cnt_update(tp->chain->block, &tp->counted, false); 455 if (sig_destroy) 456 tcf_proto_signal_destroyed(tp->chain, tp); 457 tcf_chain_put(tp->chain); 458 module_put(tp->ops->owner); 459 kfree_rcu(tp, rcu); 460 } 461 462 static void tcf_proto_put(struct tcf_proto *tp, bool rtnl_held, 463 struct netlink_ext_ack *extack) 464 { 465 if (refcount_dec_and_test(&tp->refcnt)) 466 tcf_proto_destroy(tp, rtnl_held, true, extack); 467 } 468 469 static bool tcf_proto_check_delete(struct tcf_proto *tp) 470 { 471 if (tp->ops->delete_empty) 472 return tp->ops->delete_empty(tp); 473 474 tp->deleting = true; 475 return tp->deleting; 476 } 477 478 static void tcf_proto_mark_delete(struct tcf_proto *tp) 479 { 480 spin_lock(&tp->lock); 481 tp->deleting = true; 482 spin_unlock(&tp->lock); 483 } 484 485 static bool tcf_proto_is_deleting(struct tcf_proto *tp) 486 { 487 bool deleting; 488 489 spin_lock(&tp->lock); 490 deleting = tp->deleting; 491 spin_unlock(&tp->lock); 492 493 return deleting; 494 } 495 496 #define ASSERT_BLOCK_LOCKED(block) \ 497 lockdep_assert_held(&(block)->lock) 498 499 struct tcf_filter_chain_list_item { 500 struct list_head list; 501 tcf_chain_head_change_t *chain_head_change; 502 void *chain_head_change_priv; 503 }; 504 505 static struct tcf_chain *tcf_chain_create(struct tcf_block *block, 506 u32 chain_index) 507 { 508 struct tcf_chain *chain; 509 510 ASSERT_BLOCK_LOCKED(block); 511 512 chain = kzalloc(sizeof(*chain), GFP_KERNEL); 513 if (!chain) 514 return NULL; 515 list_add_tail_rcu(&chain->list, &block->chain_list); 516 mutex_init(&chain->filter_chain_lock); 517 chain->block = block; 518 chain->index = chain_index; 519 chain->refcnt = 1; 520 if (!chain->index) 521 block->chain0.chain = chain; 522 return chain; 523 } 524 525 static void tcf_chain_head_change_item(struct tcf_filter_chain_list_item *item, 526 struct tcf_proto *tp_head) 527 { 528 if (item->chain_head_change) 529 item->chain_head_change(tp_head, item->chain_head_change_priv); 530 } 531 532 static void tcf_chain0_head_change(struct tcf_chain *chain, 533 struct tcf_proto *tp_head) 534 { 535 struct tcf_filter_chain_list_item *item; 536 struct tcf_block *block = chain->block; 537 538 if (chain->index) 539 return; 540 541 mutex_lock(&block->lock); 542 list_for_each_entry(item, &block->chain0.filter_chain_list, list) 543 tcf_chain_head_change_item(item, tp_head); 544 mutex_unlock(&block->lock); 545 } 546 547 /* Returns true if block can be safely freed. */ 548 549 static bool tcf_chain_detach(struct tcf_chain *chain) 550 { 551 struct tcf_block *block = chain->block; 552 553 ASSERT_BLOCK_LOCKED(block); 554 555 list_del_rcu(&chain->list); 556 if (!chain->index) 557 block->chain0.chain = NULL; 558 559 if (list_empty(&block->chain_list) && 560 refcount_read(&block->refcnt) == 0) 561 return true; 562 563 return false; 564 } 565 566 static void tcf_block_destroy(struct tcf_block *block) 567 { 568 mutex_destroy(&block->lock); 569 mutex_destroy(&block->proto_destroy_lock); 570 xa_destroy(&block->ports); 571 kfree_rcu(block, rcu); 572 } 573 574 static void tcf_chain_destroy(struct tcf_chain *chain, bool free_block) 575 { 576 struct tcf_block *block = chain->block; 577 578 mutex_destroy(&chain->filter_chain_lock); 579 kfree_rcu(chain, rcu); 580 if (free_block) 581 tcf_block_destroy(block); 582 } 583 584 static void tcf_chain_hold(struct tcf_chain *chain) 585 { 586 ASSERT_BLOCK_LOCKED(chain->block); 587 588 ++chain->refcnt; 589 } 590 591 static bool tcf_chain_held_by_acts_only(struct tcf_chain *chain) 592 { 593 ASSERT_BLOCK_LOCKED(chain->block); 594 595 /* In case all the references are action references, this 596 * chain should not be shown to the user. 597 */ 598 return chain->refcnt == chain->action_refcnt; 599 } 600 601 static struct tcf_chain *tcf_chain_lookup(struct tcf_block *block, 602 u32 chain_index) 603 { 604 struct tcf_chain *chain; 605 606 ASSERT_BLOCK_LOCKED(block); 607 608 list_for_each_entry(chain, &block->chain_list, list) { 609 if (chain->index == chain_index) 610 return chain; 611 } 612 return NULL; 613 } 614 615 #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT) 616 static struct tcf_chain *tcf_chain_lookup_rcu(const struct tcf_block *block, 617 u32 chain_index) 618 { 619 struct tcf_chain *chain; 620 621 list_for_each_entry_rcu(chain, &block->chain_list, list) { 622 if (chain->index == chain_index) 623 return chain; 624 } 625 return NULL; 626 } 627 #endif 628 629 static int tc_chain_notify(struct tcf_chain *chain, struct sk_buff *oskb, 630 u32 seq, u16 flags, int event, bool unicast, 631 struct netlink_ext_ack *extack); 632 633 static struct tcf_chain *__tcf_chain_get(struct tcf_block *block, 634 u32 chain_index, bool create, 635 bool by_act) 636 { 637 struct tcf_chain *chain = NULL; 638 bool is_first_reference; 639 640 mutex_lock(&block->lock); 641 chain = tcf_chain_lookup(block, chain_index); 642 if (chain) { 643 tcf_chain_hold(chain); 644 } else { 645 if (!create) 646 goto errout; 647 chain = tcf_chain_create(block, chain_index); 648 if (!chain) 649 goto errout; 650 } 651 652 if (by_act) 653 ++chain->action_refcnt; 654 is_first_reference = chain->refcnt - chain->action_refcnt == 1; 655 mutex_unlock(&block->lock); 656 657 /* Send notification only in case we got the first 658 * non-action reference. Until then, the chain acts only as 659 * a placeholder for actions pointing to it and user ought 660 * not know about them. 661 */ 662 if (is_first_reference && !by_act) 663 tc_chain_notify(chain, NULL, 0, NLM_F_CREATE | NLM_F_EXCL, 664 RTM_NEWCHAIN, false, NULL); 665 666 return chain; 667 668 errout: 669 mutex_unlock(&block->lock); 670 return chain; 671 } 672 673 static struct tcf_chain *tcf_chain_get(struct tcf_block *block, u32 chain_index, 674 bool create) 675 { 676 return __tcf_chain_get(block, chain_index, create, false); 677 } 678 679 struct tcf_chain *tcf_chain_get_by_act(struct tcf_block *block, u32 chain_index) 680 { 681 return __tcf_chain_get(block, chain_index, true, true); 682 } 683 EXPORT_SYMBOL(tcf_chain_get_by_act); 684 685 static void tc_chain_tmplt_del(const struct tcf_proto_ops *tmplt_ops, 686 void *tmplt_priv); 687 static int tc_chain_notify_delete(const struct tcf_proto_ops *tmplt_ops, 688 void *tmplt_priv, u32 chain_index, 689 struct tcf_block *block, struct sk_buff *oskb, 690 u32 seq, u16 flags); 691 692 static void __tcf_chain_put(struct tcf_chain *chain, bool by_act, 693 bool explicitly_created) 694 { 695 struct tcf_block *block = chain->block; 696 const struct tcf_proto_ops *tmplt_ops; 697 unsigned int refcnt, non_act_refcnt; 698 bool free_block = false; 699 void *tmplt_priv; 700 701 mutex_lock(&block->lock); 702 if (explicitly_created) { 703 if (!chain->explicitly_created) { 704 mutex_unlock(&block->lock); 705 return; 706 } 707 chain->explicitly_created = false; 708 } 709 710 if (by_act) 711 chain->action_refcnt--; 712 713 /* tc_chain_notify_delete can't be called while holding block lock. 714 * However, when block is unlocked chain can be changed concurrently, so 715 * save these to temporary variables. 716 */ 717 refcnt = --chain->refcnt; 718 non_act_refcnt = refcnt - chain->action_refcnt; 719 tmplt_ops = chain->tmplt_ops; 720 tmplt_priv = chain->tmplt_priv; 721 722 if (non_act_refcnt == chain->explicitly_created && !by_act) { 723 if (non_act_refcnt == 0) 724 tc_chain_notify_delete(tmplt_ops, tmplt_priv, 725 chain->index, block, NULL, 0, 0); 726 /* Last reference to chain, no need to lock. */ 727 chain->flushing = false; 728 } 729 730 if (refcnt == 0) 731 free_block = tcf_chain_detach(chain); 732 mutex_unlock(&block->lock); 733 734 if (refcnt == 0) { 735 tc_chain_tmplt_del(tmplt_ops, tmplt_priv); 736 tcf_chain_destroy(chain, free_block); 737 } 738 } 739 740 static void tcf_chain_put(struct tcf_chain *chain) 741 { 742 __tcf_chain_put(chain, false, false); 743 } 744 745 void tcf_chain_put_by_act(struct tcf_chain *chain) 746 { 747 __tcf_chain_put(chain, true, false); 748 } 749 EXPORT_SYMBOL(tcf_chain_put_by_act); 750 751 static void tcf_chain_put_explicitly_created(struct tcf_chain *chain) 752 { 753 __tcf_chain_put(chain, false, true); 754 } 755 756 static void tcf_chain_flush(struct tcf_chain *chain, bool rtnl_held) 757 { 758 struct tcf_proto *tp, *tp_next; 759 760 mutex_lock(&chain->filter_chain_lock); 761 tp = tcf_chain_dereference(chain->filter_chain, chain); 762 while (tp) { 763 tp_next = rcu_dereference_protected(tp->next, 1); 764 tcf_proto_signal_destroying(chain, tp); 765 tp = tp_next; 766 } 767 tp = tcf_chain_dereference(chain->filter_chain, chain); 768 RCU_INIT_POINTER(chain->filter_chain, NULL); 769 tcf_chain0_head_change(chain, NULL); 770 chain->flushing = true; 771 mutex_unlock(&chain->filter_chain_lock); 772 773 while (tp) { 774 tp_next = rcu_dereference_protected(tp->next, 1); 775 tcf_proto_put(tp, rtnl_held, NULL); 776 tp = tp_next; 777 } 778 } 779 780 static int tcf_block_setup(struct tcf_block *block, 781 struct flow_block_offload *bo); 782 783 static void tcf_block_offload_init(struct flow_block_offload *bo, 784 struct net_device *dev, struct Qdisc *sch, 785 enum flow_block_command command, 786 enum flow_block_binder_type binder_type, 787 struct flow_block *flow_block, 788 bool shared, struct netlink_ext_ack *extack) 789 { 790 bo->net = dev_net(dev); 791 bo->command = command; 792 bo->binder_type = binder_type; 793 bo->block = flow_block; 794 bo->block_shared = shared; 795 bo->extack = extack; 796 bo->sch = sch; 797 bo->cb_list_head = &flow_block->cb_list; 798 INIT_LIST_HEAD(&bo->cb_list); 799 } 800 801 static void tcf_block_unbind(struct tcf_block *block, 802 struct flow_block_offload *bo); 803 804 static void tc_block_indr_cleanup(struct flow_block_cb *block_cb) 805 { 806 struct tcf_block *block = block_cb->indr.data; 807 struct net_device *dev = block_cb->indr.dev; 808 struct Qdisc *sch = block_cb->indr.sch; 809 struct netlink_ext_ack extack = {}; 810 struct flow_block_offload bo = {}; 811 812 tcf_block_offload_init(&bo, dev, sch, FLOW_BLOCK_UNBIND, 813 block_cb->indr.binder_type, 814 &block->flow_block, tcf_block_shared(block), 815 &extack); 816 rtnl_lock(); 817 down_write(&block->cb_lock); 818 list_del(&block_cb->driver_list); 819 list_move(&block_cb->list, &bo.cb_list); 820 tcf_block_unbind(block, &bo); 821 up_write(&block->cb_lock); 822 rtnl_unlock(); 823 } 824 825 static bool tcf_block_offload_in_use(struct tcf_block *block) 826 { 827 return atomic_read(&block->offloadcnt); 828 } 829 830 static int tcf_block_offload_cmd(struct tcf_block *block, 831 struct net_device *dev, struct Qdisc *sch, 832 struct tcf_block_ext_info *ei, 833 enum flow_block_command command, 834 struct netlink_ext_ack *extack) 835 { 836 struct flow_block_offload bo = {}; 837 838 tcf_block_offload_init(&bo, dev, sch, command, ei->binder_type, 839 &block->flow_block, tcf_block_shared(block), 840 extack); 841 842 if (dev->netdev_ops->ndo_setup_tc) { 843 int err; 844 845 err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_BLOCK, &bo); 846 if (err < 0) { 847 if (err != -EOPNOTSUPP) 848 NL_SET_ERR_MSG(extack, "Driver ndo_setup_tc failed"); 849 return err; 850 } 851 852 return tcf_block_setup(block, &bo); 853 } 854 855 flow_indr_dev_setup_offload(dev, sch, TC_SETUP_BLOCK, block, &bo, 856 tc_block_indr_cleanup); 857 tcf_block_setup(block, &bo); 858 859 return -EOPNOTSUPP; 860 } 861 862 static int tcf_block_offload_bind(struct tcf_block *block, struct Qdisc *q, 863 struct tcf_block_ext_info *ei, 864 struct netlink_ext_ack *extack) 865 { 866 struct net_device *dev = q->dev_queue->dev; 867 int err; 868 869 down_write(&block->cb_lock); 870 871 /* If tc offload feature is disabled and the block we try to bind 872 * to already has some offloaded filters, forbid to bind. 873 */ 874 if (dev->netdev_ops->ndo_setup_tc && 875 !tc_can_offload(dev) && 876 tcf_block_offload_in_use(block)) { 877 NL_SET_ERR_MSG(extack, "Bind to offloaded block failed as dev has offload disabled"); 878 err = -EOPNOTSUPP; 879 goto err_unlock; 880 } 881 882 err = tcf_block_offload_cmd(block, dev, q, ei, FLOW_BLOCK_BIND, extack); 883 if (err == -EOPNOTSUPP) 884 goto no_offload_dev_inc; 885 if (err) 886 goto err_unlock; 887 888 up_write(&block->cb_lock); 889 return 0; 890 891 no_offload_dev_inc: 892 if (tcf_block_offload_in_use(block)) 893 goto err_unlock; 894 895 err = 0; 896 block->nooffloaddevcnt++; 897 err_unlock: 898 up_write(&block->cb_lock); 899 return err; 900 } 901 902 static void tcf_block_offload_unbind(struct tcf_block *block, struct Qdisc *q, 903 struct tcf_block_ext_info *ei) 904 { 905 struct net_device *dev = q->dev_queue->dev; 906 int err; 907 908 down_write(&block->cb_lock); 909 err = tcf_block_offload_cmd(block, dev, q, ei, FLOW_BLOCK_UNBIND, NULL); 910 if (err == -EOPNOTSUPP) 911 goto no_offload_dev_dec; 912 up_write(&block->cb_lock); 913 return; 914 915 no_offload_dev_dec: 916 WARN_ON(block->nooffloaddevcnt-- == 0); 917 up_write(&block->cb_lock); 918 } 919 920 static int 921 tcf_chain0_head_change_cb_add(struct tcf_block *block, 922 struct tcf_block_ext_info *ei, 923 struct netlink_ext_ack *extack) 924 { 925 struct tcf_filter_chain_list_item *item; 926 struct tcf_chain *chain0; 927 928 item = kmalloc(sizeof(*item), GFP_KERNEL); 929 if (!item) { 930 NL_SET_ERR_MSG(extack, "Memory allocation for head change callback item failed"); 931 return -ENOMEM; 932 } 933 item->chain_head_change = ei->chain_head_change; 934 item->chain_head_change_priv = ei->chain_head_change_priv; 935 936 mutex_lock(&block->lock); 937 chain0 = block->chain0.chain; 938 if (chain0) 939 tcf_chain_hold(chain0); 940 else 941 list_add(&item->list, &block->chain0.filter_chain_list); 942 mutex_unlock(&block->lock); 943 944 if (chain0) { 945 struct tcf_proto *tp_head; 946 947 mutex_lock(&chain0->filter_chain_lock); 948 949 tp_head = tcf_chain_dereference(chain0->filter_chain, chain0); 950 if (tp_head) 951 tcf_chain_head_change_item(item, tp_head); 952 953 mutex_lock(&block->lock); 954 list_add(&item->list, &block->chain0.filter_chain_list); 955 mutex_unlock(&block->lock); 956 957 mutex_unlock(&chain0->filter_chain_lock); 958 tcf_chain_put(chain0); 959 } 960 961 return 0; 962 } 963 964 static void 965 tcf_chain0_head_change_cb_del(struct tcf_block *block, 966 struct tcf_block_ext_info *ei) 967 { 968 struct tcf_filter_chain_list_item *item; 969 970 mutex_lock(&block->lock); 971 list_for_each_entry(item, &block->chain0.filter_chain_list, list) { 972 if ((!ei->chain_head_change && !ei->chain_head_change_priv) || 973 (item->chain_head_change == ei->chain_head_change && 974 item->chain_head_change_priv == ei->chain_head_change_priv)) { 975 if (block->chain0.chain) 976 tcf_chain_head_change_item(item, NULL); 977 list_del(&item->list); 978 mutex_unlock(&block->lock); 979 980 kfree(item); 981 return; 982 } 983 } 984 mutex_unlock(&block->lock); 985 WARN_ON(1); 986 } 987 988 struct tcf_net { 989 spinlock_t idr_lock; /* Protects idr */ 990 struct idr idr; 991 }; 992 993 static unsigned int tcf_net_id; 994 995 static int tcf_block_insert(struct tcf_block *block, struct net *net, 996 struct netlink_ext_ack *extack) 997 { 998 struct tcf_net *tn = net_generic(net, tcf_net_id); 999 int err; 1000 1001 idr_preload(GFP_KERNEL); 1002 spin_lock(&tn->idr_lock); 1003 err = idr_alloc_u32(&tn->idr, block, &block->index, block->index, 1004 GFP_NOWAIT); 1005 spin_unlock(&tn->idr_lock); 1006 idr_preload_end(); 1007 1008 return err; 1009 } 1010 1011 static void tcf_block_remove(struct tcf_block *block, struct net *net) 1012 { 1013 struct tcf_net *tn = net_generic(net, tcf_net_id); 1014 1015 spin_lock(&tn->idr_lock); 1016 idr_remove(&tn->idr, block->index); 1017 spin_unlock(&tn->idr_lock); 1018 } 1019 1020 static struct tcf_block *tcf_block_create(struct net *net, struct Qdisc *q, 1021 u32 block_index, 1022 struct netlink_ext_ack *extack) 1023 { 1024 struct tcf_block *block; 1025 1026 block = kzalloc(sizeof(*block), GFP_KERNEL); 1027 if (!block) { 1028 NL_SET_ERR_MSG(extack, "Memory allocation for block failed"); 1029 return ERR_PTR(-ENOMEM); 1030 } 1031 mutex_init(&block->lock); 1032 mutex_init(&block->proto_destroy_lock); 1033 init_rwsem(&block->cb_lock); 1034 flow_block_init(&block->flow_block); 1035 INIT_LIST_HEAD(&block->chain_list); 1036 INIT_LIST_HEAD(&block->owner_list); 1037 INIT_LIST_HEAD(&block->chain0.filter_chain_list); 1038 1039 refcount_set(&block->refcnt, 1); 1040 block->net = net; 1041 block->index = block_index; 1042 xa_init(&block->ports); 1043 1044 /* Don't store q pointer for blocks which are shared */ 1045 if (!tcf_block_shared(block)) 1046 block->q = q; 1047 return block; 1048 } 1049 1050 struct tcf_block *tcf_block_lookup(struct net *net, u32 block_index) 1051 { 1052 struct tcf_net *tn = net_generic(net, tcf_net_id); 1053 1054 return idr_find(&tn->idr, block_index); 1055 } 1056 EXPORT_SYMBOL(tcf_block_lookup); 1057 1058 static struct tcf_block *tcf_block_refcnt_get(struct net *net, u32 block_index) 1059 { 1060 struct tcf_block *block; 1061 1062 rcu_read_lock(); 1063 block = tcf_block_lookup(net, block_index); 1064 if (block && !refcount_inc_not_zero(&block->refcnt)) 1065 block = NULL; 1066 rcu_read_unlock(); 1067 1068 return block; 1069 } 1070 1071 static struct tcf_chain * 1072 __tcf_get_next_chain(struct tcf_block *block, struct tcf_chain *chain) 1073 { 1074 mutex_lock(&block->lock); 1075 if (chain) 1076 chain = list_is_last(&chain->list, &block->chain_list) ? 1077 NULL : list_next_entry(chain, list); 1078 else 1079 chain = list_first_entry_or_null(&block->chain_list, 1080 struct tcf_chain, list); 1081 1082 /* skip all action-only chains */ 1083 while (chain && tcf_chain_held_by_acts_only(chain)) 1084 chain = list_is_last(&chain->list, &block->chain_list) ? 1085 NULL : list_next_entry(chain, list); 1086 1087 if (chain) 1088 tcf_chain_hold(chain); 1089 mutex_unlock(&block->lock); 1090 1091 return chain; 1092 } 1093 1094 /* Function to be used by all clients that want to iterate over all chains on 1095 * block. It properly obtains block->lock and takes reference to chain before 1096 * returning it. Users of this function must be tolerant to concurrent chain 1097 * insertion/deletion or ensure that no concurrent chain modification is 1098 * possible. Note that all netlink dump callbacks cannot guarantee to provide 1099 * consistent dump because rtnl lock is released each time skb is filled with 1100 * data and sent to user-space. 1101 */ 1102 1103 struct tcf_chain * 1104 tcf_get_next_chain(struct tcf_block *block, struct tcf_chain *chain) 1105 { 1106 struct tcf_chain *chain_next = __tcf_get_next_chain(block, chain); 1107 1108 if (chain) 1109 tcf_chain_put(chain); 1110 1111 return chain_next; 1112 } 1113 EXPORT_SYMBOL(tcf_get_next_chain); 1114 1115 static struct tcf_proto * 1116 __tcf_get_next_proto(struct tcf_chain *chain, struct tcf_proto *tp) 1117 { 1118 u32 prio = 0; 1119 1120 ASSERT_RTNL(); 1121 mutex_lock(&chain->filter_chain_lock); 1122 1123 if (!tp) { 1124 tp = tcf_chain_dereference(chain->filter_chain, chain); 1125 } else if (tcf_proto_is_deleting(tp)) { 1126 /* 'deleting' flag is set and chain->filter_chain_lock was 1127 * unlocked, which means next pointer could be invalid. Restart 1128 * search. 1129 */ 1130 prio = tp->prio + 1; 1131 tp = tcf_chain_dereference(chain->filter_chain, chain); 1132 1133 for (; tp; tp = tcf_chain_dereference(tp->next, chain)) 1134 if (!tp->deleting && tp->prio >= prio) 1135 break; 1136 } else { 1137 tp = tcf_chain_dereference(tp->next, chain); 1138 } 1139 1140 if (tp) 1141 tcf_proto_get(tp); 1142 1143 mutex_unlock(&chain->filter_chain_lock); 1144 1145 return tp; 1146 } 1147 1148 /* Function to be used by all clients that want to iterate over all tp's on 1149 * chain. Users of this function must be tolerant to concurrent tp 1150 * insertion/deletion or ensure that no concurrent chain modification is 1151 * possible. Note that all netlink dump callbacks cannot guarantee to provide 1152 * consistent dump because rtnl lock is released each time skb is filled with 1153 * data and sent to user-space. 1154 */ 1155 1156 struct tcf_proto * 1157 tcf_get_next_proto(struct tcf_chain *chain, struct tcf_proto *tp) 1158 { 1159 struct tcf_proto *tp_next = __tcf_get_next_proto(chain, tp); 1160 1161 if (tp) 1162 tcf_proto_put(tp, true, NULL); 1163 1164 return tp_next; 1165 } 1166 EXPORT_SYMBOL(tcf_get_next_proto); 1167 1168 static void tcf_block_flush_all_chains(struct tcf_block *block, bool rtnl_held) 1169 { 1170 struct tcf_chain *chain; 1171 1172 /* Last reference to block. At this point chains cannot be added or 1173 * removed concurrently. 1174 */ 1175 for (chain = tcf_get_next_chain(block, NULL); 1176 chain; 1177 chain = tcf_get_next_chain(block, chain)) { 1178 tcf_chain_put_explicitly_created(chain); 1179 tcf_chain_flush(chain, rtnl_held); 1180 } 1181 } 1182 1183 /* Lookup Qdisc and increments its reference counter. 1184 * Set parent, if necessary. 1185 */ 1186 1187 static int __tcf_qdisc_find(struct net *net, struct Qdisc **q, 1188 u32 *parent, int ifindex, bool rtnl_held, 1189 struct netlink_ext_ack *extack) 1190 { 1191 const struct Qdisc_class_ops *cops; 1192 struct net_device *dev; 1193 int err = 0; 1194 1195 if (ifindex == TCM_IFINDEX_MAGIC_BLOCK) 1196 return 0; 1197 1198 rcu_read_lock(); 1199 1200 /* Find link */ 1201 dev = dev_get_by_index_rcu(net, ifindex); 1202 if (!dev) { 1203 rcu_read_unlock(); 1204 return -ENODEV; 1205 } 1206 1207 /* Find qdisc */ 1208 if (!*parent) { 1209 *q = rcu_dereference(dev->qdisc); 1210 *parent = (*q)->handle; 1211 } else { 1212 *q = qdisc_lookup_rcu(dev, TC_H_MAJ(*parent)); 1213 if (!*q) { 1214 NL_SET_ERR_MSG(extack, "Parent Qdisc doesn't exists"); 1215 err = -EINVAL; 1216 goto errout_rcu; 1217 } 1218 } 1219 1220 *q = qdisc_refcount_inc_nz(*q); 1221 if (!*q) { 1222 NL_SET_ERR_MSG(extack, "Parent Qdisc doesn't exists"); 1223 err = -EINVAL; 1224 goto errout_rcu; 1225 } 1226 1227 /* Is it classful? */ 1228 cops = (*q)->ops->cl_ops; 1229 if (!cops) { 1230 NL_SET_ERR_MSG(extack, "Qdisc not classful"); 1231 err = -EINVAL; 1232 goto errout_qdisc; 1233 } 1234 1235 if (!cops->tcf_block) { 1236 NL_SET_ERR_MSG(extack, "Class doesn't support blocks"); 1237 err = -EOPNOTSUPP; 1238 goto errout_qdisc; 1239 } 1240 1241 errout_rcu: 1242 /* At this point we know that qdisc is not noop_qdisc, 1243 * which means that qdisc holds a reference to net_device 1244 * and we hold a reference to qdisc, so it is safe to release 1245 * rcu read lock. 1246 */ 1247 rcu_read_unlock(); 1248 return err; 1249 1250 errout_qdisc: 1251 rcu_read_unlock(); 1252 1253 if (rtnl_held) 1254 qdisc_put(*q); 1255 else 1256 qdisc_put_unlocked(*q); 1257 *q = NULL; 1258 1259 return err; 1260 } 1261 1262 static int __tcf_qdisc_cl_find(struct Qdisc *q, u32 parent, unsigned long *cl, 1263 int ifindex, struct netlink_ext_ack *extack) 1264 { 1265 if (ifindex == TCM_IFINDEX_MAGIC_BLOCK) 1266 return 0; 1267 1268 /* Do we search for filter, attached to class? */ 1269 if (TC_H_MIN(parent)) { 1270 const struct Qdisc_class_ops *cops = q->ops->cl_ops; 1271 1272 *cl = cops->find(q, parent); 1273 if (*cl == 0) { 1274 NL_SET_ERR_MSG(extack, "Specified class doesn't exist"); 1275 return -ENOENT; 1276 } 1277 } 1278 1279 return 0; 1280 } 1281 1282 static struct tcf_block *__tcf_block_find(struct net *net, struct Qdisc *q, 1283 unsigned long cl, int ifindex, 1284 u32 block_index, 1285 struct netlink_ext_ack *extack) 1286 { 1287 struct tcf_block *block; 1288 1289 if (ifindex == TCM_IFINDEX_MAGIC_BLOCK) { 1290 block = tcf_block_refcnt_get(net, block_index); 1291 if (!block) { 1292 NL_SET_ERR_MSG(extack, "Block of given index was not found"); 1293 return ERR_PTR(-EINVAL); 1294 } 1295 } else { 1296 const struct Qdisc_class_ops *cops = q->ops->cl_ops; 1297 1298 block = cops->tcf_block(q, cl, extack); 1299 if (!block) 1300 return ERR_PTR(-EINVAL); 1301 1302 if (tcf_block_shared(block)) { 1303 NL_SET_ERR_MSG(extack, "This filter block is shared. Please use the block index to manipulate the filters"); 1304 return ERR_PTR(-EOPNOTSUPP); 1305 } 1306 1307 /* Always take reference to block in order to support execution 1308 * of rules update path of cls API without rtnl lock. Caller 1309 * must release block when it is finished using it. 'if' block 1310 * of this conditional obtain reference to block by calling 1311 * tcf_block_refcnt_get(). 1312 */ 1313 refcount_inc(&block->refcnt); 1314 } 1315 1316 return block; 1317 } 1318 1319 static void __tcf_block_put(struct tcf_block *block, struct Qdisc *q, 1320 struct tcf_block_ext_info *ei, bool rtnl_held) 1321 { 1322 if (refcount_dec_and_mutex_lock(&block->refcnt, &block->lock)) { 1323 /* Flushing/putting all chains will cause the block to be 1324 * deallocated when last chain is freed. However, if chain_list 1325 * is empty, block has to be manually deallocated. After block 1326 * reference counter reached 0, it is no longer possible to 1327 * increment it or add new chains to block. 1328 */ 1329 bool free_block = list_empty(&block->chain_list); 1330 1331 mutex_unlock(&block->lock); 1332 if (tcf_block_shared(block)) 1333 tcf_block_remove(block, block->net); 1334 1335 if (q) 1336 tcf_block_offload_unbind(block, q, ei); 1337 1338 if (free_block) 1339 tcf_block_destroy(block); 1340 else 1341 tcf_block_flush_all_chains(block, rtnl_held); 1342 } else if (q) { 1343 tcf_block_offload_unbind(block, q, ei); 1344 } 1345 } 1346 1347 static void tcf_block_refcnt_put(struct tcf_block *block, bool rtnl_held) 1348 { 1349 __tcf_block_put(block, NULL, NULL, rtnl_held); 1350 } 1351 1352 /* Find tcf block. 1353 * Set q, parent, cl when appropriate. 1354 */ 1355 1356 static struct tcf_block *tcf_block_find(struct net *net, struct Qdisc **q, 1357 u32 *parent, unsigned long *cl, 1358 int ifindex, u32 block_index, 1359 struct netlink_ext_ack *extack) 1360 { 1361 struct tcf_block *block; 1362 int err = 0; 1363 1364 ASSERT_RTNL(); 1365 1366 err = __tcf_qdisc_find(net, q, parent, ifindex, true, extack); 1367 if (err) 1368 goto errout; 1369 1370 err = __tcf_qdisc_cl_find(*q, *parent, cl, ifindex, extack); 1371 if (err) 1372 goto errout_qdisc; 1373 1374 block = __tcf_block_find(net, *q, *cl, ifindex, block_index, extack); 1375 if (IS_ERR(block)) { 1376 err = PTR_ERR(block); 1377 goto errout_qdisc; 1378 } 1379 1380 return block; 1381 1382 errout_qdisc: 1383 if (*q) 1384 qdisc_put(*q); 1385 errout: 1386 *q = NULL; 1387 return ERR_PTR(err); 1388 } 1389 1390 static void tcf_block_release(struct Qdisc *q, struct tcf_block *block, 1391 bool rtnl_held) 1392 { 1393 if (!IS_ERR_OR_NULL(block)) 1394 tcf_block_refcnt_put(block, rtnl_held); 1395 1396 if (q) { 1397 if (rtnl_held) 1398 qdisc_put(q); 1399 else 1400 qdisc_put_unlocked(q); 1401 } 1402 } 1403 1404 struct tcf_block_owner_item { 1405 struct list_head list; 1406 struct Qdisc *q; 1407 enum flow_block_binder_type binder_type; 1408 }; 1409 1410 static void 1411 tcf_block_owner_netif_keep_dst(struct tcf_block *block, 1412 struct Qdisc *q, 1413 enum flow_block_binder_type binder_type) 1414 { 1415 if (block->keep_dst && 1416 binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS && 1417 binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS) 1418 netif_keep_dst(qdisc_dev(q)); 1419 } 1420 1421 void tcf_block_netif_keep_dst(struct tcf_block *block) 1422 { 1423 struct tcf_block_owner_item *item; 1424 1425 block->keep_dst = true; 1426 list_for_each_entry(item, &block->owner_list, list) 1427 tcf_block_owner_netif_keep_dst(block, item->q, 1428 item->binder_type); 1429 } 1430 EXPORT_SYMBOL(tcf_block_netif_keep_dst); 1431 1432 static int tcf_block_owner_add(struct tcf_block *block, 1433 struct Qdisc *q, 1434 enum flow_block_binder_type binder_type) 1435 { 1436 struct tcf_block_owner_item *item; 1437 1438 item = kmalloc(sizeof(*item), GFP_KERNEL); 1439 if (!item) 1440 return -ENOMEM; 1441 item->q = q; 1442 item->binder_type = binder_type; 1443 list_add(&item->list, &block->owner_list); 1444 return 0; 1445 } 1446 1447 static void tcf_block_owner_del(struct tcf_block *block, 1448 struct Qdisc *q, 1449 enum flow_block_binder_type binder_type) 1450 { 1451 struct tcf_block_owner_item *item; 1452 1453 list_for_each_entry(item, &block->owner_list, list) { 1454 if (item->q == q && item->binder_type == binder_type) { 1455 list_del(&item->list); 1456 kfree(item); 1457 return; 1458 } 1459 } 1460 WARN_ON(1); 1461 } 1462 1463 static bool tcf_block_tracks_dev(struct tcf_block *block, 1464 struct tcf_block_ext_info *ei) 1465 { 1466 return tcf_block_shared(block) && 1467 (ei->binder_type == FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS || 1468 ei->binder_type == FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS); 1469 } 1470 1471 int tcf_block_get_ext(struct tcf_block **p_block, struct Qdisc *q, 1472 struct tcf_block_ext_info *ei, 1473 struct netlink_ext_ack *extack) 1474 { 1475 struct net_device *dev = qdisc_dev(q); 1476 struct net *net = qdisc_net(q); 1477 struct tcf_block *block = NULL; 1478 int err; 1479 1480 if (ei->block_index) 1481 /* block_index not 0 means the shared block is requested */ 1482 block = tcf_block_refcnt_get(net, ei->block_index); 1483 1484 if (!block) { 1485 block = tcf_block_create(net, q, ei->block_index, extack); 1486 if (IS_ERR(block)) 1487 return PTR_ERR(block); 1488 if (tcf_block_shared(block)) { 1489 err = tcf_block_insert(block, net, extack); 1490 if (err) 1491 goto err_block_insert; 1492 } 1493 } 1494 1495 err = tcf_block_owner_add(block, q, ei->binder_type); 1496 if (err) 1497 goto err_block_owner_add; 1498 1499 tcf_block_owner_netif_keep_dst(block, q, ei->binder_type); 1500 1501 err = tcf_chain0_head_change_cb_add(block, ei, extack); 1502 if (err) 1503 goto err_chain0_head_change_cb_add; 1504 1505 err = tcf_block_offload_bind(block, q, ei, extack); 1506 if (err) 1507 goto err_block_offload_bind; 1508 1509 if (tcf_block_tracks_dev(block, ei)) { 1510 err = xa_insert(&block->ports, dev->ifindex, dev, GFP_KERNEL); 1511 if (err) { 1512 NL_SET_ERR_MSG(extack, "block dev insert failed"); 1513 goto err_dev_insert; 1514 } 1515 } 1516 1517 *p_block = block; 1518 return 0; 1519 1520 err_dev_insert: 1521 tcf_block_offload_unbind(block, q, ei); 1522 err_block_offload_bind: 1523 tcf_chain0_head_change_cb_del(block, ei); 1524 err_chain0_head_change_cb_add: 1525 tcf_block_owner_del(block, q, ei->binder_type); 1526 err_block_owner_add: 1527 err_block_insert: 1528 tcf_block_refcnt_put(block, true); 1529 return err; 1530 } 1531 EXPORT_SYMBOL(tcf_block_get_ext); 1532 1533 static void tcf_chain_head_change_dflt(struct tcf_proto *tp_head, void *priv) 1534 { 1535 struct tcf_proto __rcu **p_filter_chain = priv; 1536 1537 rcu_assign_pointer(*p_filter_chain, tp_head); 1538 } 1539 1540 int tcf_block_get(struct tcf_block **p_block, 1541 struct tcf_proto __rcu **p_filter_chain, struct Qdisc *q, 1542 struct netlink_ext_ack *extack) 1543 { 1544 struct tcf_block_ext_info ei = { 1545 .chain_head_change = tcf_chain_head_change_dflt, 1546 .chain_head_change_priv = p_filter_chain, 1547 }; 1548 1549 WARN_ON(!p_filter_chain); 1550 return tcf_block_get_ext(p_block, q, &ei, extack); 1551 } 1552 EXPORT_SYMBOL(tcf_block_get); 1553 1554 /* XXX: Standalone actions are not allowed to jump to any chain, and bound 1555 * actions should be all removed after flushing. 1556 */ 1557 void tcf_block_put_ext(struct tcf_block *block, struct Qdisc *q, 1558 struct tcf_block_ext_info *ei) 1559 { 1560 struct net_device *dev = qdisc_dev(q); 1561 1562 if (!block) 1563 return; 1564 if (tcf_block_tracks_dev(block, ei)) 1565 xa_erase(&block->ports, dev->ifindex); 1566 tcf_chain0_head_change_cb_del(block, ei); 1567 tcf_block_owner_del(block, q, ei->binder_type); 1568 1569 __tcf_block_put(block, q, ei, true); 1570 } 1571 EXPORT_SYMBOL(tcf_block_put_ext); 1572 1573 void tcf_block_put(struct tcf_block *block) 1574 { 1575 struct tcf_block_ext_info ei = {0, }; 1576 1577 if (!block) 1578 return; 1579 tcf_block_put_ext(block, block->q, &ei); 1580 } 1581 1582 EXPORT_SYMBOL(tcf_block_put); 1583 1584 static int 1585 tcf_block_playback_offloads(struct tcf_block *block, flow_setup_cb_t *cb, 1586 void *cb_priv, bool add, bool offload_in_use, 1587 struct netlink_ext_ack *extack) 1588 { 1589 struct tcf_chain *chain, *chain_prev; 1590 struct tcf_proto *tp, *tp_prev; 1591 int err; 1592 1593 lockdep_assert_held(&block->cb_lock); 1594 1595 for (chain = __tcf_get_next_chain(block, NULL); 1596 chain; 1597 chain_prev = chain, 1598 chain = __tcf_get_next_chain(block, chain), 1599 tcf_chain_put(chain_prev)) { 1600 if (chain->tmplt_ops && add) 1601 chain->tmplt_ops->tmplt_reoffload(chain, true, cb, 1602 cb_priv); 1603 for (tp = __tcf_get_next_proto(chain, NULL); tp; 1604 tp_prev = tp, 1605 tp = __tcf_get_next_proto(chain, tp), 1606 tcf_proto_put(tp_prev, true, NULL)) { 1607 if (tp->ops->reoffload) { 1608 err = tp->ops->reoffload(tp, add, cb, cb_priv, 1609 extack); 1610 if (err && add) 1611 goto err_playback_remove; 1612 } else if (add && offload_in_use) { 1613 err = -EOPNOTSUPP; 1614 NL_SET_ERR_MSG(extack, "Filter HW offload failed - classifier without re-offloading support"); 1615 goto err_playback_remove; 1616 } 1617 } 1618 if (chain->tmplt_ops && !add) 1619 chain->tmplt_ops->tmplt_reoffload(chain, false, cb, 1620 cb_priv); 1621 } 1622 1623 return 0; 1624 1625 err_playback_remove: 1626 tcf_proto_put(tp, true, NULL); 1627 tcf_chain_put(chain); 1628 tcf_block_playback_offloads(block, cb, cb_priv, false, offload_in_use, 1629 extack); 1630 return err; 1631 } 1632 1633 static int tcf_block_bind(struct tcf_block *block, 1634 struct flow_block_offload *bo) 1635 { 1636 struct flow_block_cb *block_cb, *next; 1637 int err, i = 0; 1638 1639 lockdep_assert_held(&block->cb_lock); 1640 1641 list_for_each_entry(block_cb, &bo->cb_list, list) { 1642 err = tcf_block_playback_offloads(block, block_cb->cb, 1643 block_cb->cb_priv, true, 1644 tcf_block_offload_in_use(block), 1645 bo->extack); 1646 if (err) 1647 goto err_unroll; 1648 if (!bo->unlocked_driver_cb) 1649 block->lockeddevcnt++; 1650 1651 i++; 1652 } 1653 list_splice(&bo->cb_list, &block->flow_block.cb_list); 1654 1655 return 0; 1656 1657 err_unroll: 1658 list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) { 1659 list_del(&block_cb->driver_list); 1660 if (i-- > 0) { 1661 list_del(&block_cb->list); 1662 tcf_block_playback_offloads(block, block_cb->cb, 1663 block_cb->cb_priv, false, 1664 tcf_block_offload_in_use(block), 1665 NULL); 1666 if (!bo->unlocked_driver_cb) 1667 block->lockeddevcnt--; 1668 } 1669 flow_block_cb_free(block_cb); 1670 } 1671 1672 return err; 1673 } 1674 1675 static void tcf_block_unbind(struct tcf_block *block, 1676 struct flow_block_offload *bo) 1677 { 1678 struct flow_block_cb *block_cb, *next; 1679 1680 lockdep_assert_held(&block->cb_lock); 1681 1682 list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) { 1683 tcf_block_playback_offloads(block, block_cb->cb, 1684 block_cb->cb_priv, false, 1685 tcf_block_offload_in_use(block), 1686 NULL); 1687 list_del(&block_cb->list); 1688 flow_block_cb_free(block_cb); 1689 if (!bo->unlocked_driver_cb) 1690 block->lockeddevcnt--; 1691 } 1692 } 1693 1694 static int tcf_block_setup(struct tcf_block *block, 1695 struct flow_block_offload *bo) 1696 { 1697 int err; 1698 1699 switch (bo->command) { 1700 case FLOW_BLOCK_BIND: 1701 err = tcf_block_bind(block, bo); 1702 break; 1703 case FLOW_BLOCK_UNBIND: 1704 err = 0; 1705 tcf_block_unbind(block, bo); 1706 break; 1707 default: 1708 WARN_ON_ONCE(1); 1709 err = -EOPNOTSUPP; 1710 } 1711 1712 return err; 1713 } 1714 1715 /* Main classifier routine: scans classifier chain attached 1716 * to this qdisc, (optionally) tests for protocol and asks 1717 * specific classifiers. 1718 */ 1719 static inline int __tcf_classify(struct sk_buff *skb, 1720 const struct tcf_proto *tp, 1721 const struct tcf_proto *orig_tp, 1722 struct tcf_result *res, 1723 bool compat_mode, 1724 struct tcf_exts_miss_cookie_node *n, 1725 int act_index, 1726 u32 *last_executed_chain) 1727 { 1728 #ifdef CONFIG_NET_CLS_ACT 1729 const int max_reclassify_loop = 16; 1730 const struct tcf_proto *first_tp; 1731 int limit = 0; 1732 1733 reclassify: 1734 #endif 1735 for (; tp; tp = rcu_dereference_bh(tp->next)) { 1736 __be16 protocol = skb_protocol(skb, false); 1737 int err = 0; 1738 1739 if (n) { 1740 struct tcf_exts *exts; 1741 1742 if (n->tp_prio != tp->prio) 1743 continue; 1744 1745 /* We re-lookup the tp and chain based on index instead 1746 * of having hard refs and locks to them, so do a sanity 1747 * check if any of tp,chain,exts was replaced by the 1748 * time we got here with a cookie from hardware. 1749 */ 1750 if (unlikely(n->tp != tp || n->tp->chain != n->chain || 1751 !tp->ops->get_exts)) { 1752 tcf_set_drop_reason(skb, 1753 SKB_DROP_REASON_TC_COOKIE_ERROR); 1754 return TC_ACT_SHOT; 1755 } 1756 1757 exts = tp->ops->get_exts(tp, n->handle); 1758 if (unlikely(!exts || n->exts != exts)) { 1759 tcf_set_drop_reason(skb, 1760 SKB_DROP_REASON_TC_COOKIE_ERROR); 1761 return TC_ACT_SHOT; 1762 } 1763 1764 n = NULL; 1765 err = tcf_exts_exec_ex(skb, exts, act_index, res); 1766 } else { 1767 if (tp->protocol != protocol && 1768 tp->protocol != htons(ETH_P_ALL)) 1769 continue; 1770 1771 err = tc_classify(skb, tp, res); 1772 } 1773 #ifdef CONFIG_NET_CLS_ACT 1774 if (unlikely(err == TC_ACT_RECLASSIFY && !compat_mode)) { 1775 first_tp = orig_tp; 1776 *last_executed_chain = first_tp->chain->index; 1777 goto reset; 1778 } else if (unlikely(TC_ACT_EXT_CMP(err, TC_ACT_GOTO_CHAIN))) { 1779 first_tp = res->goto_tp; 1780 *last_executed_chain = err & TC_ACT_EXT_VAL_MASK; 1781 goto reset; 1782 } 1783 #endif 1784 if (err >= 0) 1785 return err; 1786 } 1787 1788 if (unlikely(n)) { 1789 tcf_set_drop_reason(skb, 1790 SKB_DROP_REASON_TC_COOKIE_ERROR); 1791 return TC_ACT_SHOT; 1792 } 1793 1794 return TC_ACT_UNSPEC; /* signal: continue lookup */ 1795 #ifdef CONFIG_NET_CLS_ACT 1796 reset: 1797 if (unlikely(limit++ >= max_reclassify_loop)) { 1798 net_notice_ratelimited("%u: reclassify loop, rule prio %u, protocol %02x\n", 1799 tp->chain->block->index, 1800 tp->prio & 0xffff, 1801 ntohs(tp->protocol)); 1802 tcf_set_drop_reason(skb, 1803 SKB_DROP_REASON_TC_RECLASSIFY_LOOP); 1804 return TC_ACT_SHOT; 1805 } 1806 1807 tp = first_tp; 1808 goto reclassify; 1809 #endif 1810 } 1811 1812 int tcf_classify(struct sk_buff *skb, 1813 const struct tcf_block *block, 1814 const struct tcf_proto *tp, 1815 struct tcf_result *res, bool compat_mode) 1816 { 1817 #if !IS_ENABLED(CONFIG_NET_TC_SKB_EXT) 1818 u32 last_executed_chain = 0; 1819 1820 return __tcf_classify(skb, tp, tp, res, compat_mode, NULL, 0, 1821 &last_executed_chain); 1822 #else 1823 u32 last_executed_chain = tp ? tp->chain->index : 0; 1824 struct tcf_exts_miss_cookie_node *n = NULL; 1825 const struct tcf_proto *orig_tp = tp; 1826 struct tc_skb_ext *ext; 1827 int act_index = 0; 1828 int ret; 1829 1830 if (block) { 1831 ext = skb_ext_find(skb, TC_SKB_EXT); 1832 1833 if (ext && (ext->chain || ext->act_miss)) { 1834 struct tcf_chain *fchain; 1835 u32 chain; 1836 1837 if (ext->act_miss) { 1838 n = tcf_exts_miss_cookie_lookup(ext->act_miss_cookie, 1839 &act_index); 1840 if (!n) { 1841 tcf_set_drop_reason(skb, 1842 SKB_DROP_REASON_TC_COOKIE_ERROR); 1843 return TC_ACT_SHOT; 1844 } 1845 1846 chain = n->chain_index; 1847 } else { 1848 chain = ext->chain; 1849 } 1850 1851 fchain = tcf_chain_lookup_rcu(block, chain); 1852 if (!fchain) { 1853 tcf_set_drop_reason(skb, 1854 SKB_DROP_REASON_TC_CHAIN_NOTFOUND); 1855 1856 return TC_ACT_SHOT; 1857 } 1858 1859 /* Consume, so cloned/redirect skbs won't inherit ext */ 1860 skb_ext_del(skb, TC_SKB_EXT); 1861 1862 tp = rcu_dereference_bh(fchain->filter_chain); 1863 last_executed_chain = fchain->index; 1864 } 1865 } 1866 1867 ret = __tcf_classify(skb, tp, orig_tp, res, compat_mode, n, act_index, 1868 &last_executed_chain); 1869 1870 if (tc_skb_ext_tc_enabled()) { 1871 /* If we missed on some chain */ 1872 if (ret == TC_ACT_UNSPEC && last_executed_chain) { 1873 struct tc_skb_cb *cb = tc_skb_cb(skb); 1874 1875 ext = tc_skb_ext_alloc(skb); 1876 if (WARN_ON_ONCE(!ext)) { 1877 tcf_set_drop_reason(skb, SKB_DROP_REASON_NOMEM); 1878 return TC_ACT_SHOT; 1879 } 1880 ext->chain = last_executed_chain; 1881 ext->mru = cb->mru; 1882 ext->post_ct = cb->post_ct; 1883 ext->post_ct_snat = cb->post_ct_snat; 1884 ext->post_ct_dnat = cb->post_ct_dnat; 1885 ext->zone = cb->zone; 1886 } 1887 } 1888 1889 return ret; 1890 #endif 1891 } 1892 EXPORT_SYMBOL(tcf_classify); 1893 1894 struct tcf_chain_info { 1895 struct tcf_proto __rcu **pprev; 1896 struct tcf_proto __rcu *next; 1897 }; 1898 1899 static struct tcf_proto *tcf_chain_tp_prev(struct tcf_chain *chain, 1900 struct tcf_chain_info *chain_info) 1901 { 1902 return tcf_chain_dereference(*chain_info->pprev, chain); 1903 } 1904 1905 static int tcf_chain_tp_insert(struct tcf_chain *chain, 1906 struct tcf_chain_info *chain_info, 1907 struct tcf_proto *tp) 1908 { 1909 if (chain->flushing) 1910 return -EAGAIN; 1911 1912 RCU_INIT_POINTER(tp->next, tcf_chain_tp_prev(chain, chain_info)); 1913 if (*chain_info->pprev == chain->filter_chain) 1914 tcf_chain0_head_change(chain, tp); 1915 tcf_proto_get(tp); 1916 rcu_assign_pointer(*chain_info->pprev, tp); 1917 1918 return 0; 1919 } 1920 1921 static void tcf_chain_tp_remove(struct tcf_chain *chain, 1922 struct tcf_chain_info *chain_info, 1923 struct tcf_proto *tp) 1924 { 1925 struct tcf_proto *next = tcf_chain_dereference(chain_info->next, chain); 1926 1927 tcf_proto_mark_delete(tp); 1928 if (tp == chain->filter_chain) 1929 tcf_chain0_head_change(chain, next); 1930 RCU_INIT_POINTER(*chain_info->pprev, next); 1931 } 1932 1933 static struct tcf_proto *tcf_chain_tp_find(struct tcf_chain *chain, 1934 struct tcf_chain_info *chain_info, 1935 u32 protocol, u32 prio, 1936 bool prio_allocate, 1937 struct netlink_ext_ack *extack); 1938 1939 /* Try to insert new proto. 1940 * If proto with specified priority already exists, free new proto 1941 * and return existing one. 1942 */ 1943 1944 static struct tcf_proto *tcf_chain_tp_insert_unique(struct tcf_chain *chain, 1945 struct tcf_proto *tp_new, 1946 u32 protocol, u32 prio, 1947 bool rtnl_held) 1948 { 1949 struct tcf_chain_info chain_info; 1950 struct tcf_proto *tp; 1951 int err = 0; 1952 1953 mutex_lock(&chain->filter_chain_lock); 1954 1955 if (tcf_proto_exists_destroying(chain, tp_new)) { 1956 mutex_unlock(&chain->filter_chain_lock); 1957 tcf_proto_destroy(tp_new, rtnl_held, false, NULL); 1958 return ERR_PTR(-EAGAIN); 1959 } 1960 1961 tp = tcf_chain_tp_find(chain, &chain_info, protocol, prio, false, NULL); 1962 if (!tp) 1963 err = tcf_chain_tp_insert(chain, &chain_info, tp_new); 1964 mutex_unlock(&chain->filter_chain_lock); 1965 1966 if (tp) { 1967 tcf_proto_destroy(tp_new, rtnl_held, false, NULL); 1968 tp_new = tp; 1969 } else if (err) { 1970 tcf_proto_destroy(tp_new, rtnl_held, false, NULL); 1971 tp_new = ERR_PTR(err); 1972 } 1973 1974 return tp_new; 1975 } 1976 1977 static void tcf_chain_tp_delete_empty(struct tcf_chain *chain, 1978 struct tcf_proto *tp, bool rtnl_held, 1979 struct netlink_ext_ack *extack) 1980 { 1981 struct tcf_chain_info chain_info; 1982 struct tcf_proto *tp_iter; 1983 struct tcf_proto **pprev; 1984 struct tcf_proto *next; 1985 1986 mutex_lock(&chain->filter_chain_lock); 1987 1988 /* Atomically find and remove tp from chain. */ 1989 for (pprev = &chain->filter_chain; 1990 (tp_iter = tcf_chain_dereference(*pprev, chain)); 1991 pprev = &tp_iter->next) { 1992 if (tp_iter == tp) { 1993 chain_info.pprev = pprev; 1994 chain_info.next = tp_iter->next; 1995 WARN_ON(tp_iter->deleting); 1996 break; 1997 } 1998 } 1999 /* Verify that tp still exists and no new filters were inserted 2000 * concurrently. 2001 * Mark tp for deletion if it is empty. 2002 */ 2003 if (!tp_iter || !tcf_proto_check_delete(tp)) { 2004 mutex_unlock(&chain->filter_chain_lock); 2005 return; 2006 } 2007 2008 tcf_proto_signal_destroying(chain, tp); 2009 next = tcf_chain_dereference(chain_info.next, chain); 2010 if (tp == chain->filter_chain) 2011 tcf_chain0_head_change(chain, next); 2012 RCU_INIT_POINTER(*chain_info.pprev, next); 2013 mutex_unlock(&chain->filter_chain_lock); 2014 2015 tcf_proto_put(tp, rtnl_held, extack); 2016 } 2017 2018 static struct tcf_proto *tcf_chain_tp_find(struct tcf_chain *chain, 2019 struct tcf_chain_info *chain_info, 2020 u32 protocol, u32 prio, 2021 bool prio_allocate, 2022 struct netlink_ext_ack *extack) 2023 { 2024 struct tcf_proto **pprev; 2025 struct tcf_proto *tp; 2026 2027 /* Check the chain for existence of proto-tcf with this priority */ 2028 for (pprev = &chain->filter_chain; 2029 (tp = tcf_chain_dereference(*pprev, chain)); 2030 pprev = &tp->next) { 2031 if (tp->prio >= prio) { 2032 if (tp->prio == prio) { 2033 if (prio_allocate) { 2034 NL_SET_ERR_MSG(extack, "Lowest ID from auto-alloc range already in use"); 2035 return ERR_PTR(-ENOSPC); 2036 } 2037 if (tp->protocol != protocol && protocol) { 2038 NL_SET_ERR_MSG(extack, "Protocol mismatch for filter with specified priority"); 2039 return ERR_PTR(-EINVAL); 2040 } 2041 } else { 2042 tp = NULL; 2043 } 2044 break; 2045 } 2046 } 2047 chain_info->pprev = pprev; 2048 if (tp) { 2049 chain_info->next = tp->next; 2050 tcf_proto_get(tp); 2051 } else { 2052 chain_info->next = NULL; 2053 } 2054 return tp; 2055 } 2056 2057 static int tcf_fill_node(struct net *net, struct sk_buff *skb, 2058 struct tcf_proto *tp, struct tcf_block *block, 2059 struct Qdisc *q, u32 parent, void *fh, 2060 u32 portid, u32 seq, u16 flags, int event, 2061 bool terse_dump, bool rtnl_held, 2062 struct netlink_ext_ack *extack) 2063 { 2064 struct tcmsg *tcm; 2065 struct nlmsghdr *nlh; 2066 unsigned char *b = skb_tail_pointer(skb); 2067 2068 nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags); 2069 if (!nlh) 2070 goto out_nlmsg_trim; 2071 tcm = nlmsg_data(nlh); 2072 tcm->tcm_family = AF_UNSPEC; 2073 tcm->tcm__pad1 = 0; 2074 tcm->tcm__pad2 = 0; 2075 if (q) { 2076 tcm->tcm_ifindex = qdisc_dev(q)->ifindex; 2077 tcm->tcm_parent = parent; 2078 } else { 2079 tcm->tcm_ifindex = TCM_IFINDEX_MAGIC_BLOCK; 2080 tcm->tcm_block_index = block->index; 2081 } 2082 tcm->tcm_info = TC_H_MAKE(tp->prio, tp->protocol); 2083 if (nla_put_string(skb, TCA_KIND, tp->ops->kind)) 2084 goto nla_put_failure; 2085 if (nla_put_u32(skb, TCA_CHAIN, tp->chain->index)) 2086 goto nla_put_failure; 2087 if (!fh) { 2088 tcm->tcm_handle = 0; 2089 } else if (terse_dump) { 2090 if (tp->ops->terse_dump) { 2091 if (tp->ops->terse_dump(net, tp, fh, skb, tcm, 2092 rtnl_held) < 0) 2093 goto nla_put_failure; 2094 } else { 2095 goto cls_op_not_supp; 2096 } 2097 } else { 2098 if (tp->ops->dump && 2099 tp->ops->dump(net, tp, fh, skb, tcm, rtnl_held) < 0) 2100 goto nla_put_failure; 2101 } 2102 2103 if (extack && extack->_msg && 2104 nla_put_string(skb, TCA_EXT_WARN_MSG, extack->_msg)) 2105 goto nla_put_failure; 2106 2107 nlh->nlmsg_len = skb_tail_pointer(skb) - b; 2108 2109 return skb->len; 2110 2111 out_nlmsg_trim: 2112 nla_put_failure: 2113 cls_op_not_supp: 2114 nlmsg_trim(skb, b); 2115 return -1; 2116 } 2117 2118 static int tfilter_notify(struct net *net, struct sk_buff *oskb, 2119 struct nlmsghdr *n, struct tcf_proto *tp, 2120 struct tcf_block *block, struct Qdisc *q, 2121 u32 parent, void *fh, int event, bool unicast, 2122 bool rtnl_held, struct netlink_ext_ack *extack) 2123 { 2124 struct sk_buff *skb; 2125 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0; 2126 int err = 0; 2127 2128 if (!unicast && !rtnl_notify_needed(net, n->nlmsg_flags, RTNLGRP_TC)) 2129 return 0; 2130 2131 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); 2132 if (!skb) 2133 return -ENOBUFS; 2134 2135 if (tcf_fill_node(net, skb, tp, block, q, parent, fh, portid, 2136 n->nlmsg_seq, n->nlmsg_flags, event, 2137 false, rtnl_held, extack) <= 0) { 2138 kfree_skb(skb); 2139 return -EINVAL; 2140 } 2141 2142 if (unicast) 2143 err = rtnl_unicast(skb, net, portid); 2144 else 2145 err = rtnetlink_send(skb, net, portid, RTNLGRP_TC, 2146 n->nlmsg_flags & NLM_F_ECHO); 2147 return err; 2148 } 2149 2150 static int tfilter_del_notify(struct net *net, struct sk_buff *oskb, 2151 struct nlmsghdr *n, struct tcf_proto *tp, 2152 struct tcf_block *block, struct Qdisc *q, 2153 u32 parent, void *fh, bool *last, bool rtnl_held, 2154 struct netlink_ext_ack *extack) 2155 { 2156 struct sk_buff *skb; 2157 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0; 2158 int err; 2159 2160 if (!rtnl_notify_needed(net, n->nlmsg_flags, RTNLGRP_TC)) 2161 return tp->ops->delete(tp, fh, last, rtnl_held, extack); 2162 2163 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); 2164 if (!skb) 2165 return -ENOBUFS; 2166 2167 if (tcf_fill_node(net, skb, tp, block, q, parent, fh, portid, 2168 n->nlmsg_seq, n->nlmsg_flags, RTM_DELTFILTER, 2169 false, rtnl_held, extack) <= 0) { 2170 NL_SET_ERR_MSG(extack, "Failed to build del event notification"); 2171 kfree_skb(skb); 2172 return -EINVAL; 2173 } 2174 2175 err = tp->ops->delete(tp, fh, last, rtnl_held, extack); 2176 if (err) { 2177 kfree_skb(skb); 2178 return err; 2179 } 2180 2181 err = rtnetlink_send(skb, net, portid, RTNLGRP_TC, 2182 n->nlmsg_flags & NLM_F_ECHO); 2183 if (err < 0) 2184 NL_SET_ERR_MSG(extack, "Failed to send filter delete notification"); 2185 2186 return err; 2187 } 2188 2189 static void tfilter_notify_chain(struct net *net, struct sk_buff *oskb, 2190 struct tcf_block *block, struct Qdisc *q, 2191 u32 parent, struct nlmsghdr *n, 2192 struct tcf_chain *chain, int event, 2193 struct netlink_ext_ack *extack) 2194 { 2195 struct tcf_proto *tp; 2196 2197 for (tp = tcf_get_next_proto(chain, NULL); 2198 tp; tp = tcf_get_next_proto(chain, tp)) 2199 tfilter_notify(net, oskb, n, tp, block, q, parent, NULL, 2200 event, false, true, extack); 2201 } 2202 2203 static void tfilter_put(struct tcf_proto *tp, void *fh) 2204 { 2205 if (tp->ops->put && fh) 2206 tp->ops->put(tp, fh); 2207 } 2208 2209 static bool is_qdisc_ingress(__u32 classid) 2210 { 2211 return (TC_H_MIN(classid) == TC_H_MIN(TC_H_MIN_INGRESS)); 2212 } 2213 2214 static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n, 2215 struct netlink_ext_ack *extack) 2216 { 2217 struct net *net = sock_net(skb->sk); 2218 struct nlattr *tca[TCA_MAX + 1]; 2219 char name[IFNAMSIZ]; 2220 struct tcmsg *t; 2221 u32 protocol; 2222 u32 prio; 2223 bool prio_allocate; 2224 u32 parent; 2225 u32 chain_index; 2226 struct Qdisc *q; 2227 struct tcf_chain_info chain_info; 2228 struct tcf_chain *chain; 2229 struct tcf_block *block; 2230 struct tcf_proto *tp; 2231 unsigned long cl; 2232 void *fh; 2233 int err; 2234 int tp_created; 2235 bool rtnl_held = false; 2236 u32 flags; 2237 2238 replay: 2239 tp_created = 0; 2240 2241 err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX, 2242 rtm_tca_policy, extack); 2243 if (err < 0) 2244 return err; 2245 2246 t = nlmsg_data(n); 2247 protocol = TC_H_MIN(t->tcm_info); 2248 prio = TC_H_MAJ(t->tcm_info); 2249 prio_allocate = false; 2250 parent = t->tcm_parent; 2251 tp = NULL; 2252 cl = 0; 2253 block = NULL; 2254 q = NULL; 2255 chain = NULL; 2256 flags = 0; 2257 2258 if (prio == 0) { 2259 /* If no priority is provided by the user, 2260 * we allocate one. 2261 */ 2262 if (n->nlmsg_flags & NLM_F_CREATE) { 2263 prio = TC_H_MAKE(0x80000000U, 0U); 2264 prio_allocate = true; 2265 } else { 2266 NL_SET_ERR_MSG(extack, "Invalid filter command with priority of zero"); 2267 return -ENOENT; 2268 } 2269 } 2270 2271 /* Find head of filter chain. */ 2272 2273 err = __tcf_qdisc_find(net, &q, &parent, t->tcm_ifindex, false, extack); 2274 if (err) 2275 return err; 2276 2277 if (tcf_proto_check_kind(tca[TCA_KIND], name)) { 2278 NL_SET_ERR_MSG(extack, "Specified TC filter name too long"); 2279 err = -EINVAL; 2280 goto errout; 2281 } 2282 2283 /* Take rtnl mutex if rtnl_held was set to true on previous iteration, 2284 * block is shared (no qdisc found), qdisc is not unlocked, classifier 2285 * type is not specified, classifier is not unlocked. 2286 */ 2287 if (rtnl_held || 2288 (q && !(q->ops->cl_ops->flags & QDISC_CLASS_OPS_DOIT_UNLOCKED)) || 2289 !tcf_proto_is_unlocked(name)) { 2290 rtnl_held = true; 2291 rtnl_lock(); 2292 } 2293 2294 err = __tcf_qdisc_cl_find(q, parent, &cl, t->tcm_ifindex, extack); 2295 if (err) 2296 goto errout; 2297 2298 block = __tcf_block_find(net, q, cl, t->tcm_ifindex, t->tcm_block_index, 2299 extack); 2300 if (IS_ERR(block)) { 2301 err = PTR_ERR(block); 2302 goto errout; 2303 } 2304 block->classid = parent; 2305 2306 chain_index = nla_get_u32_default(tca[TCA_CHAIN], 0); 2307 if (chain_index > TC_ACT_EXT_VAL_MASK) { 2308 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit"); 2309 err = -EINVAL; 2310 goto errout; 2311 } 2312 chain = tcf_chain_get(block, chain_index, true); 2313 if (!chain) { 2314 NL_SET_ERR_MSG(extack, "Cannot create specified filter chain"); 2315 err = -ENOMEM; 2316 goto errout; 2317 } 2318 2319 mutex_lock(&chain->filter_chain_lock); 2320 tp = tcf_chain_tp_find(chain, &chain_info, protocol, 2321 prio, prio_allocate, extack); 2322 if (IS_ERR(tp)) { 2323 err = PTR_ERR(tp); 2324 goto errout_locked; 2325 } 2326 2327 if (tp == NULL) { 2328 struct tcf_proto *tp_new = NULL; 2329 2330 if (chain->flushing) { 2331 err = -EAGAIN; 2332 goto errout_locked; 2333 } 2334 2335 /* Proto-tcf does not exist, create new one */ 2336 2337 if (tca[TCA_KIND] == NULL || !protocol) { 2338 NL_SET_ERR_MSG(extack, "Filter kind and protocol must be specified"); 2339 err = -EINVAL; 2340 goto errout_locked; 2341 } 2342 2343 if (!(n->nlmsg_flags & NLM_F_CREATE)) { 2344 NL_SET_ERR_MSG(extack, "Need both RTM_NEWTFILTER and NLM_F_CREATE to create a new filter"); 2345 err = -ENOENT; 2346 goto errout_locked; 2347 } 2348 2349 if (prio_allocate) 2350 prio = tcf_auto_prio(tcf_chain_tp_prev(chain, 2351 &chain_info)); 2352 2353 mutex_unlock(&chain->filter_chain_lock); 2354 tp_new = tcf_proto_create(name, protocol, prio, chain, 2355 rtnl_held, extack); 2356 if (IS_ERR(tp_new)) { 2357 err = PTR_ERR(tp_new); 2358 goto errout_tp; 2359 } 2360 2361 tp_created = 1; 2362 tp = tcf_chain_tp_insert_unique(chain, tp_new, protocol, prio, 2363 rtnl_held); 2364 if (IS_ERR(tp)) { 2365 err = PTR_ERR(tp); 2366 goto errout_tp; 2367 } 2368 } else { 2369 mutex_unlock(&chain->filter_chain_lock); 2370 } 2371 2372 if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) { 2373 NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one"); 2374 err = -EINVAL; 2375 goto errout; 2376 } 2377 2378 fh = tp->ops->get(tp, t->tcm_handle); 2379 2380 if (!fh) { 2381 if (!(n->nlmsg_flags & NLM_F_CREATE)) { 2382 NL_SET_ERR_MSG(extack, "Need both RTM_NEWTFILTER and NLM_F_CREATE to create a new filter"); 2383 err = -ENOENT; 2384 goto errout; 2385 } 2386 } else if (n->nlmsg_flags & NLM_F_EXCL) { 2387 tfilter_put(tp, fh); 2388 NL_SET_ERR_MSG(extack, "Filter already exists"); 2389 err = -EEXIST; 2390 goto errout; 2391 } 2392 2393 if (chain->tmplt_ops && chain->tmplt_ops != tp->ops) { 2394 tfilter_put(tp, fh); 2395 NL_SET_ERR_MSG(extack, "Chain template is set to a different filter kind"); 2396 err = -EINVAL; 2397 goto errout; 2398 } 2399 2400 if (!(n->nlmsg_flags & NLM_F_CREATE)) 2401 flags |= TCA_ACT_FLAGS_REPLACE; 2402 if (!rtnl_held) 2403 flags |= TCA_ACT_FLAGS_NO_RTNL; 2404 if (is_qdisc_ingress(parent)) 2405 flags |= TCA_ACT_FLAGS_AT_INGRESS; 2406 err = tp->ops->change(net, skb, tp, cl, t->tcm_handle, tca, &fh, 2407 flags, extack); 2408 if (err == 0) { 2409 tfilter_notify(net, skb, n, tp, block, q, parent, fh, 2410 RTM_NEWTFILTER, false, rtnl_held, extack); 2411 tfilter_put(tp, fh); 2412 tcf_block_filter_cnt_update(block, &tp->counted, true); 2413 /* q pointer is NULL for shared blocks */ 2414 if (q) 2415 q->flags &= ~TCQ_F_CAN_BYPASS; 2416 } 2417 2418 errout: 2419 if (err && tp_created) 2420 tcf_chain_tp_delete_empty(chain, tp, rtnl_held, NULL); 2421 errout_tp: 2422 if (chain) { 2423 if (tp && !IS_ERR(tp)) 2424 tcf_proto_put(tp, rtnl_held, NULL); 2425 if (!tp_created) 2426 tcf_chain_put(chain); 2427 } 2428 tcf_block_release(q, block, rtnl_held); 2429 2430 if (rtnl_held) 2431 rtnl_unlock(); 2432 2433 if (err == -EAGAIN) { 2434 /* Take rtnl lock in case EAGAIN is caused by concurrent flush 2435 * of target chain. 2436 */ 2437 rtnl_held = true; 2438 /* Replay the request. */ 2439 goto replay; 2440 } 2441 return err; 2442 2443 errout_locked: 2444 mutex_unlock(&chain->filter_chain_lock); 2445 goto errout; 2446 } 2447 2448 static int tc_del_tfilter(struct sk_buff *skb, struct nlmsghdr *n, 2449 struct netlink_ext_ack *extack) 2450 { 2451 struct net *net = sock_net(skb->sk); 2452 struct nlattr *tca[TCA_MAX + 1]; 2453 char name[IFNAMSIZ]; 2454 struct tcmsg *t; 2455 u32 protocol; 2456 u32 prio; 2457 u32 parent; 2458 u32 chain_index; 2459 struct Qdisc *q = NULL; 2460 struct tcf_chain_info chain_info; 2461 struct tcf_chain *chain = NULL; 2462 struct tcf_block *block = NULL; 2463 struct tcf_proto *tp = NULL; 2464 unsigned long cl = 0; 2465 void *fh = NULL; 2466 int err; 2467 bool rtnl_held = false; 2468 2469 err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX, 2470 rtm_tca_policy, extack); 2471 if (err < 0) 2472 return err; 2473 2474 t = nlmsg_data(n); 2475 protocol = TC_H_MIN(t->tcm_info); 2476 prio = TC_H_MAJ(t->tcm_info); 2477 parent = t->tcm_parent; 2478 2479 if (prio == 0 && (protocol || t->tcm_handle || tca[TCA_KIND])) { 2480 NL_SET_ERR_MSG(extack, "Cannot flush filters with protocol, handle or kind set"); 2481 return -ENOENT; 2482 } 2483 2484 /* Find head of filter chain. */ 2485 2486 err = __tcf_qdisc_find(net, &q, &parent, t->tcm_ifindex, false, extack); 2487 if (err) 2488 return err; 2489 2490 if (tcf_proto_check_kind(tca[TCA_KIND], name)) { 2491 NL_SET_ERR_MSG(extack, "Specified TC filter name too long"); 2492 err = -EINVAL; 2493 goto errout; 2494 } 2495 /* Take rtnl mutex if flushing whole chain, block is shared (no qdisc 2496 * found), qdisc is not unlocked, classifier type is not specified, 2497 * classifier is not unlocked. 2498 */ 2499 if (!prio || 2500 (q && !(q->ops->cl_ops->flags & QDISC_CLASS_OPS_DOIT_UNLOCKED)) || 2501 !tcf_proto_is_unlocked(name)) { 2502 rtnl_held = true; 2503 rtnl_lock(); 2504 } 2505 2506 err = __tcf_qdisc_cl_find(q, parent, &cl, t->tcm_ifindex, extack); 2507 if (err) 2508 goto errout; 2509 2510 block = __tcf_block_find(net, q, cl, t->tcm_ifindex, t->tcm_block_index, 2511 extack); 2512 if (IS_ERR(block)) { 2513 err = PTR_ERR(block); 2514 goto errout; 2515 } 2516 2517 chain_index = nla_get_u32_default(tca[TCA_CHAIN], 0); 2518 if (chain_index > TC_ACT_EXT_VAL_MASK) { 2519 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit"); 2520 err = -EINVAL; 2521 goto errout; 2522 } 2523 chain = tcf_chain_get(block, chain_index, false); 2524 if (!chain) { 2525 /* User requested flush on non-existent chain. Nothing to do, 2526 * so just return success. 2527 */ 2528 if (prio == 0) { 2529 err = 0; 2530 goto errout; 2531 } 2532 NL_SET_ERR_MSG(extack, "Cannot find specified filter chain"); 2533 err = -ENOENT; 2534 goto errout; 2535 } 2536 2537 if (prio == 0) { 2538 tfilter_notify_chain(net, skb, block, q, parent, n, 2539 chain, RTM_DELTFILTER, extack); 2540 tcf_chain_flush(chain, rtnl_held); 2541 err = 0; 2542 goto errout; 2543 } 2544 2545 mutex_lock(&chain->filter_chain_lock); 2546 tp = tcf_chain_tp_find(chain, &chain_info, protocol, 2547 prio, false, extack); 2548 if (!tp) { 2549 err = -ENOENT; 2550 NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found"); 2551 goto errout_locked; 2552 } else if (IS_ERR(tp)) { 2553 err = PTR_ERR(tp); 2554 goto errout_locked; 2555 } else if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) { 2556 NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one"); 2557 err = -EINVAL; 2558 goto errout_locked; 2559 } else if (t->tcm_handle == 0) { 2560 tcf_proto_signal_destroying(chain, tp); 2561 tcf_chain_tp_remove(chain, &chain_info, tp); 2562 mutex_unlock(&chain->filter_chain_lock); 2563 2564 tcf_proto_put(tp, rtnl_held, NULL); 2565 tfilter_notify(net, skb, n, tp, block, q, parent, fh, 2566 RTM_DELTFILTER, false, rtnl_held, extack); 2567 err = 0; 2568 goto errout; 2569 } 2570 mutex_unlock(&chain->filter_chain_lock); 2571 2572 fh = tp->ops->get(tp, t->tcm_handle); 2573 2574 if (!fh) { 2575 NL_SET_ERR_MSG(extack, "Specified filter handle not found"); 2576 err = -ENOENT; 2577 } else { 2578 bool last; 2579 2580 err = tfilter_del_notify(net, skb, n, tp, block, q, parent, fh, 2581 &last, rtnl_held, extack); 2582 2583 if (err) 2584 goto errout; 2585 if (last) 2586 tcf_chain_tp_delete_empty(chain, tp, rtnl_held, extack); 2587 } 2588 2589 errout: 2590 if (chain) { 2591 if (tp && !IS_ERR(tp)) 2592 tcf_proto_put(tp, rtnl_held, NULL); 2593 tcf_chain_put(chain); 2594 } 2595 tcf_block_release(q, block, rtnl_held); 2596 2597 if (rtnl_held) 2598 rtnl_unlock(); 2599 2600 return err; 2601 2602 errout_locked: 2603 mutex_unlock(&chain->filter_chain_lock); 2604 goto errout; 2605 } 2606 2607 static int tc_get_tfilter(struct sk_buff *skb, struct nlmsghdr *n, 2608 struct netlink_ext_ack *extack) 2609 { 2610 struct net *net = sock_net(skb->sk); 2611 struct nlattr *tca[TCA_MAX + 1]; 2612 char name[IFNAMSIZ]; 2613 struct tcmsg *t; 2614 u32 protocol; 2615 u32 prio; 2616 u32 parent; 2617 u32 chain_index; 2618 struct Qdisc *q = NULL; 2619 struct tcf_chain_info chain_info; 2620 struct tcf_chain *chain = NULL; 2621 struct tcf_block *block = NULL; 2622 struct tcf_proto *tp = NULL; 2623 unsigned long cl = 0; 2624 void *fh = NULL; 2625 int err; 2626 bool rtnl_held = false; 2627 2628 err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX, 2629 rtm_tca_policy, extack); 2630 if (err < 0) 2631 return err; 2632 2633 t = nlmsg_data(n); 2634 protocol = TC_H_MIN(t->tcm_info); 2635 prio = TC_H_MAJ(t->tcm_info); 2636 parent = t->tcm_parent; 2637 2638 if (prio == 0) { 2639 NL_SET_ERR_MSG(extack, "Invalid filter command with priority of zero"); 2640 return -ENOENT; 2641 } 2642 2643 /* Find head of filter chain. */ 2644 2645 err = __tcf_qdisc_find(net, &q, &parent, t->tcm_ifindex, false, extack); 2646 if (err) 2647 return err; 2648 2649 if (tcf_proto_check_kind(tca[TCA_KIND], name)) { 2650 NL_SET_ERR_MSG(extack, "Specified TC filter name too long"); 2651 err = -EINVAL; 2652 goto errout; 2653 } 2654 /* Take rtnl mutex if block is shared (no qdisc found), qdisc is not 2655 * unlocked, classifier type is not specified, classifier is not 2656 * unlocked. 2657 */ 2658 if ((q && !(q->ops->cl_ops->flags & QDISC_CLASS_OPS_DOIT_UNLOCKED)) || 2659 !tcf_proto_is_unlocked(name)) { 2660 rtnl_held = true; 2661 rtnl_lock(); 2662 } 2663 2664 err = __tcf_qdisc_cl_find(q, parent, &cl, t->tcm_ifindex, extack); 2665 if (err) 2666 goto errout; 2667 2668 block = __tcf_block_find(net, q, cl, t->tcm_ifindex, t->tcm_block_index, 2669 extack); 2670 if (IS_ERR(block)) { 2671 err = PTR_ERR(block); 2672 goto errout; 2673 } 2674 2675 chain_index = nla_get_u32_default(tca[TCA_CHAIN], 0); 2676 if (chain_index > TC_ACT_EXT_VAL_MASK) { 2677 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit"); 2678 err = -EINVAL; 2679 goto errout; 2680 } 2681 chain = tcf_chain_get(block, chain_index, false); 2682 if (!chain) { 2683 NL_SET_ERR_MSG(extack, "Cannot find specified filter chain"); 2684 err = -EINVAL; 2685 goto errout; 2686 } 2687 2688 mutex_lock(&chain->filter_chain_lock); 2689 tp = tcf_chain_tp_find(chain, &chain_info, protocol, 2690 prio, false, extack); 2691 mutex_unlock(&chain->filter_chain_lock); 2692 if (!tp) { 2693 err = -ENOENT; 2694 NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found"); 2695 goto errout; 2696 } else if (IS_ERR(tp)) { 2697 err = PTR_ERR(tp); 2698 goto errout; 2699 } else if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) { 2700 NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one"); 2701 err = -EINVAL; 2702 goto errout; 2703 } 2704 2705 fh = tp->ops->get(tp, t->tcm_handle); 2706 2707 if (!fh) { 2708 NL_SET_ERR_MSG(extack, "Specified filter handle not found"); 2709 err = -ENOENT; 2710 } else { 2711 err = tfilter_notify(net, skb, n, tp, block, q, parent, 2712 fh, RTM_NEWTFILTER, true, rtnl_held, NULL); 2713 if (err < 0) 2714 NL_SET_ERR_MSG(extack, "Failed to send filter notify message"); 2715 } 2716 2717 tfilter_put(tp, fh); 2718 errout: 2719 if (chain) { 2720 if (tp && !IS_ERR(tp)) 2721 tcf_proto_put(tp, rtnl_held, NULL); 2722 tcf_chain_put(chain); 2723 } 2724 tcf_block_release(q, block, rtnl_held); 2725 2726 if (rtnl_held) 2727 rtnl_unlock(); 2728 2729 return err; 2730 } 2731 2732 struct tcf_dump_args { 2733 struct tcf_walker w; 2734 struct sk_buff *skb; 2735 struct netlink_callback *cb; 2736 struct tcf_block *block; 2737 struct Qdisc *q; 2738 u32 parent; 2739 bool terse_dump; 2740 }; 2741 2742 static int tcf_node_dump(struct tcf_proto *tp, void *n, struct tcf_walker *arg) 2743 { 2744 struct tcf_dump_args *a = (void *)arg; 2745 struct net *net = sock_net(a->skb->sk); 2746 2747 return tcf_fill_node(net, a->skb, tp, a->block, a->q, a->parent, 2748 n, NETLINK_CB(a->cb->skb).portid, 2749 a->cb->nlh->nlmsg_seq, NLM_F_MULTI, 2750 RTM_NEWTFILTER, a->terse_dump, true, NULL); 2751 } 2752 2753 static bool tcf_chain_dump(struct tcf_chain *chain, struct Qdisc *q, u32 parent, 2754 struct sk_buff *skb, struct netlink_callback *cb, 2755 long index_start, long *p_index, bool terse) 2756 { 2757 struct net *net = sock_net(skb->sk); 2758 struct tcf_block *block = chain->block; 2759 struct tcmsg *tcm = nlmsg_data(cb->nlh); 2760 struct tcf_proto *tp, *tp_prev; 2761 struct tcf_dump_args arg; 2762 2763 for (tp = __tcf_get_next_proto(chain, NULL); 2764 tp; 2765 tp_prev = tp, 2766 tp = __tcf_get_next_proto(chain, tp), 2767 tcf_proto_put(tp_prev, true, NULL), 2768 (*p_index)++) { 2769 if (*p_index < index_start) 2770 continue; 2771 if (TC_H_MAJ(tcm->tcm_info) && 2772 TC_H_MAJ(tcm->tcm_info) != tp->prio) 2773 continue; 2774 if (TC_H_MIN(tcm->tcm_info) && 2775 TC_H_MIN(tcm->tcm_info) != tp->protocol) 2776 continue; 2777 if (*p_index > index_start) 2778 memset(&cb->args[1], 0, 2779 sizeof(cb->args) - sizeof(cb->args[0])); 2780 if (cb->args[1] == 0) { 2781 if (tcf_fill_node(net, skb, tp, block, q, parent, NULL, 2782 NETLINK_CB(cb->skb).portid, 2783 cb->nlh->nlmsg_seq, NLM_F_MULTI, 2784 RTM_NEWTFILTER, false, true, NULL) <= 0) 2785 goto errout; 2786 cb->args[1] = 1; 2787 } 2788 if (!tp->ops->walk) 2789 continue; 2790 arg.w.fn = tcf_node_dump; 2791 arg.skb = skb; 2792 arg.cb = cb; 2793 arg.block = block; 2794 arg.q = q; 2795 arg.parent = parent; 2796 arg.w.stop = 0; 2797 arg.w.skip = cb->args[1] - 1; 2798 arg.w.count = 0; 2799 arg.w.cookie = cb->args[2]; 2800 arg.terse_dump = terse; 2801 tp->ops->walk(tp, &arg.w, true); 2802 cb->args[2] = arg.w.cookie; 2803 cb->args[1] = arg.w.count + 1; 2804 if (arg.w.stop) 2805 goto errout; 2806 } 2807 return true; 2808 2809 errout: 2810 tcf_proto_put(tp, true, NULL); 2811 return false; 2812 } 2813 2814 static const struct nla_policy tcf_tfilter_dump_policy[TCA_MAX + 1] = { 2815 [TCA_CHAIN] = { .type = NLA_U32 }, 2816 [TCA_DUMP_FLAGS] = NLA_POLICY_BITFIELD32(TCA_DUMP_FLAGS_TERSE), 2817 }; 2818 2819 /* called with RTNL */ 2820 static int tc_dump_tfilter(struct sk_buff *skb, struct netlink_callback *cb) 2821 { 2822 struct tcf_chain *chain, *chain_prev; 2823 struct net *net = sock_net(skb->sk); 2824 struct nlattr *tca[TCA_MAX + 1]; 2825 struct Qdisc *q = NULL; 2826 struct tcf_block *block; 2827 struct tcmsg *tcm = nlmsg_data(cb->nlh); 2828 bool terse_dump = false; 2829 long index_start; 2830 long index; 2831 u32 parent; 2832 int err; 2833 2834 if (nlmsg_len(cb->nlh) < sizeof(*tcm)) 2835 return skb->len; 2836 2837 err = nlmsg_parse_deprecated(cb->nlh, sizeof(*tcm), tca, TCA_MAX, 2838 tcf_tfilter_dump_policy, cb->extack); 2839 if (err) 2840 return err; 2841 2842 if (tca[TCA_DUMP_FLAGS]) { 2843 struct nla_bitfield32 flags = 2844 nla_get_bitfield32(tca[TCA_DUMP_FLAGS]); 2845 2846 terse_dump = flags.value & TCA_DUMP_FLAGS_TERSE; 2847 } 2848 2849 if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) { 2850 block = tcf_block_refcnt_get(net, tcm->tcm_block_index); 2851 if (!block) 2852 goto out; 2853 /* If we work with block index, q is NULL and parent value 2854 * will never be used in the following code. The check 2855 * in tcf_fill_node prevents it. However, compiler does not 2856 * see that far, so set parent to zero to silence the warning 2857 * about parent being uninitialized. 2858 */ 2859 parent = 0; 2860 } else { 2861 const struct Qdisc_class_ops *cops; 2862 struct net_device *dev; 2863 unsigned long cl = 0; 2864 2865 dev = __dev_get_by_index(net, tcm->tcm_ifindex); 2866 if (!dev) 2867 return skb->len; 2868 2869 parent = tcm->tcm_parent; 2870 if (!parent) 2871 q = rtnl_dereference(dev->qdisc); 2872 else 2873 q = qdisc_lookup(dev, TC_H_MAJ(tcm->tcm_parent)); 2874 if (!q) 2875 goto out; 2876 cops = q->ops->cl_ops; 2877 if (!cops) 2878 goto out; 2879 if (!cops->tcf_block) 2880 goto out; 2881 if (TC_H_MIN(tcm->tcm_parent)) { 2882 cl = cops->find(q, tcm->tcm_parent); 2883 if (cl == 0) 2884 goto out; 2885 } 2886 block = cops->tcf_block(q, cl, NULL); 2887 if (!block) 2888 goto out; 2889 parent = block->classid; 2890 if (tcf_block_shared(block)) 2891 q = NULL; 2892 } 2893 2894 index_start = cb->args[0]; 2895 index = 0; 2896 2897 for (chain = __tcf_get_next_chain(block, NULL); 2898 chain; 2899 chain_prev = chain, 2900 chain = __tcf_get_next_chain(block, chain), 2901 tcf_chain_put(chain_prev)) { 2902 if (tca[TCA_CHAIN] && 2903 nla_get_u32(tca[TCA_CHAIN]) != chain->index) 2904 continue; 2905 if (!tcf_chain_dump(chain, q, parent, skb, cb, 2906 index_start, &index, terse_dump)) { 2907 tcf_chain_put(chain); 2908 err = -EMSGSIZE; 2909 break; 2910 } 2911 } 2912 2913 if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) 2914 tcf_block_refcnt_put(block, true); 2915 cb->args[0] = index; 2916 2917 out: 2918 /* If we did no progress, the error (EMSGSIZE) is real */ 2919 if (skb->len == 0 && err) 2920 return err; 2921 return skb->len; 2922 } 2923 2924 static int tc_chain_fill_node(const struct tcf_proto_ops *tmplt_ops, 2925 void *tmplt_priv, u32 chain_index, 2926 struct net *net, struct sk_buff *skb, 2927 struct tcf_block *block, 2928 u32 portid, u32 seq, u16 flags, int event, 2929 struct netlink_ext_ack *extack) 2930 { 2931 unsigned char *b = skb_tail_pointer(skb); 2932 const struct tcf_proto_ops *ops; 2933 struct nlmsghdr *nlh; 2934 struct tcmsg *tcm; 2935 void *priv; 2936 2937 ops = tmplt_ops; 2938 priv = tmplt_priv; 2939 2940 nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags); 2941 if (!nlh) 2942 goto out_nlmsg_trim; 2943 tcm = nlmsg_data(nlh); 2944 tcm->tcm_family = AF_UNSPEC; 2945 tcm->tcm__pad1 = 0; 2946 tcm->tcm__pad2 = 0; 2947 tcm->tcm_handle = 0; 2948 if (block->q) { 2949 tcm->tcm_ifindex = qdisc_dev(block->q)->ifindex; 2950 tcm->tcm_parent = block->q->handle; 2951 } else { 2952 tcm->tcm_ifindex = TCM_IFINDEX_MAGIC_BLOCK; 2953 tcm->tcm_block_index = block->index; 2954 } 2955 2956 if (nla_put_u32(skb, TCA_CHAIN, chain_index)) 2957 goto nla_put_failure; 2958 2959 if (ops) { 2960 if (nla_put_string(skb, TCA_KIND, ops->kind)) 2961 goto nla_put_failure; 2962 if (ops->tmplt_dump(skb, net, priv) < 0) 2963 goto nla_put_failure; 2964 } 2965 2966 if (extack && extack->_msg && 2967 nla_put_string(skb, TCA_EXT_WARN_MSG, extack->_msg)) 2968 goto out_nlmsg_trim; 2969 2970 nlh->nlmsg_len = skb_tail_pointer(skb) - b; 2971 2972 return skb->len; 2973 2974 out_nlmsg_trim: 2975 nla_put_failure: 2976 nlmsg_trim(skb, b); 2977 return -EMSGSIZE; 2978 } 2979 2980 static int tc_chain_notify(struct tcf_chain *chain, struct sk_buff *oskb, 2981 u32 seq, u16 flags, int event, bool unicast, 2982 struct netlink_ext_ack *extack) 2983 { 2984 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0; 2985 struct tcf_block *block = chain->block; 2986 struct net *net = block->net; 2987 struct sk_buff *skb; 2988 int err = 0; 2989 2990 if (!unicast && !rtnl_notify_needed(net, flags, RTNLGRP_TC)) 2991 return 0; 2992 2993 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); 2994 if (!skb) 2995 return -ENOBUFS; 2996 2997 if (tc_chain_fill_node(chain->tmplt_ops, chain->tmplt_priv, 2998 chain->index, net, skb, block, portid, 2999 seq, flags, event, extack) <= 0) { 3000 kfree_skb(skb); 3001 return -EINVAL; 3002 } 3003 3004 if (unicast) 3005 err = rtnl_unicast(skb, net, portid); 3006 else 3007 err = rtnetlink_send(skb, net, portid, RTNLGRP_TC, 3008 flags & NLM_F_ECHO); 3009 3010 return err; 3011 } 3012 3013 static int tc_chain_notify_delete(const struct tcf_proto_ops *tmplt_ops, 3014 void *tmplt_priv, u32 chain_index, 3015 struct tcf_block *block, struct sk_buff *oskb, 3016 u32 seq, u16 flags) 3017 { 3018 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0; 3019 struct net *net = block->net; 3020 struct sk_buff *skb; 3021 3022 if (!rtnl_notify_needed(net, flags, RTNLGRP_TC)) 3023 return 0; 3024 3025 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); 3026 if (!skb) 3027 return -ENOBUFS; 3028 3029 if (tc_chain_fill_node(tmplt_ops, tmplt_priv, chain_index, net, skb, 3030 block, portid, seq, flags, RTM_DELCHAIN, NULL) <= 0) { 3031 kfree_skb(skb); 3032 return -EINVAL; 3033 } 3034 3035 return rtnetlink_send(skb, net, portid, RTNLGRP_TC, flags & NLM_F_ECHO); 3036 } 3037 3038 static int tc_chain_tmplt_add(struct tcf_chain *chain, struct net *net, 3039 struct nlattr **tca, 3040 struct netlink_ext_ack *extack) 3041 { 3042 const struct tcf_proto_ops *ops; 3043 char name[IFNAMSIZ]; 3044 void *tmplt_priv; 3045 3046 /* If kind is not set, user did not specify template. */ 3047 if (!tca[TCA_KIND]) 3048 return 0; 3049 3050 if (tcf_proto_check_kind(tca[TCA_KIND], name)) { 3051 NL_SET_ERR_MSG(extack, "Specified TC chain template name too long"); 3052 return -EINVAL; 3053 } 3054 3055 ops = tcf_proto_lookup_ops(name, true, extack); 3056 if (IS_ERR(ops)) 3057 return PTR_ERR(ops); 3058 if (!ops->tmplt_create || !ops->tmplt_destroy || !ops->tmplt_dump || 3059 !ops->tmplt_reoffload) { 3060 NL_SET_ERR_MSG(extack, "Chain templates are not supported with specified classifier"); 3061 module_put(ops->owner); 3062 return -EOPNOTSUPP; 3063 } 3064 3065 tmplt_priv = ops->tmplt_create(net, chain, tca, extack); 3066 if (IS_ERR(tmplt_priv)) { 3067 module_put(ops->owner); 3068 return PTR_ERR(tmplt_priv); 3069 } 3070 chain->tmplt_ops = ops; 3071 chain->tmplt_priv = tmplt_priv; 3072 return 0; 3073 } 3074 3075 static void tc_chain_tmplt_del(const struct tcf_proto_ops *tmplt_ops, 3076 void *tmplt_priv) 3077 { 3078 /* If template ops are set, no work to do for us. */ 3079 if (!tmplt_ops) 3080 return; 3081 3082 tmplt_ops->tmplt_destroy(tmplt_priv); 3083 module_put(tmplt_ops->owner); 3084 } 3085 3086 /* Add/delete/get a chain */ 3087 3088 static int tc_ctl_chain(struct sk_buff *skb, struct nlmsghdr *n, 3089 struct netlink_ext_ack *extack) 3090 { 3091 struct net *net = sock_net(skb->sk); 3092 struct nlattr *tca[TCA_MAX + 1]; 3093 struct tcmsg *t; 3094 u32 parent; 3095 u32 chain_index; 3096 struct Qdisc *q; 3097 struct tcf_chain *chain; 3098 struct tcf_block *block; 3099 unsigned long cl; 3100 int err; 3101 3102 replay: 3103 q = NULL; 3104 err = nlmsg_parse_deprecated(n, sizeof(*t), tca, TCA_MAX, 3105 rtm_tca_policy, extack); 3106 if (err < 0) 3107 return err; 3108 3109 t = nlmsg_data(n); 3110 parent = t->tcm_parent; 3111 cl = 0; 3112 3113 block = tcf_block_find(net, &q, &parent, &cl, 3114 t->tcm_ifindex, t->tcm_block_index, extack); 3115 if (IS_ERR(block)) 3116 return PTR_ERR(block); 3117 3118 chain_index = nla_get_u32_default(tca[TCA_CHAIN], 0); 3119 if (chain_index > TC_ACT_EXT_VAL_MASK) { 3120 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit"); 3121 err = -EINVAL; 3122 goto errout_block; 3123 } 3124 3125 mutex_lock(&block->lock); 3126 chain = tcf_chain_lookup(block, chain_index); 3127 if (n->nlmsg_type == RTM_NEWCHAIN) { 3128 if (chain) { 3129 if (tcf_chain_held_by_acts_only(chain)) { 3130 /* The chain exists only because there is 3131 * some action referencing it. 3132 */ 3133 tcf_chain_hold(chain); 3134 } else { 3135 NL_SET_ERR_MSG(extack, "Filter chain already exists"); 3136 err = -EEXIST; 3137 goto errout_block_locked; 3138 } 3139 } else { 3140 if (!(n->nlmsg_flags & NLM_F_CREATE)) { 3141 NL_SET_ERR_MSG(extack, "Need both RTM_NEWCHAIN and NLM_F_CREATE to create a new chain"); 3142 err = -ENOENT; 3143 goto errout_block_locked; 3144 } 3145 chain = tcf_chain_create(block, chain_index); 3146 if (!chain) { 3147 NL_SET_ERR_MSG(extack, "Failed to create filter chain"); 3148 err = -ENOMEM; 3149 goto errout_block_locked; 3150 } 3151 } 3152 } else { 3153 if (!chain || tcf_chain_held_by_acts_only(chain)) { 3154 NL_SET_ERR_MSG(extack, "Cannot find specified filter chain"); 3155 err = -EINVAL; 3156 goto errout_block_locked; 3157 } 3158 tcf_chain_hold(chain); 3159 } 3160 3161 if (n->nlmsg_type == RTM_NEWCHAIN) { 3162 /* Modifying chain requires holding parent block lock. In case 3163 * the chain was successfully added, take a reference to the 3164 * chain. This ensures that an empty chain does not disappear at 3165 * the end of this function. 3166 */ 3167 tcf_chain_hold(chain); 3168 chain->explicitly_created = true; 3169 } 3170 mutex_unlock(&block->lock); 3171 3172 switch (n->nlmsg_type) { 3173 case RTM_NEWCHAIN: 3174 err = tc_chain_tmplt_add(chain, net, tca, extack); 3175 if (err) { 3176 tcf_chain_put_explicitly_created(chain); 3177 goto errout; 3178 } 3179 3180 tc_chain_notify(chain, NULL, 0, NLM_F_CREATE | NLM_F_EXCL, 3181 RTM_NEWCHAIN, false, extack); 3182 break; 3183 case RTM_DELCHAIN: 3184 tfilter_notify_chain(net, skb, block, q, parent, n, 3185 chain, RTM_DELTFILTER, extack); 3186 /* Flush the chain first as the user requested chain removal. */ 3187 tcf_chain_flush(chain, true); 3188 /* In case the chain was successfully deleted, put a reference 3189 * to the chain previously taken during addition. 3190 */ 3191 tcf_chain_put_explicitly_created(chain); 3192 break; 3193 case RTM_GETCHAIN: 3194 err = tc_chain_notify(chain, skb, n->nlmsg_seq, 3195 n->nlmsg_flags, n->nlmsg_type, true, extack); 3196 if (err < 0) 3197 NL_SET_ERR_MSG(extack, "Failed to send chain notify message"); 3198 break; 3199 default: 3200 err = -EOPNOTSUPP; 3201 NL_SET_ERR_MSG(extack, "Unsupported message type"); 3202 goto errout; 3203 } 3204 3205 errout: 3206 tcf_chain_put(chain); 3207 errout_block: 3208 tcf_block_release(q, block, true); 3209 if (err == -EAGAIN) 3210 /* Replay the request. */ 3211 goto replay; 3212 return err; 3213 3214 errout_block_locked: 3215 mutex_unlock(&block->lock); 3216 goto errout_block; 3217 } 3218 3219 /* called with RTNL */ 3220 static int tc_dump_chain(struct sk_buff *skb, struct netlink_callback *cb) 3221 { 3222 struct net *net = sock_net(skb->sk); 3223 struct nlattr *tca[TCA_MAX + 1]; 3224 struct Qdisc *q = NULL; 3225 struct tcf_block *block; 3226 struct tcmsg *tcm = nlmsg_data(cb->nlh); 3227 struct tcf_chain *chain; 3228 long index_start; 3229 long index; 3230 int err; 3231 3232 if (nlmsg_len(cb->nlh) < sizeof(*tcm)) 3233 return skb->len; 3234 3235 err = nlmsg_parse_deprecated(cb->nlh, sizeof(*tcm), tca, TCA_MAX, 3236 rtm_tca_policy, cb->extack); 3237 if (err) 3238 return err; 3239 3240 if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) { 3241 block = tcf_block_refcnt_get(net, tcm->tcm_block_index); 3242 if (!block) 3243 goto out; 3244 } else { 3245 const struct Qdisc_class_ops *cops; 3246 struct net_device *dev; 3247 unsigned long cl = 0; 3248 3249 dev = __dev_get_by_index(net, tcm->tcm_ifindex); 3250 if (!dev) 3251 return skb->len; 3252 3253 if (!tcm->tcm_parent) 3254 q = rtnl_dereference(dev->qdisc); 3255 else 3256 q = qdisc_lookup(dev, TC_H_MAJ(tcm->tcm_parent)); 3257 3258 if (!q) 3259 goto out; 3260 cops = q->ops->cl_ops; 3261 if (!cops) 3262 goto out; 3263 if (!cops->tcf_block) 3264 goto out; 3265 if (TC_H_MIN(tcm->tcm_parent)) { 3266 cl = cops->find(q, tcm->tcm_parent); 3267 if (cl == 0) 3268 goto out; 3269 } 3270 block = cops->tcf_block(q, cl, NULL); 3271 if (!block) 3272 goto out; 3273 if (tcf_block_shared(block)) 3274 q = NULL; 3275 } 3276 3277 index_start = cb->args[0]; 3278 index = 0; 3279 3280 mutex_lock(&block->lock); 3281 list_for_each_entry(chain, &block->chain_list, list) { 3282 if ((tca[TCA_CHAIN] && 3283 nla_get_u32(tca[TCA_CHAIN]) != chain->index)) 3284 continue; 3285 if (index < index_start) { 3286 index++; 3287 continue; 3288 } 3289 if (tcf_chain_held_by_acts_only(chain)) 3290 continue; 3291 err = tc_chain_fill_node(chain->tmplt_ops, chain->tmplt_priv, 3292 chain->index, net, skb, block, 3293 NETLINK_CB(cb->skb).portid, 3294 cb->nlh->nlmsg_seq, NLM_F_MULTI, 3295 RTM_NEWCHAIN, NULL); 3296 if (err <= 0) 3297 break; 3298 index++; 3299 } 3300 mutex_unlock(&block->lock); 3301 3302 if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) 3303 tcf_block_refcnt_put(block, true); 3304 cb->args[0] = index; 3305 3306 out: 3307 /* If we did no progress, the error (EMSGSIZE) is real */ 3308 if (skb->len == 0 && err) 3309 return err; 3310 return skb->len; 3311 } 3312 3313 int tcf_exts_init_ex(struct tcf_exts *exts, struct net *net, int action, 3314 int police, struct tcf_proto *tp, u32 handle, 3315 bool use_action_miss) 3316 { 3317 int err = 0; 3318 3319 #ifdef CONFIG_NET_CLS_ACT 3320 exts->type = 0; 3321 exts->nr_actions = 0; 3322 exts->miss_cookie_node = NULL; 3323 /* Note: we do not own yet a reference on net. 3324 * This reference might be taken later from tcf_exts_get_net(). 3325 */ 3326 exts->net = net; 3327 exts->actions = kcalloc(TCA_ACT_MAX_PRIO, sizeof(struct tc_action *), 3328 GFP_KERNEL); 3329 if (!exts->actions) 3330 return -ENOMEM; 3331 #endif 3332 3333 exts->action = action; 3334 exts->police = police; 3335 3336 if (!use_action_miss) 3337 return 0; 3338 3339 err = tcf_exts_miss_cookie_base_alloc(exts, tp, handle); 3340 if (err) 3341 goto err_miss_alloc; 3342 3343 return 0; 3344 3345 err_miss_alloc: 3346 tcf_exts_destroy(exts); 3347 #ifdef CONFIG_NET_CLS_ACT 3348 exts->actions = NULL; 3349 #endif 3350 return err; 3351 } 3352 EXPORT_SYMBOL(tcf_exts_init_ex); 3353 3354 void tcf_exts_destroy(struct tcf_exts *exts) 3355 { 3356 tcf_exts_miss_cookie_base_destroy(exts); 3357 3358 #ifdef CONFIG_NET_CLS_ACT 3359 if (exts->actions) { 3360 tcf_action_destroy(exts->actions, TCA_ACT_UNBIND); 3361 kfree(exts->actions); 3362 } 3363 exts->nr_actions = 0; 3364 #endif 3365 } 3366 EXPORT_SYMBOL(tcf_exts_destroy); 3367 3368 int tcf_exts_validate_ex(struct net *net, struct tcf_proto *tp, struct nlattr **tb, 3369 struct nlattr *rate_tlv, struct tcf_exts *exts, 3370 u32 flags, u32 fl_flags, struct netlink_ext_ack *extack) 3371 { 3372 #ifdef CONFIG_NET_CLS_ACT 3373 { 3374 int init_res[TCA_ACT_MAX_PRIO] = {}; 3375 struct tc_action *act; 3376 size_t attr_size = 0; 3377 3378 if (exts->police && tb[exts->police]) { 3379 struct tc_action_ops *a_o; 3380 3381 flags |= TCA_ACT_FLAGS_POLICE | TCA_ACT_FLAGS_BIND; 3382 a_o = tc_action_load_ops(tb[exts->police], flags, 3383 extack); 3384 if (IS_ERR(a_o)) 3385 return PTR_ERR(a_o); 3386 act = tcf_action_init_1(net, tp, tb[exts->police], 3387 rate_tlv, a_o, init_res, flags, 3388 extack); 3389 module_put(a_o->owner); 3390 if (IS_ERR(act)) 3391 return PTR_ERR(act); 3392 3393 act->type = exts->type = TCA_OLD_COMPAT; 3394 exts->actions[0] = act; 3395 exts->nr_actions = 1; 3396 tcf_idr_insert_many(exts->actions, init_res); 3397 } else if (exts->action && tb[exts->action]) { 3398 int err; 3399 3400 flags |= TCA_ACT_FLAGS_BIND; 3401 err = tcf_action_init(net, tp, tb[exts->action], 3402 rate_tlv, exts->actions, init_res, 3403 &attr_size, flags, fl_flags, 3404 extack); 3405 if (err < 0) 3406 return err; 3407 exts->nr_actions = err; 3408 } 3409 } 3410 #else 3411 if ((exts->action && tb[exts->action]) || 3412 (exts->police && tb[exts->police])) { 3413 NL_SET_ERR_MSG(extack, "Classifier actions are not supported per compile options (CONFIG_NET_CLS_ACT)"); 3414 return -EOPNOTSUPP; 3415 } 3416 #endif 3417 3418 return 0; 3419 } 3420 EXPORT_SYMBOL(tcf_exts_validate_ex); 3421 3422 int tcf_exts_validate(struct net *net, struct tcf_proto *tp, struct nlattr **tb, 3423 struct nlattr *rate_tlv, struct tcf_exts *exts, 3424 u32 flags, struct netlink_ext_ack *extack) 3425 { 3426 return tcf_exts_validate_ex(net, tp, tb, rate_tlv, exts, 3427 flags, 0, extack); 3428 } 3429 EXPORT_SYMBOL(tcf_exts_validate); 3430 3431 void tcf_exts_change(struct tcf_exts *dst, struct tcf_exts *src) 3432 { 3433 #ifdef CONFIG_NET_CLS_ACT 3434 struct tcf_exts old = *dst; 3435 3436 *dst = *src; 3437 tcf_exts_destroy(&old); 3438 #endif 3439 } 3440 EXPORT_SYMBOL(tcf_exts_change); 3441 3442 #ifdef CONFIG_NET_CLS_ACT 3443 static struct tc_action *tcf_exts_first_act(struct tcf_exts *exts) 3444 { 3445 if (exts->nr_actions == 0) 3446 return NULL; 3447 else 3448 return exts->actions[0]; 3449 } 3450 #endif 3451 3452 int tcf_exts_dump(struct sk_buff *skb, struct tcf_exts *exts) 3453 { 3454 #ifdef CONFIG_NET_CLS_ACT 3455 struct nlattr *nest; 3456 3457 if (exts->action && tcf_exts_has_actions(exts)) { 3458 /* 3459 * again for backward compatible mode - we want 3460 * to work with both old and new modes of entering 3461 * tc data even if iproute2 was newer - jhs 3462 */ 3463 if (exts->type != TCA_OLD_COMPAT) { 3464 nest = nla_nest_start_noflag(skb, exts->action); 3465 if (nest == NULL) 3466 goto nla_put_failure; 3467 3468 if (tcf_action_dump(skb, exts->actions, 0, 0, false) 3469 < 0) 3470 goto nla_put_failure; 3471 nla_nest_end(skb, nest); 3472 } else if (exts->police) { 3473 struct tc_action *act = tcf_exts_first_act(exts); 3474 nest = nla_nest_start_noflag(skb, exts->police); 3475 if (nest == NULL || !act) 3476 goto nla_put_failure; 3477 if (tcf_action_dump_old(skb, act, 0, 0) < 0) 3478 goto nla_put_failure; 3479 nla_nest_end(skb, nest); 3480 } 3481 } 3482 return 0; 3483 3484 nla_put_failure: 3485 nla_nest_cancel(skb, nest); 3486 return -1; 3487 #else 3488 return 0; 3489 #endif 3490 } 3491 EXPORT_SYMBOL(tcf_exts_dump); 3492 3493 int tcf_exts_terse_dump(struct sk_buff *skb, struct tcf_exts *exts) 3494 { 3495 #ifdef CONFIG_NET_CLS_ACT 3496 struct nlattr *nest; 3497 3498 if (!exts->action || !tcf_exts_has_actions(exts)) 3499 return 0; 3500 3501 nest = nla_nest_start_noflag(skb, exts->action); 3502 if (!nest) 3503 goto nla_put_failure; 3504 3505 if (tcf_action_dump(skb, exts->actions, 0, 0, true) < 0) 3506 goto nla_put_failure; 3507 nla_nest_end(skb, nest); 3508 return 0; 3509 3510 nla_put_failure: 3511 nla_nest_cancel(skb, nest); 3512 return -1; 3513 #else 3514 return 0; 3515 #endif 3516 } 3517 EXPORT_SYMBOL(tcf_exts_terse_dump); 3518 3519 int tcf_exts_dump_stats(struct sk_buff *skb, struct tcf_exts *exts) 3520 { 3521 #ifdef CONFIG_NET_CLS_ACT 3522 struct tc_action *a = tcf_exts_first_act(exts); 3523 if (a != NULL && tcf_action_copy_stats(skb, a, 1) < 0) 3524 return -1; 3525 #endif 3526 return 0; 3527 } 3528 EXPORT_SYMBOL(tcf_exts_dump_stats); 3529 3530 static void tcf_block_offload_inc(struct tcf_block *block, u32 *flags) 3531 { 3532 if (*flags & TCA_CLS_FLAGS_IN_HW) 3533 return; 3534 *flags |= TCA_CLS_FLAGS_IN_HW; 3535 if (tc_skip_sw(*flags)) 3536 atomic_inc(&block->skipswcnt); 3537 atomic_inc(&block->offloadcnt); 3538 } 3539 3540 static void tcf_block_offload_dec(struct tcf_block *block, u32 *flags) 3541 { 3542 if (!(*flags & TCA_CLS_FLAGS_IN_HW)) 3543 return; 3544 *flags &= ~TCA_CLS_FLAGS_IN_HW; 3545 if (tc_skip_sw(*flags)) 3546 atomic_dec(&block->skipswcnt); 3547 atomic_dec(&block->offloadcnt); 3548 } 3549 3550 static void tc_cls_offload_cnt_update(struct tcf_block *block, 3551 struct tcf_proto *tp, u32 *cnt, 3552 u32 *flags, u32 diff, bool add) 3553 { 3554 lockdep_assert_held(&block->cb_lock); 3555 3556 spin_lock(&tp->lock); 3557 if (add) { 3558 if (!*cnt) 3559 tcf_block_offload_inc(block, flags); 3560 *cnt += diff; 3561 } else { 3562 *cnt -= diff; 3563 if (!*cnt) 3564 tcf_block_offload_dec(block, flags); 3565 } 3566 spin_unlock(&tp->lock); 3567 } 3568 3569 static void 3570 tc_cls_offload_cnt_reset(struct tcf_block *block, struct tcf_proto *tp, 3571 u32 *cnt, u32 *flags) 3572 { 3573 lockdep_assert_held(&block->cb_lock); 3574 3575 spin_lock(&tp->lock); 3576 tcf_block_offload_dec(block, flags); 3577 *cnt = 0; 3578 spin_unlock(&tp->lock); 3579 } 3580 3581 static int 3582 __tc_setup_cb_call(struct tcf_block *block, enum tc_setup_type type, 3583 void *type_data, bool err_stop) 3584 { 3585 struct flow_block_cb *block_cb; 3586 int ok_count = 0; 3587 int err; 3588 3589 list_for_each_entry(block_cb, &block->flow_block.cb_list, list) { 3590 err = block_cb->cb(type, type_data, block_cb->cb_priv); 3591 if (err) { 3592 if (err_stop) 3593 return err; 3594 } else { 3595 ok_count++; 3596 } 3597 } 3598 return ok_count; 3599 } 3600 3601 int tc_setup_cb_call(struct tcf_block *block, enum tc_setup_type type, 3602 void *type_data, bool err_stop, bool rtnl_held) 3603 { 3604 bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held; 3605 int ok_count; 3606 3607 retry: 3608 if (take_rtnl) 3609 rtnl_lock(); 3610 down_read(&block->cb_lock); 3611 /* Need to obtain rtnl lock if block is bound to devs that require it. 3612 * In block bind code cb_lock is obtained while holding rtnl, so we must 3613 * obtain the locks in same order here. 3614 */ 3615 if (!rtnl_held && !take_rtnl && block->lockeddevcnt) { 3616 up_read(&block->cb_lock); 3617 take_rtnl = true; 3618 goto retry; 3619 } 3620 3621 ok_count = __tc_setup_cb_call(block, type, type_data, err_stop); 3622 3623 up_read(&block->cb_lock); 3624 if (take_rtnl) 3625 rtnl_unlock(); 3626 return ok_count; 3627 } 3628 EXPORT_SYMBOL(tc_setup_cb_call); 3629 3630 /* Non-destructive filter add. If filter that wasn't already in hardware is 3631 * successfully offloaded, increment block offloads counter. On failure, 3632 * previously offloaded filter is considered to be intact and offloads counter 3633 * is not decremented. 3634 */ 3635 3636 int tc_setup_cb_add(struct tcf_block *block, struct tcf_proto *tp, 3637 enum tc_setup_type type, void *type_data, bool err_stop, 3638 u32 *flags, unsigned int *in_hw_count, bool rtnl_held) 3639 { 3640 bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held; 3641 int ok_count; 3642 3643 retry: 3644 if (take_rtnl) 3645 rtnl_lock(); 3646 down_read(&block->cb_lock); 3647 /* Need to obtain rtnl lock if block is bound to devs that require it. 3648 * In block bind code cb_lock is obtained while holding rtnl, so we must 3649 * obtain the locks in same order here. 3650 */ 3651 if (!rtnl_held && !take_rtnl && block->lockeddevcnt) { 3652 up_read(&block->cb_lock); 3653 take_rtnl = true; 3654 goto retry; 3655 } 3656 3657 /* Make sure all netdevs sharing this block are offload-capable. */ 3658 if (block->nooffloaddevcnt && err_stop) { 3659 ok_count = -EOPNOTSUPP; 3660 goto err_unlock; 3661 } 3662 3663 ok_count = __tc_setup_cb_call(block, type, type_data, err_stop); 3664 if (ok_count < 0) 3665 goto err_unlock; 3666 3667 if (tp->ops->hw_add) 3668 tp->ops->hw_add(tp, type_data); 3669 if (ok_count > 0) 3670 tc_cls_offload_cnt_update(block, tp, in_hw_count, flags, 3671 ok_count, true); 3672 err_unlock: 3673 up_read(&block->cb_lock); 3674 if (take_rtnl) 3675 rtnl_unlock(); 3676 return min(ok_count, 0); 3677 } 3678 EXPORT_SYMBOL(tc_setup_cb_add); 3679 3680 /* Destructive filter replace. If filter that wasn't already in hardware is 3681 * successfully offloaded, increment block offload counter. On failure, 3682 * previously offloaded filter is considered to be destroyed and offload counter 3683 * is decremented. 3684 */ 3685 3686 int tc_setup_cb_replace(struct tcf_block *block, struct tcf_proto *tp, 3687 enum tc_setup_type type, void *type_data, bool err_stop, 3688 u32 *old_flags, unsigned int *old_in_hw_count, 3689 u32 *new_flags, unsigned int *new_in_hw_count, 3690 bool rtnl_held) 3691 { 3692 bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held; 3693 int ok_count; 3694 3695 retry: 3696 if (take_rtnl) 3697 rtnl_lock(); 3698 down_read(&block->cb_lock); 3699 /* Need to obtain rtnl lock if block is bound to devs that require it. 3700 * In block bind code cb_lock is obtained while holding rtnl, so we must 3701 * obtain the locks in same order here. 3702 */ 3703 if (!rtnl_held && !take_rtnl && block->lockeddevcnt) { 3704 up_read(&block->cb_lock); 3705 take_rtnl = true; 3706 goto retry; 3707 } 3708 3709 /* Make sure all netdevs sharing this block are offload-capable. */ 3710 if (block->nooffloaddevcnt && err_stop) { 3711 ok_count = -EOPNOTSUPP; 3712 goto err_unlock; 3713 } 3714 3715 tc_cls_offload_cnt_reset(block, tp, old_in_hw_count, old_flags); 3716 if (tp->ops->hw_del) 3717 tp->ops->hw_del(tp, type_data); 3718 3719 ok_count = __tc_setup_cb_call(block, type, type_data, err_stop); 3720 if (ok_count < 0) 3721 goto err_unlock; 3722 3723 if (tp->ops->hw_add) 3724 tp->ops->hw_add(tp, type_data); 3725 if (ok_count > 0) 3726 tc_cls_offload_cnt_update(block, tp, new_in_hw_count, 3727 new_flags, ok_count, true); 3728 err_unlock: 3729 up_read(&block->cb_lock); 3730 if (take_rtnl) 3731 rtnl_unlock(); 3732 return min(ok_count, 0); 3733 } 3734 EXPORT_SYMBOL(tc_setup_cb_replace); 3735 3736 /* Destroy filter and decrement block offload counter, if filter was previously 3737 * offloaded. 3738 */ 3739 3740 int tc_setup_cb_destroy(struct tcf_block *block, struct tcf_proto *tp, 3741 enum tc_setup_type type, void *type_data, bool err_stop, 3742 u32 *flags, unsigned int *in_hw_count, bool rtnl_held) 3743 { 3744 bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held; 3745 int ok_count; 3746 3747 retry: 3748 if (take_rtnl) 3749 rtnl_lock(); 3750 down_read(&block->cb_lock); 3751 /* Need to obtain rtnl lock if block is bound to devs that require it. 3752 * In block bind code cb_lock is obtained while holding rtnl, so we must 3753 * obtain the locks in same order here. 3754 */ 3755 if (!rtnl_held && !take_rtnl && block->lockeddevcnt) { 3756 up_read(&block->cb_lock); 3757 take_rtnl = true; 3758 goto retry; 3759 } 3760 3761 ok_count = __tc_setup_cb_call(block, type, type_data, err_stop); 3762 3763 tc_cls_offload_cnt_reset(block, tp, in_hw_count, flags); 3764 if (tp->ops->hw_del) 3765 tp->ops->hw_del(tp, type_data); 3766 3767 up_read(&block->cb_lock); 3768 if (take_rtnl) 3769 rtnl_unlock(); 3770 return min(ok_count, 0); 3771 } 3772 EXPORT_SYMBOL(tc_setup_cb_destroy); 3773 3774 int tc_setup_cb_reoffload(struct tcf_block *block, struct tcf_proto *tp, 3775 bool add, flow_setup_cb_t *cb, 3776 enum tc_setup_type type, void *type_data, 3777 void *cb_priv, u32 *flags, unsigned int *in_hw_count) 3778 { 3779 int err = cb(type, type_data, cb_priv); 3780 3781 if (err) { 3782 if (add && tc_skip_sw(*flags)) 3783 return err; 3784 } else { 3785 tc_cls_offload_cnt_update(block, tp, in_hw_count, flags, 1, 3786 add); 3787 } 3788 3789 return 0; 3790 } 3791 EXPORT_SYMBOL(tc_setup_cb_reoffload); 3792 3793 static int tcf_act_get_user_cookie(struct flow_action_entry *entry, 3794 const struct tc_action *act) 3795 { 3796 struct tc_cookie *user_cookie; 3797 int err = 0; 3798 3799 rcu_read_lock(); 3800 user_cookie = rcu_dereference(act->user_cookie); 3801 if (user_cookie) { 3802 entry->user_cookie = flow_action_cookie_create(user_cookie->data, 3803 user_cookie->len, 3804 GFP_ATOMIC); 3805 if (!entry->user_cookie) 3806 err = -ENOMEM; 3807 } 3808 rcu_read_unlock(); 3809 return err; 3810 } 3811 3812 static void tcf_act_put_user_cookie(struct flow_action_entry *entry) 3813 { 3814 flow_action_cookie_destroy(entry->user_cookie); 3815 } 3816 3817 void tc_cleanup_offload_action(struct flow_action *flow_action) 3818 { 3819 struct flow_action_entry *entry; 3820 int i; 3821 3822 flow_action_for_each(i, entry, flow_action) { 3823 tcf_act_put_user_cookie(entry); 3824 if (entry->destructor) 3825 entry->destructor(entry->destructor_priv); 3826 } 3827 } 3828 EXPORT_SYMBOL(tc_cleanup_offload_action); 3829 3830 static int tc_setup_offload_act(struct tc_action *act, 3831 struct flow_action_entry *entry, 3832 u32 *index_inc, 3833 struct netlink_ext_ack *extack) 3834 { 3835 #ifdef CONFIG_NET_CLS_ACT 3836 if (act->ops->offload_act_setup) { 3837 return act->ops->offload_act_setup(act, entry, index_inc, true, 3838 extack); 3839 } else { 3840 NL_SET_ERR_MSG(extack, "Action does not support offload"); 3841 return -EOPNOTSUPP; 3842 } 3843 #else 3844 return 0; 3845 #endif 3846 } 3847 3848 int tc_setup_action(struct flow_action *flow_action, 3849 struct tc_action *actions[], 3850 u32 miss_cookie_base, 3851 struct netlink_ext_ack *extack) 3852 { 3853 int i, j, k, index, err = 0; 3854 struct tc_action *act; 3855 3856 BUILD_BUG_ON(TCA_ACT_HW_STATS_ANY != FLOW_ACTION_HW_STATS_ANY); 3857 BUILD_BUG_ON(TCA_ACT_HW_STATS_IMMEDIATE != FLOW_ACTION_HW_STATS_IMMEDIATE); 3858 BUILD_BUG_ON(TCA_ACT_HW_STATS_DELAYED != FLOW_ACTION_HW_STATS_DELAYED); 3859 3860 if (!actions) 3861 return 0; 3862 3863 j = 0; 3864 tcf_act_for_each_action(i, act, actions) { 3865 struct flow_action_entry *entry; 3866 3867 entry = &flow_action->entries[j]; 3868 spin_lock_bh(&act->tcfa_lock); 3869 err = tcf_act_get_user_cookie(entry, act); 3870 if (err) 3871 goto err_out_locked; 3872 3873 index = 0; 3874 err = tc_setup_offload_act(act, entry, &index, extack); 3875 if (err) 3876 goto err_out_locked; 3877 3878 for (k = 0; k < index ; k++) { 3879 entry[k].hw_stats = tc_act_hw_stats(act->hw_stats); 3880 entry[k].hw_index = act->tcfa_index; 3881 entry[k].cookie = (unsigned long)act; 3882 entry[k].miss_cookie = 3883 tcf_exts_miss_cookie_get(miss_cookie_base, i); 3884 } 3885 3886 j += index; 3887 3888 spin_unlock_bh(&act->tcfa_lock); 3889 } 3890 3891 err_out: 3892 if (err) 3893 tc_cleanup_offload_action(flow_action); 3894 3895 return err; 3896 err_out_locked: 3897 spin_unlock_bh(&act->tcfa_lock); 3898 goto err_out; 3899 } 3900 3901 int tc_setup_offload_action(struct flow_action *flow_action, 3902 const struct tcf_exts *exts, 3903 struct netlink_ext_ack *extack) 3904 { 3905 #ifdef CONFIG_NET_CLS_ACT 3906 u32 miss_cookie_base; 3907 3908 if (!exts) 3909 return 0; 3910 3911 miss_cookie_base = exts->miss_cookie_node ? 3912 exts->miss_cookie_node->miss_cookie_base : 0; 3913 return tc_setup_action(flow_action, exts->actions, miss_cookie_base, 3914 extack); 3915 #else 3916 return 0; 3917 #endif 3918 } 3919 EXPORT_SYMBOL(tc_setup_offload_action); 3920 3921 unsigned int tcf_exts_num_actions(struct tcf_exts *exts) 3922 { 3923 unsigned int num_acts = 0; 3924 struct tc_action *act; 3925 int i; 3926 3927 tcf_exts_for_each_action(i, act, exts) { 3928 if (is_tcf_pedit(act)) 3929 num_acts += tcf_pedit_nkeys(act); 3930 else 3931 num_acts++; 3932 } 3933 return num_acts; 3934 } 3935 EXPORT_SYMBOL(tcf_exts_num_actions); 3936 3937 #ifdef CONFIG_NET_CLS_ACT 3938 static int tcf_qevent_parse_block_index(struct nlattr *block_index_attr, 3939 u32 *p_block_index, 3940 struct netlink_ext_ack *extack) 3941 { 3942 *p_block_index = nla_get_u32(block_index_attr); 3943 if (!*p_block_index) { 3944 NL_SET_ERR_MSG(extack, "Block number may not be zero"); 3945 return -EINVAL; 3946 } 3947 3948 return 0; 3949 } 3950 3951 int tcf_qevent_init(struct tcf_qevent *qe, struct Qdisc *sch, 3952 enum flow_block_binder_type binder_type, 3953 struct nlattr *block_index_attr, 3954 struct netlink_ext_ack *extack) 3955 { 3956 u32 block_index; 3957 int err; 3958 3959 if (!block_index_attr) 3960 return 0; 3961 3962 err = tcf_qevent_parse_block_index(block_index_attr, &block_index, extack); 3963 if (err) 3964 return err; 3965 3966 qe->info.binder_type = binder_type; 3967 qe->info.chain_head_change = tcf_chain_head_change_dflt; 3968 qe->info.chain_head_change_priv = &qe->filter_chain; 3969 qe->info.block_index = block_index; 3970 3971 return tcf_block_get_ext(&qe->block, sch, &qe->info, extack); 3972 } 3973 EXPORT_SYMBOL(tcf_qevent_init); 3974 3975 void tcf_qevent_destroy(struct tcf_qevent *qe, struct Qdisc *sch) 3976 { 3977 if (qe->info.block_index) 3978 tcf_block_put_ext(qe->block, sch, &qe->info); 3979 } 3980 EXPORT_SYMBOL(tcf_qevent_destroy); 3981 3982 int tcf_qevent_validate_change(struct tcf_qevent *qe, struct nlattr *block_index_attr, 3983 struct netlink_ext_ack *extack) 3984 { 3985 u32 block_index; 3986 int err; 3987 3988 if (!block_index_attr) 3989 return 0; 3990 3991 err = tcf_qevent_parse_block_index(block_index_attr, &block_index, extack); 3992 if (err) 3993 return err; 3994 3995 /* Bounce newly-configured block or change in block. */ 3996 if (block_index != qe->info.block_index) { 3997 NL_SET_ERR_MSG(extack, "Change of blocks is not supported"); 3998 return -EINVAL; 3999 } 4000 4001 return 0; 4002 } 4003 EXPORT_SYMBOL(tcf_qevent_validate_change); 4004 4005 struct sk_buff *tcf_qevent_handle(struct tcf_qevent *qe, struct Qdisc *sch, struct sk_buff *skb, 4006 struct sk_buff **to_free, int *ret) 4007 { 4008 struct tcf_result cl_res; 4009 struct tcf_proto *fl; 4010 4011 if (!qe->info.block_index) 4012 return skb; 4013 4014 fl = rcu_dereference_bh(qe->filter_chain); 4015 4016 switch (tcf_classify(skb, NULL, fl, &cl_res, false)) { 4017 case TC_ACT_SHOT: 4018 qdisc_qstats_drop(sch); 4019 __qdisc_drop(skb, to_free); 4020 *ret = __NET_XMIT_BYPASS; 4021 return NULL; 4022 case TC_ACT_STOLEN: 4023 case TC_ACT_QUEUED: 4024 case TC_ACT_TRAP: 4025 __qdisc_drop(skb, to_free); 4026 *ret = __NET_XMIT_STOLEN; 4027 return NULL; 4028 case TC_ACT_REDIRECT: 4029 skb_do_redirect(skb); 4030 *ret = __NET_XMIT_STOLEN; 4031 return NULL; 4032 } 4033 4034 return skb; 4035 } 4036 EXPORT_SYMBOL(tcf_qevent_handle); 4037 4038 int tcf_qevent_dump(struct sk_buff *skb, int attr_name, struct tcf_qevent *qe) 4039 { 4040 if (!qe->info.block_index) 4041 return 0; 4042 return nla_put_u32(skb, attr_name, qe->info.block_index); 4043 } 4044 EXPORT_SYMBOL(tcf_qevent_dump); 4045 #endif 4046 4047 static __net_init int tcf_net_init(struct net *net) 4048 { 4049 struct tcf_net *tn = net_generic(net, tcf_net_id); 4050 4051 spin_lock_init(&tn->idr_lock); 4052 idr_init(&tn->idr); 4053 return 0; 4054 } 4055 4056 static void __net_exit tcf_net_exit(struct net *net) 4057 { 4058 struct tcf_net *tn = net_generic(net, tcf_net_id); 4059 4060 idr_destroy(&tn->idr); 4061 } 4062 4063 static struct pernet_operations tcf_net_ops = { 4064 .init = tcf_net_init, 4065 .exit = tcf_net_exit, 4066 .id = &tcf_net_id, 4067 .size = sizeof(struct tcf_net), 4068 }; 4069 4070 static const struct rtnl_msg_handler tc_filter_rtnl_msg_handlers[] __initconst = { 4071 {.msgtype = RTM_NEWTFILTER, .doit = tc_new_tfilter, 4072 .flags = RTNL_FLAG_DOIT_UNLOCKED}, 4073 {.msgtype = RTM_DELTFILTER, .doit = tc_del_tfilter, 4074 .flags = RTNL_FLAG_DOIT_UNLOCKED}, 4075 {.msgtype = RTM_GETTFILTER, .doit = tc_get_tfilter, 4076 .dumpit = tc_dump_tfilter, .flags = RTNL_FLAG_DOIT_UNLOCKED}, 4077 {.msgtype = RTM_NEWCHAIN, .doit = tc_ctl_chain}, 4078 {.msgtype = RTM_DELCHAIN, .doit = tc_ctl_chain}, 4079 {.msgtype = RTM_GETCHAIN, .doit = tc_ctl_chain, 4080 .dumpit = tc_dump_chain}, 4081 }; 4082 4083 static int __init tc_filter_init(void) 4084 { 4085 int err; 4086 4087 tc_filter_wq = alloc_ordered_workqueue("tc_filter_workqueue", 0); 4088 if (!tc_filter_wq) 4089 return -ENOMEM; 4090 4091 err = register_pernet_subsys(&tcf_net_ops); 4092 if (err) 4093 goto err_register_pernet_subsys; 4094 4095 xa_init_flags(&tcf_exts_miss_cookies_xa, XA_FLAGS_ALLOC1); 4096 rtnl_register_many(tc_filter_rtnl_msg_handlers); 4097 4098 return 0; 4099 4100 err_register_pernet_subsys: 4101 destroy_workqueue(tc_filter_wq); 4102 return err; 4103 } 4104 4105 subsys_initcall(tc_filter_init); 4106