1 /* 2 * net/sched/cls_api.c Packet classifier API. 3 * 4 * This program is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU General Public License 6 * as published by the Free Software Foundation; either version 7 * 2 of the License, or (at your option) any later version. 8 * 9 * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru> 10 * 11 * Changes: 12 * 13 * Eduardo J. Blanco <ejbs@netlabs.com.uy> :990222: kmod support 14 * 15 */ 16 17 #include <linux/module.h> 18 #include <linux/types.h> 19 #include <linux/kernel.h> 20 #include <linux/string.h> 21 #include <linux/errno.h> 22 #include <linux/err.h> 23 #include <linux/skbuff.h> 24 #include <linux/init.h> 25 #include <linux/kmod.h> 26 #include <linux/slab.h> 27 #include <linux/idr.h> 28 #include <linux/rhashtable.h> 29 #include <net/net_namespace.h> 30 #include <net/sock.h> 31 #include <net/netlink.h> 32 #include <net/pkt_sched.h> 33 #include <net/pkt_cls.h> 34 #include <net/tc_act/tc_pedit.h> 35 #include <net/tc_act/tc_mirred.h> 36 #include <net/tc_act/tc_vlan.h> 37 #include <net/tc_act/tc_tunnel_key.h> 38 #include <net/tc_act/tc_csum.h> 39 #include <net/tc_act/tc_gact.h> 40 #include <net/tc_act/tc_skbedit.h> 41 42 extern const struct nla_policy rtm_tca_policy[TCA_MAX + 1]; 43 44 /* The list of all installed classifier types */ 45 static LIST_HEAD(tcf_proto_base); 46 47 /* Protects list of registered TC modules. It is pure SMP lock. */ 48 static DEFINE_RWLOCK(cls_mod_lock); 49 50 /* Find classifier type by string name */ 51 52 static const struct tcf_proto_ops *__tcf_proto_lookup_ops(const char *kind) 53 { 54 const struct tcf_proto_ops *t, *res = NULL; 55 56 if (kind) { 57 read_lock(&cls_mod_lock); 58 list_for_each_entry(t, &tcf_proto_base, head) { 59 if (strcmp(kind, t->kind) == 0) { 60 if (try_module_get(t->owner)) 61 res = t; 62 break; 63 } 64 } 65 read_unlock(&cls_mod_lock); 66 } 67 return res; 68 } 69 70 static const struct tcf_proto_ops * 71 tcf_proto_lookup_ops(const char *kind, bool rtnl_held, 72 struct netlink_ext_ack *extack) 73 { 74 const struct tcf_proto_ops *ops; 75 76 ops = __tcf_proto_lookup_ops(kind); 77 if (ops) 78 return ops; 79 #ifdef CONFIG_MODULES 80 if (rtnl_held) 81 rtnl_unlock(); 82 request_module("cls_%s", kind); 83 if (rtnl_held) 84 rtnl_lock(); 85 ops = __tcf_proto_lookup_ops(kind); 86 /* We dropped the RTNL semaphore in order to perform 87 * the module load. So, even if we succeeded in loading 88 * the module we have to replay the request. We indicate 89 * this using -EAGAIN. 90 */ 91 if (ops) { 92 module_put(ops->owner); 93 return ERR_PTR(-EAGAIN); 94 } 95 #endif 96 NL_SET_ERR_MSG(extack, "TC classifier not found"); 97 return ERR_PTR(-ENOENT); 98 } 99 100 /* Register(unregister) new classifier type */ 101 102 int register_tcf_proto_ops(struct tcf_proto_ops *ops) 103 { 104 struct tcf_proto_ops *t; 105 int rc = -EEXIST; 106 107 write_lock(&cls_mod_lock); 108 list_for_each_entry(t, &tcf_proto_base, head) 109 if (!strcmp(ops->kind, t->kind)) 110 goto out; 111 112 list_add_tail(&ops->head, &tcf_proto_base); 113 rc = 0; 114 out: 115 write_unlock(&cls_mod_lock); 116 return rc; 117 } 118 EXPORT_SYMBOL(register_tcf_proto_ops); 119 120 static struct workqueue_struct *tc_filter_wq; 121 122 int unregister_tcf_proto_ops(struct tcf_proto_ops *ops) 123 { 124 struct tcf_proto_ops *t; 125 int rc = -ENOENT; 126 127 /* Wait for outstanding call_rcu()s, if any, from a 128 * tcf_proto_ops's destroy() handler. 129 */ 130 rcu_barrier(); 131 flush_workqueue(tc_filter_wq); 132 133 write_lock(&cls_mod_lock); 134 list_for_each_entry(t, &tcf_proto_base, head) { 135 if (t == ops) { 136 list_del(&t->head); 137 rc = 0; 138 break; 139 } 140 } 141 write_unlock(&cls_mod_lock); 142 return rc; 143 } 144 EXPORT_SYMBOL(unregister_tcf_proto_ops); 145 146 bool tcf_queue_work(struct rcu_work *rwork, work_func_t func) 147 { 148 INIT_RCU_WORK(rwork, func); 149 return queue_rcu_work(tc_filter_wq, rwork); 150 } 151 EXPORT_SYMBOL(tcf_queue_work); 152 153 /* Select new prio value from the range, managed by kernel. */ 154 155 static inline u32 tcf_auto_prio(struct tcf_proto *tp) 156 { 157 u32 first = TC_H_MAKE(0xC0000000U, 0U); 158 159 if (tp) 160 first = tp->prio - 1; 161 162 return TC_H_MAJ(first); 163 } 164 165 static bool tcf_proto_is_unlocked(const char *kind) 166 { 167 const struct tcf_proto_ops *ops; 168 bool ret; 169 170 ops = tcf_proto_lookup_ops(kind, false, NULL); 171 /* On error return false to take rtnl lock. Proto lookup/create 172 * functions will perform lookup again and properly handle errors. 173 */ 174 if (IS_ERR(ops)) 175 return false; 176 177 ret = !!(ops->flags & TCF_PROTO_OPS_DOIT_UNLOCKED); 178 module_put(ops->owner); 179 return ret; 180 } 181 182 static struct tcf_proto *tcf_proto_create(const char *kind, u32 protocol, 183 u32 prio, struct tcf_chain *chain, 184 bool rtnl_held, 185 struct netlink_ext_ack *extack) 186 { 187 struct tcf_proto *tp; 188 int err; 189 190 tp = kzalloc(sizeof(*tp), GFP_KERNEL); 191 if (!tp) 192 return ERR_PTR(-ENOBUFS); 193 194 tp->ops = tcf_proto_lookup_ops(kind, rtnl_held, extack); 195 if (IS_ERR(tp->ops)) { 196 err = PTR_ERR(tp->ops); 197 goto errout; 198 } 199 tp->classify = tp->ops->classify; 200 tp->protocol = protocol; 201 tp->prio = prio; 202 tp->chain = chain; 203 spin_lock_init(&tp->lock); 204 refcount_set(&tp->refcnt, 1); 205 206 err = tp->ops->init(tp); 207 if (err) { 208 module_put(tp->ops->owner); 209 goto errout; 210 } 211 return tp; 212 213 errout: 214 kfree(tp); 215 return ERR_PTR(err); 216 } 217 218 static void tcf_proto_get(struct tcf_proto *tp) 219 { 220 refcount_inc(&tp->refcnt); 221 } 222 223 static void tcf_chain_put(struct tcf_chain *chain); 224 225 static void tcf_proto_destroy(struct tcf_proto *tp, bool rtnl_held, 226 struct netlink_ext_ack *extack) 227 { 228 tp->ops->destroy(tp, rtnl_held, extack); 229 tcf_chain_put(tp->chain); 230 module_put(tp->ops->owner); 231 kfree_rcu(tp, rcu); 232 } 233 234 static void tcf_proto_put(struct tcf_proto *tp, bool rtnl_held, 235 struct netlink_ext_ack *extack) 236 { 237 if (refcount_dec_and_test(&tp->refcnt)) 238 tcf_proto_destroy(tp, rtnl_held, extack); 239 } 240 241 static int walker_check_empty(struct tcf_proto *tp, void *fh, 242 struct tcf_walker *arg) 243 { 244 if (fh) { 245 arg->nonempty = true; 246 return -1; 247 } 248 return 0; 249 } 250 251 static bool tcf_proto_is_empty(struct tcf_proto *tp, bool rtnl_held) 252 { 253 struct tcf_walker walker = { .fn = walker_check_empty, }; 254 255 if (tp->ops->walk) { 256 tp->ops->walk(tp, &walker, rtnl_held); 257 return !walker.nonempty; 258 } 259 return true; 260 } 261 262 static bool tcf_proto_check_delete(struct tcf_proto *tp, bool rtnl_held) 263 { 264 spin_lock(&tp->lock); 265 if (tcf_proto_is_empty(tp, rtnl_held)) 266 tp->deleting = true; 267 spin_unlock(&tp->lock); 268 return tp->deleting; 269 } 270 271 static void tcf_proto_mark_delete(struct tcf_proto *tp) 272 { 273 spin_lock(&tp->lock); 274 tp->deleting = true; 275 spin_unlock(&tp->lock); 276 } 277 278 static bool tcf_proto_is_deleting(struct tcf_proto *tp) 279 { 280 bool deleting; 281 282 spin_lock(&tp->lock); 283 deleting = tp->deleting; 284 spin_unlock(&tp->lock); 285 286 return deleting; 287 } 288 289 #define ASSERT_BLOCK_LOCKED(block) \ 290 lockdep_assert_held(&(block)->lock) 291 292 struct tcf_filter_chain_list_item { 293 struct list_head list; 294 tcf_chain_head_change_t *chain_head_change; 295 void *chain_head_change_priv; 296 }; 297 298 static struct tcf_chain *tcf_chain_create(struct tcf_block *block, 299 u32 chain_index) 300 { 301 struct tcf_chain *chain; 302 303 ASSERT_BLOCK_LOCKED(block); 304 305 chain = kzalloc(sizeof(*chain), GFP_KERNEL); 306 if (!chain) 307 return NULL; 308 list_add_tail(&chain->list, &block->chain_list); 309 mutex_init(&chain->filter_chain_lock); 310 chain->block = block; 311 chain->index = chain_index; 312 chain->refcnt = 1; 313 if (!chain->index) 314 block->chain0.chain = chain; 315 return chain; 316 } 317 318 static void tcf_chain_head_change_item(struct tcf_filter_chain_list_item *item, 319 struct tcf_proto *tp_head) 320 { 321 if (item->chain_head_change) 322 item->chain_head_change(tp_head, item->chain_head_change_priv); 323 } 324 325 static void tcf_chain0_head_change(struct tcf_chain *chain, 326 struct tcf_proto *tp_head) 327 { 328 struct tcf_filter_chain_list_item *item; 329 struct tcf_block *block = chain->block; 330 331 if (chain->index) 332 return; 333 334 mutex_lock(&block->lock); 335 list_for_each_entry(item, &block->chain0.filter_chain_list, list) 336 tcf_chain_head_change_item(item, tp_head); 337 mutex_unlock(&block->lock); 338 } 339 340 /* Returns true if block can be safely freed. */ 341 342 static bool tcf_chain_detach(struct tcf_chain *chain) 343 { 344 struct tcf_block *block = chain->block; 345 346 ASSERT_BLOCK_LOCKED(block); 347 348 list_del(&chain->list); 349 if (!chain->index) 350 block->chain0.chain = NULL; 351 352 if (list_empty(&block->chain_list) && 353 refcount_read(&block->refcnt) == 0) 354 return true; 355 356 return false; 357 } 358 359 static void tcf_block_destroy(struct tcf_block *block) 360 { 361 mutex_destroy(&block->lock); 362 kfree_rcu(block, rcu); 363 } 364 365 static void tcf_chain_destroy(struct tcf_chain *chain, bool free_block) 366 { 367 struct tcf_block *block = chain->block; 368 369 mutex_destroy(&chain->filter_chain_lock); 370 kfree(chain); 371 if (free_block) 372 tcf_block_destroy(block); 373 } 374 375 static void tcf_chain_hold(struct tcf_chain *chain) 376 { 377 ASSERT_BLOCK_LOCKED(chain->block); 378 379 ++chain->refcnt; 380 } 381 382 static bool tcf_chain_held_by_acts_only(struct tcf_chain *chain) 383 { 384 ASSERT_BLOCK_LOCKED(chain->block); 385 386 /* In case all the references are action references, this 387 * chain should not be shown to the user. 388 */ 389 return chain->refcnt == chain->action_refcnt; 390 } 391 392 static struct tcf_chain *tcf_chain_lookup(struct tcf_block *block, 393 u32 chain_index) 394 { 395 struct tcf_chain *chain; 396 397 ASSERT_BLOCK_LOCKED(block); 398 399 list_for_each_entry(chain, &block->chain_list, list) { 400 if (chain->index == chain_index) 401 return chain; 402 } 403 return NULL; 404 } 405 406 static int tc_chain_notify(struct tcf_chain *chain, struct sk_buff *oskb, 407 u32 seq, u16 flags, int event, bool unicast); 408 409 static struct tcf_chain *__tcf_chain_get(struct tcf_block *block, 410 u32 chain_index, bool create, 411 bool by_act) 412 { 413 struct tcf_chain *chain = NULL; 414 bool is_first_reference; 415 416 mutex_lock(&block->lock); 417 chain = tcf_chain_lookup(block, chain_index); 418 if (chain) { 419 tcf_chain_hold(chain); 420 } else { 421 if (!create) 422 goto errout; 423 chain = tcf_chain_create(block, chain_index); 424 if (!chain) 425 goto errout; 426 } 427 428 if (by_act) 429 ++chain->action_refcnt; 430 is_first_reference = chain->refcnt - chain->action_refcnt == 1; 431 mutex_unlock(&block->lock); 432 433 /* Send notification only in case we got the first 434 * non-action reference. Until then, the chain acts only as 435 * a placeholder for actions pointing to it and user ought 436 * not know about them. 437 */ 438 if (is_first_reference && !by_act) 439 tc_chain_notify(chain, NULL, 0, NLM_F_CREATE | NLM_F_EXCL, 440 RTM_NEWCHAIN, false); 441 442 return chain; 443 444 errout: 445 mutex_unlock(&block->lock); 446 return chain; 447 } 448 449 static struct tcf_chain *tcf_chain_get(struct tcf_block *block, u32 chain_index, 450 bool create) 451 { 452 return __tcf_chain_get(block, chain_index, create, false); 453 } 454 455 struct tcf_chain *tcf_chain_get_by_act(struct tcf_block *block, u32 chain_index) 456 { 457 return __tcf_chain_get(block, chain_index, true, true); 458 } 459 EXPORT_SYMBOL(tcf_chain_get_by_act); 460 461 static void tc_chain_tmplt_del(const struct tcf_proto_ops *tmplt_ops, 462 void *tmplt_priv); 463 static int tc_chain_notify_delete(const struct tcf_proto_ops *tmplt_ops, 464 void *tmplt_priv, u32 chain_index, 465 struct tcf_block *block, struct sk_buff *oskb, 466 u32 seq, u16 flags, bool unicast); 467 468 static void __tcf_chain_put(struct tcf_chain *chain, bool by_act, 469 bool explicitly_created) 470 { 471 struct tcf_block *block = chain->block; 472 const struct tcf_proto_ops *tmplt_ops; 473 bool free_block = false; 474 unsigned int refcnt; 475 void *tmplt_priv; 476 477 mutex_lock(&block->lock); 478 if (explicitly_created) { 479 if (!chain->explicitly_created) { 480 mutex_unlock(&block->lock); 481 return; 482 } 483 chain->explicitly_created = false; 484 } 485 486 if (by_act) 487 chain->action_refcnt--; 488 489 /* tc_chain_notify_delete can't be called while holding block lock. 490 * However, when block is unlocked chain can be changed concurrently, so 491 * save these to temporary variables. 492 */ 493 refcnt = --chain->refcnt; 494 tmplt_ops = chain->tmplt_ops; 495 tmplt_priv = chain->tmplt_priv; 496 497 /* The last dropped non-action reference will trigger notification. */ 498 if (refcnt - chain->action_refcnt == 0 && !by_act) { 499 tc_chain_notify_delete(tmplt_ops, tmplt_priv, chain->index, 500 block, NULL, 0, 0, false); 501 /* Last reference to chain, no need to lock. */ 502 chain->flushing = false; 503 } 504 505 if (refcnt == 0) 506 free_block = tcf_chain_detach(chain); 507 mutex_unlock(&block->lock); 508 509 if (refcnt == 0) { 510 tc_chain_tmplt_del(tmplt_ops, tmplt_priv); 511 tcf_chain_destroy(chain, free_block); 512 } 513 } 514 515 static void tcf_chain_put(struct tcf_chain *chain) 516 { 517 __tcf_chain_put(chain, false, false); 518 } 519 520 void tcf_chain_put_by_act(struct tcf_chain *chain) 521 { 522 __tcf_chain_put(chain, true, false); 523 } 524 EXPORT_SYMBOL(tcf_chain_put_by_act); 525 526 static void tcf_chain_put_explicitly_created(struct tcf_chain *chain) 527 { 528 __tcf_chain_put(chain, false, true); 529 } 530 531 static void tcf_chain_flush(struct tcf_chain *chain, bool rtnl_held) 532 { 533 struct tcf_proto *tp, *tp_next; 534 535 mutex_lock(&chain->filter_chain_lock); 536 tp = tcf_chain_dereference(chain->filter_chain, chain); 537 RCU_INIT_POINTER(chain->filter_chain, NULL); 538 tcf_chain0_head_change(chain, NULL); 539 chain->flushing = true; 540 mutex_unlock(&chain->filter_chain_lock); 541 542 while (tp) { 543 tp_next = rcu_dereference_protected(tp->next, 1); 544 tcf_proto_put(tp, rtnl_held, NULL); 545 tp = tp_next; 546 } 547 } 548 549 static struct tcf_block *tc_dev_ingress_block(struct net_device *dev) 550 { 551 const struct Qdisc_class_ops *cops; 552 struct Qdisc *qdisc; 553 554 if (!dev_ingress_queue(dev)) 555 return NULL; 556 557 qdisc = dev_ingress_queue(dev)->qdisc_sleeping; 558 if (!qdisc) 559 return NULL; 560 561 cops = qdisc->ops->cl_ops; 562 if (!cops) 563 return NULL; 564 565 if (!cops->tcf_block) 566 return NULL; 567 568 return cops->tcf_block(qdisc, TC_H_MIN_INGRESS, NULL); 569 } 570 571 static struct rhashtable indr_setup_block_ht; 572 573 struct tc_indr_block_dev { 574 struct rhash_head ht_node; 575 struct net_device *dev; 576 unsigned int refcnt; 577 struct list_head cb_list; 578 struct tcf_block *block; 579 }; 580 581 struct tc_indr_block_cb { 582 struct list_head list; 583 void *cb_priv; 584 tc_indr_block_bind_cb_t *cb; 585 void *cb_ident; 586 }; 587 588 static const struct rhashtable_params tc_indr_setup_block_ht_params = { 589 .key_offset = offsetof(struct tc_indr_block_dev, dev), 590 .head_offset = offsetof(struct tc_indr_block_dev, ht_node), 591 .key_len = sizeof(struct net_device *), 592 }; 593 594 static struct tc_indr_block_dev * 595 tc_indr_block_dev_lookup(struct net_device *dev) 596 { 597 return rhashtable_lookup_fast(&indr_setup_block_ht, &dev, 598 tc_indr_setup_block_ht_params); 599 } 600 601 static struct tc_indr_block_dev *tc_indr_block_dev_get(struct net_device *dev) 602 { 603 struct tc_indr_block_dev *indr_dev; 604 605 indr_dev = tc_indr_block_dev_lookup(dev); 606 if (indr_dev) 607 goto inc_ref; 608 609 indr_dev = kzalloc(sizeof(*indr_dev), GFP_KERNEL); 610 if (!indr_dev) 611 return NULL; 612 613 INIT_LIST_HEAD(&indr_dev->cb_list); 614 indr_dev->dev = dev; 615 indr_dev->block = tc_dev_ingress_block(dev); 616 if (rhashtable_insert_fast(&indr_setup_block_ht, &indr_dev->ht_node, 617 tc_indr_setup_block_ht_params)) { 618 kfree(indr_dev); 619 return NULL; 620 } 621 622 inc_ref: 623 indr_dev->refcnt++; 624 return indr_dev; 625 } 626 627 static void tc_indr_block_dev_put(struct tc_indr_block_dev *indr_dev) 628 { 629 if (--indr_dev->refcnt) 630 return; 631 632 rhashtable_remove_fast(&indr_setup_block_ht, &indr_dev->ht_node, 633 tc_indr_setup_block_ht_params); 634 kfree(indr_dev); 635 } 636 637 static struct tc_indr_block_cb * 638 tc_indr_block_cb_lookup(struct tc_indr_block_dev *indr_dev, 639 tc_indr_block_bind_cb_t *cb, void *cb_ident) 640 { 641 struct tc_indr_block_cb *indr_block_cb; 642 643 list_for_each_entry(indr_block_cb, &indr_dev->cb_list, list) 644 if (indr_block_cb->cb == cb && 645 indr_block_cb->cb_ident == cb_ident) 646 return indr_block_cb; 647 return NULL; 648 } 649 650 static struct tc_indr_block_cb * 651 tc_indr_block_cb_add(struct tc_indr_block_dev *indr_dev, void *cb_priv, 652 tc_indr_block_bind_cb_t *cb, void *cb_ident) 653 { 654 struct tc_indr_block_cb *indr_block_cb; 655 656 indr_block_cb = tc_indr_block_cb_lookup(indr_dev, cb, cb_ident); 657 if (indr_block_cb) 658 return ERR_PTR(-EEXIST); 659 660 indr_block_cb = kzalloc(sizeof(*indr_block_cb), GFP_KERNEL); 661 if (!indr_block_cb) 662 return ERR_PTR(-ENOMEM); 663 664 indr_block_cb->cb_priv = cb_priv; 665 indr_block_cb->cb = cb; 666 indr_block_cb->cb_ident = cb_ident; 667 list_add(&indr_block_cb->list, &indr_dev->cb_list); 668 669 return indr_block_cb; 670 } 671 672 static void tc_indr_block_cb_del(struct tc_indr_block_cb *indr_block_cb) 673 { 674 list_del(&indr_block_cb->list); 675 kfree(indr_block_cb); 676 } 677 678 static void tc_indr_block_ing_cmd(struct tc_indr_block_dev *indr_dev, 679 struct tc_indr_block_cb *indr_block_cb, 680 enum tc_block_command command) 681 { 682 struct tc_block_offload bo = { 683 .command = command, 684 .binder_type = TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS, 685 .block = indr_dev->block, 686 }; 687 688 if (!indr_dev->block) 689 return; 690 691 indr_block_cb->cb(indr_dev->dev, indr_block_cb->cb_priv, TC_SETUP_BLOCK, 692 &bo); 693 } 694 695 int __tc_indr_block_cb_register(struct net_device *dev, void *cb_priv, 696 tc_indr_block_bind_cb_t *cb, void *cb_ident) 697 { 698 struct tc_indr_block_cb *indr_block_cb; 699 struct tc_indr_block_dev *indr_dev; 700 int err; 701 702 indr_dev = tc_indr_block_dev_get(dev); 703 if (!indr_dev) 704 return -ENOMEM; 705 706 indr_block_cb = tc_indr_block_cb_add(indr_dev, cb_priv, cb, cb_ident); 707 err = PTR_ERR_OR_ZERO(indr_block_cb); 708 if (err) 709 goto err_dev_put; 710 711 tc_indr_block_ing_cmd(indr_dev, indr_block_cb, TC_BLOCK_BIND); 712 return 0; 713 714 err_dev_put: 715 tc_indr_block_dev_put(indr_dev); 716 return err; 717 } 718 EXPORT_SYMBOL_GPL(__tc_indr_block_cb_register); 719 720 int tc_indr_block_cb_register(struct net_device *dev, void *cb_priv, 721 tc_indr_block_bind_cb_t *cb, void *cb_ident) 722 { 723 int err; 724 725 rtnl_lock(); 726 err = __tc_indr_block_cb_register(dev, cb_priv, cb, cb_ident); 727 rtnl_unlock(); 728 729 return err; 730 } 731 EXPORT_SYMBOL_GPL(tc_indr_block_cb_register); 732 733 void __tc_indr_block_cb_unregister(struct net_device *dev, 734 tc_indr_block_bind_cb_t *cb, void *cb_ident) 735 { 736 struct tc_indr_block_cb *indr_block_cb; 737 struct tc_indr_block_dev *indr_dev; 738 739 indr_dev = tc_indr_block_dev_lookup(dev); 740 if (!indr_dev) 741 return; 742 743 indr_block_cb = tc_indr_block_cb_lookup(indr_dev, cb, cb_ident); 744 if (!indr_block_cb) 745 return; 746 747 /* Send unbind message if required to free any block cbs. */ 748 tc_indr_block_ing_cmd(indr_dev, indr_block_cb, TC_BLOCK_UNBIND); 749 tc_indr_block_cb_del(indr_block_cb); 750 tc_indr_block_dev_put(indr_dev); 751 } 752 EXPORT_SYMBOL_GPL(__tc_indr_block_cb_unregister); 753 754 void tc_indr_block_cb_unregister(struct net_device *dev, 755 tc_indr_block_bind_cb_t *cb, void *cb_ident) 756 { 757 rtnl_lock(); 758 __tc_indr_block_cb_unregister(dev, cb, cb_ident); 759 rtnl_unlock(); 760 } 761 EXPORT_SYMBOL_GPL(tc_indr_block_cb_unregister); 762 763 static void tc_indr_block_call(struct tcf_block *block, struct net_device *dev, 764 struct tcf_block_ext_info *ei, 765 enum tc_block_command command, 766 struct netlink_ext_ack *extack) 767 { 768 struct tc_indr_block_cb *indr_block_cb; 769 struct tc_indr_block_dev *indr_dev; 770 struct tc_block_offload bo = { 771 .command = command, 772 .binder_type = ei->binder_type, 773 .block = block, 774 .extack = extack, 775 }; 776 777 indr_dev = tc_indr_block_dev_lookup(dev); 778 if (!indr_dev) 779 return; 780 781 indr_dev->block = command == TC_BLOCK_BIND ? block : NULL; 782 783 list_for_each_entry(indr_block_cb, &indr_dev->cb_list, list) 784 indr_block_cb->cb(dev, indr_block_cb->cb_priv, TC_SETUP_BLOCK, 785 &bo); 786 } 787 788 static bool tcf_block_offload_in_use(struct tcf_block *block) 789 { 790 return block->offloadcnt; 791 } 792 793 static int tcf_block_offload_cmd(struct tcf_block *block, 794 struct net_device *dev, 795 struct tcf_block_ext_info *ei, 796 enum tc_block_command command, 797 struct netlink_ext_ack *extack) 798 { 799 struct tc_block_offload bo = {}; 800 801 bo.command = command; 802 bo.binder_type = ei->binder_type; 803 bo.block = block; 804 bo.extack = extack; 805 return dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_BLOCK, &bo); 806 } 807 808 static int tcf_block_offload_bind(struct tcf_block *block, struct Qdisc *q, 809 struct tcf_block_ext_info *ei, 810 struct netlink_ext_ack *extack) 811 { 812 struct net_device *dev = q->dev_queue->dev; 813 int err; 814 815 if (!dev->netdev_ops->ndo_setup_tc) 816 goto no_offload_dev_inc; 817 818 /* If tc offload feature is disabled and the block we try to bind 819 * to already has some offloaded filters, forbid to bind. 820 */ 821 if (!tc_can_offload(dev) && tcf_block_offload_in_use(block)) { 822 NL_SET_ERR_MSG(extack, "Bind to offloaded block failed as dev has offload disabled"); 823 return -EOPNOTSUPP; 824 } 825 826 err = tcf_block_offload_cmd(block, dev, ei, TC_BLOCK_BIND, extack); 827 if (err == -EOPNOTSUPP) 828 goto no_offload_dev_inc; 829 if (err) 830 return err; 831 832 tc_indr_block_call(block, dev, ei, TC_BLOCK_BIND, extack); 833 return 0; 834 835 no_offload_dev_inc: 836 if (tcf_block_offload_in_use(block)) 837 return -EOPNOTSUPP; 838 block->nooffloaddevcnt++; 839 tc_indr_block_call(block, dev, ei, TC_BLOCK_BIND, extack); 840 return 0; 841 } 842 843 static void tcf_block_offload_unbind(struct tcf_block *block, struct Qdisc *q, 844 struct tcf_block_ext_info *ei) 845 { 846 struct net_device *dev = q->dev_queue->dev; 847 int err; 848 849 tc_indr_block_call(block, dev, ei, TC_BLOCK_UNBIND, NULL); 850 851 if (!dev->netdev_ops->ndo_setup_tc) 852 goto no_offload_dev_dec; 853 err = tcf_block_offload_cmd(block, dev, ei, TC_BLOCK_UNBIND, NULL); 854 if (err == -EOPNOTSUPP) 855 goto no_offload_dev_dec; 856 return; 857 858 no_offload_dev_dec: 859 WARN_ON(block->nooffloaddevcnt-- == 0); 860 } 861 862 static int 863 tcf_chain0_head_change_cb_add(struct tcf_block *block, 864 struct tcf_block_ext_info *ei, 865 struct netlink_ext_ack *extack) 866 { 867 struct tcf_filter_chain_list_item *item; 868 struct tcf_chain *chain0; 869 870 item = kmalloc(sizeof(*item), GFP_KERNEL); 871 if (!item) { 872 NL_SET_ERR_MSG(extack, "Memory allocation for head change callback item failed"); 873 return -ENOMEM; 874 } 875 item->chain_head_change = ei->chain_head_change; 876 item->chain_head_change_priv = ei->chain_head_change_priv; 877 878 mutex_lock(&block->lock); 879 chain0 = block->chain0.chain; 880 if (chain0) 881 tcf_chain_hold(chain0); 882 else 883 list_add(&item->list, &block->chain0.filter_chain_list); 884 mutex_unlock(&block->lock); 885 886 if (chain0) { 887 struct tcf_proto *tp_head; 888 889 mutex_lock(&chain0->filter_chain_lock); 890 891 tp_head = tcf_chain_dereference(chain0->filter_chain, chain0); 892 if (tp_head) 893 tcf_chain_head_change_item(item, tp_head); 894 895 mutex_lock(&block->lock); 896 list_add(&item->list, &block->chain0.filter_chain_list); 897 mutex_unlock(&block->lock); 898 899 mutex_unlock(&chain0->filter_chain_lock); 900 tcf_chain_put(chain0); 901 } 902 903 return 0; 904 } 905 906 static void 907 tcf_chain0_head_change_cb_del(struct tcf_block *block, 908 struct tcf_block_ext_info *ei) 909 { 910 struct tcf_filter_chain_list_item *item; 911 912 mutex_lock(&block->lock); 913 list_for_each_entry(item, &block->chain0.filter_chain_list, list) { 914 if ((!ei->chain_head_change && !ei->chain_head_change_priv) || 915 (item->chain_head_change == ei->chain_head_change && 916 item->chain_head_change_priv == ei->chain_head_change_priv)) { 917 if (block->chain0.chain) 918 tcf_chain_head_change_item(item, NULL); 919 list_del(&item->list); 920 mutex_unlock(&block->lock); 921 922 kfree(item); 923 return; 924 } 925 } 926 mutex_unlock(&block->lock); 927 WARN_ON(1); 928 } 929 930 struct tcf_net { 931 spinlock_t idr_lock; /* Protects idr */ 932 struct idr idr; 933 }; 934 935 static unsigned int tcf_net_id; 936 937 static int tcf_block_insert(struct tcf_block *block, struct net *net, 938 struct netlink_ext_ack *extack) 939 { 940 struct tcf_net *tn = net_generic(net, tcf_net_id); 941 int err; 942 943 idr_preload(GFP_KERNEL); 944 spin_lock(&tn->idr_lock); 945 err = idr_alloc_u32(&tn->idr, block, &block->index, block->index, 946 GFP_NOWAIT); 947 spin_unlock(&tn->idr_lock); 948 idr_preload_end(); 949 950 return err; 951 } 952 953 static void tcf_block_remove(struct tcf_block *block, struct net *net) 954 { 955 struct tcf_net *tn = net_generic(net, tcf_net_id); 956 957 spin_lock(&tn->idr_lock); 958 idr_remove(&tn->idr, block->index); 959 spin_unlock(&tn->idr_lock); 960 } 961 962 static struct tcf_block *tcf_block_create(struct net *net, struct Qdisc *q, 963 u32 block_index, 964 struct netlink_ext_ack *extack) 965 { 966 struct tcf_block *block; 967 968 block = kzalloc(sizeof(*block), GFP_KERNEL); 969 if (!block) { 970 NL_SET_ERR_MSG(extack, "Memory allocation for block failed"); 971 return ERR_PTR(-ENOMEM); 972 } 973 mutex_init(&block->lock); 974 INIT_LIST_HEAD(&block->chain_list); 975 INIT_LIST_HEAD(&block->cb_list); 976 INIT_LIST_HEAD(&block->owner_list); 977 INIT_LIST_HEAD(&block->chain0.filter_chain_list); 978 979 refcount_set(&block->refcnt, 1); 980 block->net = net; 981 block->index = block_index; 982 983 /* Don't store q pointer for blocks which are shared */ 984 if (!tcf_block_shared(block)) 985 block->q = q; 986 return block; 987 } 988 989 static struct tcf_block *tcf_block_lookup(struct net *net, u32 block_index) 990 { 991 struct tcf_net *tn = net_generic(net, tcf_net_id); 992 993 return idr_find(&tn->idr, block_index); 994 } 995 996 static struct tcf_block *tcf_block_refcnt_get(struct net *net, u32 block_index) 997 { 998 struct tcf_block *block; 999 1000 rcu_read_lock(); 1001 block = tcf_block_lookup(net, block_index); 1002 if (block && !refcount_inc_not_zero(&block->refcnt)) 1003 block = NULL; 1004 rcu_read_unlock(); 1005 1006 return block; 1007 } 1008 1009 static struct tcf_chain * 1010 __tcf_get_next_chain(struct tcf_block *block, struct tcf_chain *chain) 1011 { 1012 mutex_lock(&block->lock); 1013 if (chain) 1014 chain = list_is_last(&chain->list, &block->chain_list) ? 1015 NULL : list_next_entry(chain, list); 1016 else 1017 chain = list_first_entry_or_null(&block->chain_list, 1018 struct tcf_chain, list); 1019 1020 /* skip all action-only chains */ 1021 while (chain && tcf_chain_held_by_acts_only(chain)) 1022 chain = list_is_last(&chain->list, &block->chain_list) ? 1023 NULL : list_next_entry(chain, list); 1024 1025 if (chain) 1026 tcf_chain_hold(chain); 1027 mutex_unlock(&block->lock); 1028 1029 return chain; 1030 } 1031 1032 /* Function to be used by all clients that want to iterate over all chains on 1033 * block. It properly obtains block->lock and takes reference to chain before 1034 * returning it. Users of this function must be tolerant to concurrent chain 1035 * insertion/deletion or ensure that no concurrent chain modification is 1036 * possible. Note that all netlink dump callbacks cannot guarantee to provide 1037 * consistent dump because rtnl lock is released each time skb is filled with 1038 * data and sent to user-space. 1039 */ 1040 1041 struct tcf_chain * 1042 tcf_get_next_chain(struct tcf_block *block, struct tcf_chain *chain) 1043 { 1044 struct tcf_chain *chain_next = __tcf_get_next_chain(block, chain); 1045 1046 if (chain) 1047 tcf_chain_put(chain); 1048 1049 return chain_next; 1050 } 1051 EXPORT_SYMBOL(tcf_get_next_chain); 1052 1053 static struct tcf_proto * 1054 __tcf_get_next_proto(struct tcf_chain *chain, struct tcf_proto *tp) 1055 { 1056 u32 prio = 0; 1057 1058 ASSERT_RTNL(); 1059 mutex_lock(&chain->filter_chain_lock); 1060 1061 if (!tp) { 1062 tp = tcf_chain_dereference(chain->filter_chain, chain); 1063 } else if (tcf_proto_is_deleting(tp)) { 1064 /* 'deleting' flag is set and chain->filter_chain_lock was 1065 * unlocked, which means next pointer could be invalid. Restart 1066 * search. 1067 */ 1068 prio = tp->prio + 1; 1069 tp = tcf_chain_dereference(chain->filter_chain, chain); 1070 1071 for (; tp; tp = tcf_chain_dereference(tp->next, chain)) 1072 if (!tp->deleting && tp->prio >= prio) 1073 break; 1074 } else { 1075 tp = tcf_chain_dereference(tp->next, chain); 1076 } 1077 1078 if (tp) 1079 tcf_proto_get(tp); 1080 1081 mutex_unlock(&chain->filter_chain_lock); 1082 1083 return tp; 1084 } 1085 1086 /* Function to be used by all clients that want to iterate over all tp's on 1087 * chain. Users of this function must be tolerant to concurrent tp 1088 * insertion/deletion or ensure that no concurrent chain modification is 1089 * possible. Note that all netlink dump callbacks cannot guarantee to provide 1090 * consistent dump because rtnl lock is released each time skb is filled with 1091 * data and sent to user-space. 1092 */ 1093 1094 struct tcf_proto * 1095 tcf_get_next_proto(struct tcf_chain *chain, struct tcf_proto *tp, 1096 bool rtnl_held) 1097 { 1098 struct tcf_proto *tp_next = __tcf_get_next_proto(chain, tp); 1099 1100 if (tp) 1101 tcf_proto_put(tp, rtnl_held, NULL); 1102 1103 return tp_next; 1104 } 1105 EXPORT_SYMBOL(tcf_get_next_proto); 1106 1107 static void tcf_block_flush_all_chains(struct tcf_block *block, bool rtnl_held) 1108 { 1109 struct tcf_chain *chain; 1110 1111 /* Last reference to block. At this point chains cannot be added or 1112 * removed concurrently. 1113 */ 1114 for (chain = tcf_get_next_chain(block, NULL); 1115 chain; 1116 chain = tcf_get_next_chain(block, chain)) { 1117 tcf_chain_put_explicitly_created(chain); 1118 tcf_chain_flush(chain, rtnl_held); 1119 } 1120 } 1121 1122 /* Lookup Qdisc and increments its reference counter. 1123 * Set parent, if necessary. 1124 */ 1125 1126 static int __tcf_qdisc_find(struct net *net, struct Qdisc **q, 1127 u32 *parent, int ifindex, bool rtnl_held, 1128 struct netlink_ext_ack *extack) 1129 { 1130 const struct Qdisc_class_ops *cops; 1131 struct net_device *dev; 1132 int err = 0; 1133 1134 if (ifindex == TCM_IFINDEX_MAGIC_BLOCK) 1135 return 0; 1136 1137 rcu_read_lock(); 1138 1139 /* Find link */ 1140 dev = dev_get_by_index_rcu(net, ifindex); 1141 if (!dev) { 1142 rcu_read_unlock(); 1143 return -ENODEV; 1144 } 1145 1146 /* Find qdisc */ 1147 if (!*parent) { 1148 *q = dev->qdisc; 1149 *parent = (*q)->handle; 1150 } else { 1151 *q = qdisc_lookup_rcu(dev, TC_H_MAJ(*parent)); 1152 if (!*q) { 1153 NL_SET_ERR_MSG(extack, "Parent Qdisc doesn't exists"); 1154 err = -EINVAL; 1155 goto errout_rcu; 1156 } 1157 } 1158 1159 *q = qdisc_refcount_inc_nz(*q); 1160 if (!*q) { 1161 NL_SET_ERR_MSG(extack, "Parent Qdisc doesn't exists"); 1162 err = -EINVAL; 1163 goto errout_rcu; 1164 } 1165 1166 /* Is it classful? */ 1167 cops = (*q)->ops->cl_ops; 1168 if (!cops) { 1169 NL_SET_ERR_MSG(extack, "Qdisc not classful"); 1170 err = -EINVAL; 1171 goto errout_qdisc; 1172 } 1173 1174 if (!cops->tcf_block) { 1175 NL_SET_ERR_MSG(extack, "Class doesn't support blocks"); 1176 err = -EOPNOTSUPP; 1177 goto errout_qdisc; 1178 } 1179 1180 errout_rcu: 1181 /* At this point we know that qdisc is not noop_qdisc, 1182 * which means that qdisc holds a reference to net_device 1183 * and we hold a reference to qdisc, so it is safe to release 1184 * rcu read lock. 1185 */ 1186 rcu_read_unlock(); 1187 return err; 1188 1189 errout_qdisc: 1190 rcu_read_unlock(); 1191 1192 if (rtnl_held) 1193 qdisc_put(*q); 1194 else 1195 qdisc_put_unlocked(*q); 1196 *q = NULL; 1197 1198 return err; 1199 } 1200 1201 static int __tcf_qdisc_cl_find(struct Qdisc *q, u32 parent, unsigned long *cl, 1202 int ifindex, struct netlink_ext_ack *extack) 1203 { 1204 if (ifindex == TCM_IFINDEX_MAGIC_BLOCK) 1205 return 0; 1206 1207 /* Do we search for filter, attached to class? */ 1208 if (TC_H_MIN(parent)) { 1209 const struct Qdisc_class_ops *cops = q->ops->cl_ops; 1210 1211 *cl = cops->find(q, parent); 1212 if (*cl == 0) { 1213 NL_SET_ERR_MSG(extack, "Specified class doesn't exist"); 1214 return -ENOENT; 1215 } 1216 } 1217 1218 return 0; 1219 } 1220 1221 static struct tcf_block *__tcf_block_find(struct net *net, struct Qdisc *q, 1222 unsigned long cl, int ifindex, 1223 u32 block_index, 1224 struct netlink_ext_ack *extack) 1225 { 1226 struct tcf_block *block; 1227 1228 if (ifindex == TCM_IFINDEX_MAGIC_BLOCK) { 1229 block = tcf_block_refcnt_get(net, block_index); 1230 if (!block) { 1231 NL_SET_ERR_MSG(extack, "Block of given index was not found"); 1232 return ERR_PTR(-EINVAL); 1233 } 1234 } else { 1235 const struct Qdisc_class_ops *cops = q->ops->cl_ops; 1236 1237 block = cops->tcf_block(q, cl, extack); 1238 if (!block) 1239 return ERR_PTR(-EINVAL); 1240 1241 if (tcf_block_shared(block)) { 1242 NL_SET_ERR_MSG(extack, "This filter block is shared. Please use the block index to manipulate the filters"); 1243 return ERR_PTR(-EOPNOTSUPP); 1244 } 1245 1246 /* Always take reference to block in order to support execution 1247 * of rules update path of cls API without rtnl lock. Caller 1248 * must release block when it is finished using it. 'if' block 1249 * of this conditional obtain reference to block by calling 1250 * tcf_block_refcnt_get(). 1251 */ 1252 refcount_inc(&block->refcnt); 1253 } 1254 1255 return block; 1256 } 1257 1258 static void __tcf_block_put(struct tcf_block *block, struct Qdisc *q, 1259 struct tcf_block_ext_info *ei, bool rtnl_held) 1260 { 1261 if (refcount_dec_and_mutex_lock(&block->refcnt, &block->lock)) { 1262 /* Flushing/putting all chains will cause the block to be 1263 * deallocated when last chain is freed. However, if chain_list 1264 * is empty, block has to be manually deallocated. After block 1265 * reference counter reached 0, it is no longer possible to 1266 * increment it or add new chains to block. 1267 */ 1268 bool free_block = list_empty(&block->chain_list); 1269 1270 mutex_unlock(&block->lock); 1271 if (tcf_block_shared(block)) 1272 tcf_block_remove(block, block->net); 1273 1274 if (q) 1275 tcf_block_offload_unbind(block, q, ei); 1276 1277 if (free_block) 1278 tcf_block_destroy(block); 1279 else 1280 tcf_block_flush_all_chains(block, rtnl_held); 1281 } else if (q) { 1282 tcf_block_offload_unbind(block, q, ei); 1283 } 1284 } 1285 1286 static void tcf_block_refcnt_put(struct tcf_block *block, bool rtnl_held) 1287 { 1288 __tcf_block_put(block, NULL, NULL, rtnl_held); 1289 } 1290 1291 /* Find tcf block. 1292 * Set q, parent, cl when appropriate. 1293 */ 1294 1295 static struct tcf_block *tcf_block_find(struct net *net, struct Qdisc **q, 1296 u32 *parent, unsigned long *cl, 1297 int ifindex, u32 block_index, 1298 struct netlink_ext_ack *extack) 1299 { 1300 struct tcf_block *block; 1301 int err = 0; 1302 1303 ASSERT_RTNL(); 1304 1305 err = __tcf_qdisc_find(net, q, parent, ifindex, true, extack); 1306 if (err) 1307 goto errout; 1308 1309 err = __tcf_qdisc_cl_find(*q, *parent, cl, ifindex, extack); 1310 if (err) 1311 goto errout_qdisc; 1312 1313 block = __tcf_block_find(net, *q, *cl, ifindex, block_index, extack); 1314 if (IS_ERR(block)) { 1315 err = PTR_ERR(block); 1316 goto errout_qdisc; 1317 } 1318 1319 return block; 1320 1321 errout_qdisc: 1322 if (*q) 1323 qdisc_put(*q); 1324 errout: 1325 *q = NULL; 1326 return ERR_PTR(err); 1327 } 1328 1329 static void tcf_block_release(struct Qdisc *q, struct tcf_block *block, 1330 bool rtnl_held) 1331 { 1332 if (!IS_ERR_OR_NULL(block)) 1333 tcf_block_refcnt_put(block, rtnl_held); 1334 1335 if (q) { 1336 if (rtnl_held) 1337 qdisc_put(q); 1338 else 1339 qdisc_put_unlocked(q); 1340 } 1341 } 1342 1343 struct tcf_block_owner_item { 1344 struct list_head list; 1345 struct Qdisc *q; 1346 enum tcf_block_binder_type binder_type; 1347 }; 1348 1349 static void 1350 tcf_block_owner_netif_keep_dst(struct tcf_block *block, 1351 struct Qdisc *q, 1352 enum tcf_block_binder_type binder_type) 1353 { 1354 if (block->keep_dst && 1355 binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS && 1356 binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_EGRESS) 1357 netif_keep_dst(qdisc_dev(q)); 1358 } 1359 1360 void tcf_block_netif_keep_dst(struct tcf_block *block) 1361 { 1362 struct tcf_block_owner_item *item; 1363 1364 block->keep_dst = true; 1365 list_for_each_entry(item, &block->owner_list, list) 1366 tcf_block_owner_netif_keep_dst(block, item->q, 1367 item->binder_type); 1368 } 1369 EXPORT_SYMBOL(tcf_block_netif_keep_dst); 1370 1371 static int tcf_block_owner_add(struct tcf_block *block, 1372 struct Qdisc *q, 1373 enum tcf_block_binder_type binder_type) 1374 { 1375 struct tcf_block_owner_item *item; 1376 1377 item = kmalloc(sizeof(*item), GFP_KERNEL); 1378 if (!item) 1379 return -ENOMEM; 1380 item->q = q; 1381 item->binder_type = binder_type; 1382 list_add(&item->list, &block->owner_list); 1383 return 0; 1384 } 1385 1386 static void tcf_block_owner_del(struct tcf_block *block, 1387 struct Qdisc *q, 1388 enum tcf_block_binder_type binder_type) 1389 { 1390 struct tcf_block_owner_item *item; 1391 1392 list_for_each_entry(item, &block->owner_list, list) { 1393 if (item->q == q && item->binder_type == binder_type) { 1394 list_del(&item->list); 1395 kfree(item); 1396 return; 1397 } 1398 } 1399 WARN_ON(1); 1400 } 1401 1402 int tcf_block_get_ext(struct tcf_block **p_block, struct Qdisc *q, 1403 struct tcf_block_ext_info *ei, 1404 struct netlink_ext_ack *extack) 1405 { 1406 struct net *net = qdisc_net(q); 1407 struct tcf_block *block = NULL; 1408 int err; 1409 1410 if (ei->block_index) 1411 /* block_index not 0 means the shared block is requested */ 1412 block = tcf_block_refcnt_get(net, ei->block_index); 1413 1414 if (!block) { 1415 block = tcf_block_create(net, q, ei->block_index, extack); 1416 if (IS_ERR(block)) 1417 return PTR_ERR(block); 1418 if (tcf_block_shared(block)) { 1419 err = tcf_block_insert(block, net, extack); 1420 if (err) 1421 goto err_block_insert; 1422 } 1423 } 1424 1425 err = tcf_block_owner_add(block, q, ei->binder_type); 1426 if (err) 1427 goto err_block_owner_add; 1428 1429 tcf_block_owner_netif_keep_dst(block, q, ei->binder_type); 1430 1431 err = tcf_chain0_head_change_cb_add(block, ei, extack); 1432 if (err) 1433 goto err_chain0_head_change_cb_add; 1434 1435 err = tcf_block_offload_bind(block, q, ei, extack); 1436 if (err) 1437 goto err_block_offload_bind; 1438 1439 *p_block = block; 1440 return 0; 1441 1442 err_block_offload_bind: 1443 tcf_chain0_head_change_cb_del(block, ei); 1444 err_chain0_head_change_cb_add: 1445 tcf_block_owner_del(block, q, ei->binder_type); 1446 err_block_owner_add: 1447 err_block_insert: 1448 tcf_block_refcnt_put(block, true); 1449 return err; 1450 } 1451 EXPORT_SYMBOL(tcf_block_get_ext); 1452 1453 static void tcf_chain_head_change_dflt(struct tcf_proto *tp_head, void *priv) 1454 { 1455 struct tcf_proto __rcu **p_filter_chain = priv; 1456 1457 rcu_assign_pointer(*p_filter_chain, tp_head); 1458 } 1459 1460 int tcf_block_get(struct tcf_block **p_block, 1461 struct tcf_proto __rcu **p_filter_chain, struct Qdisc *q, 1462 struct netlink_ext_ack *extack) 1463 { 1464 struct tcf_block_ext_info ei = { 1465 .chain_head_change = tcf_chain_head_change_dflt, 1466 .chain_head_change_priv = p_filter_chain, 1467 }; 1468 1469 WARN_ON(!p_filter_chain); 1470 return tcf_block_get_ext(p_block, q, &ei, extack); 1471 } 1472 EXPORT_SYMBOL(tcf_block_get); 1473 1474 /* XXX: Standalone actions are not allowed to jump to any chain, and bound 1475 * actions should be all removed after flushing. 1476 */ 1477 void tcf_block_put_ext(struct tcf_block *block, struct Qdisc *q, 1478 struct tcf_block_ext_info *ei) 1479 { 1480 if (!block) 1481 return; 1482 tcf_chain0_head_change_cb_del(block, ei); 1483 tcf_block_owner_del(block, q, ei->binder_type); 1484 1485 __tcf_block_put(block, q, ei, true); 1486 } 1487 EXPORT_SYMBOL(tcf_block_put_ext); 1488 1489 void tcf_block_put(struct tcf_block *block) 1490 { 1491 struct tcf_block_ext_info ei = {0, }; 1492 1493 if (!block) 1494 return; 1495 tcf_block_put_ext(block, block->q, &ei); 1496 } 1497 1498 EXPORT_SYMBOL(tcf_block_put); 1499 1500 struct tcf_block_cb { 1501 struct list_head list; 1502 tc_setup_cb_t *cb; 1503 void *cb_ident; 1504 void *cb_priv; 1505 unsigned int refcnt; 1506 }; 1507 1508 void *tcf_block_cb_priv(struct tcf_block_cb *block_cb) 1509 { 1510 return block_cb->cb_priv; 1511 } 1512 EXPORT_SYMBOL(tcf_block_cb_priv); 1513 1514 struct tcf_block_cb *tcf_block_cb_lookup(struct tcf_block *block, 1515 tc_setup_cb_t *cb, void *cb_ident) 1516 { struct tcf_block_cb *block_cb; 1517 1518 list_for_each_entry(block_cb, &block->cb_list, list) 1519 if (block_cb->cb == cb && block_cb->cb_ident == cb_ident) 1520 return block_cb; 1521 return NULL; 1522 } 1523 EXPORT_SYMBOL(tcf_block_cb_lookup); 1524 1525 void tcf_block_cb_incref(struct tcf_block_cb *block_cb) 1526 { 1527 block_cb->refcnt++; 1528 } 1529 EXPORT_SYMBOL(tcf_block_cb_incref); 1530 1531 unsigned int tcf_block_cb_decref(struct tcf_block_cb *block_cb) 1532 { 1533 return --block_cb->refcnt; 1534 } 1535 EXPORT_SYMBOL(tcf_block_cb_decref); 1536 1537 static int 1538 tcf_block_playback_offloads(struct tcf_block *block, tc_setup_cb_t *cb, 1539 void *cb_priv, bool add, bool offload_in_use, 1540 struct netlink_ext_ack *extack) 1541 { 1542 struct tcf_chain *chain, *chain_prev; 1543 struct tcf_proto *tp, *tp_prev; 1544 int err; 1545 1546 for (chain = __tcf_get_next_chain(block, NULL); 1547 chain; 1548 chain_prev = chain, 1549 chain = __tcf_get_next_chain(block, chain), 1550 tcf_chain_put(chain_prev)) { 1551 for (tp = __tcf_get_next_proto(chain, NULL); tp; 1552 tp_prev = tp, 1553 tp = __tcf_get_next_proto(chain, tp), 1554 tcf_proto_put(tp_prev, true, NULL)) { 1555 if (tp->ops->reoffload) { 1556 err = tp->ops->reoffload(tp, add, cb, cb_priv, 1557 extack); 1558 if (err && add) 1559 goto err_playback_remove; 1560 } else if (add && offload_in_use) { 1561 err = -EOPNOTSUPP; 1562 NL_SET_ERR_MSG(extack, "Filter HW offload failed - classifier without re-offloading support"); 1563 goto err_playback_remove; 1564 } 1565 } 1566 } 1567 1568 return 0; 1569 1570 err_playback_remove: 1571 tcf_proto_put(tp, true, NULL); 1572 tcf_chain_put(chain); 1573 tcf_block_playback_offloads(block, cb, cb_priv, false, offload_in_use, 1574 extack); 1575 return err; 1576 } 1577 1578 struct tcf_block_cb *__tcf_block_cb_register(struct tcf_block *block, 1579 tc_setup_cb_t *cb, void *cb_ident, 1580 void *cb_priv, 1581 struct netlink_ext_ack *extack) 1582 { 1583 struct tcf_block_cb *block_cb; 1584 int err; 1585 1586 /* Replay any already present rules */ 1587 err = tcf_block_playback_offloads(block, cb, cb_priv, true, 1588 tcf_block_offload_in_use(block), 1589 extack); 1590 if (err) 1591 return ERR_PTR(err); 1592 1593 block_cb = kzalloc(sizeof(*block_cb), GFP_KERNEL); 1594 if (!block_cb) 1595 return ERR_PTR(-ENOMEM); 1596 block_cb->cb = cb; 1597 block_cb->cb_ident = cb_ident; 1598 block_cb->cb_priv = cb_priv; 1599 list_add(&block_cb->list, &block->cb_list); 1600 return block_cb; 1601 } 1602 EXPORT_SYMBOL(__tcf_block_cb_register); 1603 1604 int tcf_block_cb_register(struct tcf_block *block, 1605 tc_setup_cb_t *cb, void *cb_ident, 1606 void *cb_priv, struct netlink_ext_ack *extack) 1607 { 1608 struct tcf_block_cb *block_cb; 1609 1610 block_cb = __tcf_block_cb_register(block, cb, cb_ident, cb_priv, 1611 extack); 1612 return PTR_ERR_OR_ZERO(block_cb); 1613 } 1614 EXPORT_SYMBOL(tcf_block_cb_register); 1615 1616 void __tcf_block_cb_unregister(struct tcf_block *block, 1617 struct tcf_block_cb *block_cb) 1618 { 1619 tcf_block_playback_offloads(block, block_cb->cb, block_cb->cb_priv, 1620 false, tcf_block_offload_in_use(block), 1621 NULL); 1622 list_del(&block_cb->list); 1623 kfree(block_cb); 1624 } 1625 EXPORT_SYMBOL(__tcf_block_cb_unregister); 1626 1627 void tcf_block_cb_unregister(struct tcf_block *block, 1628 tc_setup_cb_t *cb, void *cb_ident) 1629 { 1630 struct tcf_block_cb *block_cb; 1631 1632 block_cb = tcf_block_cb_lookup(block, cb, cb_ident); 1633 if (!block_cb) 1634 return; 1635 __tcf_block_cb_unregister(block, block_cb); 1636 } 1637 EXPORT_SYMBOL(tcf_block_cb_unregister); 1638 1639 /* Main classifier routine: scans classifier chain attached 1640 * to this qdisc, (optionally) tests for protocol and asks 1641 * specific classifiers. 1642 */ 1643 int tcf_classify(struct sk_buff *skb, const struct tcf_proto *tp, 1644 struct tcf_result *res, bool compat_mode) 1645 { 1646 #ifdef CONFIG_NET_CLS_ACT 1647 const int max_reclassify_loop = 4; 1648 const struct tcf_proto *orig_tp = tp; 1649 const struct tcf_proto *first_tp; 1650 int limit = 0; 1651 1652 reclassify: 1653 #endif 1654 for (; tp; tp = rcu_dereference_bh(tp->next)) { 1655 __be16 protocol = tc_skb_protocol(skb); 1656 int err; 1657 1658 if (tp->protocol != protocol && 1659 tp->protocol != htons(ETH_P_ALL)) 1660 continue; 1661 1662 err = tp->classify(skb, tp, res); 1663 #ifdef CONFIG_NET_CLS_ACT 1664 if (unlikely(err == TC_ACT_RECLASSIFY && !compat_mode)) { 1665 first_tp = orig_tp; 1666 goto reset; 1667 } else if (unlikely(TC_ACT_EXT_CMP(err, TC_ACT_GOTO_CHAIN))) { 1668 first_tp = res->goto_tp; 1669 goto reset; 1670 } 1671 #endif 1672 if (err >= 0) 1673 return err; 1674 } 1675 1676 return TC_ACT_UNSPEC; /* signal: continue lookup */ 1677 #ifdef CONFIG_NET_CLS_ACT 1678 reset: 1679 if (unlikely(limit++ >= max_reclassify_loop)) { 1680 net_notice_ratelimited("%u: reclassify loop, rule prio %u, protocol %02x\n", 1681 tp->chain->block->index, 1682 tp->prio & 0xffff, 1683 ntohs(tp->protocol)); 1684 return TC_ACT_SHOT; 1685 } 1686 1687 tp = first_tp; 1688 goto reclassify; 1689 #endif 1690 } 1691 EXPORT_SYMBOL(tcf_classify); 1692 1693 struct tcf_chain_info { 1694 struct tcf_proto __rcu **pprev; 1695 struct tcf_proto __rcu *next; 1696 }; 1697 1698 static struct tcf_proto *tcf_chain_tp_prev(struct tcf_chain *chain, 1699 struct tcf_chain_info *chain_info) 1700 { 1701 return tcf_chain_dereference(*chain_info->pprev, chain); 1702 } 1703 1704 static int tcf_chain_tp_insert(struct tcf_chain *chain, 1705 struct tcf_chain_info *chain_info, 1706 struct tcf_proto *tp) 1707 { 1708 if (chain->flushing) 1709 return -EAGAIN; 1710 1711 if (*chain_info->pprev == chain->filter_chain) 1712 tcf_chain0_head_change(chain, tp); 1713 tcf_proto_get(tp); 1714 RCU_INIT_POINTER(tp->next, tcf_chain_tp_prev(chain, chain_info)); 1715 rcu_assign_pointer(*chain_info->pprev, tp); 1716 1717 return 0; 1718 } 1719 1720 static void tcf_chain_tp_remove(struct tcf_chain *chain, 1721 struct tcf_chain_info *chain_info, 1722 struct tcf_proto *tp) 1723 { 1724 struct tcf_proto *next = tcf_chain_dereference(chain_info->next, chain); 1725 1726 tcf_proto_mark_delete(tp); 1727 if (tp == chain->filter_chain) 1728 tcf_chain0_head_change(chain, next); 1729 RCU_INIT_POINTER(*chain_info->pprev, next); 1730 } 1731 1732 static struct tcf_proto *tcf_chain_tp_find(struct tcf_chain *chain, 1733 struct tcf_chain_info *chain_info, 1734 u32 protocol, u32 prio, 1735 bool prio_allocate); 1736 1737 /* Try to insert new proto. 1738 * If proto with specified priority already exists, free new proto 1739 * and return existing one. 1740 */ 1741 1742 static struct tcf_proto *tcf_chain_tp_insert_unique(struct tcf_chain *chain, 1743 struct tcf_proto *tp_new, 1744 u32 protocol, u32 prio, 1745 bool rtnl_held) 1746 { 1747 struct tcf_chain_info chain_info; 1748 struct tcf_proto *tp; 1749 int err = 0; 1750 1751 mutex_lock(&chain->filter_chain_lock); 1752 1753 tp = tcf_chain_tp_find(chain, &chain_info, 1754 protocol, prio, false); 1755 if (!tp) 1756 err = tcf_chain_tp_insert(chain, &chain_info, tp_new); 1757 mutex_unlock(&chain->filter_chain_lock); 1758 1759 if (tp) { 1760 tcf_proto_destroy(tp_new, rtnl_held, NULL); 1761 tp_new = tp; 1762 } else if (err) { 1763 tcf_proto_destroy(tp_new, rtnl_held, NULL); 1764 tp_new = ERR_PTR(err); 1765 } 1766 1767 return tp_new; 1768 } 1769 1770 static void tcf_chain_tp_delete_empty(struct tcf_chain *chain, 1771 struct tcf_proto *tp, bool rtnl_held, 1772 struct netlink_ext_ack *extack) 1773 { 1774 struct tcf_chain_info chain_info; 1775 struct tcf_proto *tp_iter; 1776 struct tcf_proto **pprev; 1777 struct tcf_proto *next; 1778 1779 mutex_lock(&chain->filter_chain_lock); 1780 1781 /* Atomically find and remove tp from chain. */ 1782 for (pprev = &chain->filter_chain; 1783 (tp_iter = tcf_chain_dereference(*pprev, chain)); 1784 pprev = &tp_iter->next) { 1785 if (tp_iter == tp) { 1786 chain_info.pprev = pprev; 1787 chain_info.next = tp_iter->next; 1788 WARN_ON(tp_iter->deleting); 1789 break; 1790 } 1791 } 1792 /* Verify that tp still exists and no new filters were inserted 1793 * concurrently. 1794 * Mark tp for deletion if it is empty. 1795 */ 1796 if (!tp_iter || !tcf_proto_check_delete(tp, rtnl_held)) { 1797 mutex_unlock(&chain->filter_chain_lock); 1798 return; 1799 } 1800 1801 next = tcf_chain_dereference(chain_info.next, chain); 1802 if (tp == chain->filter_chain) 1803 tcf_chain0_head_change(chain, next); 1804 RCU_INIT_POINTER(*chain_info.pprev, next); 1805 mutex_unlock(&chain->filter_chain_lock); 1806 1807 tcf_proto_put(tp, rtnl_held, extack); 1808 } 1809 1810 static struct tcf_proto *tcf_chain_tp_find(struct tcf_chain *chain, 1811 struct tcf_chain_info *chain_info, 1812 u32 protocol, u32 prio, 1813 bool prio_allocate) 1814 { 1815 struct tcf_proto **pprev; 1816 struct tcf_proto *tp; 1817 1818 /* Check the chain for existence of proto-tcf with this priority */ 1819 for (pprev = &chain->filter_chain; 1820 (tp = tcf_chain_dereference(*pprev, chain)); 1821 pprev = &tp->next) { 1822 if (tp->prio >= prio) { 1823 if (tp->prio == prio) { 1824 if (prio_allocate || 1825 (tp->protocol != protocol && protocol)) 1826 return ERR_PTR(-EINVAL); 1827 } else { 1828 tp = NULL; 1829 } 1830 break; 1831 } 1832 } 1833 chain_info->pprev = pprev; 1834 if (tp) { 1835 chain_info->next = tp->next; 1836 tcf_proto_get(tp); 1837 } else { 1838 chain_info->next = NULL; 1839 } 1840 return tp; 1841 } 1842 1843 static int tcf_fill_node(struct net *net, struct sk_buff *skb, 1844 struct tcf_proto *tp, struct tcf_block *block, 1845 struct Qdisc *q, u32 parent, void *fh, 1846 u32 portid, u32 seq, u16 flags, int event, 1847 bool rtnl_held) 1848 { 1849 struct tcmsg *tcm; 1850 struct nlmsghdr *nlh; 1851 unsigned char *b = skb_tail_pointer(skb); 1852 1853 nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags); 1854 if (!nlh) 1855 goto out_nlmsg_trim; 1856 tcm = nlmsg_data(nlh); 1857 tcm->tcm_family = AF_UNSPEC; 1858 tcm->tcm__pad1 = 0; 1859 tcm->tcm__pad2 = 0; 1860 if (q) { 1861 tcm->tcm_ifindex = qdisc_dev(q)->ifindex; 1862 tcm->tcm_parent = parent; 1863 } else { 1864 tcm->tcm_ifindex = TCM_IFINDEX_MAGIC_BLOCK; 1865 tcm->tcm_block_index = block->index; 1866 } 1867 tcm->tcm_info = TC_H_MAKE(tp->prio, tp->protocol); 1868 if (nla_put_string(skb, TCA_KIND, tp->ops->kind)) 1869 goto nla_put_failure; 1870 if (nla_put_u32(skb, TCA_CHAIN, tp->chain->index)) 1871 goto nla_put_failure; 1872 if (!fh) { 1873 tcm->tcm_handle = 0; 1874 } else { 1875 if (tp->ops->dump && 1876 tp->ops->dump(net, tp, fh, skb, tcm, rtnl_held) < 0) 1877 goto nla_put_failure; 1878 } 1879 nlh->nlmsg_len = skb_tail_pointer(skb) - b; 1880 return skb->len; 1881 1882 out_nlmsg_trim: 1883 nla_put_failure: 1884 nlmsg_trim(skb, b); 1885 return -1; 1886 } 1887 1888 static int tfilter_notify(struct net *net, struct sk_buff *oskb, 1889 struct nlmsghdr *n, struct tcf_proto *tp, 1890 struct tcf_block *block, struct Qdisc *q, 1891 u32 parent, void *fh, int event, bool unicast, 1892 bool rtnl_held) 1893 { 1894 struct sk_buff *skb; 1895 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0; 1896 1897 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); 1898 if (!skb) 1899 return -ENOBUFS; 1900 1901 if (tcf_fill_node(net, skb, tp, block, q, parent, fh, portid, 1902 n->nlmsg_seq, n->nlmsg_flags, event, 1903 rtnl_held) <= 0) { 1904 kfree_skb(skb); 1905 return -EINVAL; 1906 } 1907 1908 if (unicast) 1909 return netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT); 1910 1911 return rtnetlink_send(skb, net, portid, RTNLGRP_TC, 1912 n->nlmsg_flags & NLM_F_ECHO); 1913 } 1914 1915 static int tfilter_del_notify(struct net *net, struct sk_buff *oskb, 1916 struct nlmsghdr *n, struct tcf_proto *tp, 1917 struct tcf_block *block, struct Qdisc *q, 1918 u32 parent, void *fh, bool unicast, bool *last, 1919 bool rtnl_held, struct netlink_ext_ack *extack) 1920 { 1921 struct sk_buff *skb; 1922 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0; 1923 int err; 1924 1925 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); 1926 if (!skb) 1927 return -ENOBUFS; 1928 1929 if (tcf_fill_node(net, skb, tp, block, q, parent, fh, portid, 1930 n->nlmsg_seq, n->nlmsg_flags, RTM_DELTFILTER, 1931 rtnl_held) <= 0) { 1932 NL_SET_ERR_MSG(extack, "Failed to build del event notification"); 1933 kfree_skb(skb); 1934 return -EINVAL; 1935 } 1936 1937 err = tp->ops->delete(tp, fh, last, rtnl_held, extack); 1938 if (err) { 1939 kfree_skb(skb); 1940 return err; 1941 } 1942 1943 if (unicast) 1944 return netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT); 1945 1946 err = rtnetlink_send(skb, net, portid, RTNLGRP_TC, 1947 n->nlmsg_flags & NLM_F_ECHO); 1948 if (err < 0) 1949 NL_SET_ERR_MSG(extack, "Failed to send filter delete notification"); 1950 return err; 1951 } 1952 1953 static void tfilter_notify_chain(struct net *net, struct sk_buff *oskb, 1954 struct tcf_block *block, struct Qdisc *q, 1955 u32 parent, struct nlmsghdr *n, 1956 struct tcf_chain *chain, int event, 1957 bool rtnl_held) 1958 { 1959 struct tcf_proto *tp; 1960 1961 for (tp = tcf_get_next_proto(chain, NULL, rtnl_held); 1962 tp; tp = tcf_get_next_proto(chain, tp, rtnl_held)) 1963 tfilter_notify(net, oskb, n, tp, block, 1964 q, parent, NULL, event, false, rtnl_held); 1965 } 1966 1967 static void tfilter_put(struct tcf_proto *tp, void *fh) 1968 { 1969 if (tp->ops->put && fh) 1970 tp->ops->put(tp, fh); 1971 } 1972 1973 static int tc_new_tfilter(struct sk_buff *skb, struct nlmsghdr *n, 1974 struct netlink_ext_ack *extack) 1975 { 1976 struct net *net = sock_net(skb->sk); 1977 struct nlattr *tca[TCA_MAX + 1]; 1978 struct tcmsg *t; 1979 u32 protocol; 1980 u32 prio; 1981 bool prio_allocate; 1982 u32 parent; 1983 u32 chain_index; 1984 struct Qdisc *q = NULL; 1985 struct tcf_chain_info chain_info; 1986 struct tcf_chain *chain = NULL; 1987 struct tcf_block *block; 1988 struct tcf_proto *tp; 1989 unsigned long cl; 1990 void *fh; 1991 int err; 1992 int tp_created; 1993 bool rtnl_held = false; 1994 1995 if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN)) 1996 return -EPERM; 1997 1998 replay: 1999 tp_created = 0; 2000 2001 err = nlmsg_parse(n, sizeof(*t), tca, TCA_MAX, rtm_tca_policy, extack); 2002 if (err < 0) 2003 return err; 2004 2005 t = nlmsg_data(n); 2006 protocol = TC_H_MIN(t->tcm_info); 2007 prio = TC_H_MAJ(t->tcm_info); 2008 prio_allocate = false; 2009 parent = t->tcm_parent; 2010 tp = NULL; 2011 cl = 0; 2012 block = NULL; 2013 2014 if (prio == 0) { 2015 /* If no priority is provided by the user, 2016 * we allocate one. 2017 */ 2018 if (n->nlmsg_flags & NLM_F_CREATE) { 2019 prio = TC_H_MAKE(0x80000000U, 0U); 2020 prio_allocate = true; 2021 } else { 2022 NL_SET_ERR_MSG(extack, "Invalid filter command with priority of zero"); 2023 return -ENOENT; 2024 } 2025 } 2026 2027 /* Find head of filter chain. */ 2028 2029 err = __tcf_qdisc_find(net, &q, &parent, t->tcm_ifindex, false, extack); 2030 if (err) 2031 return err; 2032 2033 /* Take rtnl mutex if rtnl_held was set to true on previous iteration, 2034 * block is shared (no qdisc found), qdisc is not unlocked, classifier 2035 * type is not specified, classifier is not unlocked. 2036 */ 2037 if (rtnl_held || 2038 (q && !(q->ops->cl_ops->flags & QDISC_CLASS_OPS_DOIT_UNLOCKED)) || 2039 !tca[TCA_KIND] || !tcf_proto_is_unlocked(nla_data(tca[TCA_KIND]))) { 2040 rtnl_held = true; 2041 rtnl_lock(); 2042 } 2043 2044 err = __tcf_qdisc_cl_find(q, parent, &cl, t->tcm_ifindex, extack); 2045 if (err) 2046 goto errout; 2047 2048 block = __tcf_block_find(net, q, cl, t->tcm_ifindex, t->tcm_block_index, 2049 extack); 2050 if (IS_ERR(block)) { 2051 err = PTR_ERR(block); 2052 goto errout; 2053 } 2054 2055 chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0; 2056 if (chain_index > TC_ACT_EXT_VAL_MASK) { 2057 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit"); 2058 err = -EINVAL; 2059 goto errout; 2060 } 2061 chain = tcf_chain_get(block, chain_index, true); 2062 if (!chain) { 2063 NL_SET_ERR_MSG(extack, "Cannot create specified filter chain"); 2064 err = -ENOMEM; 2065 goto errout; 2066 } 2067 2068 mutex_lock(&chain->filter_chain_lock); 2069 tp = tcf_chain_tp_find(chain, &chain_info, protocol, 2070 prio, prio_allocate); 2071 if (IS_ERR(tp)) { 2072 NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found"); 2073 err = PTR_ERR(tp); 2074 goto errout_locked; 2075 } 2076 2077 if (tp == NULL) { 2078 struct tcf_proto *tp_new = NULL; 2079 2080 if (chain->flushing) { 2081 err = -EAGAIN; 2082 goto errout_locked; 2083 } 2084 2085 /* Proto-tcf does not exist, create new one */ 2086 2087 if (tca[TCA_KIND] == NULL || !protocol) { 2088 NL_SET_ERR_MSG(extack, "Filter kind and protocol must be specified"); 2089 err = -EINVAL; 2090 goto errout_locked; 2091 } 2092 2093 if (!(n->nlmsg_flags & NLM_F_CREATE)) { 2094 NL_SET_ERR_MSG(extack, "Need both RTM_NEWTFILTER and NLM_F_CREATE to create a new filter"); 2095 err = -ENOENT; 2096 goto errout_locked; 2097 } 2098 2099 if (prio_allocate) 2100 prio = tcf_auto_prio(tcf_chain_tp_prev(chain, 2101 &chain_info)); 2102 2103 mutex_unlock(&chain->filter_chain_lock); 2104 tp_new = tcf_proto_create(nla_data(tca[TCA_KIND]), 2105 protocol, prio, chain, rtnl_held, 2106 extack); 2107 if (IS_ERR(tp_new)) { 2108 err = PTR_ERR(tp_new); 2109 goto errout_tp; 2110 } 2111 2112 tp_created = 1; 2113 tp = tcf_chain_tp_insert_unique(chain, tp_new, protocol, prio, 2114 rtnl_held); 2115 if (IS_ERR(tp)) { 2116 err = PTR_ERR(tp); 2117 goto errout_tp; 2118 } 2119 } else { 2120 mutex_unlock(&chain->filter_chain_lock); 2121 } 2122 2123 if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) { 2124 NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one"); 2125 err = -EINVAL; 2126 goto errout; 2127 } 2128 2129 fh = tp->ops->get(tp, t->tcm_handle); 2130 2131 if (!fh) { 2132 if (!(n->nlmsg_flags & NLM_F_CREATE)) { 2133 NL_SET_ERR_MSG(extack, "Need both RTM_NEWTFILTER and NLM_F_CREATE to create a new filter"); 2134 err = -ENOENT; 2135 goto errout; 2136 } 2137 } else if (n->nlmsg_flags & NLM_F_EXCL) { 2138 tfilter_put(tp, fh); 2139 NL_SET_ERR_MSG(extack, "Filter already exists"); 2140 err = -EEXIST; 2141 goto errout; 2142 } 2143 2144 if (chain->tmplt_ops && chain->tmplt_ops != tp->ops) { 2145 NL_SET_ERR_MSG(extack, "Chain template is set to a different filter kind"); 2146 err = -EINVAL; 2147 goto errout; 2148 } 2149 2150 err = tp->ops->change(net, skb, tp, cl, t->tcm_handle, tca, &fh, 2151 n->nlmsg_flags & NLM_F_CREATE ? TCA_ACT_NOREPLACE : TCA_ACT_REPLACE, 2152 rtnl_held, extack); 2153 if (err == 0) { 2154 tfilter_notify(net, skb, n, tp, block, q, parent, fh, 2155 RTM_NEWTFILTER, false, rtnl_held); 2156 tfilter_put(tp, fh); 2157 } 2158 2159 errout: 2160 if (err && tp_created) 2161 tcf_chain_tp_delete_empty(chain, tp, rtnl_held, NULL); 2162 errout_tp: 2163 if (chain) { 2164 if (tp && !IS_ERR(tp)) 2165 tcf_proto_put(tp, rtnl_held, NULL); 2166 if (!tp_created) 2167 tcf_chain_put(chain); 2168 } 2169 tcf_block_release(q, block, rtnl_held); 2170 2171 if (rtnl_held) 2172 rtnl_unlock(); 2173 2174 if (err == -EAGAIN) { 2175 /* Take rtnl lock in case EAGAIN is caused by concurrent flush 2176 * of target chain. 2177 */ 2178 rtnl_held = true; 2179 /* Replay the request. */ 2180 goto replay; 2181 } 2182 return err; 2183 2184 errout_locked: 2185 mutex_unlock(&chain->filter_chain_lock); 2186 goto errout; 2187 } 2188 2189 static int tc_del_tfilter(struct sk_buff *skb, struct nlmsghdr *n, 2190 struct netlink_ext_ack *extack) 2191 { 2192 struct net *net = sock_net(skb->sk); 2193 struct nlattr *tca[TCA_MAX + 1]; 2194 struct tcmsg *t; 2195 u32 protocol; 2196 u32 prio; 2197 u32 parent; 2198 u32 chain_index; 2199 struct Qdisc *q = NULL; 2200 struct tcf_chain_info chain_info; 2201 struct tcf_chain *chain = NULL; 2202 struct tcf_block *block = NULL; 2203 struct tcf_proto *tp = NULL; 2204 unsigned long cl = 0; 2205 void *fh = NULL; 2206 int err; 2207 bool rtnl_held = false; 2208 2209 if (!netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN)) 2210 return -EPERM; 2211 2212 err = nlmsg_parse(n, sizeof(*t), tca, TCA_MAX, rtm_tca_policy, extack); 2213 if (err < 0) 2214 return err; 2215 2216 t = nlmsg_data(n); 2217 protocol = TC_H_MIN(t->tcm_info); 2218 prio = TC_H_MAJ(t->tcm_info); 2219 parent = t->tcm_parent; 2220 2221 if (prio == 0 && (protocol || t->tcm_handle || tca[TCA_KIND])) { 2222 NL_SET_ERR_MSG(extack, "Cannot flush filters with protocol, handle or kind set"); 2223 return -ENOENT; 2224 } 2225 2226 /* Find head of filter chain. */ 2227 2228 err = __tcf_qdisc_find(net, &q, &parent, t->tcm_ifindex, false, extack); 2229 if (err) 2230 return err; 2231 2232 /* Take rtnl mutex if flushing whole chain, block is shared (no qdisc 2233 * found), qdisc is not unlocked, classifier type is not specified, 2234 * classifier is not unlocked. 2235 */ 2236 if (!prio || 2237 (q && !(q->ops->cl_ops->flags & QDISC_CLASS_OPS_DOIT_UNLOCKED)) || 2238 !tca[TCA_KIND] || !tcf_proto_is_unlocked(nla_data(tca[TCA_KIND]))) { 2239 rtnl_held = true; 2240 rtnl_lock(); 2241 } 2242 2243 err = __tcf_qdisc_cl_find(q, parent, &cl, t->tcm_ifindex, extack); 2244 if (err) 2245 goto errout; 2246 2247 block = __tcf_block_find(net, q, cl, t->tcm_ifindex, t->tcm_block_index, 2248 extack); 2249 if (IS_ERR(block)) { 2250 err = PTR_ERR(block); 2251 goto errout; 2252 } 2253 2254 chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0; 2255 if (chain_index > TC_ACT_EXT_VAL_MASK) { 2256 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit"); 2257 err = -EINVAL; 2258 goto errout; 2259 } 2260 chain = tcf_chain_get(block, chain_index, false); 2261 if (!chain) { 2262 /* User requested flush on non-existent chain. Nothing to do, 2263 * so just return success. 2264 */ 2265 if (prio == 0) { 2266 err = 0; 2267 goto errout; 2268 } 2269 NL_SET_ERR_MSG(extack, "Cannot find specified filter chain"); 2270 err = -ENOENT; 2271 goto errout; 2272 } 2273 2274 if (prio == 0) { 2275 tfilter_notify_chain(net, skb, block, q, parent, n, 2276 chain, RTM_DELTFILTER, rtnl_held); 2277 tcf_chain_flush(chain, rtnl_held); 2278 err = 0; 2279 goto errout; 2280 } 2281 2282 mutex_lock(&chain->filter_chain_lock); 2283 tp = tcf_chain_tp_find(chain, &chain_info, protocol, 2284 prio, false); 2285 if (!tp || IS_ERR(tp)) { 2286 NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found"); 2287 err = tp ? PTR_ERR(tp) : -ENOENT; 2288 goto errout_locked; 2289 } else if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) { 2290 NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one"); 2291 err = -EINVAL; 2292 goto errout_locked; 2293 } else if (t->tcm_handle == 0) { 2294 tcf_chain_tp_remove(chain, &chain_info, tp); 2295 mutex_unlock(&chain->filter_chain_lock); 2296 2297 tcf_proto_put(tp, rtnl_held, NULL); 2298 tfilter_notify(net, skb, n, tp, block, q, parent, fh, 2299 RTM_DELTFILTER, false, rtnl_held); 2300 err = 0; 2301 goto errout; 2302 } 2303 mutex_unlock(&chain->filter_chain_lock); 2304 2305 fh = tp->ops->get(tp, t->tcm_handle); 2306 2307 if (!fh) { 2308 NL_SET_ERR_MSG(extack, "Specified filter handle not found"); 2309 err = -ENOENT; 2310 } else { 2311 bool last; 2312 2313 err = tfilter_del_notify(net, skb, n, tp, block, 2314 q, parent, fh, false, &last, 2315 rtnl_held, extack); 2316 2317 if (err) 2318 goto errout; 2319 if (last) 2320 tcf_chain_tp_delete_empty(chain, tp, rtnl_held, extack); 2321 } 2322 2323 errout: 2324 if (chain) { 2325 if (tp && !IS_ERR(tp)) 2326 tcf_proto_put(tp, rtnl_held, NULL); 2327 tcf_chain_put(chain); 2328 } 2329 tcf_block_release(q, block, rtnl_held); 2330 2331 if (rtnl_held) 2332 rtnl_unlock(); 2333 2334 return err; 2335 2336 errout_locked: 2337 mutex_unlock(&chain->filter_chain_lock); 2338 goto errout; 2339 } 2340 2341 static int tc_get_tfilter(struct sk_buff *skb, struct nlmsghdr *n, 2342 struct netlink_ext_ack *extack) 2343 { 2344 struct net *net = sock_net(skb->sk); 2345 struct nlattr *tca[TCA_MAX + 1]; 2346 struct tcmsg *t; 2347 u32 protocol; 2348 u32 prio; 2349 u32 parent; 2350 u32 chain_index; 2351 struct Qdisc *q = NULL; 2352 struct tcf_chain_info chain_info; 2353 struct tcf_chain *chain = NULL; 2354 struct tcf_block *block = NULL; 2355 struct tcf_proto *tp = NULL; 2356 unsigned long cl = 0; 2357 void *fh = NULL; 2358 int err; 2359 bool rtnl_held = false; 2360 2361 err = nlmsg_parse(n, sizeof(*t), tca, TCA_MAX, rtm_tca_policy, extack); 2362 if (err < 0) 2363 return err; 2364 2365 t = nlmsg_data(n); 2366 protocol = TC_H_MIN(t->tcm_info); 2367 prio = TC_H_MAJ(t->tcm_info); 2368 parent = t->tcm_parent; 2369 2370 if (prio == 0) { 2371 NL_SET_ERR_MSG(extack, "Invalid filter command with priority of zero"); 2372 return -ENOENT; 2373 } 2374 2375 /* Find head of filter chain. */ 2376 2377 err = __tcf_qdisc_find(net, &q, &parent, t->tcm_ifindex, false, extack); 2378 if (err) 2379 return err; 2380 2381 /* Take rtnl mutex if block is shared (no qdisc found), qdisc is not 2382 * unlocked, classifier type is not specified, classifier is not 2383 * unlocked. 2384 */ 2385 if ((q && !(q->ops->cl_ops->flags & QDISC_CLASS_OPS_DOIT_UNLOCKED)) || 2386 !tca[TCA_KIND] || !tcf_proto_is_unlocked(nla_data(tca[TCA_KIND]))) { 2387 rtnl_held = true; 2388 rtnl_lock(); 2389 } 2390 2391 err = __tcf_qdisc_cl_find(q, parent, &cl, t->tcm_ifindex, extack); 2392 if (err) 2393 goto errout; 2394 2395 block = __tcf_block_find(net, q, cl, t->tcm_ifindex, t->tcm_block_index, 2396 extack); 2397 if (IS_ERR(block)) { 2398 err = PTR_ERR(block); 2399 goto errout; 2400 } 2401 2402 chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0; 2403 if (chain_index > TC_ACT_EXT_VAL_MASK) { 2404 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit"); 2405 err = -EINVAL; 2406 goto errout; 2407 } 2408 chain = tcf_chain_get(block, chain_index, false); 2409 if (!chain) { 2410 NL_SET_ERR_MSG(extack, "Cannot find specified filter chain"); 2411 err = -EINVAL; 2412 goto errout; 2413 } 2414 2415 mutex_lock(&chain->filter_chain_lock); 2416 tp = tcf_chain_tp_find(chain, &chain_info, protocol, 2417 prio, false); 2418 mutex_unlock(&chain->filter_chain_lock); 2419 if (!tp || IS_ERR(tp)) { 2420 NL_SET_ERR_MSG(extack, "Filter with specified priority/protocol not found"); 2421 err = tp ? PTR_ERR(tp) : -ENOENT; 2422 goto errout; 2423 } else if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) { 2424 NL_SET_ERR_MSG(extack, "Specified filter kind does not match existing one"); 2425 err = -EINVAL; 2426 goto errout; 2427 } 2428 2429 fh = tp->ops->get(tp, t->tcm_handle); 2430 2431 if (!fh) { 2432 NL_SET_ERR_MSG(extack, "Specified filter handle not found"); 2433 err = -ENOENT; 2434 } else { 2435 err = tfilter_notify(net, skb, n, tp, block, q, parent, 2436 fh, RTM_NEWTFILTER, true, rtnl_held); 2437 if (err < 0) 2438 NL_SET_ERR_MSG(extack, "Failed to send filter notify message"); 2439 } 2440 2441 tfilter_put(tp, fh); 2442 errout: 2443 if (chain) { 2444 if (tp && !IS_ERR(tp)) 2445 tcf_proto_put(tp, rtnl_held, NULL); 2446 tcf_chain_put(chain); 2447 } 2448 tcf_block_release(q, block, rtnl_held); 2449 2450 if (rtnl_held) 2451 rtnl_unlock(); 2452 2453 return err; 2454 } 2455 2456 struct tcf_dump_args { 2457 struct tcf_walker w; 2458 struct sk_buff *skb; 2459 struct netlink_callback *cb; 2460 struct tcf_block *block; 2461 struct Qdisc *q; 2462 u32 parent; 2463 }; 2464 2465 static int tcf_node_dump(struct tcf_proto *tp, void *n, struct tcf_walker *arg) 2466 { 2467 struct tcf_dump_args *a = (void *)arg; 2468 struct net *net = sock_net(a->skb->sk); 2469 2470 return tcf_fill_node(net, a->skb, tp, a->block, a->q, a->parent, 2471 n, NETLINK_CB(a->cb->skb).portid, 2472 a->cb->nlh->nlmsg_seq, NLM_F_MULTI, 2473 RTM_NEWTFILTER, true); 2474 } 2475 2476 static bool tcf_chain_dump(struct tcf_chain *chain, struct Qdisc *q, u32 parent, 2477 struct sk_buff *skb, struct netlink_callback *cb, 2478 long index_start, long *p_index) 2479 { 2480 struct net *net = sock_net(skb->sk); 2481 struct tcf_block *block = chain->block; 2482 struct tcmsg *tcm = nlmsg_data(cb->nlh); 2483 struct tcf_proto *tp, *tp_prev; 2484 struct tcf_dump_args arg; 2485 2486 for (tp = __tcf_get_next_proto(chain, NULL); 2487 tp; 2488 tp_prev = tp, 2489 tp = __tcf_get_next_proto(chain, tp), 2490 tcf_proto_put(tp_prev, true, NULL), 2491 (*p_index)++) { 2492 if (*p_index < index_start) 2493 continue; 2494 if (TC_H_MAJ(tcm->tcm_info) && 2495 TC_H_MAJ(tcm->tcm_info) != tp->prio) 2496 continue; 2497 if (TC_H_MIN(tcm->tcm_info) && 2498 TC_H_MIN(tcm->tcm_info) != tp->protocol) 2499 continue; 2500 if (*p_index > index_start) 2501 memset(&cb->args[1], 0, 2502 sizeof(cb->args) - sizeof(cb->args[0])); 2503 if (cb->args[1] == 0) { 2504 if (tcf_fill_node(net, skb, tp, block, q, parent, NULL, 2505 NETLINK_CB(cb->skb).portid, 2506 cb->nlh->nlmsg_seq, NLM_F_MULTI, 2507 RTM_NEWTFILTER, true) <= 0) 2508 goto errout; 2509 cb->args[1] = 1; 2510 } 2511 if (!tp->ops->walk) 2512 continue; 2513 arg.w.fn = tcf_node_dump; 2514 arg.skb = skb; 2515 arg.cb = cb; 2516 arg.block = block; 2517 arg.q = q; 2518 arg.parent = parent; 2519 arg.w.stop = 0; 2520 arg.w.skip = cb->args[1] - 1; 2521 arg.w.count = 0; 2522 arg.w.cookie = cb->args[2]; 2523 tp->ops->walk(tp, &arg.w, true); 2524 cb->args[2] = arg.w.cookie; 2525 cb->args[1] = arg.w.count + 1; 2526 if (arg.w.stop) 2527 goto errout; 2528 } 2529 return true; 2530 2531 errout: 2532 tcf_proto_put(tp, true, NULL); 2533 return false; 2534 } 2535 2536 /* called with RTNL */ 2537 static int tc_dump_tfilter(struct sk_buff *skb, struct netlink_callback *cb) 2538 { 2539 struct tcf_chain *chain, *chain_prev; 2540 struct net *net = sock_net(skb->sk); 2541 struct nlattr *tca[TCA_MAX + 1]; 2542 struct Qdisc *q = NULL; 2543 struct tcf_block *block; 2544 struct tcmsg *tcm = nlmsg_data(cb->nlh); 2545 long index_start; 2546 long index; 2547 u32 parent; 2548 int err; 2549 2550 if (nlmsg_len(cb->nlh) < sizeof(*tcm)) 2551 return skb->len; 2552 2553 err = nlmsg_parse(cb->nlh, sizeof(*tcm), tca, TCA_MAX, NULL, 2554 cb->extack); 2555 if (err) 2556 return err; 2557 2558 if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) { 2559 block = tcf_block_refcnt_get(net, tcm->tcm_block_index); 2560 if (!block) 2561 goto out; 2562 /* If we work with block index, q is NULL and parent value 2563 * will never be used in the following code. The check 2564 * in tcf_fill_node prevents it. However, compiler does not 2565 * see that far, so set parent to zero to silence the warning 2566 * about parent being uninitialized. 2567 */ 2568 parent = 0; 2569 } else { 2570 const struct Qdisc_class_ops *cops; 2571 struct net_device *dev; 2572 unsigned long cl = 0; 2573 2574 dev = __dev_get_by_index(net, tcm->tcm_ifindex); 2575 if (!dev) 2576 return skb->len; 2577 2578 parent = tcm->tcm_parent; 2579 if (!parent) { 2580 q = dev->qdisc; 2581 parent = q->handle; 2582 } else { 2583 q = qdisc_lookup(dev, TC_H_MAJ(tcm->tcm_parent)); 2584 } 2585 if (!q) 2586 goto out; 2587 cops = q->ops->cl_ops; 2588 if (!cops) 2589 goto out; 2590 if (!cops->tcf_block) 2591 goto out; 2592 if (TC_H_MIN(tcm->tcm_parent)) { 2593 cl = cops->find(q, tcm->tcm_parent); 2594 if (cl == 0) 2595 goto out; 2596 } 2597 block = cops->tcf_block(q, cl, NULL); 2598 if (!block) 2599 goto out; 2600 if (tcf_block_shared(block)) 2601 q = NULL; 2602 } 2603 2604 index_start = cb->args[0]; 2605 index = 0; 2606 2607 for (chain = __tcf_get_next_chain(block, NULL); 2608 chain; 2609 chain_prev = chain, 2610 chain = __tcf_get_next_chain(block, chain), 2611 tcf_chain_put(chain_prev)) { 2612 if (tca[TCA_CHAIN] && 2613 nla_get_u32(tca[TCA_CHAIN]) != chain->index) 2614 continue; 2615 if (!tcf_chain_dump(chain, q, parent, skb, cb, 2616 index_start, &index)) { 2617 tcf_chain_put(chain); 2618 err = -EMSGSIZE; 2619 break; 2620 } 2621 } 2622 2623 if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) 2624 tcf_block_refcnt_put(block, true); 2625 cb->args[0] = index; 2626 2627 out: 2628 /* If we did no progress, the error (EMSGSIZE) is real */ 2629 if (skb->len == 0 && err) 2630 return err; 2631 return skb->len; 2632 } 2633 2634 static int tc_chain_fill_node(const struct tcf_proto_ops *tmplt_ops, 2635 void *tmplt_priv, u32 chain_index, 2636 struct net *net, struct sk_buff *skb, 2637 struct tcf_block *block, 2638 u32 portid, u32 seq, u16 flags, int event) 2639 { 2640 unsigned char *b = skb_tail_pointer(skb); 2641 const struct tcf_proto_ops *ops; 2642 struct nlmsghdr *nlh; 2643 struct tcmsg *tcm; 2644 void *priv; 2645 2646 ops = tmplt_ops; 2647 priv = tmplt_priv; 2648 2649 nlh = nlmsg_put(skb, portid, seq, event, sizeof(*tcm), flags); 2650 if (!nlh) 2651 goto out_nlmsg_trim; 2652 tcm = nlmsg_data(nlh); 2653 tcm->tcm_family = AF_UNSPEC; 2654 tcm->tcm__pad1 = 0; 2655 tcm->tcm__pad2 = 0; 2656 tcm->tcm_handle = 0; 2657 if (block->q) { 2658 tcm->tcm_ifindex = qdisc_dev(block->q)->ifindex; 2659 tcm->tcm_parent = block->q->handle; 2660 } else { 2661 tcm->tcm_ifindex = TCM_IFINDEX_MAGIC_BLOCK; 2662 tcm->tcm_block_index = block->index; 2663 } 2664 2665 if (nla_put_u32(skb, TCA_CHAIN, chain_index)) 2666 goto nla_put_failure; 2667 2668 if (ops) { 2669 if (nla_put_string(skb, TCA_KIND, ops->kind)) 2670 goto nla_put_failure; 2671 if (ops->tmplt_dump(skb, net, priv) < 0) 2672 goto nla_put_failure; 2673 } 2674 2675 nlh->nlmsg_len = skb_tail_pointer(skb) - b; 2676 return skb->len; 2677 2678 out_nlmsg_trim: 2679 nla_put_failure: 2680 nlmsg_trim(skb, b); 2681 return -EMSGSIZE; 2682 } 2683 2684 static int tc_chain_notify(struct tcf_chain *chain, struct sk_buff *oskb, 2685 u32 seq, u16 flags, int event, bool unicast) 2686 { 2687 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0; 2688 struct tcf_block *block = chain->block; 2689 struct net *net = block->net; 2690 struct sk_buff *skb; 2691 2692 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); 2693 if (!skb) 2694 return -ENOBUFS; 2695 2696 if (tc_chain_fill_node(chain->tmplt_ops, chain->tmplt_priv, 2697 chain->index, net, skb, block, portid, 2698 seq, flags, event) <= 0) { 2699 kfree_skb(skb); 2700 return -EINVAL; 2701 } 2702 2703 if (unicast) 2704 return netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT); 2705 2706 return rtnetlink_send(skb, net, portid, RTNLGRP_TC, flags & NLM_F_ECHO); 2707 } 2708 2709 static int tc_chain_notify_delete(const struct tcf_proto_ops *tmplt_ops, 2710 void *tmplt_priv, u32 chain_index, 2711 struct tcf_block *block, struct sk_buff *oskb, 2712 u32 seq, u16 flags, bool unicast) 2713 { 2714 u32 portid = oskb ? NETLINK_CB(oskb).portid : 0; 2715 struct net *net = block->net; 2716 struct sk_buff *skb; 2717 2718 skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL); 2719 if (!skb) 2720 return -ENOBUFS; 2721 2722 if (tc_chain_fill_node(tmplt_ops, tmplt_priv, chain_index, net, skb, 2723 block, portid, seq, flags, RTM_DELCHAIN) <= 0) { 2724 kfree_skb(skb); 2725 return -EINVAL; 2726 } 2727 2728 if (unicast) 2729 return netlink_unicast(net->rtnl, skb, portid, MSG_DONTWAIT); 2730 2731 return rtnetlink_send(skb, net, portid, RTNLGRP_TC, flags & NLM_F_ECHO); 2732 } 2733 2734 static int tc_chain_tmplt_add(struct tcf_chain *chain, struct net *net, 2735 struct nlattr **tca, 2736 struct netlink_ext_ack *extack) 2737 { 2738 const struct tcf_proto_ops *ops; 2739 void *tmplt_priv; 2740 2741 /* If kind is not set, user did not specify template. */ 2742 if (!tca[TCA_KIND]) 2743 return 0; 2744 2745 ops = tcf_proto_lookup_ops(nla_data(tca[TCA_KIND]), true, extack); 2746 if (IS_ERR(ops)) 2747 return PTR_ERR(ops); 2748 if (!ops->tmplt_create || !ops->tmplt_destroy || !ops->tmplt_dump) { 2749 NL_SET_ERR_MSG(extack, "Chain templates are not supported with specified classifier"); 2750 return -EOPNOTSUPP; 2751 } 2752 2753 tmplt_priv = ops->tmplt_create(net, chain, tca, extack); 2754 if (IS_ERR(tmplt_priv)) { 2755 module_put(ops->owner); 2756 return PTR_ERR(tmplt_priv); 2757 } 2758 chain->tmplt_ops = ops; 2759 chain->tmplt_priv = tmplt_priv; 2760 return 0; 2761 } 2762 2763 static void tc_chain_tmplt_del(const struct tcf_proto_ops *tmplt_ops, 2764 void *tmplt_priv) 2765 { 2766 /* If template ops are set, no work to do for us. */ 2767 if (!tmplt_ops) 2768 return; 2769 2770 tmplt_ops->tmplt_destroy(tmplt_priv); 2771 module_put(tmplt_ops->owner); 2772 } 2773 2774 /* Add/delete/get a chain */ 2775 2776 static int tc_ctl_chain(struct sk_buff *skb, struct nlmsghdr *n, 2777 struct netlink_ext_ack *extack) 2778 { 2779 struct net *net = sock_net(skb->sk); 2780 struct nlattr *tca[TCA_MAX + 1]; 2781 struct tcmsg *t; 2782 u32 parent; 2783 u32 chain_index; 2784 struct Qdisc *q = NULL; 2785 struct tcf_chain *chain = NULL; 2786 struct tcf_block *block; 2787 unsigned long cl; 2788 int err; 2789 2790 if (n->nlmsg_type != RTM_GETCHAIN && 2791 !netlink_ns_capable(skb, net->user_ns, CAP_NET_ADMIN)) 2792 return -EPERM; 2793 2794 replay: 2795 err = nlmsg_parse(n, sizeof(*t), tca, TCA_MAX, rtm_tca_policy, extack); 2796 if (err < 0) 2797 return err; 2798 2799 t = nlmsg_data(n); 2800 parent = t->tcm_parent; 2801 cl = 0; 2802 2803 block = tcf_block_find(net, &q, &parent, &cl, 2804 t->tcm_ifindex, t->tcm_block_index, extack); 2805 if (IS_ERR(block)) 2806 return PTR_ERR(block); 2807 2808 chain_index = tca[TCA_CHAIN] ? nla_get_u32(tca[TCA_CHAIN]) : 0; 2809 if (chain_index > TC_ACT_EXT_VAL_MASK) { 2810 NL_SET_ERR_MSG(extack, "Specified chain index exceeds upper limit"); 2811 err = -EINVAL; 2812 goto errout_block; 2813 } 2814 2815 mutex_lock(&block->lock); 2816 chain = tcf_chain_lookup(block, chain_index); 2817 if (n->nlmsg_type == RTM_NEWCHAIN) { 2818 if (chain) { 2819 if (tcf_chain_held_by_acts_only(chain)) { 2820 /* The chain exists only because there is 2821 * some action referencing it. 2822 */ 2823 tcf_chain_hold(chain); 2824 } else { 2825 NL_SET_ERR_MSG(extack, "Filter chain already exists"); 2826 err = -EEXIST; 2827 goto errout_block_locked; 2828 } 2829 } else { 2830 if (!(n->nlmsg_flags & NLM_F_CREATE)) { 2831 NL_SET_ERR_MSG(extack, "Need both RTM_NEWCHAIN and NLM_F_CREATE to create a new chain"); 2832 err = -ENOENT; 2833 goto errout_block_locked; 2834 } 2835 chain = tcf_chain_create(block, chain_index); 2836 if (!chain) { 2837 NL_SET_ERR_MSG(extack, "Failed to create filter chain"); 2838 err = -ENOMEM; 2839 goto errout_block_locked; 2840 } 2841 } 2842 } else { 2843 if (!chain || tcf_chain_held_by_acts_only(chain)) { 2844 NL_SET_ERR_MSG(extack, "Cannot find specified filter chain"); 2845 err = -EINVAL; 2846 goto errout_block_locked; 2847 } 2848 tcf_chain_hold(chain); 2849 } 2850 2851 if (n->nlmsg_type == RTM_NEWCHAIN) { 2852 /* Modifying chain requires holding parent block lock. In case 2853 * the chain was successfully added, take a reference to the 2854 * chain. This ensures that an empty chain does not disappear at 2855 * the end of this function. 2856 */ 2857 tcf_chain_hold(chain); 2858 chain->explicitly_created = true; 2859 } 2860 mutex_unlock(&block->lock); 2861 2862 switch (n->nlmsg_type) { 2863 case RTM_NEWCHAIN: 2864 err = tc_chain_tmplt_add(chain, net, tca, extack); 2865 if (err) { 2866 tcf_chain_put_explicitly_created(chain); 2867 goto errout; 2868 } 2869 2870 tc_chain_notify(chain, NULL, 0, NLM_F_CREATE | NLM_F_EXCL, 2871 RTM_NEWCHAIN, false); 2872 break; 2873 case RTM_DELCHAIN: 2874 tfilter_notify_chain(net, skb, block, q, parent, n, 2875 chain, RTM_DELTFILTER, true); 2876 /* Flush the chain first as the user requested chain removal. */ 2877 tcf_chain_flush(chain, true); 2878 /* In case the chain was successfully deleted, put a reference 2879 * to the chain previously taken during addition. 2880 */ 2881 tcf_chain_put_explicitly_created(chain); 2882 break; 2883 case RTM_GETCHAIN: 2884 err = tc_chain_notify(chain, skb, n->nlmsg_seq, 2885 n->nlmsg_seq, n->nlmsg_type, true); 2886 if (err < 0) 2887 NL_SET_ERR_MSG(extack, "Failed to send chain notify message"); 2888 break; 2889 default: 2890 err = -EOPNOTSUPP; 2891 NL_SET_ERR_MSG(extack, "Unsupported message type"); 2892 goto errout; 2893 } 2894 2895 errout: 2896 tcf_chain_put(chain); 2897 errout_block: 2898 tcf_block_release(q, block, true); 2899 if (err == -EAGAIN) 2900 /* Replay the request. */ 2901 goto replay; 2902 return err; 2903 2904 errout_block_locked: 2905 mutex_unlock(&block->lock); 2906 goto errout_block; 2907 } 2908 2909 /* called with RTNL */ 2910 static int tc_dump_chain(struct sk_buff *skb, struct netlink_callback *cb) 2911 { 2912 struct net *net = sock_net(skb->sk); 2913 struct nlattr *tca[TCA_MAX + 1]; 2914 struct Qdisc *q = NULL; 2915 struct tcf_block *block; 2916 struct tcmsg *tcm = nlmsg_data(cb->nlh); 2917 struct tcf_chain *chain; 2918 long index_start; 2919 long index; 2920 u32 parent; 2921 int err; 2922 2923 if (nlmsg_len(cb->nlh) < sizeof(*tcm)) 2924 return skb->len; 2925 2926 err = nlmsg_parse(cb->nlh, sizeof(*tcm), tca, TCA_MAX, rtm_tca_policy, 2927 cb->extack); 2928 if (err) 2929 return err; 2930 2931 if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) { 2932 block = tcf_block_refcnt_get(net, tcm->tcm_block_index); 2933 if (!block) 2934 goto out; 2935 /* If we work with block index, q is NULL and parent value 2936 * will never be used in the following code. The check 2937 * in tcf_fill_node prevents it. However, compiler does not 2938 * see that far, so set parent to zero to silence the warning 2939 * about parent being uninitialized. 2940 */ 2941 parent = 0; 2942 } else { 2943 const struct Qdisc_class_ops *cops; 2944 struct net_device *dev; 2945 unsigned long cl = 0; 2946 2947 dev = __dev_get_by_index(net, tcm->tcm_ifindex); 2948 if (!dev) 2949 return skb->len; 2950 2951 parent = tcm->tcm_parent; 2952 if (!parent) { 2953 q = dev->qdisc; 2954 parent = q->handle; 2955 } else { 2956 q = qdisc_lookup(dev, TC_H_MAJ(tcm->tcm_parent)); 2957 } 2958 if (!q) 2959 goto out; 2960 cops = q->ops->cl_ops; 2961 if (!cops) 2962 goto out; 2963 if (!cops->tcf_block) 2964 goto out; 2965 if (TC_H_MIN(tcm->tcm_parent)) { 2966 cl = cops->find(q, tcm->tcm_parent); 2967 if (cl == 0) 2968 goto out; 2969 } 2970 block = cops->tcf_block(q, cl, NULL); 2971 if (!block) 2972 goto out; 2973 if (tcf_block_shared(block)) 2974 q = NULL; 2975 } 2976 2977 index_start = cb->args[0]; 2978 index = 0; 2979 2980 mutex_lock(&block->lock); 2981 list_for_each_entry(chain, &block->chain_list, list) { 2982 if ((tca[TCA_CHAIN] && 2983 nla_get_u32(tca[TCA_CHAIN]) != chain->index)) 2984 continue; 2985 if (index < index_start) { 2986 index++; 2987 continue; 2988 } 2989 if (tcf_chain_held_by_acts_only(chain)) 2990 continue; 2991 err = tc_chain_fill_node(chain->tmplt_ops, chain->tmplt_priv, 2992 chain->index, net, skb, block, 2993 NETLINK_CB(cb->skb).portid, 2994 cb->nlh->nlmsg_seq, NLM_F_MULTI, 2995 RTM_NEWCHAIN); 2996 if (err <= 0) 2997 break; 2998 index++; 2999 } 3000 mutex_unlock(&block->lock); 3001 3002 if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) 3003 tcf_block_refcnt_put(block, true); 3004 cb->args[0] = index; 3005 3006 out: 3007 /* If we did no progress, the error (EMSGSIZE) is real */ 3008 if (skb->len == 0 && err) 3009 return err; 3010 return skb->len; 3011 } 3012 3013 void tcf_exts_destroy(struct tcf_exts *exts) 3014 { 3015 #ifdef CONFIG_NET_CLS_ACT 3016 tcf_action_destroy(exts->actions, TCA_ACT_UNBIND); 3017 kfree(exts->actions); 3018 exts->nr_actions = 0; 3019 #endif 3020 } 3021 EXPORT_SYMBOL(tcf_exts_destroy); 3022 3023 int tcf_exts_validate(struct net *net, struct tcf_proto *tp, struct nlattr **tb, 3024 struct nlattr *rate_tlv, struct tcf_exts *exts, bool ovr, 3025 bool rtnl_held, struct netlink_ext_ack *extack) 3026 { 3027 #ifdef CONFIG_NET_CLS_ACT 3028 { 3029 struct tc_action *act; 3030 size_t attr_size = 0; 3031 3032 if (exts->police && tb[exts->police]) { 3033 act = tcf_action_init_1(net, tp, tb[exts->police], 3034 rate_tlv, "police", ovr, 3035 TCA_ACT_BIND, rtnl_held, 3036 extack); 3037 if (IS_ERR(act)) 3038 return PTR_ERR(act); 3039 3040 act->type = exts->type = TCA_OLD_COMPAT; 3041 exts->actions[0] = act; 3042 exts->nr_actions = 1; 3043 } else if (exts->action && tb[exts->action]) { 3044 int err; 3045 3046 err = tcf_action_init(net, tp, tb[exts->action], 3047 rate_tlv, NULL, ovr, TCA_ACT_BIND, 3048 exts->actions, &attr_size, 3049 rtnl_held, extack); 3050 if (err < 0) 3051 return err; 3052 exts->nr_actions = err; 3053 } 3054 } 3055 #else 3056 if ((exts->action && tb[exts->action]) || 3057 (exts->police && tb[exts->police])) { 3058 NL_SET_ERR_MSG(extack, "Classifier actions are not supported per compile options (CONFIG_NET_CLS_ACT)"); 3059 return -EOPNOTSUPP; 3060 } 3061 #endif 3062 3063 return 0; 3064 } 3065 EXPORT_SYMBOL(tcf_exts_validate); 3066 3067 void tcf_exts_change(struct tcf_exts *dst, struct tcf_exts *src) 3068 { 3069 #ifdef CONFIG_NET_CLS_ACT 3070 struct tcf_exts old = *dst; 3071 3072 *dst = *src; 3073 tcf_exts_destroy(&old); 3074 #endif 3075 } 3076 EXPORT_SYMBOL(tcf_exts_change); 3077 3078 #ifdef CONFIG_NET_CLS_ACT 3079 static struct tc_action *tcf_exts_first_act(struct tcf_exts *exts) 3080 { 3081 if (exts->nr_actions == 0) 3082 return NULL; 3083 else 3084 return exts->actions[0]; 3085 } 3086 #endif 3087 3088 int tcf_exts_dump(struct sk_buff *skb, struct tcf_exts *exts) 3089 { 3090 #ifdef CONFIG_NET_CLS_ACT 3091 struct nlattr *nest; 3092 3093 if (exts->action && tcf_exts_has_actions(exts)) { 3094 /* 3095 * again for backward compatible mode - we want 3096 * to work with both old and new modes of entering 3097 * tc data even if iproute2 was newer - jhs 3098 */ 3099 if (exts->type != TCA_OLD_COMPAT) { 3100 nest = nla_nest_start(skb, exts->action); 3101 if (nest == NULL) 3102 goto nla_put_failure; 3103 3104 if (tcf_action_dump(skb, exts->actions, 0, 0) < 0) 3105 goto nla_put_failure; 3106 nla_nest_end(skb, nest); 3107 } else if (exts->police) { 3108 struct tc_action *act = tcf_exts_first_act(exts); 3109 nest = nla_nest_start(skb, exts->police); 3110 if (nest == NULL || !act) 3111 goto nla_put_failure; 3112 if (tcf_action_dump_old(skb, act, 0, 0) < 0) 3113 goto nla_put_failure; 3114 nla_nest_end(skb, nest); 3115 } 3116 } 3117 return 0; 3118 3119 nla_put_failure: 3120 nla_nest_cancel(skb, nest); 3121 return -1; 3122 #else 3123 return 0; 3124 #endif 3125 } 3126 EXPORT_SYMBOL(tcf_exts_dump); 3127 3128 3129 int tcf_exts_dump_stats(struct sk_buff *skb, struct tcf_exts *exts) 3130 { 3131 #ifdef CONFIG_NET_CLS_ACT 3132 struct tc_action *a = tcf_exts_first_act(exts); 3133 if (a != NULL && tcf_action_copy_stats(skb, a, 1) < 0) 3134 return -1; 3135 #endif 3136 return 0; 3137 } 3138 EXPORT_SYMBOL(tcf_exts_dump_stats); 3139 3140 int tc_setup_cb_call(struct tcf_block *block, enum tc_setup_type type, 3141 void *type_data, bool err_stop) 3142 { 3143 struct tcf_block_cb *block_cb; 3144 int ok_count = 0; 3145 int err; 3146 3147 /* Make sure all netdevs sharing this block are offload-capable. */ 3148 if (block->nooffloaddevcnt && err_stop) 3149 return -EOPNOTSUPP; 3150 3151 list_for_each_entry(block_cb, &block->cb_list, list) { 3152 err = block_cb->cb(type, type_data, block_cb->cb_priv); 3153 if (err) { 3154 if (err_stop) 3155 return err; 3156 } else { 3157 ok_count++; 3158 } 3159 } 3160 return ok_count; 3161 } 3162 EXPORT_SYMBOL(tc_setup_cb_call); 3163 3164 int tc_setup_flow_action(struct flow_action *flow_action, 3165 const struct tcf_exts *exts) 3166 { 3167 const struct tc_action *act; 3168 int i, j, k; 3169 3170 if (!exts) 3171 return 0; 3172 3173 j = 0; 3174 tcf_exts_for_each_action(i, act, exts) { 3175 struct flow_action_entry *entry; 3176 3177 entry = &flow_action->entries[j]; 3178 if (is_tcf_gact_ok(act)) { 3179 entry->id = FLOW_ACTION_ACCEPT; 3180 } else if (is_tcf_gact_shot(act)) { 3181 entry->id = FLOW_ACTION_DROP; 3182 } else if (is_tcf_gact_trap(act)) { 3183 entry->id = FLOW_ACTION_TRAP; 3184 } else if (is_tcf_gact_goto_chain(act)) { 3185 entry->id = FLOW_ACTION_GOTO; 3186 entry->chain_index = tcf_gact_goto_chain_index(act); 3187 } else if (is_tcf_mirred_egress_redirect(act)) { 3188 entry->id = FLOW_ACTION_REDIRECT; 3189 entry->dev = tcf_mirred_dev(act); 3190 } else if (is_tcf_mirred_egress_mirror(act)) { 3191 entry->id = FLOW_ACTION_MIRRED; 3192 entry->dev = tcf_mirred_dev(act); 3193 } else if (is_tcf_vlan(act)) { 3194 switch (tcf_vlan_action(act)) { 3195 case TCA_VLAN_ACT_PUSH: 3196 entry->id = FLOW_ACTION_VLAN_PUSH; 3197 entry->vlan.vid = tcf_vlan_push_vid(act); 3198 entry->vlan.proto = tcf_vlan_push_proto(act); 3199 entry->vlan.prio = tcf_vlan_push_prio(act); 3200 break; 3201 case TCA_VLAN_ACT_POP: 3202 entry->id = FLOW_ACTION_VLAN_POP; 3203 break; 3204 case TCA_VLAN_ACT_MODIFY: 3205 entry->id = FLOW_ACTION_VLAN_MANGLE; 3206 entry->vlan.vid = tcf_vlan_push_vid(act); 3207 entry->vlan.proto = tcf_vlan_push_proto(act); 3208 entry->vlan.prio = tcf_vlan_push_prio(act); 3209 break; 3210 default: 3211 goto err_out; 3212 } 3213 } else if (is_tcf_tunnel_set(act)) { 3214 entry->id = FLOW_ACTION_TUNNEL_ENCAP; 3215 entry->tunnel = tcf_tunnel_info(act); 3216 } else if (is_tcf_tunnel_release(act)) { 3217 entry->id = FLOW_ACTION_TUNNEL_DECAP; 3218 entry->tunnel = tcf_tunnel_info(act); 3219 } else if (is_tcf_pedit(act)) { 3220 for (k = 0; k < tcf_pedit_nkeys(act); k++) { 3221 switch (tcf_pedit_cmd(act, k)) { 3222 case TCA_PEDIT_KEY_EX_CMD_SET: 3223 entry->id = FLOW_ACTION_MANGLE; 3224 break; 3225 case TCA_PEDIT_KEY_EX_CMD_ADD: 3226 entry->id = FLOW_ACTION_ADD; 3227 break; 3228 default: 3229 goto err_out; 3230 } 3231 entry->mangle.htype = tcf_pedit_htype(act, k); 3232 entry->mangle.mask = tcf_pedit_mask(act, k); 3233 entry->mangle.val = tcf_pedit_val(act, k); 3234 entry->mangle.offset = tcf_pedit_offset(act, k); 3235 entry = &flow_action->entries[++j]; 3236 } 3237 } else if (is_tcf_csum(act)) { 3238 entry->id = FLOW_ACTION_CSUM; 3239 entry->csum_flags = tcf_csum_update_flags(act); 3240 } else if (is_tcf_skbedit_mark(act)) { 3241 entry->id = FLOW_ACTION_MARK; 3242 entry->mark = tcf_skbedit_mark(act); 3243 } else { 3244 goto err_out; 3245 } 3246 3247 if (!is_tcf_pedit(act)) 3248 j++; 3249 } 3250 return 0; 3251 err_out: 3252 return -EOPNOTSUPP; 3253 } 3254 EXPORT_SYMBOL(tc_setup_flow_action); 3255 3256 unsigned int tcf_exts_num_actions(struct tcf_exts *exts) 3257 { 3258 unsigned int num_acts = 0; 3259 struct tc_action *act; 3260 int i; 3261 3262 tcf_exts_for_each_action(i, act, exts) { 3263 if (is_tcf_pedit(act)) 3264 num_acts += tcf_pedit_nkeys(act); 3265 else 3266 num_acts++; 3267 } 3268 return num_acts; 3269 } 3270 EXPORT_SYMBOL(tcf_exts_num_actions); 3271 3272 static __net_init int tcf_net_init(struct net *net) 3273 { 3274 struct tcf_net *tn = net_generic(net, tcf_net_id); 3275 3276 spin_lock_init(&tn->idr_lock); 3277 idr_init(&tn->idr); 3278 return 0; 3279 } 3280 3281 static void __net_exit tcf_net_exit(struct net *net) 3282 { 3283 struct tcf_net *tn = net_generic(net, tcf_net_id); 3284 3285 idr_destroy(&tn->idr); 3286 } 3287 3288 static struct pernet_operations tcf_net_ops = { 3289 .init = tcf_net_init, 3290 .exit = tcf_net_exit, 3291 .id = &tcf_net_id, 3292 .size = sizeof(struct tcf_net), 3293 }; 3294 3295 static int __init tc_filter_init(void) 3296 { 3297 int err; 3298 3299 tc_filter_wq = alloc_ordered_workqueue("tc_filter_workqueue", 0); 3300 if (!tc_filter_wq) 3301 return -ENOMEM; 3302 3303 err = register_pernet_subsys(&tcf_net_ops); 3304 if (err) 3305 goto err_register_pernet_subsys; 3306 3307 err = rhashtable_init(&indr_setup_block_ht, 3308 &tc_indr_setup_block_ht_params); 3309 if (err) 3310 goto err_rhash_setup_block_ht; 3311 3312 rtnl_register(PF_UNSPEC, RTM_NEWTFILTER, tc_new_tfilter, NULL, 3313 RTNL_FLAG_DOIT_UNLOCKED); 3314 rtnl_register(PF_UNSPEC, RTM_DELTFILTER, tc_del_tfilter, NULL, 3315 RTNL_FLAG_DOIT_UNLOCKED); 3316 rtnl_register(PF_UNSPEC, RTM_GETTFILTER, tc_get_tfilter, 3317 tc_dump_tfilter, RTNL_FLAG_DOIT_UNLOCKED); 3318 rtnl_register(PF_UNSPEC, RTM_NEWCHAIN, tc_ctl_chain, NULL, 0); 3319 rtnl_register(PF_UNSPEC, RTM_DELCHAIN, tc_ctl_chain, NULL, 0); 3320 rtnl_register(PF_UNSPEC, RTM_GETCHAIN, tc_ctl_chain, 3321 tc_dump_chain, 0); 3322 3323 return 0; 3324 3325 err_rhash_setup_block_ht: 3326 unregister_pernet_subsys(&tcf_net_ops); 3327 err_register_pernet_subsys: 3328 destroy_workqueue(tc_filter_wq); 3329 return err; 3330 } 3331 3332 subsys_initcall(tc_filter_init); 3333