Lines Matching +full:array +full:- +full:nest
1 // SPDX-License-Identifier: GPL-2.0-or-later
65 * action index in the exts tc actions array.
84 if (WARN_ON(!handle || !tp->ops->get_exts)) in tcf_exts_miss_cookie_base_alloc()
85 return -EINVAL; in tcf_exts_miss_cookie_base_alloc()
89 return -ENOMEM; in tcf_exts_miss_cookie_base_alloc()
91 n->chain_index = tp->chain->index; in tcf_exts_miss_cookie_base_alloc()
92 n->chain = tp->chain; in tcf_exts_miss_cookie_base_alloc()
93 n->tp_prio = tp->prio; in tcf_exts_miss_cookie_base_alloc()
94 n->tp = tp; in tcf_exts_miss_cookie_base_alloc()
95 n->exts = exts; in tcf_exts_miss_cookie_base_alloc()
96 n->handle = handle; in tcf_exts_miss_cookie_base_alloc()
98 err = xa_alloc_cyclic(&tcf_exts_miss_cookies_xa, &n->miss_cookie_base, in tcf_exts_miss_cookie_base_alloc()
103 exts->miss_cookie_node = n; in tcf_exts_miss_cookie_base_alloc()
115 if (!exts->miss_cookie_node) in tcf_exts_miss_cookie_base_destroy()
118 n = exts->miss_cookie_node; in tcf_exts_miss_cookie_base_destroy()
119 xa_erase(&tcf_exts_miss_cookies_xa, n->miss_cookie_base); in tcf_exts_miss_cookie_base_destroy()
174 return jhash_3words(tp->chain->index, tp->prio, in destroy_obj_hashfn()
175 (__force __u32)tp->protocol, 0); in destroy_obj_hashfn()
181 struct tcf_block *block = chain->block; in tcf_proto_signal_destroying()
183 mutex_lock(&block->proto_destroy_lock); in tcf_proto_signal_destroying()
184 hash_add_rcu(block->proto_destroy_ht, &tp->destroy_ht_node, in tcf_proto_signal_destroying()
186 mutex_unlock(&block->proto_destroy_lock); in tcf_proto_signal_destroying()
192 return tp1->chain->index == tp2->chain->index && in tcf_proto_cmp()
193 tp1->prio == tp2->prio && in tcf_proto_cmp()
194 tp1->protocol == tp2->protocol; in tcf_proto_cmp()
205 hash_for_each_possible_rcu(chain->block->proto_destroy_ht, iter, in tcf_proto_exists_destroying()
220 struct tcf_block *block = chain->block; in tcf_proto_signal_destroyed()
222 mutex_lock(&block->proto_destroy_lock); in tcf_proto_signal_destroyed()
223 if (hash_hashed(&tp->destroy_ht_node)) in tcf_proto_signal_destroyed()
224 hash_del_rcu(&tp->destroy_ht_node); in tcf_proto_signal_destroyed()
225 mutex_unlock(&block->proto_destroy_lock); in tcf_proto_signal_destroyed()
237 if (strcmp(kind, t->kind) == 0) { in __tcf_proto_lookup_ops()
238 if (try_module_get(t->owner)) in __tcf_proto_lookup_ops()
267 * this using -EAGAIN. in tcf_proto_lookup_ops()
270 module_put(ops->owner); in tcf_proto_lookup_ops()
271 return ERR_PTR(-EAGAIN); in tcf_proto_lookup_ops()
275 return ERR_PTR(-ENOENT); in tcf_proto_lookup_ops()
283 int rc = -EEXIST; in register_tcf_proto_ops()
287 if (!strcmp(ops->kind, t->kind)) in register_tcf_proto_ops()
290 list_add_tail(&ops->head, &tcf_proto_base); in register_tcf_proto_ops()
303 int rc = -ENOENT; in unregister_tcf_proto_ops()
314 list_del(&t->head); in unregister_tcf_proto_ops()
321 WARN(rc, "unregister tc filter kind(%s) failed %d\n", ops->kind, rc); in unregister_tcf_proto_ops()
339 first = tp->prio - 1; in tcf_auto_prio()
367 ret = !!(ops->flags & TCF_PROTO_OPS_DOIT_UNLOCKED); in tcf_proto_is_unlocked()
368 module_put(ops->owner); in tcf_proto_is_unlocked()
382 return ERR_PTR(-ENOBUFS); in tcf_proto_create()
384 tp->ops = tcf_proto_lookup_ops(kind, rtnl_held, extack); in tcf_proto_create()
385 if (IS_ERR(tp->ops)) { in tcf_proto_create()
386 err = PTR_ERR(tp->ops); in tcf_proto_create()
389 tp->classify = tp->ops->classify; in tcf_proto_create()
390 tp->protocol = protocol; in tcf_proto_create()
391 tp->prio = prio; in tcf_proto_create()
392 tp->chain = chain; in tcf_proto_create()
393 tp->usesw = !tp->ops->reoffload; in tcf_proto_create()
394 spin_lock_init(&tp->lock); in tcf_proto_create()
395 refcount_set(&tp->refcnt, 1); in tcf_proto_create()
397 err = tp->ops->init(tp); in tcf_proto_create()
399 module_put(tp->ops->owner); in tcf_proto_create()
411 refcount_inc(&tp->refcnt); in tcf_proto_get()
417 struct tcf_block *block = tp->chain->block; in tcf_proto_count_usesw()
421 if (tp->usesw && tp->counted) { in tcf_proto_count_usesw()
422 if (!atomic_dec_return(&block->useswcnt)) in tcf_proto_count_usesw()
424 tp->counted = false; in tcf_proto_count_usesw()
429 spin_lock(&tp->lock); in tcf_proto_count_usesw()
430 if (tp->usesw && !tp->counted) { in tcf_proto_count_usesw()
432 tp->counted = true; in tcf_proto_count_usesw()
434 spin_unlock(&tp->lock); in tcf_proto_count_usesw()
436 if (counted && atomic_inc_return(&block->useswcnt) == 1) in tcf_proto_count_usesw()
446 tp->ops->destroy(tp, rtnl_held, extack); in tcf_proto_destroy()
449 tcf_proto_signal_destroyed(tp->chain, tp); in tcf_proto_destroy()
450 tcf_chain_put(tp->chain); in tcf_proto_destroy()
451 module_put(tp->ops->owner); in tcf_proto_destroy()
458 if (refcount_dec_and_test(&tp->refcnt)) in tcf_proto_put()
464 if (tp->ops->delete_empty) in tcf_proto_check_delete()
465 return tp->ops->delete_empty(tp); in tcf_proto_check_delete()
467 tp->deleting = true; in tcf_proto_check_delete()
468 return tp->deleting; in tcf_proto_check_delete()
473 spin_lock(&tp->lock); in tcf_proto_mark_delete()
474 tp->deleting = true; in tcf_proto_mark_delete()
475 spin_unlock(&tp->lock); in tcf_proto_mark_delete()
482 spin_lock(&tp->lock); in tcf_proto_is_deleting()
483 deleting = tp->deleting; in tcf_proto_is_deleting()
484 spin_unlock(&tp->lock); in tcf_proto_is_deleting()
490 lockdep_assert_held(&(block)->lock)
508 list_add_tail_rcu(&chain->list, &block->chain_list); in tcf_chain_create()
509 mutex_init(&chain->filter_chain_lock); in tcf_chain_create()
510 chain->block = block; in tcf_chain_create()
511 chain->index = chain_index; in tcf_chain_create()
512 chain->refcnt = 1; in tcf_chain_create()
513 if (!chain->index) in tcf_chain_create()
514 block->chain0.chain = chain; in tcf_chain_create()
521 if (item->chain_head_change) in tcf_chain_head_change_item()
522 item->chain_head_change(tp_head, item->chain_head_change_priv); in tcf_chain_head_change_item()
529 struct tcf_block *block = chain->block; in tcf_chain0_head_change()
531 if (chain->index) in tcf_chain0_head_change()
534 mutex_lock(&block->lock); in tcf_chain0_head_change()
535 list_for_each_entry(item, &block->chain0.filter_chain_list, list) in tcf_chain0_head_change()
537 mutex_unlock(&block->lock); in tcf_chain0_head_change()
544 struct tcf_block *block = chain->block; in tcf_chain_detach()
548 list_del_rcu(&chain->list); in tcf_chain_detach()
549 if (!chain->index) in tcf_chain_detach()
550 block->chain0.chain = NULL; in tcf_chain_detach()
552 if (list_empty(&block->chain_list) && in tcf_chain_detach()
553 refcount_read(&block->refcnt) == 0) in tcf_chain_detach()
561 mutex_destroy(&block->lock); in tcf_block_destroy()
562 mutex_destroy(&block->proto_destroy_lock); in tcf_block_destroy()
563 xa_destroy(&block->ports); in tcf_block_destroy()
569 struct tcf_block *block = chain->block; in tcf_chain_destroy()
571 mutex_destroy(&chain->filter_chain_lock); in tcf_chain_destroy()
579 ASSERT_BLOCK_LOCKED(chain->block); in tcf_chain_hold()
581 ++chain->refcnt; in tcf_chain_hold()
586 ASSERT_BLOCK_LOCKED(chain->block); in tcf_chain_held_by_acts_only()
591 return chain->refcnt == chain->action_refcnt; in tcf_chain_held_by_acts_only()
601 list_for_each_entry(chain, &block->chain_list, list) { in tcf_chain_lookup()
602 if (chain->index == chain_index) in tcf_chain_lookup()
614 list_for_each_entry_rcu(chain, &block->chain_list, list) { in tcf_chain_lookup_rcu()
615 if (chain->index == chain_index) in tcf_chain_lookup_rcu()
633 mutex_lock(&block->lock); in __tcf_chain_get()
646 ++chain->action_refcnt; in __tcf_chain_get()
647 is_first_reference = chain->refcnt - chain->action_refcnt == 1; in __tcf_chain_get()
648 mutex_unlock(&block->lock); in __tcf_chain_get()
651 * non-action reference. Until then, the chain acts only as in __tcf_chain_get()
662 mutex_unlock(&block->lock); in __tcf_chain_get()
688 struct tcf_block *block = chain->block; in __tcf_chain_put()
694 mutex_lock(&block->lock); in __tcf_chain_put()
696 if (!chain->explicitly_created) { in __tcf_chain_put()
697 mutex_unlock(&block->lock); in __tcf_chain_put()
700 chain->explicitly_created = false; in __tcf_chain_put()
704 chain->action_refcnt--; in __tcf_chain_put()
710 refcnt = --chain->refcnt; in __tcf_chain_put()
711 non_act_refcnt = refcnt - chain->action_refcnt; in __tcf_chain_put()
712 tmplt_ops = chain->tmplt_ops; in __tcf_chain_put()
713 tmplt_priv = chain->tmplt_priv; in __tcf_chain_put()
715 if (non_act_refcnt == chain->explicitly_created && !by_act) { in __tcf_chain_put()
718 chain->index, block, NULL, 0, 0); in __tcf_chain_put()
720 chain->flushing = false; in __tcf_chain_put()
725 mutex_unlock(&block->lock); in __tcf_chain_put()
753 mutex_lock(&chain->filter_chain_lock); in tcf_chain_flush()
754 tp = tcf_chain_dereference(chain->filter_chain, chain); in tcf_chain_flush()
756 tp_next = rcu_dereference_protected(tp->next, 1); in tcf_chain_flush()
760 tp = tcf_chain_dereference(chain->filter_chain, chain); in tcf_chain_flush()
761 RCU_INIT_POINTER(chain->filter_chain, NULL); in tcf_chain_flush()
763 chain->flushing = true; in tcf_chain_flush()
764 mutex_unlock(&chain->filter_chain_lock); in tcf_chain_flush()
767 tp_next = rcu_dereference_protected(tp->next, 1); in tcf_chain_flush()
783 bo->net = dev_net(dev); in tcf_block_offload_init()
784 bo->command = command; in tcf_block_offload_init()
785 bo->binder_type = binder_type; in tcf_block_offload_init()
786 bo->block = flow_block; in tcf_block_offload_init()
787 bo->block_shared = shared; in tcf_block_offload_init()
788 bo->extack = extack; in tcf_block_offload_init()
789 bo->sch = sch; in tcf_block_offload_init()
790 bo->cb_list_head = &flow_block->cb_list; in tcf_block_offload_init()
791 INIT_LIST_HEAD(&bo->cb_list); in tcf_block_offload_init()
799 struct tcf_block *block = block_cb->indr.data; in tc_block_indr_cleanup()
800 struct net_device *dev = block_cb->indr.dev; in tc_block_indr_cleanup()
801 struct Qdisc *sch = block_cb->indr.sch; in tc_block_indr_cleanup()
806 block_cb->indr.binder_type, in tc_block_indr_cleanup()
807 &block->flow_block, tcf_block_shared(block), in tc_block_indr_cleanup()
810 down_write(&block->cb_lock); in tc_block_indr_cleanup()
811 list_del(&block_cb->driver_list); in tc_block_indr_cleanup()
812 list_move(&block_cb->list, &bo.cb_list); in tc_block_indr_cleanup()
814 up_write(&block->cb_lock); in tc_block_indr_cleanup()
820 return atomic_read(&block->offloadcnt); in tcf_block_offload_in_use()
831 tcf_block_offload_init(&bo, dev, sch, command, ei->binder_type, in tcf_block_offload_cmd()
832 &block->flow_block, tcf_block_shared(block), in tcf_block_offload_cmd()
835 if (dev->netdev_ops->ndo_setup_tc) { in tcf_block_offload_cmd()
838 err = dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_BLOCK, &bo); in tcf_block_offload_cmd()
840 if (err != -EOPNOTSUPP) in tcf_block_offload_cmd()
852 return -EOPNOTSUPP; in tcf_block_offload_cmd()
859 struct net_device *dev = q->dev_queue->dev; in tcf_block_offload_bind()
862 down_write(&block->cb_lock); in tcf_block_offload_bind()
867 if (dev->netdev_ops->ndo_setup_tc && in tcf_block_offload_bind()
871 err = -EOPNOTSUPP; in tcf_block_offload_bind()
876 if (err == -EOPNOTSUPP) in tcf_block_offload_bind()
881 up_write(&block->cb_lock); in tcf_block_offload_bind()
889 block->nooffloaddevcnt++; in tcf_block_offload_bind()
891 up_write(&block->cb_lock); in tcf_block_offload_bind()
898 struct net_device *dev = q->dev_queue->dev; in tcf_block_offload_unbind()
901 down_write(&block->cb_lock); in tcf_block_offload_unbind()
903 if (err == -EOPNOTSUPP) in tcf_block_offload_unbind()
905 up_write(&block->cb_lock); in tcf_block_offload_unbind()
909 WARN_ON(block->nooffloaddevcnt-- == 0); in tcf_block_offload_unbind()
910 up_write(&block->cb_lock); in tcf_block_offload_unbind()
924 return -ENOMEM; in tcf_chain0_head_change_cb_add()
926 item->chain_head_change = ei->chain_head_change; in tcf_chain0_head_change_cb_add()
927 item->chain_head_change_priv = ei->chain_head_change_priv; in tcf_chain0_head_change_cb_add()
929 mutex_lock(&block->lock); in tcf_chain0_head_change_cb_add()
930 chain0 = block->chain0.chain; in tcf_chain0_head_change_cb_add()
934 list_add(&item->list, &block->chain0.filter_chain_list); in tcf_chain0_head_change_cb_add()
935 mutex_unlock(&block->lock); in tcf_chain0_head_change_cb_add()
940 mutex_lock(&chain0->filter_chain_lock); in tcf_chain0_head_change_cb_add()
942 tp_head = tcf_chain_dereference(chain0->filter_chain, chain0); in tcf_chain0_head_change_cb_add()
946 mutex_lock(&block->lock); in tcf_chain0_head_change_cb_add()
947 list_add(&item->list, &block->chain0.filter_chain_list); in tcf_chain0_head_change_cb_add()
948 mutex_unlock(&block->lock); in tcf_chain0_head_change_cb_add()
950 mutex_unlock(&chain0->filter_chain_lock); in tcf_chain0_head_change_cb_add()
963 mutex_lock(&block->lock); in tcf_chain0_head_change_cb_del()
964 list_for_each_entry(item, &block->chain0.filter_chain_list, list) { in tcf_chain0_head_change_cb_del()
965 if ((!ei->chain_head_change && !ei->chain_head_change_priv) || in tcf_chain0_head_change_cb_del()
966 (item->chain_head_change == ei->chain_head_change && in tcf_chain0_head_change_cb_del()
967 item->chain_head_change_priv == ei->chain_head_change_priv)) { in tcf_chain0_head_change_cb_del()
968 if (block->chain0.chain) in tcf_chain0_head_change_cb_del()
970 list_del(&item->list); in tcf_chain0_head_change_cb_del()
971 mutex_unlock(&block->lock); in tcf_chain0_head_change_cb_del()
977 mutex_unlock(&block->lock); in tcf_chain0_head_change_cb_del()
995 spin_lock(&tn->idr_lock); in tcf_block_insert()
996 err = idr_alloc_u32(&tn->idr, block, &block->index, block->index, in tcf_block_insert()
998 spin_unlock(&tn->idr_lock); in tcf_block_insert()
1008 spin_lock(&tn->idr_lock); in tcf_block_remove()
1009 idr_remove(&tn->idr, block->index); in tcf_block_remove()
1010 spin_unlock(&tn->idr_lock); in tcf_block_remove()
1022 return ERR_PTR(-ENOMEM); in tcf_block_create()
1024 mutex_init(&block->lock); in tcf_block_create()
1025 mutex_init(&block->proto_destroy_lock); in tcf_block_create()
1026 init_rwsem(&block->cb_lock); in tcf_block_create()
1027 flow_block_init(&block->flow_block); in tcf_block_create()
1028 INIT_LIST_HEAD(&block->chain_list); in tcf_block_create()
1029 INIT_LIST_HEAD(&block->owner_list); in tcf_block_create()
1030 INIT_LIST_HEAD(&block->chain0.filter_chain_list); in tcf_block_create()
1032 refcount_set(&block->refcnt, 1); in tcf_block_create()
1033 block->net = net; in tcf_block_create()
1034 block->index = block_index; in tcf_block_create()
1035 xa_init(&block->ports); in tcf_block_create()
1039 block->q = q; in tcf_block_create()
1047 return idr_find(&tn->idr, block_index); in tcf_block_lookup()
1057 if (block && !refcount_inc_not_zero(&block->refcnt)) in tcf_block_refcnt_get()
1067 mutex_lock(&block->lock); in __tcf_get_next_chain()
1069 chain = list_is_last(&chain->list, &block->chain_list) ? in __tcf_get_next_chain()
1072 chain = list_first_entry_or_null(&block->chain_list, in __tcf_get_next_chain()
1075 /* skip all action-only chains */ in __tcf_get_next_chain()
1077 chain = list_is_last(&chain->list, &block->chain_list) ? in __tcf_get_next_chain()
1082 mutex_unlock(&block->lock); in __tcf_get_next_chain()
1088 * block. It properly obtains block->lock and takes reference to chain before
1093 * data and sent to user-space.
1114 mutex_lock(&chain->filter_chain_lock); in __tcf_get_next_proto()
1117 tp = tcf_chain_dereference(chain->filter_chain, chain); in __tcf_get_next_proto()
1119 /* 'deleting' flag is set and chain->filter_chain_lock was in __tcf_get_next_proto()
1123 prio = tp->prio + 1; in __tcf_get_next_proto()
1124 tp = tcf_chain_dereference(chain->filter_chain, chain); in __tcf_get_next_proto()
1126 for (; tp; tp = tcf_chain_dereference(tp->next, chain)) in __tcf_get_next_proto()
1127 if (!tp->deleting && tp->prio >= prio) in __tcf_get_next_proto()
1130 tp = tcf_chain_dereference(tp->next, chain); in __tcf_get_next_proto()
1136 mutex_unlock(&chain->filter_chain_lock); in __tcf_get_next_proto()
1146 * data and sent to user-space.
1197 return -ENODEV; in __tcf_qdisc_find()
1202 *q = rcu_dereference(dev->qdisc); in __tcf_qdisc_find()
1203 *parent = (*q)->handle; in __tcf_qdisc_find()
1208 err = -EINVAL; in __tcf_qdisc_find()
1216 err = -EINVAL; in __tcf_qdisc_find()
1221 cops = (*q)->ops->cl_ops; in __tcf_qdisc_find()
1224 err = -EINVAL; in __tcf_qdisc_find()
1228 if (!cops->tcf_block) { in __tcf_qdisc_find()
1230 err = -EOPNOTSUPP; in __tcf_qdisc_find()
1263 const struct Qdisc_class_ops *cops = q->ops->cl_ops; in __tcf_qdisc_cl_find()
1265 *cl = cops->find(q, parent); in __tcf_qdisc_cl_find()
1268 return -ENOENT; in __tcf_qdisc_cl_find()
1286 return ERR_PTR(-EINVAL); in __tcf_block_find()
1289 const struct Qdisc_class_ops *cops = q->ops->cl_ops; in __tcf_block_find()
1291 block = cops->tcf_block(q, cl, extack); in __tcf_block_find()
1293 return ERR_PTR(-EINVAL); in __tcf_block_find()
1297 return ERR_PTR(-EOPNOTSUPP); in __tcf_block_find()
1306 refcount_inc(&block->refcnt); in __tcf_block_find()
1315 if (refcount_dec_and_mutex_lock(&block->refcnt, &block->lock)) { in __tcf_block_put()
1322 bool free_block = list_empty(&block->chain_list); in __tcf_block_put()
1324 mutex_unlock(&block->lock); in __tcf_block_put()
1326 tcf_block_remove(block, block->net); in __tcf_block_put()
1408 if (block->keep_dst && in tcf_block_owner_netif_keep_dst()
1418 block->keep_dst = true; in tcf_block_netif_keep_dst()
1419 list_for_each_entry(item, &block->owner_list, list) in tcf_block_netif_keep_dst()
1420 tcf_block_owner_netif_keep_dst(block, item->q, in tcf_block_netif_keep_dst()
1421 item->binder_type); in tcf_block_netif_keep_dst()
1433 return -ENOMEM; in tcf_block_owner_add()
1434 item->q = q; in tcf_block_owner_add()
1435 item->binder_type = binder_type; in tcf_block_owner_add()
1436 list_add(&item->list, &block->owner_list); in tcf_block_owner_add()
1446 list_for_each_entry(item, &block->owner_list, list) { in tcf_block_owner_del()
1447 if (item->q == q && item->binder_type == binder_type) { in tcf_block_owner_del()
1448 list_del(&item->list); in tcf_block_owner_del()
1460 (ei->binder_type == FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS || in tcf_block_tracks_dev()
1461 ei->binder_type == FLOW_BLOCK_BINDER_TYPE_CLSACT_EGRESS); in tcf_block_tracks_dev()
1473 if (ei->block_index) in tcf_block_get_ext()
1475 block = tcf_block_refcnt_get(net, ei->block_index); in tcf_block_get_ext()
1478 block = tcf_block_create(net, q, ei->block_index, extack); in tcf_block_get_ext()
1488 err = tcf_block_owner_add(block, q, ei->binder_type); in tcf_block_get_ext()
1492 tcf_block_owner_netif_keep_dst(block, q, ei->binder_type); in tcf_block_get_ext()
1503 err = xa_insert(&block->ports, dev->ifindex, dev, GFP_KERNEL); in tcf_block_get_ext()
1518 tcf_block_owner_del(block, q, ei->binder_type); in tcf_block_get_ext()
1558 xa_erase(&block->ports, dev->ifindex); in tcf_block_put_ext()
1560 tcf_block_owner_del(block, q, ei->binder_type); in tcf_block_put_ext()
1572 tcf_block_put_ext(block, block->q, &ei); in tcf_block_put()
1586 lockdep_assert_held(&block->cb_lock); in tcf_block_playback_offloads()
1593 if (chain->tmplt_ops && add) in tcf_block_playback_offloads()
1594 chain->tmplt_ops->tmplt_reoffload(chain, true, cb, in tcf_block_playback_offloads()
1600 if (tp->ops->reoffload) { in tcf_block_playback_offloads()
1601 err = tp->ops->reoffload(tp, add, cb, cb_priv, in tcf_block_playback_offloads()
1606 err = -EOPNOTSUPP; in tcf_block_playback_offloads()
1607 NL_SET_ERR_MSG(extack, "Filter HW offload failed - classifier without re-offloading support"); in tcf_block_playback_offloads()
1611 if (chain->tmplt_ops && !add) in tcf_block_playback_offloads()
1612 chain->tmplt_ops->tmplt_reoffload(chain, false, cb, in tcf_block_playback_offloads()
1632 lockdep_assert_held(&block->cb_lock); in tcf_block_bind()
1634 list_for_each_entry(block_cb, &bo->cb_list, list) { in tcf_block_bind()
1635 err = tcf_block_playback_offloads(block, block_cb->cb, in tcf_block_bind()
1636 block_cb->cb_priv, true, in tcf_block_bind()
1638 bo->extack); in tcf_block_bind()
1641 if (!bo->unlocked_driver_cb) in tcf_block_bind()
1642 block->lockeddevcnt++; in tcf_block_bind()
1646 list_splice(&bo->cb_list, &block->flow_block.cb_list); in tcf_block_bind()
1651 list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) { in tcf_block_bind()
1652 list_del(&block_cb->driver_list); in tcf_block_bind()
1653 if (i-- > 0) { in tcf_block_bind()
1654 list_del(&block_cb->list); in tcf_block_bind()
1655 tcf_block_playback_offloads(block, block_cb->cb, in tcf_block_bind()
1656 block_cb->cb_priv, false, in tcf_block_bind()
1659 if (!bo->unlocked_driver_cb) in tcf_block_bind()
1660 block->lockeddevcnt--; in tcf_block_bind()
1673 lockdep_assert_held(&block->cb_lock); in tcf_block_unbind()
1675 list_for_each_entry_safe(block_cb, next, &bo->cb_list, list) { in tcf_block_unbind()
1676 tcf_block_playback_offloads(block, block_cb->cb, in tcf_block_unbind()
1677 block_cb->cb_priv, false, in tcf_block_unbind()
1680 list_del(&block_cb->list); in tcf_block_unbind()
1682 if (!bo->unlocked_driver_cb) in tcf_block_unbind()
1683 block->lockeddevcnt--; in tcf_block_unbind()
1692 switch (bo->command) { in tcf_block_setup()
1702 err = -EOPNOTSUPP; in tcf_block_setup()
1728 for (; tp; tp = rcu_dereference_bh(tp->next)) { in __tcf_classify()
1735 if (n->tp_prio != tp->prio) in __tcf_classify()
1738 /* We re-lookup the tp and chain based on index instead in __tcf_classify()
1743 if (unlikely(n->tp != tp || n->tp->chain != n->chain || in __tcf_classify()
1744 !tp->ops->get_exts)) { in __tcf_classify()
1750 exts = tp->ops->get_exts(tp, n->handle); in __tcf_classify()
1751 if (unlikely(!exts || n->exts != exts)) { in __tcf_classify()
1760 if (tp->protocol != protocol && in __tcf_classify()
1761 tp->protocol != htons(ETH_P_ALL)) in __tcf_classify()
1769 *last_executed_chain = first_tp->chain->index; in __tcf_classify()
1772 first_tp = res->goto_tp; in __tcf_classify()
1792 tp->chain->block->index, in __tcf_classify()
1793 tp->prio & 0xffff, in __tcf_classify()
1794 ntohs(tp->protocol)); in __tcf_classify()
1816 u32 last_executed_chain = tp ? tp->chain->index : 0; in tcf_classify()
1826 if (ext && (ext->chain || ext->act_miss)) { in tcf_classify()
1830 if (ext->act_miss) { in tcf_classify()
1831 n = tcf_exts_miss_cookie_lookup(ext->act_miss_cookie, in tcf_classify()
1839 chain = n->chain_index; in tcf_classify()
1841 chain = ext->chain; in tcf_classify()
1855 tp = rcu_dereference_bh(fchain->filter_chain); in tcf_classify()
1856 last_executed_chain = fchain->index; in tcf_classify()
1873 ext->chain = last_executed_chain; in tcf_classify()
1874 ext->mru = cb->mru; in tcf_classify()
1875 ext->post_ct = cb->post_ct; in tcf_classify()
1876 ext->post_ct_snat = cb->post_ct_snat; in tcf_classify()
1877 ext->post_ct_dnat = cb->post_ct_dnat; in tcf_classify()
1878 ext->zone = cb->zone; in tcf_classify()
1895 return tcf_chain_dereference(*chain_info->pprev, chain); in tcf_chain_tp_prev()
1902 if (chain->flushing) in tcf_chain_tp_insert()
1903 return -EAGAIN; in tcf_chain_tp_insert()
1905 RCU_INIT_POINTER(tp->next, tcf_chain_tp_prev(chain, chain_info)); in tcf_chain_tp_insert()
1906 if (*chain_info->pprev == chain->filter_chain) in tcf_chain_tp_insert()
1909 rcu_assign_pointer(*chain_info->pprev, tp); in tcf_chain_tp_insert()
1918 struct tcf_proto *next = tcf_chain_dereference(chain_info->next, chain); in tcf_chain_tp_remove()
1921 if (tp == chain->filter_chain) in tcf_chain_tp_remove()
1923 RCU_INIT_POINTER(*chain_info->pprev, next); in tcf_chain_tp_remove()
1946 mutex_lock(&chain->filter_chain_lock); in tcf_chain_tp_insert_unique()
1949 mutex_unlock(&chain->filter_chain_lock); in tcf_chain_tp_insert_unique()
1951 return ERR_PTR(-EAGAIN); in tcf_chain_tp_insert_unique()
1957 mutex_unlock(&chain->filter_chain_lock); in tcf_chain_tp_insert_unique()
1979 mutex_lock(&chain->filter_chain_lock); in tcf_chain_tp_delete_empty()
1982 for (pprev = &chain->filter_chain; in tcf_chain_tp_delete_empty()
1984 pprev = &tp_iter->next) { in tcf_chain_tp_delete_empty()
1987 chain_info.next = tp_iter->next; in tcf_chain_tp_delete_empty()
1988 WARN_ON(tp_iter->deleting); in tcf_chain_tp_delete_empty()
1997 mutex_unlock(&chain->filter_chain_lock); in tcf_chain_tp_delete_empty()
2003 if (tp == chain->filter_chain) in tcf_chain_tp_delete_empty()
2006 mutex_unlock(&chain->filter_chain_lock); in tcf_chain_tp_delete_empty()
2020 /* Check the chain for existence of proto-tcf with this priority */ in tcf_chain_tp_find()
2021 for (pprev = &chain->filter_chain; in tcf_chain_tp_find()
2023 pprev = &tp->next) { in tcf_chain_tp_find()
2024 if (tp->prio >= prio) { in tcf_chain_tp_find()
2025 if (tp->prio == prio) { in tcf_chain_tp_find()
2027 NL_SET_ERR_MSG(extack, "Lowest ID from auto-alloc range already in use"); in tcf_chain_tp_find()
2028 return ERR_PTR(-ENOSPC); in tcf_chain_tp_find()
2030 if (tp->protocol != protocol && protocol) { in tcf_chain_tp_find()
2032 return ERR_PTR(-EINVAL); in tcf_chain_tp_find()
2040 chain_info->pprev = pprev; in tcf_chain_tp_find()
2042 chain_info->next = tp->next; in tcf_chain_tp_find()
2045 chain_info->next = NULL; in tcf_chain_tp_find()
2065 tcm->tcm_family = AF_UNSPEC; in tcf_fill_node()
2066 tcm->tcm__pad1 = 0; in tcf_fill_node()
2067 tcm->tcm__pad2 = 0; in tcf_fill_node()
2069 tcm->tcm_ifindex = qdisc_dev(q)->ifindex; in tcf_fill_node()
2070 tcm->tcm_parent = parent; in tcf_fill_node()
2072 tcm->tcm_ifindex = TCM_IFINDEX_MAGIC_BLOCK; in tcf_fill_node()
2073 tcm->tcm_block_index = block->index; in tcf_fill_node()
2075 tcm->tcm_info = TC_H_MAKE(tp->prio, tp->protocol); in tcf_fill_node()
2076 if (nla_put_string(skb, TCA_KIND, tp->ops->kind)) in tcf_fill_node()
2078 if (nla_put_u32(skb, TCA_CHAIN, tp->chain->index)) in tcf_fill_node()
2081 tcm->tcm_handle = 0; in tcf_fill_node()
2083 if (tp->ops->terse_dump) { in tcf_fill_node()
2084 if (tp->ops->terse_dump(net, tp, fh, skb, tcm, in tcf_fill_node()
2091 if (tp->ops->dump && in tcf_fill_node()
2092 tp->ops->dump(net, tp, fh, skb, tcm, rtnl_held) < 0) in tcf_fill_node()
2096 if (extack && extack->_msg && in tcf_fill_node()
2097 nla_put_string(skb, TCA_EXT_WARN_MSG, extack->_msg)) in tcf_fill_node()
2100 nlh->nlmsg_len = skb_tail_pointer(skb) - b; in tcf_fill_node()
2102 return skb->len; in tcf_fill_node()
2108 return -1; in tcf_fill_node()
2121 if (!unicast && !rtnl_notify_needed(net, n->nlmsg_flags, RTNLGRP_TC)) in tfilter_notify()
2126 return -ENOBUFS; in tfilter_notify()
2129 n->nlmsg_seq, n->nlmsg_flags, event, in tfilter_notify()
2132 return -EINVAL; in tfilter_notify()
2139 n->nlmsg_flags & NLM_F_ECHO); in tfilter_notify()
2153 if (!rtnl_notify_needed(net, n->nlmsg_flags, RTNLGRP_TC)) in tfilter_del_notify()
2154 return tp->ops->delete(tp, fh, last, rtnl_held, extack); in tfilter_del_notify()
2158 return -ENOBUFS; in tfilter_del_notify()
2161 n->nlmsg_seq, n->nlmsg_flags, RTM_DELTFILTER, in tfilter_del_notify()
2165 return -EINVAL; in tfilter_del_notify()
2168 err = tp->ops->delete(tp, fh, last, rtnl_held, extack); in tfilter_del_notify()
2175 n->nlmsg_flags & NLM_F_ECHO); in tfilter_del_notify()
2198 if (tp->ops->put && fh) in tfilter_put()
2199 tp->ops->put(tp, fh); in tfilter_put()
2210 struct net *net = sock_net(skb->sk); in tc_new_tfilter()
2240 protocol = TC_H_MIN(t->tcm_info); in tc_new_tfilter()
2241 prio = TC_H_MAJ(t->tcm_info); in tc_new_tfilter()
2243 parent = t->tcm_parent; in tc_new_tfilter()
2255 if (n->nlmsg_flags & NLM_F_CREATE) { in tc_new_tfilter()
2260 return -ENOENT; in tc_new_tfilter()
2266 err = __tcf_qdisc_find(net, &q, &parent, t->tcm_ifindex, false, extack); in tc_new_tfilter()
2272 err = -EINVAL; in tc_new_tfilter()
2281 (q && !(q->ops->cl_ops->flags & QDISC_CLASS_OPS_DOIT_UNLOCKED)) || in tc_new_tfilter()
2287 err = __tcf_qdisc_cl_find(q, parent, &cl, t->tcm_ifindex, extack); in tc_new_tfilter()
2291 block = __tcf_block_find(net, q, cl, t->tcm_ifindex, t->tcm_block_index, in tc_new_tfilter()
2297 block->classid = parent; in tc_new_tfilter()
2302 err = -EINVAL; in tc_new_tfilter()
2308 err = -ENOMEM; in tc_new_tfilter()
2312 mutex_lock(&chain->filter_chain_lock); in tc_new_tfilter()
2323 if (chain->flushing) { in tc_new_tfilter()
2324 err = -EAGAIN; in tc_new_tfilter()
2328 /* Proto-tcf does not exist, create new one */ in tc_new_tfilter()
2332 err = -EINVAL; in tc_new_tfilter()
2336 if (!(n->nlmsg_flags & NLM_F_CREATE)) { in tc_new_tfilter()
2338 err = -ENOENT; in tc_new_tfilter()
2346 mutex_unlock(&chain->filter_chain_lock); in tc_new_tfilter()
2362 mutex_unlock(&chain->filter_chain_lock); in tc_new_tfilter()
2365 if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) { in tc_new_tfilter()
2367 err = -EINVAL; in tc_new_tfilter()
2371 fh = tp->ops->get(tp, t->tcm_handle); in tc_new_tfilter()
2374 if (!(n->nlmsg_flags & NLM_F_CREATE)) { in tc_new_tfilter()
2376 err = -ENOENT; in tc_new_tfilter()
2379 } else if (n->nlmsg_flags & NLM_F_EXCL) { in tc_new_tfilter()
2382 err = -EEXIST; in tc_new_tfilter()
2386 if (chain->tmplt_ops && chain->tmplt_ops != tp->ops) { in tc_new_tfilter()
2389 err = -EINVAL; in tc_new_tfilter()
2393 if (!(n->nlmsg_flags & NLM_F_CREATE)) in tc_new_tfilter()
2399 err = tp->ops->change(net, skb, tp, cl, t->tcm_handle, tca, &fh, in tc_new_tfilter()
2408 q->flags &= ~TCQ_F_CAN_BYPASS; in tc_new_tfilter()
2426 if (err == -EAGAIN) { in tc_new_tfilter()
2437 mutex_unlock(&chain->filter_chain_lock); in tc_new_tfilter()
2444 struct net *net = sock_net(skb->sk); in tc_del_tfilter()
2468 protocol = TC_H_MIN(t->tcm_info); in tc_del_tfilter()
2469 prio = TC_H_MAJ(t->tcm_info); in tc_del_tfilter()
2470 parent = t->tcm_parent; in tc_del_tfilter()
2472 if (prio == 0 && (protocol || t->tcm_handle || tca[TCA_KIND])) { in tc_del_tfilter()
2474 return -ENOENT; in tc_del_tfilter()
2479 err = __tcf_qdisc_find(net, &q, &parent, t->tcm_ifindex, false, extack); in tc_del_tfilter()
2485 err = -EINVAL; in tc_del_tfilter()
2493 (q && !(q->ops->cl_ops->flags & QDISC_CLASS_OPS_DOIT_UNLOCKED)) || in tc_del_tfilter()
2499 err = __tcf_qdisc_cl_find(q, parent, &cl, t->tcm_ifindex, extack); in tc_del_tfilter()
2503 block = __tcf_block_find(net, q, cl, t->tcm_ifindex, t->tcm_block_index, in tc_del_tfilter()
2513 err = -EINVAL; in tc_del_tfilter()
2518 /* User requested flush on non-existent chain. Nothing to do, in tc_del_tfilter()
2526 err = -ENOENT; in tc_del_tfilter()
2538 mutex_lock(&chain->filter_chain_lock); in tc_del_tfilter()
2542 err = -ENOENT; in tc_del_tfilter()
2548 } else if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) { in tc_del_tfilter()
2550 err = -EINVAL; in tc_del_tfilter()
2552 } else if (t->tcm_handle == 0) { in tc_del_tfilter()
2555 mutex_unlock(&chain->filter_chain_lock); in tc_del_tfilter()
2563 mutex_unlock(&chain->filter_chain_lock); in tc_del_tfilter()
2565 fh = tp->ops->get(tp, t->tcm_handle); in tc_del_tfilter()
2569 err = -ENOENT; in tc_del_tfilter()
2596 mutex_unlock(&chain->filter_chain_lock); in tc_del_tfilter()
2603 struct net *net = sock_net(skb->sk); in tc_get_tfilter()
2627 protocol = TC_H_MIN(t->tcm_info); in tc_get_tfilter()
2628 prio = TC_H_MAJ(t->tcm_info); in tc_get_tfilter()
2629 parent = t->tcm_parent; in tc_get_tfilter()
2633 return -ENOENT; in tc_get_tfilter()
2638 err = __tcf_qdisc_find(net, &q, &parent, t->tcm_ifindex, false, extack); in tc_get_tfilter()
2644 err = -EINVAL; in tc_get_tfilter()
2651 if ((q && !(q->ops->cl_ops->flags & QDISC_CLASS_OPS_DOIT_UNLOCKED)) || in tc_get_tfilter()
2657 err = __tcf_qdisc_cl_find(q, parent, &cl, t->tcm_ifindex, extack); in tc_get_tfilter()
2661 block = __tcf_block_find(net, q, cl, t->tcm_ifindex, t->tcm_block_index, in tc_get_tfilter()
2671 err = -EINVAL; in tc_get_tfilter()
2677 err = -EINVAL; in tc_get_tfilter()
2681 mutex_lock(&chain->filter_chain_lock); in tc_get_tfilter()
2684 mutex_unlock(&chain->filter_chain_lock); in tc_get_tfilter()
2686 err = -ENOENT; in tc_get_tfilter()
2692 } else if (tca[TCA_KIND] && nla_strcmp(tca[TCA_KIND], tp->ops->kind)) { in tc_get_tfilter()
2694 err = -EINVAL; in tc_get_tfilter()
2698 fh = tp->ops->get(tp, t->tcm_handle); in tc_get_tfilter()
2702 err = -ENOENT; in tc_get_tfilter()
2738 struct net *net = sock_net(a->skb->sk); in tcf_node_dump()
2740 return tcf_fill_node(net, a->skb, tp, a->block, a->q, a->parent, in tcf_node_dump()
2741 n, NETLINK_CB(a->cb->skb).portid, in tcf_node_dump()
2742 a->cb->nlh->nlmsg_seq, NLM_F_MULTI, in tcf_node_dump()
2743 RTM_NEWTFILTER, a->terse_dump, true, NULL); in tcf_node_dump()
2750 struct net *net = sock_net(skb->sk); in tcf_chain_dump()
2751 struct tcf_block *block = chain->block; in tcf_chain_dump()
2752 struct tcmsg *tcm = nlmsg_data(cb->nlh); in tcf_chain_dump()
2764 if (TC_H_MAJ(tcm->tcm_info) && in tcf_chain_dump()
2765 TC_H_MAJ(tcm->tcm_info) != tp->prio) in tcf_chain_dump()
2767 if (TC_H_MIN(tcm->tcm_info) && in tcf_chain_dump()
2768 TC_H_MIN(tcm->tcm_info) != tp->protocol) in tcf_chain_dump()
2771 memset(&cb->args[1], 0, in tcf_chain_dump()
2772 sizeof(cb->args) - sizeof(cb->args[0])); in tcf_chain_dump()
2773 if (cb->args[1] == 0) { in tcf_chain_dump()
2775 NETLINK_CB(cb->skb).portid, in tcf_chain_dump()
2776 cb->nlh->nlmsg_seq, NLM_F_MULTI, in tcf_chain_dump()
2779 cb->args[1] = 1; in tcf_chain_dump()
2781 if (!tp->ops->walk) in tcf_chain_dump()
2790 arg.w.skip = cb->args[1] - 1; in tcf_chain_dump()
2792 arg.w.cookie = cb->args[2]; in tcf_chain_dump()
2794 tp->ops->walk(tp, &arg.w, true); in tcf_chain_dump()
2795 cb->args[2] = arg.w.cookie; in tcf_chain_dump()
2796 cb->args[1] = arg.w.count + 1; in tcf_chain_dump()
2816 struct net *net = sock_net(skb->sk); in tc_dump_tfilter()
2820 struct tcmsg *tcm = nlmsg_data(cb->nlh); in tc_dump_tfilter()
2827 if (nlmsg_len(cb->nlh) < sizeof(*tcm)) in tc_dump_tfilter()
2828 return skb->len; in tc_dump_tfilter()
2830 err = nlmsg_parse_deprecated(cb->nlh, sizeof(*tcm), tca, TCA_MAX, in tc_dump_tfilter()
2831 tcf_tfilter_dump_policy, cb->extack); in tc_dump_tfilter()
2842 if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) { in tc_dump_tfilter()
2843 block = tcf_block_refcnt_get(net, tcm->tcm_block_index); in tc_dump_tfilter()
2858 dev = __dev_get_by_index(net, tcm->tcm_ifindex); in tc_dump_tfilter()
2860 return skb->len; in tc_dump_tfilter()
2862 parent = tcm->tcm_parent; in tc_dump_tfilter()
2864 q = rtnl_dereference(dev->qdisc); in tc_dump_tfilter()
2866 q = qdisc_lookup(dev, TC_H_MAJ(tcm->tcm_parent)); in tc_dump_tfilter()
2869 cops = q->ops->cl_ops; in tc_dump_tfilter()
2872 if (!cops->tcf_block) in tc_dump_tfilter()
2874 if (TC_H_MIN(tcm->tcm_parent)) { in tc_dump_tfilter()
2875 cl = cops->find(q, tcm->tcm_parent); in tc_dump_tfilter()
2879 block = cops->tcf_block(q, cl, NULL); in tc_dump_tfilter()
2882 parent = block->classid; in tc_dump_tfilter()
2887 index_start = cb->args[0]; in tc_dump_tfilter()
2896 nla_get_u32(tca[TCA_CHAIN]) != chain->index) in tc_dump_tfilter()
2901 err = -EMSGSIZE; in tc_dump_tfilter()
2906 if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) in tc_dump_tfilter()
2908 cb->args[0] = index; in tc_dump_tfilter()
2912 if (skb->len == 0 && err) in tc_dump_tfilter()
2914 return skb->len; in tc_dump_tfilter()
2937 tcm->tcm_family = AF_UNSPEC; in tc_chain_fill_node()
2938 tcm->tcm__pad1 = 0; in tc_chain_fill_node()
2939 tcm->tcm__pad2 = 0; in tc_chain_fill_node()
2940 tcm->tcm_handle = 0; in tc_chain_fill_node()
2941 if (block->q) { in tc_chain_fill_node()
2942 tcm->tcm_ifindex = qdisc_dev(block->q)->ifindex; in tc_chain_fill_node()
2943 tcm->tcm_parent = block->q->handle; in tc_chain_fill_node()
2945 tcm->tcm_ifindex = TCM_IFINDEX_MAGIC_BLOCK; in tc_chain_fill_node()
2946 tcm->tcm_block_index = block->index; in tc_chain_fill_node()
2953 if (nla_put_string(skb, TCA_KIND, ops->kind)) in tc_chain_fill_node()
2955 if (ops->tmplt_dump(skb, net, priv) < 0) in tc_chain_fill_node()
2959 if (extack && extack->_msg && in tc_chain_fill_node()
2960 nla_put_string(skb, TCA_EXT_WARN_MSG, extack->_msg)) in tc_chain_fill_node()
2963 nlh->nlmsg_len = skb_tail_pointer(skb) - b; in tc_chain_fill_node()
2965 return skb->len; in tc_chain_fill_node()
2970 return -EMSGSIZE; in tc_chain_fill_node()
2978 struct tcf_block *block = chain->block; in tc_chain_notify()
2979 struct net *net = block->net; in tc_chain_notify()
2988 return -ENOBUFS; in tc_chain_notify()
2990 if (tc_chain_fill_node(chain->tmplt_ops, chain->tmplt_priv, in tc_chain_notify()
2991 chain->index, net, skb, block, portid, in tc_chain_notify()
2994 return -EINVAL; in tc_chain_notify()
3012 struct net *net = block->net; in tc_chain_notify_delete()
3020 return -ENOBUFS; in tc_chain_notify_delete()
3025 return -EINVAL; in tc_chain_notify_delete()
3045 return -EINVAL; in tc_chain_tmplt_add()
3051 if (!ops->tmplt_create || !ops->tmplt_destroy || !ops->tmplt_dump || in tc_chain_tmplt_add()
3052 !ops->tmplt_reoffload) { in tc_chain_tmplt_add()
3054 module_put(ops->owner); in tc_chain_tmplt_add()
3055 return -EOPNOTSUPP; in tc_chain_tmplt_add()
3058 tmplt_priv = ops->tmplt_create(net, chain, tca, extack); in tc_chain_tmplt_add()
3060 module_put(ops->owner); in tc_chain_tmplt_add()
3063 chain->tmplt_ops = ops; in tc_chain_tmplt_add()
3064 chain->tmplt_priv = tmplt_priv; in tc_chain_tmplt_add()
3075 tmplt_ops->tmplt_destroy(tmplt_priv); in tc_chain_tmplt_del()
3076 module_put(tmplt_ops->owner); in tc_chain_tmplt_del()
3084 struct net *net = sock_net(skb->sk); in tc_ctl_chain()
3103 parent = t->tcm_parent; in tc_ctl_chain()
3107 t->tcm_ifindex, t->tcm_block_index, extack); in tc_ctl_chain()
3114 err = -EINVAL; in tc_ctl_chain()
3118 mutex_lock(&block->lock); in tc_ctl_chain()
3120 if (n->nlmsg_type == RTM_NEWCHAIN) { in tc_ctl_chain()
3129 err = -EEXIST; in tc_ctl_chain()
3133 if (!(n->nlmsg_flags & NLM_F_CREATE)) { in tc_ctl_chain()
3135 err = -ENOENT; in tc_ctl_chain()
3141 err = -ENOMEM; in tc_ctl_chain()
3148 err = -EINVAL; in tc_ctl_chain()
3154 if (n->nlmsg_type == RTM_NEWCHAIN) { in tc_ctl_chain()
3161 chain->explicitly_created = true; in tc_ctl_chain()
3163 mutex_unlock(&block->lock); in tc_ctl_chain()
3165 switch (n->nlmsg_type) { in tc_ctl_chain()
3187 err = tc_chain_notify(chain, skb, n->nlmsg_seq, in tc_ctl_chain()
3188 n->nlmsg_flags, n->nlmsg_type, true, extack); in tc_ctl_chain()
3193 err = -EOPNOTSUPP; in tc_ctl_chain()
3202 if (err == -EAGAIN) in tc_ctl_chain()
3208 mutex_unlock(&block->lock); in tc_ctl_chain()
3215 struct net *net = sock_net(skb->sk); in tc_dump_chain()
3219 struct tcmsg *tcm = nlmsg_data(cb->nlh); in tc_dump_chain()
3225 if (nlmsg_len(cb->nlh) < sizeof(*tcm)) in tc_dump_chain()
3226 return skb->len; in tc_dump_chain()
3228 err = nlmsg_parse_deprecated(cb->nlh, sizeof(*tcm), tca, TCA_MAX, in tc_dump_chain()
3229 rtm_tca_policy, cb->extack); in tc_dump_chain()
3233 if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) { in tc_dump_chain()
3234 block = tcf_block_refcnt_get(net, tcm->tcm_block_index); in tc_dump_chain()
3242 dev = __dev_get_by_index(net, tcm->tcm_ifindex); in tc_dump_chain()
3244 return skb->len; in tc_dump_chain()
3246 if (!tcm->tcm_parent) in tc_dump_chain()
3247 q = rtnl_dereference(dev->qdisc); in tc_dump_chain()
3249 q = qdisc_lookup(dev, TC_H_MAJ(tcm->tcm_parent)); in tc_dump_chain()
3253 cops = q->ops->cl_ops; in tc_dump_chain()
3256 if (!cops->tcf_block) in tc_dump_chain()
3258 if (TC_H_MIN(tcm->tcm_parent)) { in tc_dump_chain()
3259 cl = cops->find(q, tcm->tcm_parent); in tc_dump_chain()
3263 block = cops->tcf_block(q, cl, NULL); in tc_dump_chain()
3270 index_start = cb->args[0]; in tc_dump_chain()
3273 mutex_lock(&block->lock); in tc_dump_chain()
3274 list_for_each_entry(chain, &block->chain_list, list) { in tc_dump_chain()
3276 nla_get_u32(tca[TCA_CHAIN]) != chain->index)) in tc_dump_chain()
3284 err = tc_chain_fill_node(chain->tmplt_ops, chain->tmplt_priv, in tc_dump_chain()
3285 chain->index, net, skb, block, in tc_dump_chain()
3286 NETLINK_CB(cb->skb).portid, in tc_dump_chain()
3287 cb->nlh->nlmsg_seq, NLM_F_MULTI, in tc_dump_chain()
3293 mutex_unlock(&block->lock); in tc_dump_chain()
3295 if (tcm->tcm_ifindex == TCM_IFINDEX_MAGIC_BLOCK) in tc_dump_chain()
3297 cb->args[0] = index; in tc_dump_chain()
3301 if (skb->len == 0 && err) in tc_dump_chain()
3303 return skb->len; in tc_dump_chain()
3313 exts->type = 0; in tcf_exts_init_ex()
3314 exts->nr_actions = 0; in tcf_exts_init_ex()
3315 exts->miss_cookie_node = NULL; in tcf_exts_init_ex()
3319 exts->net = net; in tcf_exts_init_ex()
3320 exts->actions = kcalloc(TCA_ACT_MAX_PRIO, sizeof(struct tc_action *), in tcf_exts_init_ex()
3322 if (!exts->actions) in tcf_exts_init_ex()
3323 return -ENOMEM; in tcf_exts_init_ex()
3326 exts->action = action; in tcf_exts_init_ex()
3327 exts->police = police; in tcf_exts_init_ex()
3341 exts->actions = NULL; in tcf_exts_init_ex()
3352 if (exts->actions) { in tcf_exts_destroy()
3353 tcf_action_destroy(exts->actions, TCA_ACT_UNBIND); in tcf_exts_destroy()
3354 kfree(exts->actions); in tcf_exts_destroy()
3356 exts->nr_actions = 0; in tcf_exts_destroy()
3371 if (exts->police && tb[exts->police]) { in tcf_exts_validate_ex()
3375 a_o = tc_action_load_ops(tb[exts->police], flags, in tcf_exts_validate_ex()
3379 act = tcf_action_init_1(net, tp, tb[exts->police], in tcf_exts_validate_ex()
3382 module_put(a_o->owner); in tcf_exts_validate_ex()
3386 act->type = exts->type = TCA_OLD_COMPAT; in tcf_exts_validate_ex()
3387 exts->actions[0] = act; in tcf_exts_validate_ex()
3388 exts->nr_actions = 1; in tcf_exts_validate_ex()
3389 tcf_idr_insert_many(exts->actions, init_res); in tcf_exts_validate_ex()
3390 } else if (exts->action && tb[exts->action]) { in tcf_exts_validate_ex()
3394 err = tcf_action_init(net, tp, tb[exts->action], in tcf_exts_validate_ex()
3395 rate_tlv, exts->actions, init_res, in tcf_exts_validate_ex()
3400 exts->nr_actions = err; in tcf_exts_validate_ex()
3404 if ((exts->action && tb[exts->action]) || in tcf_exts_validate_ex()
3405 (exts->police && tb[exts->police])) { in tcf_exts_validate_ex()
3407 return -EOPNOTSUPP; in tcf_exts_validate_ex()
3438 if (exts->nr_actions == 0) in tcf_exts_first_act()
3441 return exts->actions[0]; in tcf_exts_first_act()
3448 struct nlattr *nest; in tcf_exts_dump() local
3450 if (exts->action && tcf_exts_has_actions(exts)) { in tcf_exts_dump()
3452 * again for backward compatible mode - we want in tcf_exts_dump()
3454 * tc data even if iproute2 was newer - jhs in tcf_exts_dump()
3456 if (exts->type != TCA_OLD_COMPAT) { in tcf_exts_dump()
3457 nest = nla_nest_start_noflag(skb, exts->action); in tcf_exts_dump()
3458 if (nest == NULL) in tcf_exts_dump()
3461 if (tcf_action_dump(skb, exts->actions, 0, 0, false) in tcf_exts_dump()
3464 nla_nest_end(skb, nest); in tcf_exts_dump()
3465 } else if (exts->police) { in tcf_exts_dump()
3467 nest = nla_nest_start_noflag(skb, exts->police); in tcf_exts_dump()
3468 if (nest == NULL || !act) in tcf_exts_dump()
3472 nla_nest_end(skb, nest); in tcf_exts_dump()
3478 nla_nest_cancel(skb, nest); in tcf_exts_dump()
3479 return -1; in tcf_exts_dump()
3489 struct nlattr *nest; in tcf_exts_terse_dump() local
3491 if (!exts->action || !tcf_exts_has_actions(exts)) in tcf_exts_terse_dump()
3494 nest = nla_nest_start_noflag(skb, exts->action); in tcf_exts_terse_dump()
3495 if (!nest) in tcf_exts_terse_dump()
3498 if (tcf_action_dump(skb, exts->actions, 0, 0, true) < 0) in tcf_exts_terse_dump()
3500 nla_nest_end(skb, nest); in tcf_exts_terse_dump()
3504 nla_nest_cancel(skb, nest); in tcf_exts_terse_dump()
3505 return -1; in tcf_exts_terse_dump()
3517 return -1; in tcf_exts_dump_stats()
3528 atomic_inc(&block->offloadcnt); in tcf_block_offload_inc()
3536 atomic_dec(&block->offloadcnt); in tcf_block_offload_dec()
3543 lockdep_assert_held(&block->cb_lock); in tc_cls_offload_cnt_update()
3545 spin_lock(&tp->lock); in tc_cls_offload_cnt_update()
3551 *cnt -= diff; in tc_cls_offload_cnt_update()
3555 spin_unlock(&tp->lock); in tc_cls_offload_cnt_update()
3562 lockdep_assert_held(&block->cb_lock); in tc_cls_offload_cnt_reset()
3564 spin_lock(&tp->lock); in tc_cls_offload_cnt_reset()
3567 spin_unlock(&tp->lock); in tc_cls_offload_cnt_reset()
3578 list_for_each_entry(block_cb, &block->flow_block.cb_list, list) { in __tc_setup_cb_call()
3579 err = block_cb->cb(type, type_data, block_cb->cb_priv); in __tc_setup_cb_call()
3593 bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held; in tc_setup_cb_call()
3599 down_read(&block->cb_lock); in tc_setup_cb_call()
3604 if (!rtnl_held && !take_rtnl && block->lockeddevcnt) { in tc_setup_cb_call()
3605 up_read(&block->cb_lock); in tc_setup_cb_call()
3612 up_read(&block->cb_lock); in tc_setup_cb_call()
3619 /* Non-destructive filter add. If filter that wasn't already in hardware is
3629 bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held; in tc_setup_cb_add()
3635 down_read(&block->cb_lock); in tc_setup_cb_add()
3640 if (!rtnl_held && !take_rtnl && block->lockeddevcnt) { in tc_setup_cb_add()
3641 up_read(&block->cb_lock); in tc_setup_cb_add()
3646 /* Make sure all netdevs sharing this block are offload-capable. */ in tc_setup_cb_add()
3647 if (block->nooffloaddevcnt && err_stop) { in tc_setup_cb_add()
3648 ok_count = -EOPNOTSUPP; in tc_setup_cb_add()
3656 if (tp->ops->hw_add) in tc_setup_cb_add()
3657 tp->ops->hw_add(tp, type_data); in tc_setup_cb_add()
3662 up_read(&block->cb_lock); in tc_setup_cb_add()
3681 bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held; in tc_setup_cb_replace()
3687 down_read(&block->cb_lock); in tc_setup_cb_replace()
3692 if (!rtnl_held && !take_rtnl && block->lockeddevcnt) { in tc_setup_cb_replace()
3693 up_read(&block->cb_lock); in tc_setup_cb_replace()
3698 /* Make sure all netdevs sharing this block are offload-capable. */ in tc_setup_cb_replace()
3699 if (block->nooffloaddevcnt && err_stop) { in tc_setup_cb_replace()
3700 ok_count = -EOPNOTSUPP; in tc_setup_cb_replace()
3705 if (tp->ops->hw_del) in tc_setup_cb_replace()
3706 tp->ops->hw_del(tp, type_data); in tc_setup_cb_replace()
3712 if (tp->ops->hw_add) in tc_setup_cb_replace()
3713 tp->ops->hw_add(tp, type_data); in tc_setup_cb_replace()
3718 up_read(&block->cb_lock); in tc_setup_cb_replace()
3733 bool take_rtnl = READ_ONCE(block->lockeddevcnt) && !rtnl_held; in tc_setup_cb_destroy()
3739 down_read(&block->cb_lock); in tc_setup_cb_destroy()
3744 if (!rtnl_held && !take_rtnl && block->lockeddevcnt) { in tc_setup_cb_destroy()
3745 up_read(&block->cb_lock); in tc_setup_cb_destroy()
3753 if (tp->ops->hw_del) in tc_setup_cb_destroy()
3754 tp->ops->hw_del(tp, type_data); in tc_setup_cb_destroy()
3756 up_read(&block->cb_lock); in tc_setup_cb_destroy()
3789 user_cookie = rcu_dereference(act->user_cookie); in tcf_act_get_user_cookie()
3791 entry->user_cookie = flow_action_cookie_create(user_cookie->data, in tcf_act_get_user_cookie()
3792 user_cookie->len, in tcf_act_get_user_cookie()
3794 if (!entry->user_cookie) in tcf_act_get_user_cookie()
3795 err = -ENOMEM; in tcf_act_get_user_cookie()
3803 flow_action_cookie_destroy(entry->user_cookie); in tcf_act_put_user_cookie()
3813 if (entry->destructor) in tc_cleanup_offload_action()
3814 entry->destructor(entry->destructor_priv); in tc_cleanup_offload_action()
3825 if (act->ops->offload_act_setup) { in tc_setup_offload_act()
3826 return act->ops->offload_act_setup(act, entry, index_inc, true, in tc_setup_offload_act()
3830 return -EOPNOTSUPP; in tc_setup_offload_act()
3856 entry = &flow_action->entries[j]; in tc_setup_action()
3857 spin_lock_bh(&act->tcfa_lock); in tc_setup_action()
3868 entry[k].hw_stats = tc_act_hw_stats(act->hw_stats); in tc_setup_action()
3869 entry[k].hw_index = act->tcfa_index; in tc_setup_action()
3877 spin_unlock_bh(&act->tcfa_lock); in tc_setup_action()
3886 spin_unlock_bh(&act->tcfa_lock); in tc_setup_action()
3900 miss_cookie_base = exts->miss_cookie_node ? in tc_setup_offload_action()
3901 exts->miss_cookie_node->miss_cookie_base : 0; in tc_setup_offload_action()
3902 return tc_setup_action(flow_action, exts->actions, miss_cookie_base, in tc_setup_offload_action()
3934 return -EINVAL; in tcf_qevent_parse_block_index()
3955 qe->info.binder_type = binder_type; in tcf_qevent_init()
3956 qe->info.chain_head_change = tcf_chain_head_change_dflt; in tcf_qevent_init()
3957 qe->info.chain_head_change_priv = &qe->filter_chain; in tcf_qevent_init()
3958 qe->info.block_index = block_index; in tcf_qevent_init()
3960 return tcf_block_get_ext(&qe->block, sch, &qe->info, extack); in tcf_qevent_init()
3966 if (qe->info.block_index) in tcf_qevent_destroy()
3967 tcf_block_put_ext(qe->block, sch, &qe->info); in tcf_qevent_destroy()
3984 /* Bounce newly-configured block or change in block. */ in tcf_qevent_validate_change()
3985 if (block_index != qe->info.block_index) { in tcf_qevent_validate_change()
3987 return -EINVAL; in tcf_qevent_validate_change()
4000 if (!qe->info.block_index) in tcf_qevent_handle()
4003 fl = rcu_dereference_bh(qe->filter_chain); in tcf_qevent_handle()
4029 if (!qe->info.block_index) in tcf_qevent_dump()
4031 return nla_put_u32(skb, attr_name, qe->info.block_index); in tcf_qevent_dump()
4040 spin_lock_init(&tn->idr_lock); in tcf_net_init()
4041 idr_init(&tn->idr); in tcf_net_init()
4049 idr_destroy(&tn->idr); in tcf_net_exit()
4078 return -ENOMEM; in tc_filter_init()