Lines Matching +full:max +full:- +full:adj
1 // SPDX-License-Identifier: GPL-2.0-or-later
11 * Florian la Roche <rzsfl@rz.uni-sb.de>
19 * D.J. Barrow : Fixed bug where dev->refcnt gets set
64 * Pekka Riikonen : Netdev boot-time settings code
66 * indefinitely on dev->refcnt
67 * J Hadi Salim : - Backlog queue sampling
68 * - netif_rx() feedback
169 #include "net-sysfs.h"
189 unsigned int val = net->dev_base_seq + 1; in dev_base_seq_inc()
191 WRITE_ONCE(net->dev_base_seq, val ?: 1); in dev_base_seq_inc()
198 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)]; in dev_name_hash()
203 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)]; in dev_index_hash()
235 spin_lock_irqsave(&sd->input_pkt_queue.lock, *flags); in backlog_lock_irq_save()
243 spin_lock_irq(&sd->input_pkt_queue.lock); in backlog_lock_irq_disable()
252 spin_unlock_irqrestore(&sd->input_pkt_queue.lock, *flags); in backlog_unlock_irq_restore()
260 spin_unlock_irq(&sd->input_pkt_queue.lock); in backlog_unlock_irq_enable()
273 INIT_HLIST_NODE(&name_node->hlist); in netdev_name_node_alloc()
274 name_node->dev = dev; in netdev_name_node_alloc()
275 name_node->name = name; in netdev_name_node_alloc()
284 name_node = netdev_name_node_alloc(dev, dev->name); in netdev_name_node_head_alloc()
287 INIT_LIST_HEAD(&name_node->list); in netdev_name_node_head_alloc()
299 hlist_add_head_rcu(&name_node->hlist, in netdev_name_node_add()
300 dev_name_hash(net, name_node->name)); in netdev_name_node_add()
305 hlist_del_rcu(&name_node->hlist); in netdev_name_node_del()
315 if (!strcmp(name_node->name, name)) in netdev_name_node_lookup()
327 if (!strcmp(name_node->name, name)) in netdev_name_node_lookup_rcu()
345 return -EEXIST; in netdev_name_node_alt_create()
348 return -ENOMEM; in netdev_name_node_alt_create()
350 /* The node that holds dev->name acts as a head of per-device list. */ in netdev_name_node_alt_create()
351 list_add_tail_rcu(&name_node->list, &dev->name_node->list); in netdev_name_node_alt_create()
361 kfree(name_node->name); in netdev_name_node_alt_free()
368 list_del(&name_node->list); in __netdev_name_node_alt_destroy()
369 call_rcu(&name_node->rcu, netdev_name_node_alt_free); in __netdev_name_node_alt_destroy()
379 return -ENOENT; in netdev_name_node_alt_destroy()
383 if (name_node == dev->name_node || name_node->dev != dev) in netdev_name_node_alt_destroy()
384 return -EINVAL; in netdev_name_node_alt_destroy()
394 list_for_each_entry_safe(name_node, tmp, &dev->name_node->list, list) { in netdev_name_node_alt_flush()
395 list_del(&name_node->list); in netdev_name_node_alt_flush()
396 netdev_name_node_alt_free(&name_node->rcu); in netdev_name_node_alt_flush()
408 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head); in list_netdevice()
409 netdev_name_node_add(net, dev->name_node); in list_netdevice()
410 hlist_add_head_rcu(&dev->index_hlist, in list_netdevice()
411 dev_index_hash(net, dev->ifindex)); in list_netdevice()
417 WARN_ON(xa_store(&net->dev_by_index, dev->ifindex, dev, GFP_KERNEL)); in list_netdevice()
432 xa_erase(&net->dev_by_index, dev->ifindex); in unlist_netdevice()
438 list_del_rcu(&dev->dev_list); in unlist_netdevice()
439 netdev_name_node_del(dev->name_node); in unlist_netdevice()
440 hlist_del_rcu(&dev->index_hlist); in unlist_netdevice()
471 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
472 * according to dev->type
519 return ARRAY_SIZE(netdev_lock_type) - 1; in netdev_lock_pos()
536 i = netdev_lock_pos(dev->type); in netdev_set_addr_lockdep_class()
537 lockdep_set_class_and_name(&dev->addr_list_lock, in netdev_set_addr_lockdep_class()
570 * is cloned and should be copied-on-write, so that it will
572 * --ANK (980803)
577 if (pt->type == htons(ETH_P_ALL)) { in ptype_head()
578 if (!pt->af_packet_net && !pt->dev) in ptype_head()
581 return pt->dev ? &pt->dev->ptype_all : in ptype_head()
582 &pt->af_packet_net->ptype_all; in ptype_head()
585 if (pt->dev) in ptype_head()
586 return &pt->dev->ptype_specific; in ptype_head()
588 return pt->af_packet_net ? &pt->af_packet_net->ptype_specific : in ptype_head()
589 &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK]; in ptype_head()
593 * dev_add_pack - add packet handler
613 list_add_rcu(&pt->list, head); in dev_add_pack()
619 * __dev_remove_pack - remove packet handler
643 list_del_rcu(&pt->list); in __dev_remove_pack()
655 * dev_remove_pack - remove packet handler
682 * dev_get_iflink - get 'iflink' value of a interface
691 if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink) in dev_get_iflink()
692 return dev->netdev_ops->ndo_get_iflink(dev); in dev_get_iflink()
694 return READ_ONCE(dev->ifindex); in dev_get_iflink()
699 * dev_fill_metadata_dst - Retrieve tunnel egress information.
711 if (!dev->netdev_ops || !dev->netdev_ops->ndo_fill_metadata_dst) in dev_fill_metadata_dst()
712 return -EINVAL; in dev_fill_metadata_dst()
716 return -ENOMEM; in dev_fill_metadata_dst()
717 if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX))) in dev_fill_metadata_dst()
718 return -EINVAL; in dev_fill_metadata_dst()
720 return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb); in dev_fill_metadata_dst()
726 int k = stack->num_paths++; in dev_fwd_path()
731 return &stack->path[k]; in dev_fwd_path()
745 stack->num_paths = 0; in dev_fill_forward_path()
746 while (ctx.dev && ctx.dev->netdev_ops->ndo_fill_forward_path) { in dev_fill_forward_path()
750 return -1; in dev_fill_forward_path()
753 ret = ctx.dev->netdev_ops->ndo_fill_forward_path(&ctx, path); in dev_fill_forward_path()
755 return -1; in dev_fill_forward_path()
758 return -1; in dev_fill_forward_path()
766 return -1; in dev_fill_forward_path()
767 path->type = DEV_PATH_ETHERNET; in dev_fill_forward_path()
768 path->dev = ctx.dev; in dev_fill_forward_path()
781 if (napi->napi_id == napi_id) in napi_by_id()
797 if (WARN_ON_ONCE(!napi->dev)) in netdev_napi_by_id()
799 if (!net_eq(net, dev_net(napi->dev))) in netdev_napi_by_id()
806 * netdev_napi_by_id_lock() - find a device by NAPI ID and lock it
824 if (!napi || READ_ONCE(napi->dev->reg_state) != NETREG_REGISTERED) { in netdev_napi_by_id_lock()
829 dev = napi->dev; in netdev_napi_by_id_lock()
839 if (napi && napi->dev != dev) in netdev_napi_by_id_lock()
849 * __dev_get_by_name - find a device by its name
865 return node_name ? node_name->dev : NULL; in __dev_get_by_name()
870 * dev_get_by_name_rcu - find a device by its name
886 return node_name ? node_name->dev : NULL; in dev_get_by_name_rcu()
904 * netdev_get_by_name() - find a device by its name
929 * __dev_get_by_index - find a device by its ifindex
945 if (dev->ifindex == ifindex) in __dev_get_by_index()
953 * dev_get_by_index_rcu - find a device by its ifindex
969 if (dev->ifindex == ifindex) in dev_get_by_index_rcu()
990 * netdev_get_by_index() - find a device by its ifindex
1014 * dev_get_by_napi_id - find a device by napi_id
1033 return napi ? napi->dev : NULL; in dev_get_by_napi_id()
1047 if (dev->reg_state > NETREG_REGISTERED || in __netdev_put_lock()
1048 dev->moving_ns || !net_eq(dev_net(dev), net)) { in __netdev_put_lock()
1061 if (dev->reg_state > NETREG_REGISTERED || in __netdev_put_lock_ops_compat()
1062 dev->moving_ns || !net_eq(dev_net(dev), net)) { in __netdev_put_lock_ops_compat()
1072 * netdev_get_by_index_lock() - find a device by its ifindex
1077 * with @ifindex is found it will be returned with netdev->lock held.
1114 dev = xa_find(&net->dev_by_index, index, ULONG_MAX, XA_PRESENT); in netdev_xa_find_lock()
1139 dev = xa_find(&net->dev_by_index, index, ULONG_MAX, XA_PRESENT); in netdev_xa_find_lock_ops_compat()
1163 strscpy(name, dev->name, IFNAMSIZ); in netdev_copy_name()
1168 * netdev_get_name - get a netdevice name, knowing its ifindex.
1182 ret = -ENODEV; in netdev_get_name()
1197 return dev->type == type && !memcmp(dev->dev_addr, ha, dev->addr_len); in dev_addr_cmp()
1201 * dev_getbyhwaddr_rcu - find a device by its hardware address
1228 * dev_getbyhwaddr() - find a device by its hardware address
1259 if (dev->type == type) { in dev_getfirstbyhwtype()
1270 * __dev_get_by_flags - find any device with given flags
1289 if (((dev->flags ^ if_flags) & mask) == 0) { in __dev_get_by_flags()
1299 * dev_valid_name - check if name is okay for network device
1325 * __dev_alloc_name - allocate a name for a device
1330 * Passed a format string - eg "lt%d" it will try and find a suitable
1353 return -EINVAL; in __dev_alloc_name()
1358 return -ENOMEM; in __dev_alloc_name()
1364 if (!sscanf(name_node->name, name, &i)) in __dev_alloc_name()
1371 if (!strncmp(buf, name_node->name, IFNAMSIZ)) in __dev_alloc_name()
1374 if (!sscanf(d->name, name, &i)) in __dev_alloc_name()
1381 if (!strncmp(buf, d->name, IFNAMSIZ)) in __dev_alloc_name()
1388 return -ENFILE; in __dev_alloc_name()
1402 return -EINVAL; in dev_prep_valid_name()
1408 return -dup_errno; in dev_prep_valid_name()
1415 * dev_alloc_name - allocate a name for a device
1419 * Passed a format string - eg "lt%d" it will try and find a suitable
1430 return dev_prep_valid_name(dev_net(dev), dev, name, dev->name, ENFILE); in dev_alloc_name()
1439 ret = dev_prep_valid_name(net, dev, name, dev->name, EEXIST); in dev_get_valid_name()
1453 if (!strncmp(newname, dev->name, IFNAMSIZ)) in netif_change_name()
1456 memcpy(oldname, dev->name, IFNAMSIZ); in netif_change_name()
1467 dev->flags & IFF_UP ? " (while UP)" : ""); in netif_change_name()
1469 old_assign_type = dev->name_assign_type; in netif_change_name()
1470 WRITE_ONCE(dev->name_assign_type, NET_NAME_RENAMED); in netif_change_name()
1473 ret = device_rename(&dev->dev, dev->name); in netif_change_name()
1476 memcpy(dev->name, oldname, IFNAMSIZ); in netif_change_name()
1478 WRITE_ONCE(dev->name_assign_type, old_assign_type); in netif_change_name()
1484 netdev_name_node_del(dev->name_node); in netif_change_name()
1488 netdev_name_node_add(net, dev->name_node); in netif_change_name()
1498 memcpy(dev->name, oldname, IFNAMSIZ); in netif_change_name()
1501 WRITE_ONCE(dev->name_assign_type, old_assign_type); in netif_change_name()
1518 return -EINVAL; in netif_set_alias()
1523 return -ENOMEM; in netif_set_alias()
1525 memcpy(new_alias->ifalias, alias, len); in netif_set_alias()
1526 new_alias->ifalias[len] = 0; in netif_set_alias()
1530 new_alias = rcu_replace_pointer(dev->ifalias, new_alias, in netif_set_alias()
1541 * dev_get_alias - get ifalias of a device
1555 alias = rcu_dereference(dev->ifalias); in dev_get_alias()
1557 ret = snprintf(name, len, "%s", alias->ifalias); in dev_get_alias()
1564 * netdev_features_change - device changes features
1579 if (dev->flags & IFF_UP) { in netif_state_change()
1591 * __netdev_notify_peers - notify network peers about existence of @dev,
1610 * netdev_notify_peers - notify network peers about existence of @dev
1637 n->thread = kthread_run(napi_threaded_poll, n, "napi/%s-%d", in napi_kthread_create()
1638 n->dev->name, n->napi_id); in napi_kthread_create()
1639 if (IS_ERR(n->thread)) { in napi_kthread_create()
1640 err = PTR_ERR(n->thread); in napi_kthread_create()
1642 n->thread = NULL; in napi_kthread_create()
1650 const struct net_device_ops *ops = dev->netdev_ops; in __dev_open()
1657 /* may be detached because parent is runtime-suspended */ in __dev_open()
1658 if (dev->dev.parent) in __dev_open()
1659 pm_runtime_resume(dev->dev.parent); in __dev_open()
1661 return -ENODEV; in __dev_open()
1675 set_bit(__LINK_STATE_START, &dev->state); in __dev_open()
1679 if (ops->ndo_validate_addr) in __dev_open()
1680 ret = ops->ndo_validate_addr(dev); in __dev_open()
1682 if (!ret && ops->ndo_open) in __dev_open()
1683 ret = ops->ndo_open(dev); in __dev_open()
1688 clear_bit(__LINK_STATE_START, &dev->state); in __dev_open()
1693 add_device_randomness(dev->dev_addr, dev->addr_len); in __dev_open()
1703 if (dev->flags & IFF_UP) in netif_open()
1729 clear_bit(__LINK_STATE_START, &dev->state); in __dev_close_many()
1734 * dev->stop() will invoke napi_disable() on all of it's in __dev_close_many()
1743 const struct net_device_ops *ops = dev->netdev_ops; in __dev_close_many()
1749 * We allow it to be called even after a DETACH hot-plug in __dev_close_many()
1755 if (ops->ndo_stop) in __dev_close_many()
1756 ops->ndo_stop(dev); in __dev_close_many()
1767 list_add(&dev->close_list, &single); in __dev_close()
1778 if (!(dev->flags & IFF_UP)) in dev_close_many()
1779 list_del_init(&dev->close_list); in dev_close_many()
1787 list_del_init(&dev->close_list); in dev_close_many()
1794 if (dev->flags & IFF_UP) { in netif_close()
1797 list_add(&dev->close_list, &single); in netif_close()
1809 dev->wanted_features &= ~NETIF_F_LRO; in netif_disable_lro()
1812 if (unlikely(dev->features & NETIF_F_LRO)) in netif_disable_lro()
1824 * dev_disable_gro_hw - disable HW Generic Receive Offload on a device
1833 dev->wanted_features &= ~NETIF_F_GRO_HW; in dev_disable_gro_hw()
1836 if (unlikely(dev->features & NETIF_F_GRO_HW)) in dev_disable_gro_hw()
1871 return nb->notifier_call(nb, val, &info); in call_netdevice_notifier()
1884 if (!(dev->flags & IFF_UP)) in call_netdevice_register_notifiers()
1894 if (dev->flags & IFF_UP) { in call_netdevice_unregister_notifiers()
1935 * register_netdevice_notifier - register a network notifier block
1990 * unregister_netdevice_notifier - unregister a network notifier block
2034 err = raw_notifier_chain_register(&net->netdev_chain, nb); in __register_netdevice_notifier_net()
2047 raw_notifier_chain_unregister(&net->netdev_chain, nb); in __register_netdevice_notifier_net()
2056 err = raw_notifier_chain_unregister(&net->netdev_chain, nb); in __unregister_netdevice_notifier_net()
2065 * register_netdevice_notifier_net - register a per-netns network notifier block
2092 * unregister_netdevice_notifier_net - unregister a per-netns
2147 if (!net_eq(net, rcu_access_pointer(dev->nd_net.net))) { in rtnl_net_dev_lock()
2173 nn->nb = nb; in register_netdevice_notifier_dev_net()
2174 list_add(&nn->list, &dev->net_notifier_list); in register_netdevice_notifier_dev_net()
2189 list_del(&nn->list); in unregister_netdevice_notifier_dev_net()
2202 list_for_each_entry(nn, &dev->net_notifier_list, list) in move_netdevice_notifiers_dev_net()
2203 __move_netdevice_notifier_net(dev_net(dev), net, nn->nb); in move_netdevice_notifiers_dev_net()
2207 * call_netdevice_notifiers_info - call all network notifier blocks
2218 struct net *net = dev_net(info->dev); in call_netdevice_notifiers_info()
2223 /* Run per-netns notifier block chain first, then run the global one. in call_netdevice_notifiers_info()
2225 * all notifier block registrators get converted to be per-netns. in call_netdevice_notifiers_info()
2227 ret = raw_notifier_call_chain(&net->netdev_chain, val, info); in call_netdevice_notifiers_info()
2234 * call_netdevice_notifiers_info_robust - call per-netns notifier blocks
2241 * Call all per-netns network notifier blocks, but not notifier blocks on
2251 struct net *net = dev_net(info->dev); in call_netdevice_notifiers_info_robust()
2255 return raw_notifier_call_chain_robust(&net->netdev_chain, in call_netdevice_notifiers_info_robust()
2272 * call_netdevice_notifiers - call all network notifier blocks
2287 * call_netdevice_notifiers_mtu - call all network notifier blocks
2387 if (atomic_try_cmpxchg(&netstamp_wanted, &wanted, wanted - 1)) in net_disable_timestamp()
2400 skb->tstamp = 0; in net_timestamp_set()
2401 skb->tstamp_type = SKB_CLOCK_REALTIME; in net_timestamp_set()
2403 skb->tstamp = ktime_get_real(); in net_timestamp_set()
2408 if ((COND) && !(SKB)->tstamp) \
2409 (SKB)->tstamp = ktime_get_real(); \
2424 skb->protocol = eth_type_trans(skb, dev); in __dev_forward_skb2()
2438 * dev_forward_skb - loopback an skb to another netif
2471 return -ENOMEM; in deliver_skb()
2472 refcount_inc(&skb->users); in deliver_skb()
2473 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev); in deliver_skb()
2485 if (ptype->type != type) in deliver_ptype_list_skb()
2496 if (!ptype->af_packet_priv || !skb->sk) in skb_loop_sk()
2499 if (ptype->id_match) in skb_loop_sk()
2500 return ptype->id_match(ptype, skb->sk); in skb_loop_sk()
2501 else if ((struct sock *)ptype->af_packet_priv == skb->sk) in skb_loop_sk()
2508 * dev_nit_active_rcu - return true if any network interface taps are in use
2519 return !list_empty(&dev_net(dev)->ptype_all) || in dev_nit_active_rcu()
2520 !list_empty(&dev->ptype_all); in dev_nit_active_rcu()
2536 ptype_list = &dev_net_rcu(dev)->ptype_all; in dev_queue_xmit_nit()
2539 if (READ_ONCE(ptype->ignore_outgoing)) in dev_queue_xmit_nit()
2543 * they originated from - MvS (miquels@drinkel.ow.org) in dev_queue_xmit_nit()
2549 deliver_skb(skb2, pt_prev, skb->dev); in dev_queue_xmit_nit()
2561 /* skb->nh should be correctly in dev_queue_xmit_nit()
2567 if (skb_network_header(skb2) < skb2->data || in dev_queue_xmit_nit()
2570 ntohs(skb2->protocol), in dev_queue_xmit_nit()
2571 dev->name); in dev_queue_xmit_nit()
2575 skb2->transport_header = skb2->network_header; in dev_queue_xmit_nit()
2576 skb2->pkt_type = PACKET_OUTGOING; in dev_queue_xmit_nit()
2580 if (ptype_list != &dev->ptype_all) { in dev_queue_xmit_nit()
2581 ptype_list = &dev->ptype_all; in dev_queue_xmit_nit()
2587 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev); in dev_queue_xmit_nit()
2596 * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
2611 struct netdev_tc_txq *tc = &dev->tc_to_txq[0]; in netif_setup_tc()
2614 if (tc->offset + tc->count > txq) { in netif_setup_tc()
2616 dev->num_tc = 0; in netif_setup_tc()
2624 tc = &dev->tc_to_txq[q]; in netif_setup_tc()
2625 if (tc->offset + tc->count > txq) { in netif_setup_tc()
2635 if (dev->num_tc) { in netdev_txq_to_tc()
2636 struct netdev_tc_txq *tc = &dev->tc_to_txq[0]; in netdev_txq_to_tc()
2641 if ((txq - tc->offset) < tc->count) in netdev_txq_to_tc()
2645 /* didn't find it, just return -1 to indicate no match */ in netdev_txq_to_tc()
2646 return -1; in netdev_txq_to_tc()
2666 map = xmap_dereference(dev_maps->attr_map[tci]); in remove_xps_queue()
2670 for (pos = map->len; pos--;) { in remove_xps_queue()
2671 if (map->queues[pos] != index) in remove_xps_queue()
2674 if (map->len > 1) { in remove_xps_queue()
2675 map->queues[pos] = map->queues[--map->len]; in remove_xps_queue()
2680 RCU_INIT_POINTER(old_maps->attr_map[tci], NULL); in remove_xps_queue()
2681 RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL); in remove_xps_queue()
2693 int num_tc = dev_maps->num_tc; in remove_xps_queue_cpu()
2697 for (tci = cpu * num_tc; num_tc--; tci++) { in remove_xps_queue_cpu()
2700 for (i = count, j = offset; i--; j++) { in remove_xps_queue_cpu()
2719 RCU_INIT_POINTER(dev->xps_maps[type], NULL); in reset_xps_maps()
2731 dev_maps = xmap_dereference(dev->xps_maps[type]); in clean_xps_maps()
2735 for (j = 0; j < dev_maps->nr_ids; j++) in clean_xps_maps()
2741 for (i = offset + (count - 1); count--; i--) in clean_xps_maps()
2767 netif_reset_xps_queues(dev, index, dev->num_tx_queues - index); in netif_reset_xps_queues_gt()
2777 for (pos = 0; map && pos < map->len; pos++) { in expand_xps_map()
2778 if (map->queues[pos] != index) in expand_xps_map()
2783 /* Need to add tx-queue to this CPU's/rx-queue's existing map */ in expand_xps_map()
2785 if (pos < map->alloc_len) in expand_xps_map()
2788 alloc_len = map->alloc_len * 2; in expand_xps_map()
2791 /* Need to allocate new map to store tx-queue on this CPU's/rx-queue's in expand_xps_map()
2803 new_map->queues[i] = map->queues[i]; in expand_xps_map()
2804 new_map->alloc_len = alloc_len; in expand_xps_map()
2805 new_map->len = pos; in expand_xps_map()
2815 int i, tci = index * dev_maps->num_tc; in xps_copy_dev_maps()
2819 for (i = 0; i < dev_maps->num_tc; i++, tci++) { in xps_copy_dev_maps()
2824 map = xmap_dereference(dev_maps->attr_map[tci]); in xps_copy_dev_maps()
2825 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map); in xps_copy_dev_maps()
2836 int i, j, tci, numa_node_id = -2; in __netif_set_xps_queue()
2841 WARN_ON_ONCE(index >= dev->num_tx_queues); in __netif_set_xps_queue()
2843 if (dev->num_tc) { in __netif_set_xps_queue()
2845 num_tc = dev->num_tc; in __netif_set_xps_queue()
2847 return -EINVAL; in __netif_set_xps_queue()
2850 dev = netdev_get_tx_queue(dev, index)->sb_dev ? : dev; in __netif_set_xps_queue()
2854 return -EINVAL; in __netif_set_xps_queue()
2859 dev_maps = xmap_dereference(dev->xps_maps[type]); in __netif_set_xps_queue()
2861 maps_sz = XPS_RXQ_DEV_MAPS_SIZE(num_tc, dev->num_rx_queues); in __netif_set_xps_queue()
2862 nr_ids = dev->num_rx_queues; in __netif_set_xps_queue()
2874 * setting up now, as dev->num_tc or nr_ids could have been updated in in __netif_set_xps_queue()
2879 dev_maps->num_tc == num_tc && dev_maps->nr_ids == nr_ids) in __netif_set_xps_queue()
2883 for (j = -1; j = netif_attrmask_next_and(j, online_mask, mask, nr_ids), in __netif_set_xps_queue()
2889 return -ENOMEM; in __netif_set_xps_queue()
2892 new_dev_maps->nr_ids = nr_ids; in __netif_set_xps_queue()
2893 new_dev_maps->num_tc = num_tc; in __netif_set_xps_queue()
2897 map = copy ? xmap_dereference(dev_maps->attr_map[tci]) : NULL; in __netif_set_xps_queue()
2903 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map); in __netif_set_xps_queue()
2922 /* add tx-queue to CPU/rx-queue maps */ in __netif_set_xps_queue()
2927 map = xmap_dereference(new_dev_maps->attr_map[tci]); in __netif_set_xps_queue()
2928 while ((pos < map->len) && (map->queues[pos] != index)) in __netif_set_xps_queue()
2931 if (pos == map->len) in __netif_set_xps_queue()
2932 map->queues[map->len++] = index; in __netif_set_xps_queue()
2935 if (numa_node_id == -2) in __netif_set_xps_queue()
2938 numa_node_id = -1; in __netif_set_xps_queue()
2948 rcu_assign_pointer(dev->xps_maps[type], new_dev_maps); in __netif_set_xps_queue()
2954 for (j = 0; j < dev_maps->nr_ids; j++) { in __netif_set_xps_queue()
2955 for (i = num_tc, tci = j * dev_maps->num_tc; i--; tci++) { in __netif_set_xps_queue()
2956 map = xmap_dereference(dev_maps->attr_map[tci]); in __netif_set_xps_queue()
2961 new_map = xmap_dereference(new_dev_maps->attr_map[tci]); in __netif_set_xps_queue()
2966 RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL); in __netif_set_xps_queue()
2987 /* removes tx-queue from unused CPUs/rx-queues */ in __netif_set_xps_queue()
2988 for (j = 0; j < dev_maps->nr_ids; j++) { in __netif_set_xps_queue()
2989 tci = j * dev_maps->num_tc; in __netif_set_xps_queue()
2991 for (i = 0; i < dev_maps->num_tc; i++, tci++) { in __netif_set_xps_queue()
2993 netif_attr_test_mask(j, mask, dev_maps->nr_ids) && in __netif_set_xps_queue()
2994 netif_attr_test_online(j, online_mask, dev_maps->nr_ids)) in __netif_set_xps_queue()
3017 for (i = num_tc, tci = j * num_tc; i--; tci++) { in __netif_set_xps_queue()
3018 new_map = xmap_dereference(new_dev_maps->attr_map[tci]); in __netif_set_xps_queue()
3020 xmap_dereference(dev_maps->attr_map[tci]) : in __netif_set_xps_queue()
3030 return -ENOMEM; in __netif_set_xps_queue()
3050 struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues]; in netdev_unbind_all_sb_channels()
3053 while (txq-- != &dev->_tx[0]) { in netdev_unbind_all_sb_channels()
3054 if (txq->sb_dev) in netdev_unbind_all_sb_channels()
3055 netdev_unbind_sb_channel(dev, txq->sb_dev); in netdev_unbind_all_sb_channels()
3067 dev->num_tc = 0; in netdev_reset_tc()
3068 memset(dev->tc_to_txq, 0, sizeof(dev->tc_to_txq)); in netdev_reset_tc()
3069 memset(dev->prio_tc_map, 0, sizeof(dev->prio_tc_map)); in netdev_reset_tc()
3075 if (tc >= dev->num_tc) in netdev_set_tc_queue()
3076 return -EINVAL; in netdev_set_tc_queue()
3081 dev->tc_to_txq[tc].count = count; in netdev_set_tc_queue()
3082 dev->tc_to_txq[tc].offset = offset; in netdev_set_tc_queue()
3090 return -EINVAL; in netdev_set_num_tc()
3097 dev->num_tc = num_tc; in netdev_set_num_tc()
3105 struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues]; in netdev_unbind_sb_channel()
3110 memset(sb_dev->tc_to_txq, 0, sizeof(sb_dev->tc_to_txq)); in netdev_unbind_sb_channel()
3111 memset(sb_dev->prio_tc_map, 0, sizeof(sb_dev->prio_tc_map)); in netdev_unbind_sb_channel()
3113 while (txq-- != &dev->_tx[0]) { in netdev_unbind_sb_channel()
3114 if (txq->sb_dev == sb_dev) in netdev_unbind_sb_channel()
3115 txq->sb_dev = NULL; in netdev_unbind_sb_channel()
3125 if (sb_dev->num_tc >= 0 || tc >= dev->num_tc) in netdev_bind_sb_channel_queue()
3126 return -EINVAL; in netdev_bind_sb_channel_queue()
3129 if ((offset + count) > dev->real_num_tx_queues) in netdev_bind_sb_channel_queue()
3130 return -EINVAL; in netdev_bind_sb_channel_queue()
3133 sb_dev->tc_to_txq[tc].count = count; in netdev_bind_sb_channel_queue()
3134 sb_dev->tc_to_txq[tc].offset = offset; in netdev_bind_sb_channel_queue()
3139 while (count--) in netdev_bind_sb_channel_queue()
3140 netdev_get_tx_queue(dev, count + offset)->sb_dev = sb_dev; in netdev_bind_sb_channel_queue()
3150 return -ENODEV; in netdev_set_sb_channel()
3152 /* We allow channels 1 - 32767 to be used for subordinate channels. in netdev_set_sb_channel()
3158 return -EINVAL; in netdev_set_sb_channel()
3160 dev->num_tc = -channel; in netdev_set_sb_channel()
3175 disabling = txq < dev->real_num_tx_queues; in netif_set_real_num_tx_queues()
3177 if (txq < 1 || txq > dev->num_tx_queues) in netif_set_real_num_tx_queues()
3178 return -EINVAL; in netif_set_real_num_tx_queues()
3180 if (dev->reg_state == NETREG_REGISTERED || in netif_set_real_num_tx_queues()
3181 dev->reg_state == NETREG_UNREGISTERING) { in netif_set_real_num_tx_queues()
3185 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues, in netif_set_real_num_tx_queues()
3190 if (dev->num_tc) in netif_set_real_num_tx_queues()
3197 dev->real_num_tx_queues = txq; in netif_set_real_num_tx_queues()
3207 dev->real_num_tx_queues = txq; in netif_set_real_num_tx_queues()
3215 * netif_set_real_num_rx_queues - set actual number of RX queues used
3228 if (rxq < 1 || rxq > dev->num_rx_queues) in netif_set_real_num_rx_queues()
3229 return -EINVAL; in netif_set_real_num_rx_queues()
3231 if (dev->reg_state == NETREG_REGISTERED) { in netif_set_real_num_rx_queues()
3235 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues, in netif_set_real_num_rx_queues()
3241 dev->real_num_rx_queues = rxq; in netif_set_real_num_rx_queues()
3247 * netif_set_real_num_queues - set actual number of RX and TX queues used
3258 unsigned int old_rxq = dev->real_num_rx_queues; in netif_set_real_num_queues()
3261 if (txq < 1 || txq > dev->num_tx_queues || in netif_set_real_num_queues()
3262 rxq < 1 || rxq > dev->num_rx_queues) in netif_set_real_num_queues()
3263 return -EINVAL; in netif_set_real_num_queues()
3265 /* Start from increases, so the error path only does decreases - in netif_set_real_num_queues()
3268 if (rxq > dev->real_num_rx_queues) { in netif_set_real_num_queues()
3273 if (txq > dev->real_num_tx_queues) { in netif_set_real_num_queues()
3278 if (rxq < dev->real_num_rx_queues) in netif_set_real_num_queues()
3280 if (txq < dev->real_num_tx_queues) in netif_set_real_num_queues()
3291 * netif_set_tso_max_size() - set the max size of TSO frames supported
3293 * @size: max skb->len of a TSO frame
3295 * Set the limit on the size of TSO super-frames the device can handle.
3301 dev->tso_max_size = min(GSO_MAX_SIZE, size); in netif_set_tso_max_size()
3302 if (size < READ_ONCE(dev->gso_max_size)) in netif_set_tso_max_size()
3304 if (size < READ_ONCE(dev->gso_ipv4_max_size)) in netif_set_tso_max_size()
3310 * netif_set_tso_max_segs() - set the max number of segs supported for TSO
3312 * @segs: max number of TCP segments
3315 * a single TSO super-frame.
3320 dev->tso_max_segs = segs; in netif_set_tso_max_segs()
3321 if (segs < READ_ONCE(dev->gso_max_segs)) in netif_set_tso_max_segs()
3327 * netif_inherit_tso_max() - copy all TSO limits from a lower device to an upper
3333 netif_set_tso_max_size(to, from->tso_max_size); in netif_inherit_tso_max()
3334 netif_set_tso_max_segs(to, from->tso_max_segs); in netif_inherit_tso_max()
3339 * netif_get_num_default_rss_queues - default number of RSS queues
3370 q->next_sched = NULL; in __netif_reschedule()
3371 *sd->output_queue_tailp = q; in __netif_reschedule()
3372 sd->output_queue_tailp = &q->next_sched; in __netif_reschedule()
3379 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state)) in __netif_schedule()
3390 return (struct dev_kfree_skb_cb *)skb->cb; in get_kfree_skb_cb()
3397 struct Qdisc *q = rcu_dereference(txq->qdisc); in netif_schedule_queue()
3407 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) { in netif_tx_wake_queue()
3411 q = rcu_dereference(dev_queue->qdisc); in netif_tx_wake_queue()
3425 if (likely(refcount_read(&skb->users) == 1)) { in dev_kfree_skb_irq_reason()
3427 refcount_set(&skb->users, 0); in dev_kfree_skb_irq_reason()
3428 } else if (likely(!refcount_dec_and_test(&skb->users))) { in dev_kfree_skb_irq_reason()
3431 get_kfree_skb_cb(skb)->reason = reason; in dev_kfree_skb_irq_reason()
3433 skb->next = __this_cpu_read(softnet_data.completion_queue); in dev_kfree_skb_irq_reason()
3451 * netif_device_detach - mark device as removed
3458 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) && in netif_device_detach()
3466 * netif_device_attach - mark device as attached
3473 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) && in netif_device_attach()
3491 u16 qcount = dev->real_num_tx_queues; in skb_tx_hash()
3493 if (dev->num_tc) { in skb_tx_hash()
3494 u8 tc = netdev_get_prio_tc_map(dev, skb->priority); in skb_tx_hash()
3496 qoffset = sb_dev->tc_to_txq[tc].offset; in skb_tx_hash()
3497 qcount = sb_dev->tc_to_txq[tc].count; in skb_tx_hash()
3500 sb_dev->name, qoffset, tc); in skb_tx_hash()
3502 qcount = dev->real_num_tx_queues; in skb_tx_hash()
3510 hash -= qoffset; in skb_tx_hash()
3512 hash -= qcount; in skb_tx_hash()
3522 struct net_device *dev = skb->dev; in skb_warn_bad_offload()
3529 if (dev->dev.parent) in skb_warn_bad_offload()
3530 name = dev_driver_string(dev->dev.parent); in skb_warn_bad_offload()
3536 name, dev ? &dev->features : &null_features, in skb_warn_bad_offload()
3537 skb->sk ? &skb->sk->sk_route_caps : &null_features); in skb_warn_bad_offload()
3549 if (skb->ip_summed == CHECKSUM_COMPLETE) in skb_checksum_help()
3554 return -EINVAL; in skb_checksum_help()
3558 return -EFAULT; in skb_checksum_help()
3571 ret = -EINVAL; in skb_checksum_help()
3578 csum = skb_checksum(skb, offset, skb->len - offset, 0); in skb_checksum_help()
3580 offset += skb->csum_offset; in skb_checksum_help()
3591 *(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0; in skb_checksum_help()
3593 skb->ip_summed = CHECKSUM_NONE; in skb_checksum_help()
3605 if (skb->ip_summed != CHECKSUM_PARTIAL) in skb_crc32c_csum_help()
3622 ret = -EINVAL; in skb_crc32c_csum_help()
3630 crc = ~skb_crc32c(skb, start, skb->len - start, ~0); in skb_crc32c_csum_help()
3631 *(__le32 *)(skb->data + offset) = cpu_to_le32(crc); in skb_crc32c_csum_help()
3641 __be16 type = skb->protocol; in skb_network_protocol()
3650 eth = (struct ethhdr *)skb->data; in skb_network_protocol()
3651 type = eth->h_proto; in skb_network_protocol()
3680 if (!(dev->features & NETIF_F_HIGHDMA)) { in illegal_highdma()
3681 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { in illegal_highdma()
3682 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; in illegal_highdma()
3702 features &= skb->dev->mpls_features; in net_mpls_features()
3723 if (skb->ip_summed != CHECKSUM_NONE && in harmonize_features()
3727 if (illegal_highdma(skb->dev, skb)) in harmonize_features()
3752 u16 gso_segs = skb_shinfo(skb)->gso_segs; in gso_features_check()
3754 if (gso_segs > READ_ONCE(dev->gso_max_segs)) in gso_features_check()
3757 if (unlikely(skb->len >= netif_get_gso_max_size(dev, skb))) in gso_features_check()
3760 if (!skb_shinfo(skb)->gso_type) { in gso_features_check()
3771 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL)) in gso_features_check()
3772 features &= ~dev->gso_partial_features; in gso_features_check()
3777 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) { in gso_features_check()
3778 struct iphdr *iph = skb->encapsulation ? in gso_features_check()
3781 if (!(iph->frag_off & htons(IP_DF))) in gso_features_check()
3790 struct net_device *dev = skb->dev; in netif_skb_features()
3791 netdev_features_t features = dev->features; in netif_skb_features()
3800 if (skb->encapsulation) in netif_skb_features()
3801 features &= dev->hw_enc_features; in netif_skb_features()
3805 dev->vlan_features | in netif_skb_features()
3809 if (dev->netdev_ops->ndo_features_check) in netif_skb_features()
3810 features &= dev->netdev_ops->ndo_features_check(skb, dev, in netif_skb_features()
3828 len = skb->len; in xmit_one()
3843 struct sk_buff *next = skb->next; in dev_hard_start_xmit()
3848 skb->next = next; in dev_hard_start_xmit()
3868 !vlan_hw_offload_capable(features, skb->vlan_proto)) in validate_xmit_vlan()
3889 switch (skb->csum_offset) { in skb_csum_hwoffload_help()
3910 if (!dev->netmem_tx) in validate_xmit_unreadable_skb()
3915 if (shinfo->nr_frags > 0) { in validate_xmit_unreadable_skb()
3916 niov = netmem_to_net_iov(skb_frag_netmem(&shinfo->frags[0])); in validate_xmit_unreadable_skb()
3918 net_devmem_iov_binding(niov)->dev != dev) in validate_xmit_unreadable_skb()
3966 if (skb->ip_summed == CHECKSUM_PARTIAL) { in validate_xmit_skb()
3967 if (skb->encapsulation) in validate_xmit_skb()
3994 next = skb->next; in validate_xmit_skb_list()
3998 skb->prev = skb; in validate_xmit_skb_list()
4007 tail->next = skb; in validate_xmit_skb_list()
4008 /* If skb was segmented, skb->prev points to in validate_xmit_skb_list()
4011 tail = skb->prev; in validate_xmit_skb_list()
4021 qdisc_skb_cb(skb)->pkt_len = skb->len; in qdisc_pkt_len_init()
4026 if (shinfo->gso_size && skb_transport_header_was_set(skb)) { in qdisc_pkt_len_init()
4027 u16 gso_segs = shinfo->gso_segs; in qdisc_pkt_len_init()
4034 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) { in qdisc_pkt_len_init()
4042 } else if (shinfo->gso_type & SKB_GSO_UDP_L4) { in qdisc_pkt_len_init()
4050 if (unlikely(shinfo->gso_type & SKB_GSO_DODGY)) { in qdisc_pkt_len_init()
4051 int payload = skb->len - hdr_len; in qdisc_pkt_len_init()
4056 gso_segs = DIV_ROUND_UP(payload, shinfo->gso_size); in qdisc_pkt_len_init()
4058 qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len; in qdisc_pkt_len_init()
4068 rc = q->enqueue(skb, q, to_free) & NET_XMIT_MASK; in dev_qdisc_enqueue()
4087 if (q->flags & TCQ_F_NOLOCK) { in __dev_xmit_skb()
4088 if (q->flags & TCQ_F_CAN_BYPASS && nolock_qdisc_is_empty(q) && in __dev_xmit_skb()
4091 * of q->seqlock to protect from racing with requeuing. in __dev_xmit_skb()
4120 if (unlikely(READ_ONCE(q->owner) == smp_processor_id())) { in __dev_xmit_skb()
4127 * This permits qdisc->running owner to get the lock more in __dev_xmit_skb()
4136 spin_lock(&q->busylock); in __dev_xmit_skb()
4139 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) { in __dev_xmit_skb()
4142 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) && in __dev_xmit_skb()
4145 * This is a work-conserving queue; there are no old skbs in __dev_xmit_skb()
4146 * waiting to be sent out; and the qdisc is not running - in __dev_xmit_skb()
4154 spin_unlock(&q->busylock); in __dev_xmit_skb()
4163 WRITE_ONCE(q->owner, smp_processor_id()); in __dev_xmit_skb()
4165 WRITE_ONCE(q->owner, -1); in __dev_xmit_skb()
4168 spin_unlock(&q->busylock); in __dev_xmit_skb()
4180 spin_unlock(&q->busylock); in __dev_xmit_skb()
4191 if (skb->priority) in skb_update_prio()
4193 map = rcu_dereference_bh(skb->dev->priomap); in skb_update_prio()
4200 prioidx = sock_cgroup_prioidx(&sk->sk_cgrp_data); in skb_update_prio()
4202 if (prioidx < map->priomap_len) in skb_update_prio()
4203 skb->priority = map->priomap[prioidx]; in skb_update_prio()
4210 * dev_loopback_xmit - loop back @skb
4219 skb->pkt_type = PACKET_LOOPBACK; in dev_loopback_xmit()
4220 if (skb->ip_summed == CHECKSUM_NONE) in dev_loopback_xmit()
4221 skb->ip_summed = CHECKSUM_UNNECESSARY; in dev_loopback_xmit()
4253 return current->net_xmit.skip_txqueue; in netdev_xmit_txqueue_skipped()
4258 current->net_xmit.skip_txqueue = skip; in netdev_xmit_skip_txqueue()
4270 struct mini_Qdisc *miniq = rcu_dereference_bh(entry->miniq); in tc_run()
4280 /* Block-wise bypass */ in tc_run()
4281 if (tcf_block_bypass_sw(miniq->block)) in tc_run()
4284 tc_skb_cb(skb)->mru = 0; in tc_run()
4285 tc_skb_cb(skb)->post_ct = false; in tc_run()
4289 ret = tcf_classify(skb, miniq->block, miniq->filter_list, &res, false); in tc_run()
4298 skb->tc_index = TC_H_MIN(res.classid); in tc_run()
4326 __skb_push(skb, skb->mac_len); in tcx_run()
4334 __skb_pull(skb, skb->mac_len); in tcx_run()
4342 struct bpf_mprog_entry *entry = rcu_dereference_bh(skb->dev->tcx_ingress); in sch_handle_ingress()
4356 qdisc_skb_cb(skb)->pkt_len = skb->len; in sch_handle_ingress()
4372 __skb_push(skb, skb->mac_len); in sch_handle_ingress()
4373 if (skb_do_redirect(skb) == -EAGAIN) { in sch_handle_ingress()
4374 __skb_pull(skb, skb->mac_len); in sch_handle_ingress()
4405 struct bpf_mprog_entry *entry = rcu_dereference_bh(dev->tcx_egress); in sch_handle_egress()
4415 /* qdisc_skb_cb(skb)->pkt_len & tcx_set_ingress() was in sch_handle_egress()
4471 int tc = netdev_get_prio_tc_map(dev, skb->priority); in __get_xps_queue_idx()
4473 int queue_index = -1; in __get_xps_queue_idx()
4475 if (tc >= dev_maps->num_tc || tci >= dev_maps->nr_ids) in __get_xps_queue_idx()
4478 tci *= dev_maps->num_tc; in __get_xps_queue_idx()
4481 map = rcu_dereference(dev_maps->attr_map[tci]); in __get_xps_queue_idx()
4483 if (map->len == 1) in __get_xps_queue_idx()
4484 queue_index = map->queues[0]; in __get_xps_queue_idx()
4486 queue_index = map->queues[reciprocal_scale( in __get_xps_queue_idx()
4487 skb_get_hash(skb), map->len)]; in __get_xps_queue_idx()
4488 if (unlikely(queue_index >= dev->real_num_tx_queues)) in __get_xps_queue_idx()
4489 queue_index = -1; in __get_xps_queue_idx()
4500 struct sock *sk = skb->sk; in get_xps_queue()
4501 int queue_index = -1; in get_xps_queue()
4504 return -1; in get_xps_queue()
4510 dev_maps = rcu_dereference(sb_dev->xps_maps[XPS_RXQS]); in get_xps_queue()
4521 dev_maps = rcu_dereference(sb_dev->xps_maps[XPS_CPUS]); in get_xps_queue()
4523 unsigned int tci = skb->sender_cpu - 1; in get_xps_queue()
4533 return -1; in get_xps_queue()
4547 struct sock *sk = skb->sk; in netdev_pick_tx()
4552 if (queue_index < 0 || skb->ooo_okay || in netdev_pick_tx()
4553 queue_index >= dev->real_num_tx_queues) { in netdev_pick_tx()
4561 rcu_access_pointer(sk->sk_dst_cache)) in netdev_pick_tx()
4578 u32 sender_cpu = skb->sender_cpu - 1; in netdev_core_pick_tx()
4581 skb->sender_cpu = raw_smp_processor_id() + 1; in netdev_core_pick_tx()
4584 if (dev->real_num_tx_queues != 1) { in netdev_core_pick_tx()
4585 const struct net_device_ops *ops = dev->netdev_ops; in netdev_core_pick_tx()
4587 if (ops->ndo_select_queue) in netdev_core_pick_tx()
4588 queue_index = ops->ndo_select_queue(dev, skb, sb_dev); in netdev_core_pick_tx()
4600 * __dev_queue_xmit() - transmit a buffer
4616 * * 0 - buffer successfully transmitted
4617 * * positive qdisc return code - NET_XMIT_DROP etc.
4618 * * negative errno - other errors
4622 struct net_device *dev = skb->dev; in __dev_queue_xmit()
4625 int rc = -ENOMEM; in __dev_queue_xmit()
4631 if (unlikely(skb_shinfo(skb)->tx_flags & in __dev_queue_xmit()
4633 __skb_tstamp_tx(skb, NULL, NULL, skb->sk, SCM_TSTAMP_SCHED); in __dev_queue_xmit()
4664 /* If device/qdisc don't need skb->dst, release it right now while in __dev_queue_xmit()
4667 if (dev->priv_flags & IFF_XMIT_DST_RELEASE) in __dev_queue_xmit()
4675 q = rcu_dereference_bh(txq->qdisc); in __dev_queue_xmit()
4678 if (q->enqueue) { in __dev_queue_xmit()
4695 if (dev->flags & IFF_UP) { in __dev_queue_xmit()
4698 /* Other cpus might concurrently change txq->xmit_lock_owner in __dev_queue_xmit()
4699 * to -1 or to their cpu id, but not to our id. in __dev_queue_xmit()
4701 if (READ_ONCE(txq->xmit_lock_owner) != cpu) { in __dev_queue_xmit()
4722 dev->name); in __dev_queue_xmit()
4729 dev->name); in __dev_queue_xmit()
4733 rc = -ENETDOWN; in __dev_queue_xmit()
4747 struct net_device *dev = skb->dev; in __dev_direct_xmit()
4799 if (test_bit(NAPI_STATE_THREADED, &napi->state)) { in ____napi_schedule()
4803 * read on napi->thread. Only call in ____napi_schedule()
4806 thread = READ_ONCE(napi->thread); in ____napi_schedule()
4811 set_bit(NAPI_STATE_SCHED_THREADED, &napi->state); in ____napi_schedule()
4818 DEBUG_NET_WARN_ON_ONCE(!list_empty(&napi->poll_list)); in ____napi_schedule()
4819 list_add_tail(&napi->poll_list, &sd->poll_list); in ____napi_schedule()
4820 WRITE_ONCE(napi->list_owner, smp_processor_id()); in ____napi_schedule()
4824 if (!sd->in_net_rx_action) in ____napi_schedule()
4837 return hash_32(hash, flow_table->log); in rfs_slot()
4855 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap || in set_rps_cpu()
4856 !(dev->features & NETIF_F_NTUPLE)) in set_rps_cpu()
4858 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu); in set_rps_cpu()
4862 rxqueue = dev->_rx + rxq_index; in set_rps_cpu()
4863 flow_table = rcu_dereference(rxqueue->rps_flow_table); in set_rps_cpu()
4867 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb, in set_rps_cpu()
4872 rflow = &flow_table->flows[flow_id]; in set_rps_cpu()
4873 WRITE_ONCE(rflow->filter, rc); in set_rps_cpu()
4874 if (old_rflow->filter == rc) in set_rps_cpu()
4875 WRITE_ONCE(old_rflow->filter, RPS_NO_FILTER); in set_rps_cpu()
4879 rps_input_queue_tail_save(&rflow->last_qtail, head); in set_rps_cpu()
4882 WRITE_ONCE(rflow->cpu, next_cpu); in set_rps_cpu()
4895 struct netdev_rx_queue *rxqueue = dev->_rx; in get_rps_cpu()
4898 int cpu = -1; in get_rps_cpu()
4905 if (unlikely(index >= dev->real_num_rx_queues)) { in get_rps_cpu()
4906 WARN_ONCE(dev->real_num_rx_queues > 1, in get_rps_cpu()
4909 dev->name, index, dev->real_num_rx_queues); in get_rps_cpu()
4917 flow_table = rcu_dereference(rxqueue->rps_flow_table); in get_rps_cpu()
4918 map = rcu_dereference(rxqueue->rps_map); in get_rps_cpu()
4936 ident = READ_ONCE(sock_flow_table->ents[hash & sock_flow_table->mask]); in get_rps_cpu()
4945 rflow = &flow_table->flows[rfs_slot(hash, flow_table)]; in get_rps_cpu()
4946 tcpu = rflow->cpu; in get_rps_cpu()
4950 * different from current CPU (one in the rx-queue flow in get_rps_cpu()
4952 * - Current CPU is unset (>= nr_cpu_ids). in get_rps_cpu()
4953 * - Current CPU is offline. in get_rps_cpu()
4954 * - The current CPU's queue tail has advanced beyond the in get_rps_cpu()
4961 ((int)(READ_ONCE(per_cpu(softnet_data, tcpu).input_queue_head) - in get_rps_cpu()
4962 rflow->last_qtail)) >= 0)) { in get_rps_cpu()
4977 tcpu = map->cpus[reciprocal_scale(hash, map->len)]; in get_rps_cpu()
4991 * rps_may_expire_flow - check whether an RFS hardware filter may be removed
5004 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index; in rps_may_expire_flow()
5011 flow_table = rcu_dereference(rxqueue->rps_flow_table); in rps_may_expire_flow()
5012 if (flow_table && flow_id < (1UL << flow_table->log)) { in rps_may_expire_flow()
5013 rflow = &flow_table->flows[flow_id]; in rps_may_expire_flow()
5014 cpu = READ_ONCE(rflow->cpu); in rps_may_expire_flow()
5015 if (READ_ONCE(rflow->filter) == filter_id && cpu < nr_cpu_ids && in rps_may_expire_flow()
5016 ((int)(READ_ONCE(per_cpu(softnet_data, cpu).input_queue_head) - in rps_may_expire_flow()
5017 READ_ONCE(rflow->last_qtail)) < in rps_may_expire_flow()
5018 (int)(10 << flow_table->log))) in rps_may_expire_flow()
5033 ____napi_schedule(sd, &sd->backlog); in rps_trigger_softirq()
5035 WRITE_ONCE(sd->received_rps, sd->received_rps + 1); in rps_trigger_softirq()
5046 smp_store_release(&sd->defer_ipi_scheduled, 0); in trigger_rx_softirq()
5050 * After we queued a packet into sd->input_pkt_queue,
5053 * - If this is another cpu queue, link it to our rps_ipi_list,
5056 * - If this is our own queue, NAPI schedule our backlog.
5066 __napi_schedule_irqoff(&sd->backlog); in napi_schedule_rps()
5070 sd->rps_ipi_next = mysd->rps_ipi_list; in napi_schedule_rps()
5071 mysd->rps_ipi_list = sd; in napi_schedule_rps()
5076 if (!mysd->in_net_rx_action && !mysd->in_napi_threaded_poll) in napi_schedule_rps()
5081 __napi_schedule_irqoff(&mysd->backlog); in napi_schedule_rps()
5091 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) in kick_defer_list_purge()
5092 __napi_schedule_irqoff(&sd->backlog); in kick_defer_list_purge()
5096 } else if (!cmpxchg(&sd->defer_ipi_scheduled, 0, 1)) { in kick_defer_list_purge()
5097 smp_call_function_single_async(cpu, &sd->defer_csd); in kick_defer_list_purge()
5118 fl = rcu_dereference(sd->flow_limit); in skb_flow_limit()
5120 new_flow = hash_32(skb_get_hash(skb), fl->log_buckets); in skb_flow_limit()
5121 old_flow = fl->history[fl->history_head]; in skb_flow_limit()
5122 fl->history[fl->history_head] = new_flow; in skb_flow_limit()
5124 fl->history_head++; in skb_flow_limit()
5125 fl->history_head &= FLOW_LIMIT_HISTORY - 1; in skb_flow_limit()
5127 if (likely(fl->buckets[old_flow])) in skb_flow_limit()
5128 fl->buckets[old_flow]--; in skb_flow_limit()
5130 if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) { in skb_flow_limit()
5132 WRITE_ONCE(fl->count, fl->count + 1); in skb_flow_limit()
5157 if (!netif_running(skb->dev)) in enqueue_to_backlog()
5163 qlen = skb_queue_len_lockless(&sd->input_pkt_queue); in enqueue_to_backlog()
5168 qlen = skb_queue_len(&sd->input_pkt_queue); in enqueue_to_backlog()
5175 &sd->backlog.state)) in enqueue_to_backlog()
5178 __skb_queue_tail(&sd->input_pkt_queue, skb); in enqueue_to_backlog()
5190 atomic_inc(&sd->dropped); in enqueue_to_backlog()
5192 dev_core_stats_rx_dropped_inc(skb->dev); in enqueue_to_backlog()
5199 struct net_device *dev = skb->dev; in netif_get_rxqueue()
5202 rxqueue = dev->_rx; in netif_get_rxqueue()
5207 if (unlikely(index >= dev->real_num_rx_queues)) { in netif_get_rxqueue()
5208 WARN_ONCE(dev->real_num_rx_queues > 1, in netif_get_rxqueue()
5211 dev->name, index, dev->real_num_rx_queues); in netif_get_rxqueue()
5235 mac_len = skb->data - skb_mac_header(skb); in bpf_prog_run_generic_xdp()
5236 hard_start = skb->data - skb_headroom(skb); in bpf_prog_run_generic_xdp()
5239 frame_sz = (void *)skb_end_pointer(skb) - hard_start; in bpf_prog_run_generic_xdp()
5243 xdp_init_buff(xdp, frame_sz, &rxqueue->xdp_rxq); in bpf_prog_run_generic_xdp()
5244 xdp_prepare_buff(xdp, hard_start, skb_headroom(skb) - mac_len, in bpf_prog_run_generic_xdp()
5247 skb_shinfo(skb)->xdp_frags_size = skb->data_len; in bpf_prog_run_generic_xdp()
5253 orig_data_end = xdp->data_end; in bpf_prog_run_generic_xdp()
5254 orig_data = xdp->data; in bpf_prog_run_generic_xdp()
5255 eth = (struct ethhdr *)xdp->data; in bpf_prog_run_generic_xdp()
5256 orig_host = ether_addr_equal_64bits(eth->h_dest, skb->dev->dev_addr); in bpf_prog_run_generic_xdp()
5257 orig_bcast = is_multicast_ether_addr_64bits(eth->h_dest); in bpf_prog_run_generic_xdp()
5258 orig_eth_type = eth->h_proto; in bpf_prog_run_generic_xdp()
5263 off = xdp->data - orig_data; in bpf_prog_run_generic_xdp()
5268 __skb_push(skb, -off); in bpf_prog_run_generic_xdp()
5270 skb->mac_header += off; in bpf_prog_run_generic_xdp()
5275 off = xdp->data_end - orig_data_end; in bpf_prog_run_generic_xdp()
5277 skb_set_tail_pointer(skb, xdp->data_end - xdp->data); in bpf_prog_run_generic_xdp()
5278 skb->len += off; /* positive on grow, negative on shrink */ in bpf_prog_run_generic_xdp()
5285 skb->data_len = skb_shinfo(skb)->xdp_frags_size; in bpf_prog_run_generic_xdp()
5287 skb->data_len = 0; in bpf_prog_run_generic_xdp()
5290 eth = (struct ethhdr *)xdp->data; in bpf_prog_run_generic_xdp()
5291 if ((orig_eth_type != eth->h_proto) || in bpf_prog_run_generic_xdp()
5292 (orig_host != ether_addr_equal_64bits(eth->h_dest, in bpf_prog_run_generic_xdp()
5293 skb->dev->dev_addr)) || in bpf_prog_run_generic_xdp()
5294 (orig_bcast != is_multicast_ether_addr_64bits(eth->h_dest))) { in bpf_prog_run_generic_xdp()
5296 skb->pkt_type = PACKET_HOST; in bpf_prog_run_generic_xdp()
5297 skb->protocol = eth_type_trans(skb, skb->dev); in bpf_prog_run_generic_xdp()
5313 metalen = xdp->data - xdp->data_meta; in bpf_prog_run_generic_xdp()
5337 hroom = XDP_PACKET_HEADROOM - skb_headroom(skb); in netif_skb_check_for_xdp()
5338 troom = skb->tail + skb->data_len - skb->end; in netif_skb_check_for_xdp()
5365 mac_len = skb->data - skb_mac_header(skb); in netif_receive_generic_xdp()
5383 bpf_warn_invalid_xdp_action((*pskb)->dev, xdp_prog, act); in netif_receive_generic_xdp()
5386 trace_xdp_exception((*pskb)->dev, xdp_prog, act); in netif_receive_generic_xdp()
5398 * network taps in order to match in-driver-XDP behavior. This also means
5400 * and DDOS attacks will be more effective. In-driver-XDP use dedicated TX
5405 struct net_device *dev = skb->dev; in generic_xdp_tx()
5442 err = xdp_do_generic_redirect((*pskb)->dev, *pskb, in do_xdp_generic()
5479 cpu = get_rps_cpu(skb->dev, skb, &rflow); in netif_rx_internal()
5483 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail); in netif_rx_internal()
5497 * __netif_rx - Slightly optimized version of netif_rx
5518 * netif_rx - post buffer to the network code
5556 if (sd->completion_queue) { in net_tx_action()
5560 clist = sd->completion_queue; in net_tx_action()
5561 sd->completion_queue = NULL; in net_tx_action()
5567 clist = clist->next; in net_tx_action()
5569 WARN_ON(refcount_read(&skb->users)); in net_tx_action()
5570 if (likely(get_kfree_skb_cb(skb)->reason == SKB_CONSUMED)) in net_tx_action()
5574 get_kfree_skb_cb(skb)->reason, NULL); in net_tx_action()
5576 if (skb->fclone != SKB_FCLONE_UNAVAILABLE) in net_tx_action()
5580 get_kfree_skb_cb(skb)->reason); in net_tx_action()
5584 if (sd->output_queue) { in net_tx_action()
5588 head = sd->output_queue; in net_tx_action()
5589 sd->output_queue = NULL; in net_tx_action()
5590 sd->output_queue_tailp = &sd->output_queue; in net_tx_action()
5599 head = head->next_sched; in net_tx_action()
5601 /* We need to make sure head->next_sched is read in net_tx_action()
5606 if (!(q->flags & TCQ_F_NOLOCK)) { in net_tx_action()
5610 &q->state))) { in net_tx_action()
5619 clear_bit(__QDISC_STATE_SCHED, &q->state); in net_tx_action()
5623 clear_bit(__QDISC_STATE_SCHED, &q->state); in net_tx_action()
5643 * netdev_is_rx_handler_busy - check if receive handler is registered
5654 return dev && rtnl_dereference(dev->rx_handler); in netdev_is_rx_handler_busy()
5659 * netdev_rx_handler_register - register receive handler
5677 return -EBUSY; in netdev_rx_handler_register()
5679 if (dev->priv_flags & IFF_NO_RX_HANDLER) in netdev_rx_handler_register()
5680 return -EINVAL; in netdev_rx_handler_register()
5683 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data); in netdev_rx_handler_register()
5684 rcu_assign_pointer(dev->rx_handler, rx_handler); in netdev_rx_handler_register()
5691 * netdev_rx_handler_unregister - unregister receive handler
5702 RCU_INIT_POINTER(dev->rx_handler, NULL); in netdev_rx_handler_unregister()
5708 RCU_INIT_POINTER(dev->rx_handler_data, NULL); in netdev_rx_handler_unregister()
5718 switch (skb->protocol) { in skb_pfmemalloc_protocol()
5764 orig_dev = skb->dev; in __netif_receive_skb_core()
5780 skb->skb_iif = skb->dev->ifindex; in __netif_receive_skb_core()
5788 ret2 = do_xdp_generic(rcu_dereference(skb->dev->xdp_prog), in __netif_receive_skb_core()
5798 if (eth_type_vlan(skb->protocol)) { in __netif_receive_skb_core()
5810 list_for_each_entry_rcu(ptype, &dev_net_rcu(skb->dev)->ptype_all, in __netif_receive_skb_core()
5817 list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) { in __netif_receive_skb_core()
5857 rx_handler = rcu_dereference(skb->dev->rx_handler); in __netif_receive_skb_core()
5879 if (unlikely(skb_vlan_tag_present(skb)) && !netdev_uses_dsa(skb->dev)) { in __netif_receive_skb_core()
5885 skb->pkt_type = PACKET_OTHERHOST; in __netif_receive_skb_core()
5886 } else if (eth_type_vlan(skb->protocol)) { in __netif_receive_skb_core()
5910 * and set skb->priority like in vlan_do_receive() in __netif_receive_skb_core()
5916 type = skb->protocol; in __netif_receive_skb_core()
5924 /* orig_dev and skb->dev could belong to different netns; in __netif_receive_skb_core()
5926 * coming from skb->dev, as the ptype owner (packet socket) in __netif_receive_skb_core()
5927 * will use dev_net(skb->dev) to do namespace filtering. in __netif_receive_skb_core()
5930 &dev_net_rcu(skb->dev)->ptype_specific); in __netif_receive_skb_core()
5934 &orig_dev->ptype_specific); in __netif_receive_skb_core()
5936 if (unlikely(skb->dev != orig_dev)) { in __netif_receive_skb_core()
5938 &skb->dev->ptype_specific); in __netif_receive_skb_core()
5948 dev_core_stats_rx_dropped_inc(skb->dev); in __netif_receive_skb_core()
5950 dev_core_stats_rx_nohandler_inc(skb->dev); in __netif_receive_skb_core()
5953 * me how you were going to use this. :-) in __netif_receive_skb_core()
5960 * then skb should also be non-NULL. in __netif_receive_skb_core()
5971 struct net_device *orig_dev = skb->dev; in __netif_receive_skb_one_core()
5977 ret = INDIRECT_CALL_INET(pt_prev->func, ipv6_rcv, ip_rcv, skb, in __netif_receive_skb_one_core()
5978 skb->dev, pt_prev, orig_dev); in __netif_receive_skb_one_core()
5983 * netif_receive_skb_core - special purpose version of netif_receive_skb
6019 if (pt_prev->list_func != NULL) in __netif_receive_skb_list_ptype()
6020 INDIRECT_CALL_INET(pt_prev->list_func, ipv6_list_rcv, in __netif_receive_skb_list_ptype()
6025 pt_prev->func(skb, skb->dev, pt_prev, orig_dev); in __netif_receive_skb_list_ptype()
6031 /* Fast-path assumptions: in __netif_receive_skb_list_core()
6032 * - There is no RX handler. in __netif_receive_skb_list_core()
6033 * - Only one packet_type matches. in __netif_receive_skb_list_core()
6034 * If either of these fails, we will end up doing some per-packet in __netif_receive_skb_list_core()
6035 * processing in-line, then handling the 'last ptype' for the whole in __netif_receive_skb_list_core()
6036 * sublist. This can't cause out-of-order delivery to any single ptype, in __netif_receive_skb_list_core()
6038 * other ptypes are handled per-packet. in __netif_receive_skb_list_core()
6048 struct net_device *orig_dev = skb->dev; in __netif_receive_skb_list_core()
6063 list_add_tail(&skb->list, &sublist); in __netif_receive_skb_list_core()
6079 * - be delivered to SOCK_MEMALLOC sockets only in __netif_receive_skb()
6080 * - stay away from userspace in __netif_receive_skb()
6081 * - have bounded memory usage in __netif_receive_skb()
6106 list_cut_before(&sublist, head, &skb->list); in __netif_receive_skb_list()
6127 struct bpf_prog *old = rtnl_dereference(dev->xdp_prog); in generic_xdp_install()
6128 struct bpf_prog *new = xdp->prog; in generic_xdp_install()
6131 switch (xdp->command) { in generic_xdp_install()
6133 rcu_assign_pointer(dev->xdp_prog, new); in generic_xdp_install()
6147 ret = -EINVAL; in generic_xdp_install()
6167 int cpu = get_rps_cpu(skb->dev, skb, &rflow); in netif_receive_skb_internal()
6170 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail); in netif_receive_skb_internal()
6191 list_add_tail(&skb->list, &sublist); in netif_receive_skb_list_internal()
6200 int cpu = get_rps_cpu(skb->dev, skb, &rflow); in netif_receive_skb_list_internal()
6205 enqueue_to_backlog(skb, cpu, &rflow->last_qtail); in netif_receive_skb_list_internal()
6215 * netif_receive_skb - process receive buffer from network
6243 * netif_receive_skb_list - process many receive buffers from network
6279 skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) { in flush_backlog()
6280 if (READ_ONCE(skb->dev->reg_state) == NETREG_UNREGISTERING) { in flush_backlog()
6281 __skb_unlink(skb, &sd->input_pkt_queue); in flush_backlog()
6289 skb_queue_walk_safe(&sd->process_queue, skb, tmp) { in flush_backlog()
6290 if (READ_ONCE(skb->dev->reg_state) == NETREG_UNREGISTERING) { in flush_backlog()
6291 __skb_unlink(skb, &sd->process_queue); in flush_backlog()
6313 do_flush = !skb_queue_empty(&sd->input_pkt_queue) || in flush_required()
6314 !skb_queue_empty_lockless(&sd->process_queue); in flush_required()
6321 * input_pkt_queue and process_queue even if the latter could end-up in flush_required()
6350 cpumask_clear(&ptr->flush_cpus); in flush_all_backlogs()
6356 INIT_WORK(&ptr->w[cpu], flush_backlog); in flush_all_backlogs()
6357 queue_work_on(cpu, system_highpri_wq, &ptr->w[cpu]); in flush_all_backlogs()
6358 __cpumask_set_cpu(cpu, &ptr->flush_cpus); in flush_all_backlogs()
6366 for_each_cpu(cpu, &ptr->flush_cpus) in flush_all_backlogs()
6367 flush_work(&ptr->w[cpu]); in flush_all_backlogs()
6381 struct softnet_data *next = remsd->rps_ipi_next; in net_rps_send_ipi()
6383 if (cpu_online(remsd->cpu)) in net_rps_send_ipi()
6384 smp_call_function_single_async(remsd->cpu, &remsd->csd); in net_rps_send_ipi()
6397 struct softnet_data *remsd = sd->rps_ipi_list; in net_rps_action_and_irq_enable()
6400 sd->rps_ipi_list = NULL; in net_rps_action_and_irq_enable()
6414 return !use_backlog_threads() && sd->rps_ipi_list; in sd_has_rps_ipi_waiting()
6434 napi->weight = READ_ONCE(net_hotdata.dev_rx_weight); in process_backlog()
6439 while ((skb = __skb_dequeue(&sd->process_queue))) { in process_backlog()
6454 if (skb_queue_empty(&sd->input_pkt_queue)) { in process_backlog()
6463 napi->state &= NAPIF_STATE_THREADED; in process_backlog()
6467 skb_queue_splice_tail_init(&sd->input_pkt_queue, in process_backlog()
6468 &sd->process_queue); in process_backlog()
6480 * __napi_schedule - schedule for receive
6497 * napi_schedule_prep - check if napi can be scheduled
6507 unsigned long new, val = READ_ONCE(n->state); in napi_schedule_prep()
6522 } while (!try_cmpxchg(&n->state, &val, new)); in napi_schedule_prep()
6529 * __napi_schedule_irqoff - schedule for receive
6536 * due to force-threaded interrupts and spinlock substitution.
6558 if (unlikely(n->state & (NAPIF_STATE_NPSVC | in napi_complete_done()
6563 if (n->gro.bitmask) in napi_complete_done()
6565 n->defer_hard_irqs_count = napi_get_defer_hard_irqs(n); in napi_complete_done()
6567 if (n->defer_hard_irqs_count > 0) { in napi_complete_done()
6568 n->defer_hard_irqs_count--; in napi_complete_done()
6579 gro_flush(&n->gro, !!timeout); in napi_complete_done()
6580 gro_normal_list(&n->gro); in napi_complete_done()
6582 if (unlikely(!list_empty(&n->poll_list))) { in napi_complete_done()
6583 /* If n->poll_list is not empty, we need to mask irqs */ in napi_complete_done()
6585 list_del_init(&n->poll_list); in napi_complete_done()
6588 WRITE_ONCE(n->list_owner, -1); in napi_complete_done()
6590 val = READ_ONCE(n->state); in napi_complete_done()
6599 * because we will call napi->poll() one more time. in napi_complete_done()
6604 } while (!try_cmpxchg(&n->state, &val, new)); in napi_complete_done()
6612 hrtimer_start(&n->timer, ns_to_ktime(timeout), in napi_complete_done()
6623 if (!READ_ONCE(sd->defer_list)) in skb_defer_free_flush()
6626 spin_lock(&sd->defer_lock); in skb_defer_free_flush()
6627 skb = sd->defer_list; in skb_defer_free_flush()
6628 sd->defer_list = NULL; in skb_defer_free_flush()
6629 sd->defer_count = 0; in skb_defer_free_flush()
6630 spin_unlock(&sd->defer_lock); in skb_defer_free_flush()
6633 next = skb->next; in skb_defer_free_flush()
6644 gro_normal_list(&napi->gro); in __busy_poll_stop()
6650 gro_flush(&napi->gro, HZ >= 1000); in __busy_poll_stop()
6651 gro_normal_list(&napi->gro); in __busy_poll_stop()
6653 clear_bit(NAPI_STATE_SCHED, &napi->state); in __busy_poll_stop()
6672 * Since we are about to call napi->poll() once more, we can safely in busy_poll_stop()
6678 clear_bit(NAPI_STATE_MISSED, &napi->state); in busy_poll_stop()
6679 clear_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state); in busy_poll_stop()
6685 napi->defer_hard_irqs_count = napi_get_defer_hard_irqs(napi); in busy_poll_stop()
6687 if (napi->defer_hard_irqs_count && timeout) { in busy_poll_stop()
6688 hrtimer_start(&napi->timer, ns_to_ktime(timeout), HRTIMER_MODE_REL_PINNED); in busy_poll_stop()
6693 /* All we really want here is to re-enable device interrupts. in busy_poll_stop()
6696 rc = napi->poll(napi, budget); in busy_poll_stop()
6697 /* We can't gro_normal_list() here, because napi->poll() might have in busy_poll_stop()
6736 unsigned long val = READ_ONCE(napi->state); in __napi_busy_loop()
6739 * we avoid dirtying napi->state as much as we can. in __napi_busy_loop()
6744 set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state); in __napi_busy_loop()
6747 if (cmpxchg(&napi->state, val, in __napi_busy_loop()
6751 set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state); in __napi_busy_loop()
6755 napi_poll = napi->poll; in __napi_busy_loop()
6759 gro_normal_list(&napi->gro); in __napi_busy_loop()
6762 __NET_ADD_STATS(dev_net(napi->dev), in __napi_busy_loop()
6827 hrtimer_start(&napi->timer, ns_to_ktime(timeout), in napi_suspend_irqs()
6859 napi->gro.cached_napi_id = napi_id; in __napi_hash_add_with_id()
6861 WRITE_ONCE(napi->napi_id, napi_id); in __napi_hash_add_with_id()
6862 hlist_add_head_rcu(&napi->napi_hash_node, in __napi_hash_add_with_id()
6863 &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]); in __napi_hash_add_with_id()
6881 if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state)) in napi_hash_add()
6906 hlist_del_init_rcu(&napi->napi_hash_node); in napi_hash_del()
6921 !test_and_set_bit(NAPI_STATE_SCHED, &napi->state)) { in napi_watchdog()
6922 clear_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state); in napi_watchdog()
6936 if (dev->threaded == threaded) in dev_set_threaded()
6940 list_for_each_entry(napi, &dev->napi_list, dev_list) { in dev_set_threaded()
6941 if (!napi->thread) { in dev_set_threaded()
6951 WRITE_ONCE(dev->threaded, threaded); in dev_set_threaded()
6964 list_for_each_entry(napi, &dev->napi_list, dev_list) in dev_set_threaded()
6965 assign_bit(NAPI_STATE_THREADED, &napi->state, threaded); in dev_set_threaded()
6972 * netif_queue_set_napi - Associate queue with the napi
6979 * registering the NAPI handler for the queue-vector and the queues have been
6988 if (WARN_ON_ONCE(napi && !napi->dev)) in netif_queue_set_napi()
6995 rxq->napi = napi; in netif_queue_set_napi()
6999 txq->napi = napi; in netif_queue_set_napi()
7014 struct cpu_rmap *rmap = napi->dev->rx_cpu_rmap; in netif_napi_irq_notify()
7018 if (napi->config && napi->dev->irq_affinity_auto) in netif_napi_irq_notify()
7019 cpumask_copy(&napi->config->affinity_mask, mask); in netif_napi_irq_notify()
7022 if (napi->dev->rx_cpu_rmap_auto) { in netif_napi_irq_notify()
7023 err = cpu_rmap_update(rmap, napi->napi_rmap_idx, mask); in netif_napi_irq_notify()
7025 netdev_warn(napi->dev, "RMAP update failed (%d)\n", in netif_napi_irq_notify()
7036 struct cpu_rmap *rmap = napi->dev->rx_cpu_rmap; in netif_napi_affinity_release()
7038 netdev_assert_locked(napi->dev); in netif_napi_affinity_release()
7040 &napi->state)); in netif_napi_affinity_release()
7042 if (!napi->dev->rx_cpu_rmap_auto) in netif_napi_affinity_release()
7044 rmap->obj[napi->napi_rmap_idx] = NULL; in netif_napi_affinity_release()
7045 napi->napi_rmap_idx = -1; in netif_napi_affinity_release()
7051 if (dev->rx_cpu_rmap_auto) in netif_enable_cpu_rmap()
7054 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(num_irqs); in netif_enable_cpu_rmap()
7055 if (!dev->rx_cpu_rmap) in netif_enable_cpu_rmap()
7056 return -ENOMEM; in netif_enable_cpu_rmap()
7058 dev->rx_cpu_rmap_auto = true; in netif_enable_cpu_rmap()
7065 struct cpu_rmap *rmap = dev->rx_cpu_rmap; in netif_del_cpu_rmap()
7067 if (!dev->rx_cpu_rmap_auto) in netif_del_cpu_rmap()
7072 dev->rx_cpu_rmap = NULL; in netif_del_cpu_rmap()
7073 dev->rx_cpu_rmap_auto = false; in netif_del_cpu_rmap()
7096 maxqs = max(dev->num_tx_queues, dev->num_rx_queues); in netif_set_affinity_auto()
7097 numa = dev_to_node(&dev->dev); in netif_set_affinity_auto()
7101 &dev->napi_config[i].affinity_mask); in netif_set_affinity_auto()
7103 dev->irq_affinity_auto = true; in netif_set_affinity_auto()
7111 netdev_assert_locked_or_invisible(napi->dev); in netif_napi_set_irq_locked()
7113 if (napi->irq == irq) in netif_napi_set_irq_locked()
7117 if (test_and_clear_bit(NAPI_STATE_HAS_NOTIFIER, &napi->state)) in netif_napi_set_irq_locked()
7118 irq_set_affinity_notifier(napi->irq, NULL); in netif_napi_set_irq_locked()
7120 napi->irq = irq; in netif_napi_set_irq_locked()
7122 (!napi->dev->rx_cpu_rmap_auto && !napi->dev->irq_affinity_auto)) in netif_napi_set_irq_locked()
7126 if (napi->dev->irq_affinity_auto && WARN_ON_ONCE(!napi->config)) in netif_napi_set_irq_locked()
7130 if (napi->dev->rx_cpu_rmap_auto) { in netif_napi_set_irq_locked()
7131 rc = cpu_rmap_add(napi->dev->rx_cpu_rmap, napi); in netif_napi_set_irq_locked()
7135 cpu_rmap_get(napi->dev->rx_cpu_rmap); in netif_napi_set_irq_locked()
7136 napi->napi_rmap_idx = rc; in netif_napi_set_irq_locked()
7141 napi->notify.notify = netif_napi_irq_notify; in netif_napi_set_irq_locked()
7142 napi->notify.release = netif_napi_affinity_release; in netif_napi_set_irq_locked()
7143 rc = irq_set_affinity_notifier(irq, &napi->notify); in netif_napi_set_irq_locked()
7145 netdev_warn(napi->dev, "Unable to set IRQ notifier (%d)\n", in netif_napi_set_irq_locked()
7150 set_bit(NAPI_STATE_HAS_NOTIFIER, &napi->state); in netif_napi_set_irq_locked()
7155 if (napi->dev->rx_cpu_rmap_auto) { in netif_napi_set_irq_locked()
7156 napi->dev->rx_cpu_rmap->obj[napi->napi_rmap_idx] = NULL; in netif_napi_set_irq_locked()
7157 cpu_rmap_put(napi->dev->rx_cpu_rmap); in netif_napi_set_irq_locked()
7158 napi->napi_rmap_idx = -1; in netif_napi_set_irq_locked()
7161 napi->notify.notify = NULL; in netif_napi_set_irq_locked()
7162 napi->notify.release = NULL; in netif_napi_set_irq_locked()
7168 n->defer_hard_irqs = n->config->defer_hard_irqs; in napi_restore_config()
7169 n->gro_flush_timeout = n->config->gro_flush_timeout; in napi_restore_config()
7170 n->irq_suspend_timeout = n->config->irq_suspend_timeout; in napi_restore_config()
7172 if (n->dev->irq_affinity_auto && in napi_restore_config()
7173 test_bit(NAPI_STATE_HAS_NOTIFIER, &n->state)) in napi_restore_config()
7174 irq_set_affinity(n->irq, &n->config->affinity_mask); in napi_restore_config()
7179 if (n->config->napi_id) { in napi_restore_config()
7180 napi_hash_add_with_id(n, n->config->napi_id); in napi_restore_config()
7183 n->config->napi_id = n->napi_id; in napi_restore_config()
7189 n->config->defer_hard_irqs = n->defer_hard_irqs; in napi_save_config()
7190 n->config->gro_flush_timeout = n->gro_flush_timeout; in napi_save_config()
7191 n->config->irq_suspend_timeout = n->irq_suspend_timeout; in napi_save_config()
7206 if (napi->config && napi->config->napi_id) in netif_napi_dev_list_add()
7207 new_id = napi->config->napi_id; in netif_napi_dev_list_add()
7209 higher = &dev->napi_list; in netif_napi_dev_list_add()
7210 list_for_each_entry(pos, &dev->napi_list, dev_list) { in netif_napi_dev_list_add()
7211 if (napi_id_valid(pos->napi_id)) in netif_napi_dev_list_add()
7212 pos_id = pos->napi_id; in netif_napi_dev_list_add()
7213 else if (pos->config) in netif_napi_dev_list_add()
7214 pos_id = pos->config->napi_id; in netif_napi_dev_list_add()
7220 higher = &pos->dev_list; in netif_napi_dev_list_add()
7222 list_add_rcu(&napi->dev_list, higher); /* adds after higher */ in netif_napi_dev_list_add()
7226 * skb->head being backed by slab, not a page fragment.
7228 * ("net: avoid 32 x truesize under-estimation for tiny skbs")
7237 WARN_ON_ONCE(skb && skb->head_frag); in napi_get_frags_check()
7248 if (WARN_ON(test_and_set_bit(NAPI_STATE_LISTED, &napi->state))) in netif_napi_add_weight_locked()
7251 INIT_LIST_HEAD(&napi->poll_list); in netif_napi_add_weight_locked()
7252 INIT_HLIST_NODE(&napi->napi_hash_node); in netif_napi_add_weight_locked()
7253 hrtimer_setup(&napi->timer, napi_watchdog, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); in netif_napi_add_weight_locked()
7254 gro_init(&napi->gro); in netif_napi_add_weight_locked()
7255 napi->skb = NULL; in netif_napi_add_weight_locked()
7256 napi->poll = poll; in netif_napi_add_weight_locked()
7260 napi->weight = weight; in netif_napi_add_weight_locked()
7261 napi->dev = dev; in netif_napi_add_weight_locked()
7263 napi->poll_owner = -1; in netif_napi_add_weight_locked()
7265 napi->list_owner = -1; in netif_napi_add_weight_locked()
7266 set_bit(NAPI_STATE_SCHED, &napi->state); in netif_napi_add_weight_locked()
7267 set_bit(NAPI_STATE_NPSVC, &napi->state); in netif_napi_add_weight_locked()
7270 /* default settings from sysfs are applied to all NAPIs. any per-NAPI in netif_napi_add_weight_locked()
7273 napi_set_defer_hard_irqs(napi, READ_ONCE(dev->napi_defer_hard_irqs)); in netif_napi_add_weight_locked()
7274 napi_set_gro_flush_timeout(napi, READ_ONCE(dev->gro_flush_timeout)); in netif_napi_add_weight_locked()
7277 /* Create kthread for this napi if dev->threaded is set. in netif_napi_add_weight_locked()
7278 * Clear dev->threaded if kthread creation failed so that in netif_napi_add_weight_locked()
7281 if (dev->threaded && napi_kthread_create(napi)) in netif_napi_add_weight_locked()
7282 dev->threaded = false; in netif_napi_add_weight_locked()
7283 netif_napi_set_irq_locked(napi, -1); in netif_napi_add_weight_locked()
7292 netdev_assert_locked(n->dev); in napi_disable_locked()
7294 set_bit(NAPI_STATE_DISABLE, &n->state); in napi_disable_locked()
7296 val = READ_ONCE(n->state); in napi_disable_locked()
7300 val = READ_ONCE(n->state); in napi_disable_locked()
7305 } while (!try_cmpxchg(&n->state, &val, new)); in napi_disable_locked()
7307 hrtimer_cancel(&n->timer); in napi_disable_locked()
7309 if (n->config) in napi_disable_locked()
7314 clear_bit(NAPI_STATE_DISABLE, &n->state); in napi_disable_locked()
7319 * napi_disable() - prevent NAPI from scheduling
7328 netdev_lock(n->dev); in napi_disable()
7330 netdev_unlock(n->dev); in napi_disable()
7336 unsigned long new, val = READ_ONCE(n->state); in napi_enable_locked()
7338 if (n->config) in napi_enable_locked()
7347 if (n->dev->threaded && n->thread) in napi_enable_locked()
7349 } while (!try_cmpxchg(&n->state, &val, new)); in napi_enable_locked()
7354 * napi_enable() - enable NAPI scheduling
7363 netdev_lock(n->dev); in napi_enable()
7365 netdev_unlock(n->dev); in napi_enable()
7372 netdev_assert_locked(napi->dev); in __netif_napi_del_locked()
7374 if (!test_and_clear_bit(NAPI_STATE_LISTED, &napi->state)) in __netif_napi_del_locked()
7378 WARN_ON(!test_bit(NAPI_STATE_SCHED, &napi->state)); in __netif_napi_del_locked()
7380 if (test_and_clear_bit(NAPI_STATE_HAS_NOTIFIER, &napi->state)) in __netif_napi_del_locked()
7381 irq_set_affinity_notifier(napi->irq, NULL); in __netif_napi_del_locked()
7383 if (napi->config) { in __netif_napi_del_locked()
7384 napi->index = -1; in __netif_napi_del_locked()
7385 napi->config = NULL; in __netif_napi_del_locked()
7388 list_del_rcu(&napi->dev_list); in __netif_napi_del_locked()
7391 gro_cleanup(&napi->gro); in __netif_napi_del_locked()
7393 if (napi->thread) { in __netif_napi_del_locked()
7394 kthread_stop(napi->thread); in __netif_napi_del_locked()
7395 napi->thread = NULL; in __netif_napi_del_locked()
7404 weight = n->weight; in __napi_poll()
7409 * actually make the ->poll() call. Therefore we avoid in __napi_poll()
7410 * accidentally calling ->poll() when NAPI is not scheduled. in __napi_poll()
7414 work = n->poll(n, weight); in __napi_poll()
7421 netdev_err_once(n->dev, "NAPI poll function %pS returned %d, exceeding its budget of %d.\n", in __napi_poll()
7422 n->poll, work, weight); in __napi_poll()
7430 * move the instance around on the list at-will. in __napi_poll()
7437 /* The NAPI context has more processing work, but busy-polling in __napi_poll()
7443 * that the NAPI is re-scheduled. in __napi_poll()
7451 gro_flush(&n->gro, HZ >= 1000); in __napi_poll()
7452 gro_normal_list(&n->gro); in __napi_poll()
7457 if (unlikely(!list_empty(&n->poll_list))) { in __napi_poll()
7459 n->dev ? n->dev->name : "backlog"); in __napi_poll()
7474 list_del_init(&n->poll_list); in napi_poll()
7484 n->dev->name, n->poll); in napi_poll()
7486 list_add_tail(&n->poll_list, repoll); in napi_poll()
7503 if (test_bit(NAPI_STATE_SCHED_THREADED, &napi->state)) { in napi_thread_wait()
7504 WARN_ON(!list_empty(&napi->poll_list)); in napi_thread_wait()
7514 return -1; in napi_thread_wait()
7531 sd->in_napi_threaded_poll = true; in napi_threaded_poll_loop()
7537 sd->in_napi_threaded_poll = false; in napi_threaded_poll_loop()
7578 sd->in_net_rx_action = true; in net_rx_action()
7580 list_splice_init(&sd->poll_list, &list); in net_rx_action()
7590 sd->in_net_rx_action = false; in net_rx_action()
7594 * sd->in_net_rx_action was true. in net_rx_action()
7596 if (!list_empty(&sd->poll_list)) in net_rx_action()
7605 budget -= napi_poll(n, &repoll); in net_rx_action()
7614 WRITE_ONCE(sd->time_squeeze, sd->time_squeeze + 1); in net_rx_action()
7621 list_splice_tail_init(&sd->poll_list, &list); in net_rx_action()
7623 list_splice(&list, &sd->poll_list); in net_rx_action()
7624 if (!list_empty(&sd->poll_list)) in net_rx_action()
7627 sd->in_net_rx_action = false; in net_rx_action()
7657 struct netdev_adjacent *adj; in __netdev_find_adj() local
7659 list_for_each_entry(adj, adj_list, list) { in __netdev_find_adj()
7660 if (adj->dev == adj_dev) in __netdev_find_adj()
7661 return adj; in __netdev_find_adj()
7669 struct net_device *dev = (struct net_device *)priv->data; in ____netdev_has_upper_dev()
7675 * netdev_has_upper_dev - Check if device is linked to an upper device
7698 * netdev_has_upper_dev_all_rcu - Check if device is linked to an upper device
7720 * netdev_has_any_upper_dev - Check if device is linked to some device
7730 return !list_empty(&dev->adj_list.upper); in netdev_has_any_upper_dev()
7735 * netdev_master_upper_dev_get - Get master upper device
7747 if (list_empty(&dev->adj_list.upper)) in netdev_master_upper_dev_get()
7750 upper = list_first_entry(&dev->adj_list.upper, in netdev_master_upper_dev_get()
7752 if (likely(upper->master)) in netdev_master_upper_dev_get()
7753 return upper->dev; in netdev_master_upper_dev_get()
7764 if (list_empty(&dev->adj_list.upper)) in __netdev_master_upper_dev_get()
7767 upper = list_first_entry(&dev->adj_list.upper, in __netdev_master_upper_dev_get()
7769 if (likely(upper->master) && !upper->ignore) in __netdev_master_upper_dev_get()
7770 return upper->dev; in __netdev_master_upper_dev_get()
7775 * netdev_has_any_lower_dev - Check if device is linked to some device
7785 return !list_empty(&dev->adj_list.lower); in netdev_has_any_lower_dev()
7790 struct netdev_adjacent *adj; in netdev_adjacent_get_private() local
7792 adj = list_entry(adj_list, struct netdev_adjacent, list); in netdev_adjacent_get_private()
7794 return adj->private; in netdev_adjacent_get_private()
7799 * netdev_upper_get_next_dev_rcu - Get the next dev from upper list
7813 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); in netdev_upper_get_next_dev_rcu()
7815 if (&upper->list == &dev->adj_list.upper) in netdev_upper_get_next_dev_rcu()
7818 *iter = &upper->list; in netdev_upper_get_next_dev_rcu()
7820 return upper->dev; in netdev_upper_get_next_dev_rcu()
7830 upper = list_entry((*iter)->next, struct netdev_adjacent, list); in __netdev_next_upper_dev()
7832 if (&upper->list == &dev->adj_list.upper) in __netdev_next_upper_dev()
7835 *iter = &upper->list; in __netdev_next_upper_dev()
7836 *ignore = upper->ignore; in __netdev_next_upper_dev()
7838 return upper->dev; in __netdev_next_upper_dev()
7848 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); in netdev_next_upper_dev_rcu()
7850 if (&upper->list == &dev->adj_list.upper) in netdev_next_upper_dev_rcu()
7853 *iter = &upper->list; in netdev_next_upper_dev_rcu()
7855 return upper->dev; in netdev_next_upper_dev_rcu()
7869 iter = &dev->adj_list.upper; in __netdev_walk_all_upper_dev()
7887 niter = &udev->adj_list.upper; in __netdev_walk_all_upper_dev()
7896 next = dev_stack[--cur]; in __netdev_walk_all_upper_dev()
7917 iter = &dev->adj_list.upper; in netdev_walk_all_upper_dev_rcu()
7933 niter = &udev->adj_list.upper; in netdev_walk_all_upper_dev_rcu()
7942 next = dev_stack[--cur]; in netdev_walk_all_upper_dev_rcu()
7969 * netdev_lower_get_next_private - Get the next ->private from the
7974 * Gets the next netdev_adjacent->private from the dev's lower neighbour
7986 if (&lower->list == &dev->adj_list.lower) in netdev_lower_get_next_private()
7989 *iter = lower->list.next; in netdev_lower_get_next_private()
7991 return lower->private; in netdev_lower_get_next_private()
7996 * netdev_lower_get_next_private_rcu - Get the next ->private from the
8002 * Gets the next netdev_adjacent->private from the dev's lower neighbour
8012 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); in netdev_lower_get_next_private_rcu()
8014 if (&lower->list == &dev->adj_list.lower) in netdev_lower_get_next_private_rcu()
8017 *iter = &lower->list; in netdev_lower_get_next_private_rcu()
8019 return lower->private; in netdev_lower_get_next_private_rcu()
8024 * netdev_lower_get_next - Get the next device from the lower neighbour
8040 if (&lower->list == &dev->adj_list.lower) in netdev_lower_get_next()
8043 *iter = lower->list.next; in netdev_lower_get_next()
8045 return lower->dev; in netdev_lower_get_next()
8054 lower = list_entry((*iter)->next, struct netdev_adjacent, list); in netdev_next_lower_dev()
8056 if (&lower->list == &dev->adj_list.lower) in netdev_next_lower_dev()
8059 *iter = &lower->list; in netdev_next_lower_dev()
8061 return lower->dev; in netdev_next_lower_dev()
8070 lower = list_entry((*iter)->next, struct netdev_adjacent, list); in __netdev_next_lower_dev()
8072 if (&lower->list == &dev->adj_list.lower) in __netdev_next_lower_dev()
8075 *iter = &lower->list; in __netdev_next_lower_dev()
8076 *ignore = lower->ignore; in __netdev_next_lower_dev()
8078 return lower->dev; in __netdev_next_lower_dev()
8091 iter = &dev->adj_list.lower; in netdev_walk_all_lower_dev()
8107 niter = &ldev->adj_list.lower; in netdev_walk_all_lower_dev()
8116 next = dev_stack[--cur]; in netdev_walk_all_lower_dev()
8139 iter = &dev->adj_list.lower; in __netdev_walk_all_lower_dev()
8157 niter = &ldev->adj_list.lower; in __netdev_walk_all_lower_dev()
8166 next = dev_stack[--cur]; in __netdev_walk_all_lower_dev()
8182 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); in netdev_next_lower_dev_rcu()
8183 if (&lower->list == &dev->adj_list.lower) in netdev_next_lower_dev_rcu()
8186 *iter = &lower->list; in netdev_next_lower_dev_rcu()
8188 return lower->dev; in netdev_next_lower_dev_rcu()
8199 for (iter = &dev->adj_list.upper, in __netdev_upper_depth()
8205 if (max_depth < udev->upper_level) in __netdev_upper_depth()
8206 max_depth = udev->upper_level; in __netdev_upper_depth()
8219 for (iter = &dev->adj_list.lower, in __netdev_lower_depth()
8225 if (max_depth < ldev->lower_level) in __netdev_lower_depth()
8226 max_depth = ldev->lower_level; in __netdev_lower_depth()
8235 dev->upper_level = __netdev_upper_depth(dev) + 1; in __netdev_update_upper_level()
8244 if (list_empty(&dev->unlink_list)) in net_unlink_todo()
8245 list_add_tail(&dev->unlink_list, &net_unlink_list); in net_unlink_todo()
8252 dev->lower_level = __netdev_lower_depth(dev) + 1; in __netdev_update_lower_level()
8258 if (priv->flags & NESTED_SYNC_IMM) in __netdev_update_lower_level()
8259 dev->nested_level = dev->lower_level - 1; in __netdev_update_lower_level()
8260 if (priv->flags & NESTED_SYNC_TODO) in __netdev_update_lower_level()
8276 iter = &dev->adj_list.lower; in netdev_walk_all_lower_dev_rcu()
8292 niter = &ldev->adj_list.lower; in netdev_walk_all_lower_dev_rcu()
8301 next = dev_stack[--cur]; in netdev_walk_all_lower_dev_rcu()
8314 * netdev_lower_get_first_private_rcu - Get the first ->private from the
8319 * Gets the first netdev_adjacent->private from the dev's lower neighbour
8326 lower = list_first_or_null_rcu(&dev->adj_list.lower, in netdev_lower_get_first_private_rcu()
8329 return lower->private; in netdev_lower_get_first_private_rcu()
8335 * netdev_master_upper_dev_get_rcu - Get master upper device
8345 upper = list_first_or_null_rcu(&dev->adj_list.upper, in netdev_master_upper_dev_get_rcu()
8347 if (upper && likely(upper->master)) in netdev_master_upper_dev_get_rcu()
8348 return upper->dev; in netdev_master_upper_dev_get_rcu()
8359 sprintf(linkname, dev_list == &dev->adj_list.upper ? in netdev_adjacent_sysfs_add()
8360 "upper_%s" : "lower_%s", adj_dev->name); in netdev_adjacent_sysfs_add()
8361 return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj), in netdev_adjacent_sysfs_add()
8370 sprintf(linkname, dev_list == &dev->adj_list.upper ? in netdev_adjacent_sysfs_del()
8372 sysfs_remove_link(&(dev->dev.kobj), linkname); in netdev_adjacent_sysfs_del()
8379 return (dev_list == &dev->adj_list.upper || in netdev_adjacent_is_neigh_list()
8380 dev_list == &dev->adj_list.lower) && in netdev_adjacent_is_neigh_list()
8389 struct netdev_adjacent *adj; in __netdev_adjacent_dev_insert() local
8392 adj = __netdev_find_adj(adj_dev, dev_list); in __netdev_adjacent_dev_insert()
8394 if (adj) { in __netdev_adjacent_dev_insert()
8395 adj->ref_nr += 1; in __netdev_adjacent_dev_insert()
8396 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d\n", in __netdev_adjacent_dev_insert()
8397 dev->name, adj_dev->name, adj->ref_nr); in __netdev_adjacent_dev_insert()
8402 adj = kmalloc(sizeof(*adj), GFP_KERNEL); in __netdev_adjacent_dev_insert()
8403 if (!adj) in __netdev_adjacent_dev_insert()
8404 return -ENOMEM; in __netdev_adjacent_dev_insert()
8406 adj->dev = adj_dev; in __netdev_adjacent_dev_insert()
8407 adj->master = master; in __netdev_adjacent_dev_insert()
8408 adj->ref_nr = 1; in __netdev_adjacent_dev_insert()
8409 adj->private = private; in __netdev_adjacent_dev_insert()
8410 adj->ignore = false; in __netdev_adjacent_dev_insert()
8411 netdev_hold(adj_dev, &adj->dev_tracker, GFP_KERNEL); in __netdev_adjacent_dev_insert()
8413 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d; dev_hold on %s\n", in __netdev_adjacent_dev_insert()
8414 dev->name, adj_dev->name, adj->ref_nr, adj_dev->name); in __netdev_adjacent_dev_insert()
8424 ret = sysfs_create_link(&(dev->dev.kobj), in __netdev_adjacent_dev_insert()
8425 &(adj_dev->dev.kobj), "master"); in __netdev_adjacent_dev_insert()
8429 list_add_rcu(&adj->list, dev_list); in __netdev_adjacent_dev_insert()
8431 list_add_tail_rcu(&adj->list, dev_list); in __netdev_adjacent_dev_insert()
8438 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list); in __netdev_adjacent_dev_insert()
8440 netdev_put(adj_dev, &adj->dev_tracker); in __netdev_adjacent_dev_insert()
8441 kfree(adj); in __netdev_adjacent_dev_insert()
8451 struct netdev_adjacent *adj; in __netdev_adjacent_dev_remove() local
8454 dev->name, adj_dev->name, ref_nr); in __netdev_adjacent_dev_remove()
8456 adj = __netdev_find_adj(adj_dev, dev_list); in __netdev_adjacent_dev_remove()
8458 if (!adj) { in __netdev_adjacent_dev_remove()
8460 dev->name, adj_dev->name); in __netdev_adjacent_dev_remove()
8465 if (adj->ref_nr > ref_nr) { in __netdev_adjacent_dev_remove()
8466 pr_debug("adjacency: %s to %s ref_nr - %d = %d\n", in __netdev_adjacent_dev_remove()
8467 dev->name, adj_dev->name, ref_nr, in __netdev_adjacent_dev_remove()
8468 adj->ref_nr - ref_nr); in __netdev_adjacent_dev_remove()
8469 adj->ref_nr -= ref_nr; in __netdev_adjacent_dev_remove()
8473 if (adj->master) in __netdev_adjacent_dev_remove()
8474 sysfs_remove_link(&(dev->dev.kobj), "master"); in __netdev_adjacent_dev_remove()
8477 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list); in __netdev_adjacent_dev_remove()
8479 list_del_rcu(&adj->list); in __netdev_adjacent_dev_remove()
8481 adj_dev->name, dev->name, adj_dev->name); in __netdev_adjacent_dev_remove()
8482 netdev_put(adj_dev, &adj->dev_tracker); in __netdev_adjacent_dev_remove()
8483 kfree_rcu(adj, rcu); in __netdev_adjacent_dev_remove()
8524 &dev->adj_list.upper, in __netdev_adjacent_dev_link_neighbour()
8525 &upper_dev->adj_list.lower, in __netdev_adjacent_dev_link_neighbour()
8533 &dev->adj_list.upper, in __netdev_adjacent_dev_unlink_neighbour()
8534 &upper_dev->adj_list.lower); in __netdev_adjacent_dev_unlink_neighbour()
8559 return -EBUSY; in __netdev_upper_dev_link()
8563 return -EBUSY; in __netdev_upper_dev_link()
8565 if ((dev->lower_level + upper_dev->upper_level) > MAX_NEST_DEV) in __netdev_upper_dev_link()
8566 return -EMLINK; in __netdev_upper_dev_link()
8570 return -EEXIST; in __netdev_upper_dev_link()
8574 return master_dev == upper_dev ? -EEXIST : -EBUSY; in __netdev_upper_dev_link()
8610 * netdev_upper_dev_link - Add a link to the upper device
8635 * netdev_master_upper_dev_link - Add a master link to the upper device
8643 * one master upper device can be linked, although other non-master devices
8696 * netdev_upper_dev_unlink - Removes a link to upper device
8719 struct netdev_adjacent *adj; in __netdev_adjacent_dev_set() local
8721 adj = __netdev_find_adj(lower_dev, &upper_dev->adj_list.lower); in __netdev_adjacent_dev_set()
8722 if (adj) in __netdev_adjacent_dev_set()
8723 adj->ignore = val; in __netdev_adjacent_dev_set()
8725 adj = __netdev_find_adj(upper_dev, &lower_dev->adj_list.upper); in __netdev_adjacent_dev_set()
8726 if (adj) in __netdev_adjacent_dev_set()
8727 adj->ignore = val; in __netdev_adjacent_dev_set()
8810 * netdev_bonding_info_change - Dispatch event about slave change
8842 dev->offload_xstats_l3 = kzalloc(sizeof(*dev->offload_xstats_l3), in netdev_offload_xstats_enable_l3()
8844 if (!dev->offload_xstats_l3) in netdev_offload_xstats_enable_l3()
8845 return -ENOMEM; in netdev_offload_xstats_enable_l3()
8857 kfree(dev->offload_xstats_l3); in netdev_offload_xstats_enable_l3()
8858 dev->offload_xstats_l3 = NULL; in netdev_offload_xstats_enable_l3()
8869 return -EALREADY; in netdev_offload_xstats_enable()
8877 return -EINVAL; in netdev_offload_xstats_enable()
8890 kfree(dev->offload_xstats_l3); in netdev_offload_xstats_disable_l3()
8891 dev->offload_xstats_l3 = NULL; in netdev_offload_xstats_disable_l3()
8900 return -EALREADY; in netdev_offload_xstats_disable()
8909 return -EINVAL; in netdev_offload_xstats_disable()
8924 return dev->offload_xstats_l3; in netdev_offload_xstats_get_ptr()
8952 dest->rx_packets += src->rx_packets; in netdev_hw_stats64_add()
8953 dest->tx_packets += src->tx_packets; in netdev_hw_stats64_add()
8954 dest->rx_bytes += src->rx_bytes; in netdev_hw_stats64_add()
8955 dest->tx_bytes += src->tx_bytes; in netdev_hw_stats64_add()
8956 dest->rx_errors += src->rx_errors; in netdev_hw_stats64_add()
8957 dest->tx_errors += src->tx_errors; in netdev_hw_stats64_add()
8958 dest->rx_dropped += src->rx_dropped; in netdev_hw_stats64_add()
8959 dest->tx_dropped += src->tx_dropped; in netdev_hw_stats64_add()
8960 dest->multicast += src->multicast; in netdev_hw_stats64_add()
9002 return -EINVAL; in netdev_offload_xstats_get_stats()
9039 report_delta->used = true; in netdev_offload_xstats_report_delta()
9040 netdev_hw_stats64_add(&report_delta->stats, stats); in netdev_offload_xstats_report_delta()
9047 report_used->used = true; in netdev_offload_xstats_report_used()
9068 * netdev_get_xmit_slave - Get the xmit slave of master device
9082 const struct net_device_ops *ops = dev->netdev_ops; in netdev_get_xmit_slave()
9084 if (!ops->ndo_get_xmit_slave) in netdev_get_xmit_slave()
9086 return ops->ndo_get_xmit_slave(dev, skb, all_slaves); in netdev_get_xmit_slave()
9093 const struct net_device_ops *ops = dev->netdev_ops; in netdev_sk_get_lower_dev()
9095 if (!ops->ndo_sk_get_lower_dev) in netdev_sk_get_lower_dev()
9097 return ops->ndo_sk_get_lower_dev(dev, sk); in netdev_sk_get_lower_dev()
9101 * netdev_sk_get_lowest_dev - Get the lowest device in chain given device and socket
9129 list_for_each_entry(iter, &dev->adj_list.upper, list) { in netdev_adjacent_add_links()
9130 if (!net_eq(net, dev_net(iter->dev))) in netdev_adjacent_add_links()
9132 netdev_adjacent_sysfs_add(iter->dev, dev, in netdev_adjacent_add_links()
9133 &iter->dev->adj_list.lower); in netdev_adjacent_add_links()
9134 netdev_adjacent_sysfs_add(dev, iter->dev, in netdev_adjacent_add_links()
9135 &dev->adj_list.upper); in netdev_adjacent_add_links()
9138 list_for_each_entry(iter, &dev->adj_list.lower, list) { in netdev_adjacent_add_links()
9139 if (!net_eq(net, dev_net(iter->dev))) in netdev_adjacent_add_links()
9141 netdev_adjacent_sysfs_add(iter->dev, dev, in netdev_adjacent_add_links()
9142 &iter->dev->adj_list.upper); in netdev_adjacent_add_links()
9143 netdev_adjacent_sysfs_add(dev, iter->dev, in netdev_adjacent_add_links()
9144 &dev->adj_list.lower); in netdev_adjacent_add_links()
9154 list_for_each_entry(iter, &dev->adj_list.upper, list) { in netdev_adjacent_del_links()
9155 if (!net_eq(net, dev_net(iter->dev))) in netdev_adjacent_del_links()
9157 netdev_adjacent_sysfs_del(iter->dev, dev->name, in netdev_adjacent_del_links()
9158 &iter->dev->adj_list.lower); in netdev_adjacent_del_links()
9159 netdev_adjacent_sysfs_del(dev, iter->dev->name, in netdev_adjacent_del_links()
9160 &dev->adj_list.upper); in netdev_adjacent_del_links()
9163 list_for_each_entry(iter, &dev->adj_list.lower, list) { in netdev_adjacent_del_links()
9164 if (!net_eq(net, dev_net(iter->dev))) in netdev_adjacent_del_links()
9166 netdev_adjacent_sysfs_del(iter->dev, dev->name, in netdev_adjacent_del_links()
9167 &iter->dev->adj_list.upper); in netdev_adjacent_del_links()
9168 netdev_adjacent_sysfs_del(dev, iter->dev->name, in netdev_adjacent_del_links()
9169 &dev->adj_list.lower); in netdev_adjacent_del_links()
9179 list_for_each_entry(iter, &dev->adj_list.upper, list) { in netdev_adjacent_rename_links()
9180 if (!net_eq(net, dev_net(iter->dev))) in netdev_adjacent_rename_links()
9182 netdev_adjacent_sysfs_del(iter->dev, oldname, in netdev_adjacent_rename_links()
9183 &iter->dev->adj_list.lower); in netdev_adjacent_rename_links()
9184 netdev_adjacent_sysfs_add(iter->dev, dev, in netdev_adjacent_rename_links()
9185 &iter->dev->adj_list.lower); in netdev_adjacent_rename_links()
9188 list_for_each_entry(iter, &dev->adj_list.lower, list) { in netdev_adjacent_rename_links()
9189 if (!net_eq(net, dev_net(iter->dev))) in netdev_adjacent_rename_links()
9191 netdev_adjacent_sysfs_del(iter->dev, oldname, in netdev_adjacent_rename_links()
9192 &iter->dev->adj_list.upper); in netdev_adjacent_rename_links()
9193 netdev_adjacent_sysfs_add(iter->dev, dev, in netdev_adjacent_rename_links()
9194 &iter->dev->adj_list.upper); in netdev_adjacent_rename_links()
9205 lower = __netdev_find_adj(lower_dev, &dev->adj_list.lower); in netdev_lower_dev_get_private()
9209 return lower->private; in netdev_lower_dev_get_private()
9215 * netdev_lower_state_changed - Dispatch event about lower device state change
9238 const struct net_device_ops *ops = dev->netdev_ops; in dev_change_rx_flags()
9240 if (ops->ndo_change_rx_flags) in dev_change_rx_flags()
9241 ops->ndo_change_rx_flags(dev, flags); in dev_change_rx_flags()
9246 unsigned int old_flags = dev->flags; in __dev_set_promiscuity()
9253 promiscuity = dev->promiscuity + inc; in __dev_set_promiscuity()
9261 return -EOVERFLOW; in __dev_set_promiscuity()
9267 WRITE_ONCE(dev->promiscuity, promiscuity); in __dev_set_promiscuity()
9269 WRITE_ONCE(dev->flags, flags); in __dev_set_promiscuity()
9271 dev->flags & IFF_PROMISC ? "entered" : "left"); in __dev_set_promiscuity()
9277 dev->name, (dev->flags & IFF_PROMISC), in __dev_set_promiscuity()
9302 unsigned int old_flags = dev->flags; in netif_set_promiscuity()
9308 if (dev->flags != old_flags) in netif_set_promiscuity()
9315 unsigned int old_flags = dev->flags, old_gflags = dev->gflags; in netif_set_allmulti()
9320 allmulti = dev->allmulti + inc; in netif_set_allmulti()
9328 return -EOVERFLOW; in netif_set_allmulti()
9334 WRITE_ONCE(dev->allmulti, allmulti); in netif_set_allmulti()
9336 WRITE_ONCE(dev->flags, flags); in netif_set_allmulti()
9338 dev->flags & IFF_ALLMULTI ? "entered" : "left"); in netif_set_allmulti()
9343 dev->gflags ^ old_gflags, 0, NULL); in netif_set_allmulti()
9356 const struct net_device_ops *ops = dev->netdev_ops; in __dev_set_rx_mode()
9359 if (!(dev->flags&IFF_UP)) in __dev_set_rx_mode()
9365 if (!(dev->priv_flags & IFF_UNICAST_FLT)) { in __dev_set_rx_mode()
9369 if (!netdev_uc_empty(dev) && !dev->uc_promisc) { in __dev_set_rx_mode()
9371 dev->uc_promisc = true; in __dev_set_rx_mode()
9372 } else if (netdev_uc_empty(dev) && dev->uc_promisc) { in __dev_set_rx_mode()
9373 __dev_set_promiscuity(dev, -1, false); in __dev_set_rx_mode()
9374 dev->uc_promisc = false; in __dev_set_rx_mode()
9378 if (ops->ndo_set_rx_mode) in __dev_set_rx_mode()
9379 ops->ndo_set_rx_mode(dev); in __dev_set_rx_mode()
9390 * dev_get_flags - get flags reported to userspace
9399 flags = (READ_ONCE(dev->flags) & ~(IFF_PROMISC | in dev_get_flags()
9404 (READ_ONCE(dev->gflags) & (IFF_PROMISC | in dev_get_flags()
9423 unsigned int old_flags = dev->flags; in __dev_change_flags()
9432 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP | in __dev_change_flags()
9435 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC | in __dev_change_flags()
9461 if ((flags ^ dev->gflags) & IFF_PROMISC) { in __dev_change_flags()
9462 int inc = (flags & IFF_PROMISC) ? 1 : -1; in __dev_change_flags()
9463 old_flags = dev->flags; in __dev_change_flags()
9465 dev->gflags ^= IFF_PROMISC; in __dev_change_flags()
9468 if (dev->flags != old_flags) in __dev_change_flags()
9476 if ((flags ^ dev->gflags) & IFF_ALLMULTI) { in __dev_change_flags()
9477 int inc = (flags & IFF_ALLMULTI) ? 1 : -1; in __dev_change_flags()
9479 dev->gflags ^= IFF_ALLMULTI; in __dev_change_flags()
9490 unsigned int changes = dev->flags ^ old_flags; in __dev_notify_flags()
9496 if (dev->flags & IFF_UP) in __dev_notify_flags()
9502 if (dev->flags & IFF_UP && in __dev_notify_flags()
9519 unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags; in netif_change_flags()
9525 changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags); in netif_change_flags()
9532 const struct net_device_ops *ops = dev->netdev_ops; in __dev_set_mtu()
9534 if (ops->ndo_change_mtu) in __dev_set_mtu()
9535 return ops->ndo_change_mtu(dev, new_mtu); in __dev_set_mtu()
9537 /* Pairs with all the lockless reads of dev->mtu in the stack */ in __dev_set_mtu()
9538 WRITE_ONCE(dev->mtu, new_mtu); in __dev_set_mtu()
9547 if (new_mtu < 0 || new_mtu < dev->min_mtu) { in dev_validate_mtu()
9549 return -EINVAL; in dev_validate_mtu()
9552 if (dev->max_mtu > 0 && new_mtu > dev->max_mtu) { in dev_validate_mtu()
9554 return -EINVAL; in dev_validate_mtu()
9560 * netif_set_mtu_ext - Change maximum transfer unit
9572 if (new_mtu == dev->mtu) in netif_set_mtu_ext()
9580 return -ENODEV; in netif_set_mtu_ext()
9587 orig_mtu = dev->mtu; in netif_set_mtu_ext()
9614 net_err_ratelimited("%s: %s\n", dev->name, extack._msg); in netif_set_mtu()
9621 unsigned int orig_len = dev->tx_queue_len; in netif_change_tx_queue_len()
9625 return -ERANGE; in netif_change_tx_queue_len()
9628 WRITE_ONCE(dev->tx_queue_len, new_len); in netif_change_tx_queue_len()
9642 WRITE_ONCE(dev->tx_queue_len, orig_len); in netif_change_tx_queue_len()
9648 dev->group = new_group; in netif_set_group()
9652 * dev_pre_changeaddr_notify - Call NETDEV_PRE_CHANGEADDR.
9675 const struct net_device_ops *ops = dev->netdev_ops; in netif_set_mac_address()
9678 if (!ops->ndo_set_mac_address) in netif_set_mac_address()
9679 return -EOPNOTSUPP; in netif_set_mac_address()
9680 if (ss->ss_family != dev->type) in netif_set_mac_address()
9681 return -EINVAL; in netif_set_mac_address()
9683 return -ENODEV; in netif_set_mac_address()
9684 err = dev_pre_changeaddr_notify(dev, ss->__data, extack); in netif_set_mac_address()
9687 if (memcmp(dev->dev_addr, ss->__data, dev->addr_len)) { in netif_set_mac_address()
9688 err = ops->ndo_set_mac_address(dev, ss); in netif_set_mac_address()
9692 dev->addr_assign_type = NET_ADDR_SET; in netif_set_mac_address()
9694 add_device_randomness(dev->dev_addr, dev->addr_len); in netif_set_mac_address()
9703 size_t size = sizeof(sa->sa_data_min); in dev_get_mac_address()
9712 ret = -ENODEV; in dev_get_mac_address()
9715 if (!dev->addr_len) in dev_get_mac_address()
9716 memset(sa->sa_data, 0, size); in dev_get_mac_address()
9718 memcpy(sa->sa_data, dev->dev_addr, in dev_get_mac_address()
9719 min_t(size_t, size, dev->addr_len)); in dev_get_mac_address()
9720 sa->sa_family = dev->type; in dev_get_mac_address()
9731 const struct net_device_ops *ops = dev->netdev_ops; in netif_change_carrier()
9733 if (!ops->ndo_change_carrier) in netif_change_carrier()
9734 return -EOPNOTSUPP; in netif_change_carrier()
9736 return -ENODEV; in netif_change_carrier()
9737 return ops->ndo_change_carrier(dev, new_carrier); in netif_change_carrier()
9741 * dev_get_phys_port_id - Get device physical port ID
9750 const struct net_device_ops *ops = dev->netdev_ops; in dev_get_phys_port_id()
9752 if (!ops->ndo_get_phys_port_id) in dev_get_phys_port_id()
9753 return -EOPNOTSUPP; in dev_get_phys_port_id()
9754 return ops->ndo_get_phys_port_id(dev, ppid); in dev_get_phys_port_id()
9758 * dev_get_phys_port_name - Get device physical port name
9768 const struct net_device_ops *ops = dev->netdev_ops; in dev_get_phys_port_name()
9771 if (ops->ndo_get_phys_port_name) { in dev_get_phys_port_name()
9772 err = ops->ndo_get_phys_port_name(dev, name, len); in dev_get_phys_port_name()
9773 if (err != -EOPNOTSUPP) in dev_get_phys_port_name()
9780 * dev_get_port_parent_id - Get the device's port parent identifier
9791 const struct net_device_ops *ops = dev->netdev_ops; in dev_get_port_parent_id()
9797 if (ops->ndo_get_port_parent_id) { in dev_get_port_parent_id()
9798 err = ops->ndo_get_port_parent_id(dev, ppid); in dev_get_port_parent_id()
9799 if (err != -EOPNOTSUPP) in dev_get_port_parent_id()
9804 if (!recurse || err != -EOPNOTSUPP) in dev_get_port_parent_id()
9814 return -EOPNOTSUPP; in dev_get_port_parent_id()
9822 * netdev_port_same_parent_id - Indicate if two network devices have
9842 if (!dev->change_proto_down) in netif_change_proto_down()
9843 return -EOPNOTSUPP; in netif_change_proto_down()
9845 return -ENODEV; in netif_change_proto_down()
9850 WRITE_ONCE(dev->proto_down, proto_down); in netif_change_proto_down()
9855 * netdev_change_proto_down_reason_locked - proto down reason
9870 proto_down_reason = dev->proto_down_reason; in netdev_change_proto_down_reason_locked()
9878 WRITE_ONCE(dev->proto_down_reason, proto_down_reason); in netdev_change_proto_down_reason_locked()
9895 return dev->netdev_ops->ndo_bpf ? XDP_MODE_DRV : XDP_MODE_SKB; in dev_xdp_mode()
9905 return dev->netdev_ops->ndo_bpf; in dev_xdp_bpf_op()
9914 return dev->xdp_state[mode].link; in dev_xdp_link()
9923 return link->link.prog; in dev_xdp_prog()
9924 return dev->xdp_state[mode].prog; in dev_xdp_prog()
9933 if (dev->xdp_state[i].prog || dev->xdp_state[i].link) in dev_xdp_prog_count()
9945 if (dev->xdp_state[i].prog && in dev_xdp_sb_prog_count()
9946 !dev->xdp_state[i].prog->aux->xdp_has_frags) in dev_xdp_sb_prog_count()
9953 if (!dev->netdev_ops->ndo_bpf) in netif_xdp_propagate()
9954 return -EOPNOTSUPP; in netif_xdp_propagate()
9956 if (dev->cfg->hds_config == ETHTOOL_TCP_DATA_SPLIT_ENABLED && in netif_xdp_propagate()
9957 bpf->command == XDP_SETUP_PROG && in netif_xdp_propagate()
9958 bpf->prog && !bpf->prog->aux->xdp_has_frags) { in netif_xdp_propagate()
9959 NL_SET_ERR_MSG(bpf->extack, in netif_xdp_propagate()
9960 "unable to propagate XDP to device using tcp-data-split"); in netif_xdp_propagate()
9961 return -EBUSY; in netif_xdp_propagate()
9965 NL_SET_ERR_MSG(bpf->extack, "unable to propagate XDP to device using memory provider"); in netif_xdp_propagate()
9966 return -EBUSY; in netif_xdp_propagate()
9969 return dev->netdev_ops->ndo_bpf(dev, bpf); in netif_xdp_propagate()
9977 return prog ? prog->aux->id : 0; in dev_xdp_prog_id()
9983 dev->xdp_state[mode].link = link; in dev_xdp_set_link()
9984 dev->xdp_state[mode].prog = NULL; in dev_xdp_set_link()
9990 dev->xdp_state[mode].link = NULL; in dev_xdp_set_prog()
9991 dev->xdp_state[mode].prog = prog; in dev_xdp_set_prog()
10003 if (dev->cfg->hds_config == ETHTOOL_TCP_DATA_SPLIT_ENABLED && in dev_xdp_install()
10004 prog && !prog->aux->xdp_has_frags) { in dev_xdp_install()
10005 NL_SET_ERR_MSG(extack, "unable to install XDP to device using tcp-data-split"); in dev_xdp_install()
10006 return -EBUSY; in dev_xdp_install()
10011 return -EBUSY; in dev_xdp_install()
10061 /* auto-detach link from net device */ in dev_xdp_uninstall()
10064 link->dev = NULL; in dev_xdp_uninstall()
10088 return -EINVAL; in dev_xdp_attach()
10092 return -EINVAL; in dev_xdp_attach()
10097 return -EINVAL; in dev_xdp_attach()
10103 return -EINVAL; in dev_xdp_attach()
10108 return -EINVAL; in dev_xdp_attach()
10115 return -EBUSY; in dev_xdp_attach()
10122 return -EEXIST; in dev_xdp_attach()
10130 return -EBUSY; in dev_xdp_attach()
10134 return -EEXIST; in dev_xdp_attach()
10139 new_prog = link->link.prog; in dev_xdp_attach()
10148 return -EBUSY; in dev_xdp_attach()
10152 return -EEXIST; in dev_xdp_attach()
10154 if (!offload && bpf_prog_is_offloaded(new_prog->aux)) { in dev_xdp_attach()
10156 return -EINVAL; in dev_xdp_attach()
10158 if (bpf_prog_is_dev_bound(new_prog->aux) && !bpf_offload_dev_match(new_prog, dev)) { in dev_xdp_attach()
10160 return -EINVAL; in dev_xdp_attach()
10162 if (bpf_prog_is_dev_bound(new_prog->aux) && mode == XDP_MODE_SKB) { in dev_xdp_attach()
10163 NL_SET_ERR_MSG(extack, "Can't attach device-bound programs in generic mode"); in dev_xdp_attach()
10164 return -EINVAL; in dev_xdp_attach()
10166 if (new_prog->expected_attach_type == BPF_XDP_DEVMAP) { in dev_xdp_attach()
10168 return -EINVAL; in dev_xdp_attach()
10170 if (new_prog->expected_attach_type == BPF_XDP_CPUMAP) { in dev_xdp_attach()
10172 return -EINVAL; in dev_xdp_attach()
10181 return -EOPNOTSUPP; in dev_xdp_attach()
10203 return dev_xdp_attach(dev, extack, link, NULL, NULL, link->flags); in dev_xdp_attach_link()
10215 mode = dev_xdp_mode(dev, link->flags); in dev_xdp_detach_link()
10217 return -EINVAL; in dev_xdp_detach_link()
10231 /* if racing with net_device's tear down, xdp_link->dev might be in bpf_xdp_link_release()
10232 * already NULL, in which case link was already auto-detached in bpf_xdp_link_release()
10234 if (xdp_link->dev) { in bpf_xdp_link_release()
10235 netdev_lock_ops(xdp_link->dev); in bpf_xdp_link_release()
10236 WARN_ON(dev_xdp_detach_link(xdp_link->dev, NULL, xdp_link)); in bpf_xdp_link_release()
10237 netdev_unlock_ops(xdp_link->dev); in bpf_xdp_link_release()
10238 xdp_link->dev = NULL; in bpf_xdp_link_release()
10264 if (xdp_link->dev) in bpf_xdp_link_show_fdinfo()
10265 ifindex = xdp_link->dev->ifindex; in bpf_xdp_link_show_fdinfo()
10278 if (xdp_link->dev) in bpf_xdp_link_fill_link_info()
10279 ifindex = xdp_link->dev->ifindex; in bpf_xdp_link_fill_link_info()
10282 info->xdp.ifindex = ifindex; in bpf_xdp_link_fill_link_info()
10296 /* link might have been auto-released already, so fail */ in bpf_xdp_link_update()
10297 if (!xdp_link->dev) { in bpf_xdp_link_update()
10298 err = -ENOLINK; in bpf_xdp_link_update()
10302 if (old_prog && link->prog != old_prog) { in bpf_xdp_link_update()
10303 err = -EPERM; in bpf_xdp_link_update()
10306 old_prog = link->prog; in bpf_xdp_link_update()
10307 if (old_prog->type != new_prog->type || in bpf_xdp_link_update()
10308 old_prog->expected_attach_type != new_prog->expected_attach_type) { in bpf_xdp_link_update()
10309 err = -EINVAL; in bpf_xdp_link_update()
10314 /* no-op, don't disturb drivers */ in bpf_xdp_link_update()
10319 netdev_lock_ops(xdp_link->dev); in bpf_xdp_link_update()
10320 mode = dev_xdp_mode(xdp_link->dev, xdp_link->flags); in bpf_xdp_link_update()
10321 bpf_op = dev_xdp_bpf_op(xdp_link->dev, mode); in bpf_xdp_link_update()
10322 err = dev_xdp_install(xdp_link->dev, mode, bpf_op, NULL, in bpf_xdp_link_update()
10323 xdp_link->flags, new_prog); in bpf_xdp_link_update()
10324 netdev_unlock_ops(xdp_link->dev); in bpf_xdp_link_update()
10328 old_prog = xchg(&link->prog, new_prog); in bpf_xdp_link_update()
10347 struct net *net = current->nsproxy->net_ns; in bpf_xdp_link_attach()
10355 dev = dev_get_by_index(net, attr->link_create.target_ifindex); in bpf_xdp_link_attach()
10358 return -EINVAL; in bpf_xdp_link_attach()
10363 err = -ENOMEM; in bpf_xdp_link_attach()
10367 bpf_link_init(&link->link, BPF_LINK_TYPE_XDP, &bpf_xdp_link_lops, prog); in bpf_xdp_link_attach()
10368 link->dev = dev; in bpf_xdp_link_attach()
10369 link->flags = attr->link_create.flags; in bpf_xdp_link_attach()
10371 err = bpf_link_prime(&link->link, &link_primer); in bpf_xdp_link_attach()
10383 link->dev = NULL; in bpf_xdp_link_attach()
10403 * dev_change_xdp_fd - set or clear a bpf program for a device rx path
10408 * @flags: xdp-related flags
10454 for (i = dev->real_num_rx_queues - 1; i >= 0; i--) in dev_get_min_mp_channel_count()
10455 if (dev->_rx[i].mp_params.mp_priv) in dev_get_min_mp_channel_count()
10463 * dev_index_reserve() - allocate an ifindex in a namespace
10471 * Return: a suitable unique value for a new device interface number or -errno.
10479 return -EINVAL; in dev_index_reserve()
10483 err = xa_alloc_cyclic(&net->dev_by_index, &ifindex, NULL, in dev_index_reserve()
10484 xa_limit_31b, &net->ifindex, GFP_KERNEL); in dev_index_reserve()
10486 err = xa_insert(&net->dev_by_index, ifindex, NULL, GFP_KERNEL); in dev_index_reserve()
10496 WARN_ON(xa_erase(&net->dev_by_index, ifindex)); in dev_index_release()
10515 list_add_tail(&dev->todo_list, &net_todo_list); in net_set_todo()
10527 if (!(upper->wanted_features & feature) in netdev_sync_upper_features()
10530 &feature, upper->name); in netdev_sync_upper_features()
10547 if (!(features & feature) && (lower->features & feature)) { in netdev_sync_lower_features()
10549 &feature, lower->name); in netdev_sync_lower_features()
10551 lower->wanted_features &= ~feature; in netdev_sync_lower_features()
10554 if (unlikely(lower->features & feature)) in netdev_sync_lower_features()
10556 &feature, lower->name); in netdev_sync_lower_features()
10617 if ((features & dev->gso_partial_features) && in netdev_fix_features()
10621 features &= ~dev->gso_partial_features; in netdev_fix_features()
10636 /* LRO/HW-GRO features cannot be combined with RX-FCS */ in netdev_fix_features()
10639 netdev_dbg(dev, "Dropping LRO feature since RX-FCS is requested.\n"); in netdev_fix_features()
10644 netdev_dbg(dev, "Dropping HW-GRO feature since RX-FCS is requested.\n"); in netdev_fix_features()
10650 netdev_dbg(dev, "Dropping LRO feature since HW-GRO is requested.\n"); in netdev_fix_features()
10677 int err = -1; in __netdev_update_features()
10684 if (dev->netdev_ops->ndo_fix_features) in __netdev_update_features()
10685 features = dev->netdev_ops->ndo_fix_features(dev, features); in __netdev_update_features()
10694 if (dev->features == features) in __netdev_update_features()
10697 netdev_dbg(dev, "Features changed: %pNF -> %pNF\n", in __netdev_update_features()
10698 &dev->features, &features); in __netdev_update_features()
10700 if (dev->netdev_ops->ndo_set_features) in __netdev_update_features()
10701 err = dev->netdev_ops->ndo_set_features(dev, features); in __netdev_update_features()
10708 err, &features, &dev->features); in __netdev_update_features()
10709 /* return non-0 since some features might have changed and in __netdev_update_features()
10712 return -1; in __netdev_update_features()
10723 netdev_features_t diff = features ^ dev->features; in __netdev_update_features()
10729 * Thus we need to update dev->features in __netdev_update_features()
10734 dev->features = features; in __netdev_update_features()
10743 dev->features = features; in __netdev_update_features()
10752 dev->features = features; in __netdev_update_features()
10759 dev->features = features; in __netdev_update_features()
10766 * netdev_update_features - recalculate device features
10769 * Recalculate dev->features set and send notifications if it
10781 * netdev_change_features - recalculate device features
10784 * Recalculate dev->features set and send notifications even
10786 * netdev_update_features() if also dev->vlan_features might
10798 * netif_stacked_transfer_operstate - transfer operstate
10809 if (rootdev->operstate == IF_OPER_DORMANT) in netif_stacked_transfer_operstate()
10814 if (rootdev->operstate == IF_OPER_TESTING) in netif_stacked_transfer_operstate()
10828 unsigned int i, count = dev->num_rx_queues; in netif_alloc_rx_queues()
10837 return -ENOMEM; in netif_alloc_rx_queues()
10839 dev->_rx = rx; in netif_alloc_rx_queues()
10844 /* XDP RX-queue setup */ in netif_alloc_rx_queues()
10853 while (i--) in netif_alloc_rx_queues()
10855 kvfree(dev->_rx); in netif_alloc_rx_queues()
10856 dev->_rx = NULL; in netif_alloc_rx_queues()
10862 unsigned int i, count = dev->num_rx_queues; in netif_free_rx_queues()
10865 if (!dev->_rx) in netif_free_rx_queues()
10869 xdp_rxq_info_unreg(&dev->_rx[i].xdp_rxq); in netif_free_rx_queues()
10871 kvfree(dev->_rx); in netif_free_rx_queues()
10878 spin_lock_init(&queue->_xmit_lock); in netdev_init_one_queue()
10879 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type); in netdev_init_one_queue()
10880 queue->xmit_lock_owner = -1; in netdev_init_one_queue()
10882 queue->dev = dev; in netdev_init_one_queue()
10884 dql_init(&queue->dql, HZ); in netdev_init_one_queue()
10890 kvfree(dev->_tx); in netif_free_tx_queues()
10895 unsigned int count = dev->num_tx_queues; in netif_alloc_netdev_queues()
10900 return -EINVAL; in netif_alloc_netdev_queues()
10904 return -ENOMEM; in netif_alloc_netdev_queues()
10906 dev->_tx = tx; in netif_alloc_netdev_queues()
10909 spin_lock_init(&dev->tx_global_lock); in netif_alloc_netdev_queues()
10918 for (i = 0; i < dev->num_tx_queues; i++) { in netif_tx_stop_all_queues()
10934 if (dev->netdev_ops->ndo_get_peer_dev && in netdev_do_alloc_pcpu_stats()
10935 dev->pcpu_stat_type != NETDEV_PCPU_STAT_TSTATS) in netdev_do_alloc_pcpu_stats()
10936 return -EOPNOTSUPP; in netdev_do_alloc_pcpu_stats()
10938 switch (dev->pcpu_stat_type) { in netdev_do_alloc_pcpu_stats()
10942 v = dev->lstats = netdev_alloc_pcpu_stats(struct pcpu_lstats); in netdev_do_alloc_pcpu_stats()
10945 v = dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats); in netdev_do_alloc_pcpu_stats()
10948 v = dev->dstats = netdev_alloc_pcpu_stats(struct pcpu_dstats); in netdev_do_alloc_pcpu_stats()
10951 return -EINVAL; in netdev_do_alloc_pcpu_stats()
10954 return v ? 0 : -ENOMEM; in netdev_do_alloc_pcpu_stats()
10959 switch (dev->pcpu_stat_type) { in netdev_do_free_pcpu_stats()
10963 free_percpu(dev->lstats); in netdev_do_free_pcpu_stats()
10966 free_percpu(dev->tstats); in netdev_do_free_pcpu_stats()
10969 free_percpu(dev->dstats); in netdev_do_free_pcpu_stats()
10976 struct phy_link_topology *topo = dev->link_topo; in netdev_free_phy_link_topology()
10979 xa_destroy(&topo->phys); in netdev_free_phy_link_topology()
10981 dev->link_topo = NULL; in netdev_free_phy_link_topology()
10986 * register_netdevice() - register a network device
10991 * Callers must hold the rtnl lock - you may want register_netdev()
11007 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED); in register_netdevice()
11010 ret = ethtool_check_ops(dev->ethtool_ops); in register_netdevice()
11015 xa_init_flags(&dev->ethtool->rss_ctx, XA_FLAGS_ALLOC1); in register_netdevice()
11016 mutex_init(&dev->ethtool->rss_lock); in register_netdevice()
11018 spin_lock_init(&dev->addr_list_lock); in register_netdevice()
11021 ret = dev_get_valid_name(net, dev, dev->name); in register_netdevice()
11025 ret = -ENOMEM; in register_netdevice()
11026 dev->name_node = netdev_name_node_head_alloc(dev); in register_netdevice()
11027 if (!dev->name_node) in register_netdevice()
11031 if (dev->netdev_ops->ndo_init) { in register_netdevice()
11032 ret = dev->netdev_ops->ndo_init(dev); in register_netdevice()
11035 ret = -EIO; in register_netdevice()
11040 if (((dev->hw_features | dev->features) & in register_netdevice()
11042 (!dev->netdev_ops->ndo_vlan_rx_add_vid || in register_netdevice()
11043 !dev->netdev_ops->ndo_vlan_rx_kill_vid)) { in register_netdevice()
11045 ret = -EINVAL; in register_netdevice()
11053 ret = dev_index_reserve(net, dev->ifindex); in register_netdevice()
11056 dev->ifindex = ret; in register_netdevice()
11061 dev->hw_features |= (NETIF_F_SOFT_FEATURES | NETIF_F_SOFT_FEATURES_OFF); in register_netdevice()
11062 dev->features |= NETIF_F_SOFT_FEATURES; in register_netdevice()
11064 if (dev->udp_tunnel_nic_info) { in register_netdevice()
11065 dev->features |= NETIF_F_RX_UDP_TUNNEL_PORT; in register_netdevice()
11066 dev->hw_features |= NETIF_F_RX_UDP_TUNNEL_PORT; in register_netdevice()
11069 dev->wanted_features = dev->features & dev->hw_features; in register_netdevice()
11071 if (!(dev->flags & IFF_LOOPBACK)) in register_netdevice()
11072 dev->hw_features |= NETIF_F_NOCACHE_COPY; in register_netdevice()
11079 if (dev->hw_features & NETIF_F_TSO) in register_netdevice()
11080 dev->hw_features |= NETIF_F_TSO_MANGLEID; in register_netdevice()
11081 if (dev->vlan_features & NETIF_F_TSO) in register_netdevice()
11082 dev->vlan_features |= NETIF_F_TSO_MANGLEID; in register_netdevice()
11083 if (dev->mpls_features & NETIF_F_TSO) in register_netdevice()
11084 dev->mpls_features |= NETIF_F_TSO_MANGLEID; in register_netdevice()
11085 if (dev->hw_enc_features & NETIF_F_TSO) in register_netdevice()
11086 dev->hw_enc_features |= NETIF_F_TSO_MANGLEID; in register_netdevice()
11090 dev->vlan_features |= NETIF_F_HIGHDMA; in register_netdevice()
11094 dev->hw_enc_features |= NETIF_F_SG | NETIF_F_GSO_PARTIAL; in register_netdevice()
11098 dev->mpls_features |= NETIF_F_SG; in register_netdevice()
11108 WRITE_ONCE(dev->reg_state, ret ? NETREG_UNREGISTERED : NETREG_REGISTERED); in register_netdevice()
11123 set_bit(__LINK_STATE_PRESENT, &dev->state); in register_netdevice()
11129 netdev_hold(dev, &dev->dev_registered_tracker, GFP_KERNEL); in register_netdevice()
11132 add_device_randomness(dev->dev_addr, dev->addr_len); in register_netdevice()
11138 if (dev->addr_assign_type == NET_ADDR_PERM) in register_netdevice()
11139 memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len); in register_netdevice()
11148 dev->needs_free_netdev = false; in register_netdevice()
11156 if (!(dev->rtnl_link_ops && dev->rtnl_link_initializing)) in register_netdevice()
11165 dev_index_release(net, dev->ifindex); in register_netdevice()
11169 if (dev->netdev_ops->ndo_uninit) in register_netdevice()
11170 dev->netdev_ops->ndo_uninit(dev); in register_netdevice()
11171 if (dev->priv_destructor) in register_netdevice()
11172 dev->priv_destructor(dev); in register_netdevice()
11174 netdev_name_node_free(dev->name_node); in register_netdevice()
11188 dev->reg_state = NETREG_DUMMY; in init_dummy_netdev()
11191 set_bit(__LINK_STATE_PRESENT, &dev->state); in init_dummy_netdev()
11192 set_bit(__LINK_STATE_START, &dev->state); in init_dummy_netdev()
11201 * register_netdev - register a network device
11219 return -EINTR; in register_netdev()
11235 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i); in netdev_refcnt_read()
11238 return refcount_read(&dev->dev_refcnt); in netdev_refcnt_read()
11248 * netdev_wait_allrefs_any - wait until all references are gone.
11285 &dev->state)) { in netdev_wait_allrefs_any()
11318 dev->name, netdev_refcnt_read(dev)); in netdev_wait_allrefs_any()
11319 ref_tracker_dir_print(&dev->refcnt_tracker, 10); in netdev_wait_allrefs_any()
11364 list_del_init(&dev->unlink_list); in netdev_run_todo()
11365 dev->nested_level = dev->lower_level - 1; in netdev_run_todo()
11379 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) { in netdev_run_todo()
11381 list_del(&dev->todo_list); in netdev_run_todo()
11386 WRITE_ONCE(dev->reg_state, NETREG_UNREGISTERED); in netdev_run_todo()
11394 list_del(&dev->todo_list); in netdev_run_todo()
11398 BUG_ON(!list_empty(&dev->ptype_all)); in netdev_run_todo()
11399 BUG_ON(!list_empty(&dev->ptype_specific)); in netdev_run_todo()
11400 WARN_ON(rcu_access_pointer(dev->ip_ptr)); in netdev_run_todo()
11401 WARN_ON(rcu_access_pointer(dev->ip6_ptr)); in netdev_run_todo()
11404 if (dev->priv_destructor) in netdev_run_todo()
11405 dev->priv_destructor(dev); in netdev_run_todo()
11406 if (dev->needs_free_netdev) in netdev_run_todo()
11412 kobject_put(&dev->dev.kobj); in netdev_run_todo()
11418 /* Collate per-cpu network dstats statistics
11420 * Read per-cpu network statistics from dev->dstats and populate the related
11436 start = u64_stats_fetch_begin(&stats->syncp); in dev_fetch_dstats()
11437 rx_packets = u64_stats_read(&stats->rx_packets); in dev_fetch_dstats()
11438 rx_bytes = u64_stats_read(&stats->rx_bytes); in dev_fetch_dstats()
11439 rx_drops = u64_stats_read(&stats->rx_drops); in dev_fetch_dstats()
11440 tx_packets = u64_stats_read(&stats->tx_packets); in dev_fetch_dstats()
11441 tx_bytes = u64_stats_read(&stats->tx_bytes); in dev_fetch_dstats()
11442 tx_drops = u64_stats_read(&stats->tx_drops); in dev_fetch_dstats()
11443 } while (u64_stats_fetch_retry(&stats->syncp, start)); in dev_fetch_dstats()
11445 s->rx_packets += rx_packets; in dev_fetch_dstats()
11446 s->rx_bytes += rx_bytes; in dev_fetch_dstats()
11447 s->rx_dropped += rx_drops; in dev_fetch_dstats()
11448 s->tx_packets += tx_packets; in dev_fetch_dstats()
11449 s->tx_bytes += tx_bytes; in dev_fetch_dstats()
11450 s->tx_dropped += tx_drops; in dev_fetch_dstats()
11454 /* ndo_get_stats64 implementation for dtstats-based accounting.
11456 * Populate @s from dev->stats and dev->dstats. This is used internally by the
11457 * core for NETDEV_PCPU_STAT_DSTAT-type stats collection.
11462 netdev_stats_to_stats64(s, &dev->stats); in dev_get_dstats64()
11463 dev_fetch_dstats(s, dev->dstats); in dev_get_dstats64()
11483 sizeof(*stats64) - n * sizeof(u64)); in netdev_stats_to_stats64()
11495 if (p && cmpxchg(&dev->core_stats, NULL, p)) in netdev_core_stats_alloc()
11499 return READ_ONCE(dev->core_stats); in netdev_core_stats_alloc()
11505 struct net_device_core_stats __percpu *p = READ_ONCE(dev->core_stats); in netdev_core_stats_inc()
11520 * dev_get_stats - get network device statistics
11526 * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
11532 const struct net_device_ops *ops = dev->netdev_ops; in dev_get_stats()
11549 if (ops->ndo_get_stats64) { in dev_get_stats()
11551 ops->ndo_get_stats64(dev, storage); in dev_get_stats()
11552 } else if (ops->ndo_get_stats) { in dev_get_stats()
11553 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev)); in dev_get_stats()
11554 } else if (dev->pcpu_stat_type == NETDEV_PCPU_STAT_TSTATS) { in dev_get_stats()
11556 } else if (dev->pcpu_stat_type == NETDEV_PCPU_STAT_DSTATS) { in dev_get_stats()
11559 netdev_stats_to_stats64(storage, &dev->stats); in dev_get_stats()
11563 p = READ_ONCE(dev->core_stats); in dev_get_stats()
11570 storage->rx_dropped += READ_ONCE(core_stats->rx_dropped); in dev_get_stats()
11571 storage->tx_dropped += READ_ONCE(core_stats->tx_dropped); in dev_get_stats()
11572 storage->rx_nohandler += READ_ONCE(core_stats->rx_nohandler); in dev_get_stats()
11573 storage->rx_otherhost_dropped += READ_ONCE(core_stats->rx_otherhost_dropped); in dev_get_stats()
11581 * dev_fetch_sw_netstats - get per-cpu network device statistics
11583 * @netstats: per-cpu network stats to read from
11585 * Read per-cpu network statistics and populate the related fields in @s.
11599 start = u64_stats_fetch_begin(&stats->syncp); in dev_fetch_sw_netstats()
11600 rx_packets = u64_stats_read(&stats->rx_packets); in dev_fetch_sw_netstats()
11601 rx_bytes = u64_stats_read(&stats->rx_bytes); in dev_fetch_sw_netstats()
11602 tx_packets = u64_stats_read(&stats->tx_packets); in dev_fetch_sw_netstats()
11603 tx_bytes = u64_stats_read(&stats->tx_bytes); in dev_fetch_sw_netstats()
11604 } while (u64_stats_fetch_retry(&stats->syncp, start)); in dev_fetch_sw_netstats()
11606 s->rx_packets += rx_packets; in dev_fetch_sw_netstats()
11607 s->rx_bytes += rx_bytes; in dev_fetch_sw_netstats()
11608 s->tx_packets += tx_packets; in dev_fetch_sw_netstats()
11609 s->tx_bytes += tx_bytes; in dev_fetch_sw_netstats()
11615 * dev_get_tstats64 - ndo_get_stats64 implementation
11619 * Populate @s from dev->stats and dev->tstats. Can be used as
11624 netdev_stats_to_stats64(s, &dev->stats); in dev_get_tstats64()
11625 dev_fetch_sw_netstats(s, dev->tstats); in dev_get_tstats64()
11640 RCU_INIT_POINTER(queue->qdisc, &noop_qdisc); in dev_ingress_queue_create()
11641 RCU_INIT_POINTER(queue->qdisc_sleeping, &noop_qdisc); in dev_ingress_queue_create()
11642 rcu_assign_pointer(dev->ingress_queue, queue); in dev_ingress_queue_create()
11652 if (dev->ethtool_ops == &default_ethtool_ops) in netdev_set_default_ethtool_ops()
11653 dev->ethtool_ops = ops; in netdev_set_default_ethtool_ops()
11658 * netdev_sw_irq_coalesce_default_on() - enable SW IRQ coalescing by default
11666 WARN_ON(dev->reg_state == NETREG_REGISTERED); in netdev_sw_irq_coalesce_default_on()
11676 * alloc_netdev_mqs - allocate network device
11697 BUG_ON(strlen(name) >= sizeof(dev->name)); in alloc_netdev_mqs()
11709 maxqs = max(txqs, rxqs); in alloc_netdev_mqs()
11716 dev->priv_len = sizeof_priv; in alloc_netdev_mqs()
11718 ref_tracker_dir_init(&dev->refcnt_tracker, 128, name); in alloc_netdev_mqs()
11720 dev->pcpu_refcnt = alloc_percpu(int); in alloc_netdev_mqs()
11721 if (!dev->pcpu_refcnt) in alloc_netdev_mqs()
11725 refcount_set(&dev->dev_refcnt, 1); in alloc_netdev_mqs()
11736 dev->gso_max_size = GSO_LEGACY_MAX_SIZE; in alloc_netdev_mqs()
11737 dev->xdp_zc_max_segs = 1; in alloc_netdev_mqs()
11738 dev->gso_max_segs = GSO_MAX_SEGS; in alloc_netdev_mqs()
11739 dev->gro_max_size = GRO_LEGACY_MAX_SIZE; in alloc_netdev_mqs()
11740 dev->gso_ipv4_max_size = GSO_LEGACY_MAX_SIZE; in alloc_netdev_mqs()
11741 dev->gro_ipv4_max_size = GRO_LEGACY_MAX_SIZE; in alloc_netdev_mqs()
11742 dev->tso_max_size = TSO_LEGACY_MAX_SIZE; in alloc_netdev_mqs()
11743 dev->tso_max_segs = TSO_MAX_SEGS; in alloc_netdev_mqs()
11744 dev->upper_level = 1; in alloc_netdev_mqs()
11745 dev->lower_level = 1; in alloc_netdev_mqs()
11747 dev->nested_level = 0; in alloc_netdev_mqs()
11748 INIT_LIST_HEAD(&dev->unlink_list); in alloc_netdev_mqs()
11751 INIT_LIST_HEAD(&dev->napi_list); in alloc_netdev_mqs()
11752 INIT_LIST_HEAD(&dev->unreg_list); in alloc_netdev_mqs()
11753 INIT_LIST_HEAD(&dev->close_list); in alloc_netdev_mqs()
11754 INIT_LIST_HEAD(&dev->link_watch_list); in alloc_netdev_mqs()
11755 INIT_LIST_HEAD(&dev->adj_list.upper); in alloc_netdev_mqs()
11756 INIT_LIST_HEAD(&dev->adj_list.lower); in alloc_netdev_mqs()
11757 INIT_LIST_HEAD(&dev->ptype_all); in alloc_netdev_mqs()
11758 INIT_LIST_HEAD(&dev->ptype_specific); in alloc_netdev_mqs()
11759 INIT_LIST_HEAD(&dev->net_notifier_list); in alloc_netdev_mqs()
11761 hash_init(dev->qdisc_hash); in alloc_netdev_mqs()
11764 mutex_init(&dev->lock); in alloc_netdev_mqs()
11766 dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM; in alloc_netdev_mqs()
11769 if (!dev->tx_queue_len) { in alloc_netdev_mqs()
11770 dev->priv_flags |= IFF_NO_QUEUE; in alloc_netdev_mqs()
11771 dev->tx_queue_len = DEFAULT_TX_QUEUE_LEN; in alloc_netdev_mqs()
11774 dev->num_tx_queues = txqs; in alloc_netdev_mqs()
11775 dev->real_num_tx_queues = txqs; in alloc_netdev_mqs()
11779 dev->num_rx_queues = rxqs; in alloc_netdev_mqs()
11780 dev->real_num_rx_queues = rxqs; in alloc_netdev_mqs()
11783 dev->ethtool = kzalloc(sizeof(*dev->ethtool), GFP_KERNEL_ACCOUNT); in alloc_netdev_mqs()
11784 if (!dev->ethtool) in alloc_netdev_mqs()
11787 dev->cfg = kzalloc(sizeof(*dev->cfg), GFP_KERNEL_ACCOUNT); in alloc_netdev_mqs()
11788 if (!dev->cfg) in alloc_netdev_mqs()
11790 dev->cfg_pending = dev->cfg; in alloc_netdev_mqs()
11792 napi_config_sz = array_size(maxqs, sizeof(*dev->napi_config)); in alloc_netdev_mqs()
11793 dev->napi_config = kvzalloc(napi_config_sz, GFP_KERNEL_ACCOUNT); in alloc_netdev_mqs()
11794 if (!dev->napi_config) in alloc_netdev_mqs()
11797 strscpy(dev->name, name); in alloc_netdev_mqs()
11798 dev->name_assign_type = name_assign_type; in alloc_netdev_mqs()
11799 dev->group = INIT_NETDEV_GROUP; in alloc_netdev_mqs()
11800 if (!dev->ethtool_ops) in alloc_netdev_mqs()
11801 dev->ethtool_ops = &default_ethtool_ops; in alloc_netdev_mqs()
11813 free_percpu(dev->pcpu_refcnt); in alloc_netdev_mqs()
11823 if (!list_empty(&dev->napi_list)) { in netdev_napi_exit()
11827 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list) in netdev_napi_exit()
11834 kvfree(dev->napi_config); in netdev_napi_exit()
11838 * free_netdev - free network device
11854 if (dev->reg_state == NETREG_UNREGISTERING) { in free_netdev()
11856 dev->needs_free_netdev = true; in free_netdev()
11860 WARN_ON(dev->cfg != dev->cfg_pending); in free_netdev()
11861 kfree(dev->cfg); in free_netdev()
11862 kfree(dev->ethtool); in free_netdev()
11866 kfree(rcu_dereference_protected(dev->ingress_queue, 1)); in free_netdev()
11875 ref_tracker_dir_exit(&dev->refcnt_tracker); in free_netdev()
11877 free_percpu(dev->pcpu_refcnt); in free_netdev()
11878 dev->pcpu_refcnt = NULL; in free_netdev()
11880 free_percpu(dev->core_stats); in free_netdev()
11881 dev->core_stats = NULL; in free_netdev()
11882 free_percpu(dev->xdp_bulkq); in free_netdev()
11883 dev->xdp_bulkq = NULL; in free_netdev()
11887 mutex_destroy(&dev->lock); in free_netdev()
11890 if (dev->reg_state == NETREG_UNINITIALIZED || in free_netdev()
11891 dev->reg_state == NETREG_DUMMY) { in free_netdev()
11896 BUG_ON(dev->reg_state != NETREG_UNREGISTERED); in free_netdev()
11897 WRITE_ONCE(dev->reg_state, NETREG_RELEASED); in free_netdev()
11900 put_device(&dev->dev); in free_netdev()
11905 * alloc_netdev_dummy - Allocate and initialize a dummy net device.
11918 * synchronize_net - Synchronize with packet receive processing
11938 mutex_lock(&dev->ethtool->rss_lock); in netdev_rss_contexts_free()
11939 xa_for_each(&dev->ethtool->rss_ctx, context, ctx) { in netdev_rss_contexts_free()
11944 rxfh.hfunc = ctx->hfunc; in netdev_rss_contexts_free()
11945 rxfh.input_xfrm = ctx->input_xfrm; in netdev_rss_contexts_free()
11949 xa_erase(&dev->ethtool->rss_ctx, context); in netdev_rss_contexts_free()
11950 if (dev->ethtool_ops->create_rxfh_context) in netdev_rss_contexts_free()
11951 dev->ethtool_ops->remove_rxfh_context(dev, ctx, in netdev_rss_contexts_free()
11954 dev->ethtool_ops->set_rxfh(dev, &rxfh, NULL); in netdev_rss_contexts_free()
11957 xa_destroy(&dev->ethtool->rss_ctx); in netdev_rss_contexts_free()
11958 mutex_unlock(&dev->ethtool->rss_lock); in netdev_rss_contexts_free()
11962 * unregister_netdevice_queue - remove device from the kernel
11979 list_move_tail(&dev->unreg_list, head); in unregister_netdevice_queue()
11983 list_add(&dev->unreg_list, &single); in unregister_netdevice_queue()
11993 for (i = 0; i < dev->real_num_rx_queues; i++) { in dev_memory_provider_uninstall()
11994 struct netdev_rx_queue *rxq = &dev->_rx[i]; in dev_memory_provider_uninstall()
11995 struct pp_memory_provider_params *p = &rxq->mp_params; in dev_memory_provider_uninstall()
11997 if (p->mp_ops && p->mp_ops->uninstall) in dev_memory_provider_uninstall()
11998 p->mp_ops->uninstall(rxq->mp_params.mp_priv, rxq); in dev_memory_provider_uninstall()
12020 if (dev->reg_state == NETREG_UNINITIALIZED) { in unregister_netdevice_many_notify()
12022 dev->name, dev); in unregister_netdevice_many_notify()
12025 list_del(&dev->unreg_list); in unregister_netdevice_many_notify()
12028 dev->dismantle = true; in unregister_netdevice_many_notify()
12029 BUG_ON(dev->reg_state != NETREG_REGISTERED); in unregister_netdevice_many_notify()
12035 list_add_tail(&dev->close_list, &close_head); in unregister_netdevice_many_notify()
12045 list_add_tail(&dev->close_list, &close_head); in unregister_netdevice_many_notify()
12053 WRITE_ONCE(dev->reg_state, NETREG_UNREGISTERING); in unregister_netdevice_many_notify()
12079 if (!(dev->rtnl_link_ops && dev->rtnl_link_initializing)) in unregister_netdevice_many_notify()
12091 netdev_name_node_free(dev->name_node); in unregister_netdevice_many_notify()
12097 if (dev->netdev_ops->ndo_uninit) in unregister_netdevice_many_notify()
12098 dev->netdev_ops->ndo_uninit(dev); in unregister_netdevice_many_notify()
12100 mutex_destroy(&dev->ethtool->rss_lock); in unregister_netdevice_many_notify()
12122 netdev_put(dev, &dev->dev_registered_tracker); in unregister_netdevice_many_notify()
12132 * unregister_netdevice_many - unregister many devices
12145 * unregister_netdev - remove device from the kernel
12175 err = -EINVAL; in __dev_change_net_namespace()
12176 if (dev->netns_immutable) { in __dev_change_net_namespace()
12182 if (dev->reg_state != NETREG_REGISTERED) { in __dev_change_net_namespace()
12195 err = -EEXIST; in __dev_change_net_namespace()
12196 if (netdev_name_in_use(net, dev->name)) { in __dev_change_net_namespace()
12212 err = -EEXIST; in __dev_change_net_namespace()
12214 if (netdev_name_in_use(net, name_node->name)) { in __dev_change_net_namespace()
12217 name_node->name); in __dev_change_net_namespace()
12233 err = dev_index_reserve(net, dev->ifindex); in __dev_change_net_namespace()
12234 if (err == -EBUSY) in __dev_change_net_namespace()
12256 dev->moving_ns = true; in __dev_change_net_namespace()
12269 * Note that dev->reg_state stays at NETREG_REGISTERED. in __dev_change_net_namespace()
12287 /* Send a netdev-removed uevent to the old namespace */ in __dev_change_net_namespace()
12288 kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE); in __dev_change_net_namespace()
12291 /* Move per-net netdevice notifiers that are following the netdevice */ in __dev_change_net_namespace()
12298 dev->ifindex = new_ifindex; in __dev_change_net_namespace()
12303 strscpy(dev->name, new_name, IFNAMSIZ); in __dev_change_net_namespace()
12308 dev_set_uevent_suppress(&dev->dev, 1); in __dev_change_net_namespace()
12309 err = device_rename(&dev->dev, dev->name); in __dev_change_net_namespace()
12310 dev_set_uevent_suppress(&dev->dev, 0); in __dev_change_net_namespace()
12313 /* Send a netdev-add uevent to the new namespace */ in __dev_change_net_namespace()
12314 kobject_uevent(&dev->dev.kobj, KOBJ_ADD); in __dev_change_net_namespace()
12324 dev->moving_ns = false; in __dev_change_net_namespace()
12359 list_skb = &sd->completion_queue; in dev_cpu_dead()
12361 list_skb = &(*list_skb)->next; in dev_cpu_dead()
12363 *list_skb = oldsd->completion_queue; in dev_cpu_dead()
12364 oldsd->completion_queue = NULL; in dev_cpu_dead()
12367 if (oldsd->output_queue) { in dev_cpu_dead()
12368 *sd->output_queue_tailp = oldsd->output_queue; in dev_cpu_dead()
12369 sd->output_queue_tailp = oldsd->output_queue_tailp; in dev_cpu_dead()
12370 oldsd->output_queue = NULL; in dev_cpu_dead()
12371 oldsd->output_queue_tailp = &oldsd->output_queue; in dev_cpu_dead()
12377 while (!list_empty(&oldsd->poll_list)) { in dev_cpu_dead()
12378 struct napi_struct *napi = list_first_entry(&oldsd->poll_list, in dev_cpu_dead()
12382 list_del_init(&napi->poll_list); in dev_cpu_dead()
12383 if (napi->poll == process_backlog) in dev_cpu_dead()
12384 napi->state &= NAPIF_STATE_THREADED; in dev_cpu_dead()
12394 remsd = oldsd->rps_ipi_list; in dev_cpu_dead()
12395 oldsd->rps_ipi_list = NULL; in dev_cpu_dead()
12402 while ((skb = __skb_dequeue(&oldsd->process_queue))) { in dev_cpu_dead()
12406 while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) { in dev_cpu_dead()
12415 * netdev_increment_features - increment feature set by one
12461 INIT_LIST_HEAD(&net->dev_base_head); in netdev_init()
12463 net->dev_name_head = netdev_create_hash(); in netdev_init()
12464 if (net->dev_name_head == NULL) in netdev_init()
12467 net->dev_index_head = netdev_create_hash(); in netdev_init()
12468 if (net->dev_index_head == NULL) in netdev_init()
12471 xa_init_flags(&net->dev_by_index, XA_FLAGS_ALLOC1); in netdev_init()
12473 RAW_INIT_NOTIFIER_HEAD(&net->netdev_chain); in netdev_init()
12478 kfree(net->dev_name_head); in netdev_init()
12480 return -ENOMEM; in netdev_init()
12484 * netdev_drivername - network driver for the device
12495 parent = dev->dev.parent; in netdev_drivername()
12499 driver = parent->driver; in netdev_drivername()
12500 if (driver && driver->name) in netdev_drivername()
12501 return driver->name; in netdev_drivername()
12508 if (dev && dev->dev.parent) { in __netdev_printk()
12509 dev_printk_emit(level[1] - '0', in __netdev_printk()
12510 dev->dev.parent, in __netdev_printk()
12512 dev_driver_string(dev->dev.parent), in __netdev_printk()
12513 dev_name(dev->dev.parent), in __netdev_printk()
12568 kfree(net->dev_name_head); in netdev_exit()
12569 kfree(net->dev_index_head); in netdev_exit()
12570 xa_destroy(&net->dev_by_index); in netdev_exit()
12572 WARN_ON_ONCE(!list_empty(&net->dev_base_head)); in netdev_exit()
12594 if (dev->netns_immutable) in default_device_exit_net()
12598 if (dev->rtnl_link_ops && !dev->rtnl_link_ops->netns_refund) in default_device_exit_net()
12602 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex); in default_device_exit_net()
12607 if (netdev_name_in_use(&init_net, name_node->name)) in default_device_exit_net()
12613 __func__, dev->name, err); in default_device_exit_net()
12638 if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink) in default_device_exit_batch()
12639 dev->rtnl_link_ops->dellink(dev, &dev_kill_list); in default_device_exit_batch()
12654 /* TX read-mostly hotpath */ in net_dev_struct_check()
12679 /* TXRX read-mostly hotpath */ in net_dev_struct_check()
12688 /* RX read-mostly hotpath */ in net_dev_struct_check()
12730 return -ENOMEM; in net_page_pool_create()
12746 struct napi_struct *napi = &sd->backlog; in backlog_napi_should_run()
12748 return test_bit(NAPI_STATE_SCHED_THREADED, &napi->state); in backlog_napi_should_run()
12755 napi_threaded_poll_loop(&sd->backlog); in run_backlog_napi()
12761 struct napi_struct *napi = &sd->backlog; in backlog_napi_setup()
12763 napi->thread = this_cpu_read(backlog_napi); in backlog_napi_setup()
12764 set_bit(NAPI_STATE_THREADED, &napi->state); in backlog_napi_setup()
12781 int i, rc = -ENOMEM; in net_dev_init()
12810 skb_queue_head_init(&sd->input_pkt_queue); in net_dev_init()
12811 skb_queue_head_init(&sd->process_queue); in net_dev_init()
12813 skb_queue_head_init(&sd->xfrm_backlog); in net_dev_init()
12815 INIT_LIST_HEAD(&sd->poll_list); in net_dev_init()
12816 sd->output_queue_tailp = &sd->output_queue; in net_dev_init()
12818 INIT_CSD(&sd->csd, rps_trigger_softirq, sd); in net_dev_init()
12819 sd->cpu = i; in net_dev_init()
12821 INIT_CSD(&sd->defer_csd, trigger_rx_softirq, sd); in net_dev_init()
12822 spin_lock_init(&sd->defer_lock); in net_dev_init()
12824 gro_init(&sd->backlog.gro); in net_dev_init()
12825 sd->backlog.poll = process_backlog; in net_dev_init()
12826 sd->backlog.weight = weight_p; in net_dev_init()
12827 INIT_LIST_HEAD(&sd->backlog.poll_list); in net_dev_init()