Lines Matching +full:ext +full:- +full:bhs +full:- +full:reg
1 // SPDX-License-Identifier: GPL-2.0-or-later
11 * Florian la Roche <rzsfl@rz.uni-sb.de>
19 * D.J. Barrow : Fixed bug where dev->refcnt gets set
64 * Pekka Riikonen : Netdev boot-time settings code
66 * indefinitely on dev->refcnt
67 * J Hadi Salim : - Backlog queue sampling
68 * - netif_rx() feedback
169 #include "net-sysfs.h"
189 unsigned int val = net->dev_base_seq + 1; in dev_base_seq_inc()
191 WRITE_ONCE(net->dev_base_seq, val ?: 1); in dev_base_seq_inc()
198 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)]; in dev_name_hash()
203 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)]; in dev_index_hash()
235 spin_lock_irqsave(&sd->input_pkt_queue.lock, *flags); in backlog_lock_irq_save()
243 spin_lock_irq(&sd->input_pkt_queue.lock); in backlog_lock_irq_disable()
252 spin_unlock_irqrestore(&sd->input_pkt_queue.lock, *flags); in backlog_unlock_irq_restore()
260 spin_unlock_irq(&sd->input_pkt_queue.lock); in backlog_unlock_irq_enable()
273 INIT_HLIST_NODE(&name_node->hlist); in netdev_name_node_alloc()
274 name_node->dev = dev; in netdev_name_node_alloc()
275 name_node->name = name; in netdev_name_node_alloc()
284 name_node = netdev_name_node_alloc(dev, dev->name); in netdev_name_node_head_alloc()
287 INIT_LIST_HEAD(&name_node->list); in netdev_name_node_head_alloc()
299 hlist_add_head_rcu(&name_node->hlist, in netdev_name_node_add()
300 dev_name_hash(net, name_node->name)); in netdev_name_node_add()
305 hlist_del_rcu(&name_node->hlist); in netdev_name_node_del()
315 if (!strcmp(name_node->name, name)) in netdev_name_node_lookup()
327 if (!strcmp(name_node->name, name)) in netdev_name_node_lookup_rcu()
345 return -EEXIST; in netdev_name_node_alt_create()
348 return -ENOMEM; in netdev_name_node_alt_create()
350 /* The node that holds dev->name acts as a head of per-device list. */ in netdev_name_node_alt_create()
351 list_add_tail_rcu(&name_node->list, &dev->name_node->list); in netdev_name_node_alt_create()
361 kfree(name_node->name); in netdev_name_node_alt_free()
368 list_del(&name_node->list); in __netdev_name_node_alt_destroy()
369 call_rcu(&name_node->rcu, netdev_name_node_alt_free); in __netdev_name_node_alt_destroy()
379 return -ENOENT; in netdev_name_node_alt_destroy()
383 if (name_node == dev->name_node || name_node->dev != dev) in netdev_name_node_alt_destroy()
384 return -EINVAL; in netdev_name_node_alt_destroy()
394 list_for_each_entry_safe(name_node, tmp, &dev->name_node->list, list) { in netdev_name_node_alt_flush()
395 list_del(&name_node->list); in netdev_name_node_alt_flush()
396 netdev_name_node_alt_free(&name_node->rcu); in netdev_name_node_alt_flush()
408 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head); in list_netdevice()
409 netdev_name_node_add(net, dev->name_node); in list_netdevice()
410 hlist_add_head_rcu(&dev->index_hlist, in list_netdevice()
411 dev_index_hash(net, dev->ifindex)); in list_netdevice()
417 WARN_ON(xa_store(&net->dev_by_index, dev->ifindex, dev, GFP_KERNEL)); in list_netdevice()
432 xa_erase(&net->dev_by_index, dev->ifindex); in unlist_netdevice()
438 list_del_rcu(&dev->dev_list); in unlist_netdevice()
439 netdev_name_node_del(dev->name_node); in unlist_netdevice()
440 hlist_del_rcu(&dev->index_hlist); in unlist_netdevice()
471 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
472 * according to dev->type
519 return ARRAY_SIZE(netdev_lock_type) - 1; in netdev_lock_pos()
536 i = netdev_lock_pos(dev->type); in netdev_set_addr_lockdep_class()
537 lockdep_set_class_and_name(&dev->addr_list_lock, in netdev_set_addr_lockdep_class()
570 * is cloned and should be copied-on-write, so that it will
572 * --ANK (980803)
577 if (pt->type == htons(ETH_P_ALL)) { in ptype_head()
578 if (!pt->af_packet_net && !pt->dev) in ptype_head()
581 return pt->dev ? &pt->dev->ptype_all : in ptype_head()
582 &pt->af_packet_net->ptype_all; in ptype_head()
585 if (pt->dev) in ptype_head()
586 return &pt->dev->ptype_specific; in ptype_head()
588 return pt->af_packet_net ? &pt->af_packet_net->ptype_specific : in ptype_head()
589 &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK]; in ptype_head()
593 * dev_add_pack - add packet handler
613 list_add_rcu(&pt->list, head); in dev_add_pack()
619 * __dev_remove_pack - remove packet handler
643 list_del_rcu(&pt->list); in __dev_remove_pack()
655 * dev_remove_pack - remove packet handler
682 * dev_get_iflink - get 'iflink' value of a interface
691 if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink) in dev_get_iflink()
692 return dev->netdev_ops->ndo_get_iflink(dev); in dev_get_iflink()
694 return READ_ONCE(dev->ifindex); in dev_get_iflink()
699 * dev_fill_metadata_dst - Retrieve tunnel egress information.
711 if (!dev->netdev_ops || !dev->netdev_ops->ndo_fill_metadata_dst) in dev_fill_metadata_dst()
712 return -EINVAL; in dev_fill_metadata_dst()
716 return -ENOMEM; in dev_fill_metadata_dst()
717 if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX))) in dev_fill_metadata_dst()
718 return -EINVAL; in dev_fill_metadata_dst()
720 return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb); in dev_fill_metadata_dst()
726 int k = stack->num_paths++; in dev_fwd_path()
731 return &stack->path[k]; in dev_fwd_path()
745 stack->num_paths = 0; in dev_fill_forward_path()
746 while (ctx.dev && ctx.dev->netdev_ops->ndo_fill_forward_path) { in dev_fill_forward_path()
750 return -1; in dev_fill_forward_path()
753 ret = ctx.dev->netdev_ops->ndo_fill_forward_path(&ctx, path); in dev_fill_forward_path()
755 return -1; in dev_fill_forward_path()
758 return -1; in dev_fill_forward_path()
766 return -1; in dev_fill_forward_path()
767 path->type = DEV_PATH_ETHERNET; in dev_fill_forward_path()
768 path->dev = ctx.dev; in dev_fill_forward_path()
781 if (napi->napi_id == napi_id) in napi_by_id()
797 if (WARN_ON_ONCE(!napi->dev)) in netdev_napi_by_id()
799 if (!net_eq(net, dev_net(napi->dev))) in netdev_napi_by_id()
806 * netdev_napi_by_id_lock() - find a device by NAPI ID and lock it
824 if (!napi || READ_ONCE(napi->dev->reg_state) != NETREG_REGISTERED) { in netdev_napi_by_id_lock()
829 dev = napi->dev; in netdev_napi_by_id_lock()
839 if (napi && napi->dev != dev) in netdev_napi_by_id_lock()
849 * __dev_get_by_name - find a device by its name
865 return node_name ? node_name->dev : NULL; in __dev_get_by_name()
870 * dev_get_by_name_rcu - find a device by its name
886 return node_name ? node_name->dev : NULL; in dev_get_by_name_rcu()
904 * netdev_get_by_name() - find a device by its name
929 * __dev_get_by_index - find a device by its ifindex
945 if (dev->ifindex == ifindex) in __dev_get_by_index()
953 * dev_get_by_index_rcu - find a device by its ifindex
969 if (dev->ifindex == ifindex) in dev_get_by_index_rcu()
990 * netdev_get_by_index() - find a device by its ifindex
1014 * dev_get_by_napi_id - find a device by napi_id
1033 return napi ? napi->dev : NULL; in dev_get_by_napi_id()
1047 if (dev->reg_state > NETREG_REGISTERED || in __netdev_put_lock()
1048 dev->moving_ns || !net_eq(dev_net(dev), net)) { in __netdev_put_lock()
1061 if (dev->reg_state > NETREG_REGISTERED || in __netdev_put_lock_ops_compat()
1062 dev->moving_ns || !net_eq(dev_net(dev), net)) { in __netdev_put_lock_ops_compat()
1072 * netdev_get_by_index_lock() - find a device by its ifindex
1077 * with @ifindex is found it will be returned with netdev->lock held.
1114 dev = xa_find(&net->dev_by_index, index, ULONG_MAX, XA_PRESENT); in netdev_xa_find_lock()
1139 dev = xa_find(&net->dev_by_index, index, ULONG_MAX, XA_PRESENT); in netdev_xa_find_lock_ops_compat()
1163 strscpy(name, dev->name, IFNAMSIZ); in netdev_copy_name()
1168 * netdev_get_name - get a netdevice name, knowing its ifindex.
1182 ret = -ENODEV; in netdev_get_name()
1197 return dev->type == type && !memcmp(dev->dev_addr, ha, dev->addr_len); in dev_addr_cmp()
1201 * dev_getbyhwaddr_rcu - find a device by its hardware address
1228 * dev_getbyhwaddr() - find a device by its hardware address
1259 if (dev->type == type) { in dev_getfirstbyhwtype()
1270 * netdev_get_by_flags_rcu - find any device with given flags
1287 if (((READ_ONCE(dev->flags) ^ if_flags) & mask) == 0) { in netdev_get_by_flags_rcu()
1298 * dev_valid_name - check if name is okay for network device
1324 * __dev_alloc_name - allocate a name for a device
1329 * Passed a format string - eg "lt%d" it will try and find a suitable
1352 return -EINVAL; in __dev_alloc_name()
1357 return -ENOMEM; in __dev_alloc_name()
1363 if (!sscanf(name_node->name, name, &i)) in __dev_alloc_name()
1370 if (!strncmp(buf, name_node->name, IFNAMSIZ)) in __dev_alloc_name()
1373 if (!sscanf(d->name, name, &i)) in __dev_alloc_name()
1380 if (!strncmp(buf, d->name, IFNAMSIZ)) in __dev_alloc_name()
1387 return -ENFILE; in __dev_alloc_name()
1401 return -EINVAL; in dev_prep_valid_name()
1407 return -dup_errno; in dev_prep_valid_name()
1414 * dev_alloc_name - allocate a name for a device
1418 * Passed a format string - eg "lt%d" it will try and find a suitable
1429 return dev_prep_valid_name(dev_net(dev), dev, name, dev->name, ENFILE); in dev_alloc_name()
1438 ret = dev_prep_valid_name(net, dev, name, dev->name, EEXIST); in dev_get_valid_name()
1452 if (!strncmp(newname, dev->name, IFNAMSIZ)) in netif_change_name()
1455 memcpy(oldname, dev->name, IFNAMSIZ); in netif_change_name()
1466 dev->flags & IFF_UP ? " (while UP)" : ""); in netif_change_name()
1468 old_assign_type = dev->name_assign_type; in netif_change_name()
1469 WRITE_ONCE(dev->name_assign_type, NET_NAME_RENAMED); in netif_change_name()
1472 ret = device_rename(&dev->dev, dev->name); in netif_change_name()
1475 memcpy(dev->name, oldname, IFNAMSIZ); in netif_change_name()
1477 WRITE_ONCE(dev->name_assign_type, old_assign_type); in netif_change_name()
1483 netdev_name_node_del(dev->name_node); in netif_change_name()
1487 netdev_name_node_add(net, dev->name_node); in netif_change_name()
1497 memcpy(dev->name, oldname, IFNAMSIZ); in netif_change_name()
1500 WRITE_ONCE(dev->name_assign_type, old_assign_type); in netif_change_name()
1517 return -EINVAL; in netif_set_alias()
1522 return -ENOMEM; in netif_set_alias()
1524 memcpy(new_alias->ifalias, alias, len); in netif_set_alias()
1525 new_alias->ifalias[len] = 0; in netif_set_alias()
1529 new_alias = rcu_replace_pointer(dev->ifalias, new_alias, in netif_set_alias()
1540 * dev_get_alias - get ifalias of a device
1554 alias = rcu_dereference(dev->ifalias); in dev_get_alias()
1556 ret = snprintf(name, len, "%s", alias->ifalias); in dev_get_alias()
1563 * netdev_features_change - device changes features
1578 if (dev->flags & IFF_UP) { in netif_state_change()
1590 * __netdev_notify_peers - notify network peers about existence of @dev,
1609 * netdev_notify_peers - notify network peers about existence of @dev
1636 n->thread = kthread_run(napi_threaded_poll, n, "napi/%s-%d", in napi_kthread_create()
1637 n->dev->name, n->napi_id); in napi_kthread_create()
1638 if (IS_ERR(n->thread)) { in napi_kthread_create()
1639 err = PTR_ERR(n->thread); in napi_kthread_create()
1641 n->thread = NULL; in napi_kthread_create()
1649 const struct net_device_ops *ops = dev->netdev_ops; in __dev_open()
1656 /* may be detached because parent is runtime-suspended */ in __dev_open()
1657 if (dev->dev.parent) in __dev_open()
1658 pm_runtime_resume(dev->dev.parent); in __dev_open()
1660 return -ENODEV; in __dev_open()
1674 set_bit(__LINK_STATE_START, &dev->state); in __dev_open()
1678 if (ops->ndo_validate_addr) in __dev_open()
1679 ret = ops->ndo_validate_addr(dev); in __dev_open()
1681 if (!ret && ops->ndo_open) in __dev_open()
1682 ret = ops->ndo_open(dev); in __dev_open()
1687 clear_bit(__LINK_STATE_START, &dev->state); in __dev_open()
1692 add_device_randomness(dev->dev_addr, dev->addr_len); in __dev_open()
1702 if (dev->flags & IFF_UP) in netif_open()
1728 clear_bit(__LINK_STATE_START, &dev->state); in __dev_close_many()
1733 * dev->stop() will invoke napi_disable() on all of it's in __dev_close_many()
1742 const struct net_device_ops *ops = dev->netdev_ops; in __dev_close_many()
1748 * We allow it to be called even after a DETACH hot-plug in __dev_close_many()
1754 if (ops->ndo_stop) in __dev_close_many()
1755 ops->ndo_stop(dev); in __dev_close_many()
1766 list_add(&dev->close_list, &single); in __dev_close()
1777 if (!(dev->flags & IFF_UP)) in netif_close_many()
1778 list_del_init(&dev->close_list); in netif_close_many()
1786 list_del_init(&dev->close_list); in netif_close_many()
1793 if (dev->flags & IFF_UP) { in netif_close()
1796 list_add(&dev->close_list, &single); in netif_close()
1808 dev->wanted_features &= ~NETIF_F_LRO; in netif_disable_lro()
1811 if (unlikely(dev->features & NETIF_F_LRO)) in netif_disable_lro()
1823 * dev_disable_gro_hw - disable HW Generic Receive Offload on a device
1832 dev->wanted_features &= ~NETIF_F_GRO_HW; in dev_disable_gro_hw()
1835 if (unlikely(dev->features & NETIF_F_GRO_HW)) in dev_disable_gro_hw()
1870 return nb->notifier_call(nb, val, &info); in call_netdevice_notifier()
1883 if (!(dev->flags & IFF_UP)) in call_netdevice_register_notifiers()
1893 if (dev->flags & IFF_UP) { in call_netdevice_unregister_notifiers()
1934 * register_netdevice_notifier - register a network notifier block
1989 * unregister_netdevice_notifier - unregister a network notifier block
2033 err = raw_notifier_chain_register(&net->netdev_chain, nb); in __register_netdevice_notifier_net()
2046 raw_notifier_chain_unregister(&net->netdev_chain, nb); in __register_netdevice_notifier_net()
2055 err = raw_notifier_chain_unregister(&net->netdev_chain, nb); in __unregister_netdevice_notifier_net()
2064 * register_netdevice_notifier_net - register a per-netns network notifier block
2091 * unregister_netdevice_notifier_net - unregister a per-netns
2146 if (!net_eq(net, rcu_access_pointer(dev->nd_net.net))) { in rtnl_net_dev_lock()
2172 nn->nb = nb; in register_netdevice_notifier_dev_net()
2173 list_add(&nn->list, &dev->net_notifier_list); in register_netdevice_notifier_dev_net()
2188 list_del(&nn->list); in unregister_netdevice_notifier_dev_net()
2201 list_for_each_entry(nn, &dev->net_notifier_list, list) in move_netdevice_notifiers_dev_net()
2202 __move_netdevice_notifier_net(dev_net(dev), net, nn->nb); in move_netdevice_notifiers_dev_net()
2206 * call_netdevice_notifiers_info - call all network notifier blocks
2217 struct net *net = dev_net(info->dev); in call_netdevice_notifiers_info()
2222 /* Run per-netns notifier block chain first, then run the global one. in call_netdevice_notifiers_info()
2224 * all notifier block registrators get converted to be per-netns. in call_netdevice_notifiers_info()
2226 ret = raw_notifier_call_chain(&net->netdev_chain, val, info); in call_netdevice_notifiers_info()
2233 * call_netdevice_notifiers_info_robust - call per-netns notifier blocks
2240 * Call all per-netns network notifier blocks, but not notifier blocks on
2250 struct net *net = dev_net(info->dev); in call_netdevice_notifiers_info_robust()
2254 return raw_notifier_call_chain_robust(&net->netdev_chain, in call_netdevice_notifiers_info_robust()
2271 * call_netdevice_notifiers - call all network notifier blocks
2286 * call_netdevice_notifiers_mtu - call all network notifier blocks
2299 .ext.mtu = arg, in call_netdevice_notifiers_mtu()
2386 if (atomic_try_cmpxchg(&netstamp_wanted, &wanted, wanted - 1)) in net_disable_timestamp()
2399 skb->tstamp = 0; in net_timestamp_set()
2400 skb->tstamp_type = SKB_CLOCK_REALTIME; in net_timestamp_set()
2402 skb->tstamp = ktime_get_real(); in net_timestamp_set()
2407 if ((COND) && !(SKB)->tstamp) \
2408 (SKB)->tstamp = ktime_get_real(); \
2423 skb->protocol = eth_type_trans(skb, dev); in __dev_forward_skb2()
2437 * dev_forward_skb - loopback an skb to another netif
2470 return -ENOMEM; in deliver_skb()
2471 refcount_inc(&skb->users); in deliver_skb()
2472 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev); in deliver_skb()
2484 if (ptype->type != type) in deliver_ptype_list_skb()
2495 if (!ptype->af_packet_priv || !skb->sk) in skb_loop_sk()
2498 if (ptype->id_match) in skb_loop_sk()
2499 return ptype->id_match(ptype, skb->sk); in skb_loop_sk()
2500 else if ((struct sock *)ptype->af_packet_priv == skb->sk) in skb_loop_sk()
2507 * dev_nit_active_rcu - return true if any network interface taps are in use
2518 return !list_empty(&dev_net(dev)->ptype_all) || in dev_nit_active_rcu()
2519 !list_empty(&dev->ptype_all); in dev_nit_active_rcu()
2535 ptype_list = &dev_net_rcu(dev)->ptype_all; in dev_queue_xmit_nit()
2538 if (READ_ONCE(ptype->ignore_outgoing)) in dev_queue_xmit_nit()
2542 * they originated from - MvS (miquels@drinkel.ow.org) in dev_queue_xmit_nit()
2548 deliver_skb(skb2, pt_prev, skb->dev); in dev_queue_xmit_nit()
2560 /* skb->nh should be correctly in dev_queue_xmit_nit()
2566 if (skb_network_header(skb2) < skb2->data || in dev_queue_xmit_nit()
2569 ntohs(skb2->protocol), in dev_queue_xmit_nit()
2570 dev->name); in dev_queue_xmit_nit()
2574 skb2->transport_header = skb2->network_header; in dev_queue_xmit_nit()
2575 skb2->pkt_type = PACKET_OUTGOING; in dev_queue_xmit_nit()
2579 if (ptype_list != &dev->ptype_all) { in dev_queue_xmit_nit()
2580 ptype_list = &dev->ptype_all; in dev_queue_xmit_nit()
2586 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev); in dev_queue_xmit_nit()
2595 * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
2610 struct netdev_tc_txq *tc = &dev->tc_to_txq[0]; in netif_setup_tc()
2613 if (tc->offset + tc->count > txq) { in netif_setup_tc()
2615 dev->num_tc = 0; in netif_setup_tc()
2623 tc = &dev->tc_to_txq[q]; in netif_setup_tc()
2624 if (tc->offset + tc->count > txq) { in netif_setup_tc()
2634 if (dev->num_tc) { in netdev_txq_to_tc()
2635 struct netdev_tc_txq *tc = &dev->tc_to_txq[0]; in netdev_txq_to_tc()
2640 if ((txq - tc->offset) < tc->count) in netdev_txq_to_tc()
2644 /* didn't find it, just return -1 to indicate no match */ in netdev_txq_to_tc()
2645 return -1; in netdev_txq_to_tc()
2665 map = xmap_dereference(dev_maps->attr_map[tci]); in remove_xps_queue()
2669 for (pos = map->len; pos--;) { in remove_xps_queue()
2670 if (map->queues[pos] != index) in remove_xps_queue()
2673 if (map->len > 1) { in remove_xps_queue()
2674 map->queues[pos] = map->queues[--map->len]; in remove_xps_queue()
2679 RCU_INIT_POINTER(old_maps->attr_map[tci], NULL); in remove_xps_queue()
2680 RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL); in remove_xps_queue()
2692 int num_tc = dev_maps->num_tc; in remove_xps_queue_cpu()
2696 for (tci = cpu * num_tc; num_tc--; tci++) { in remove_xps_queue_cpu()
2699 for (i = count, j = offset; i--; j++) { in remove_xps_queue_cpu()
2718 RCU_INIT_POINTER(dev->xps_maps[type], NULL); in reset_xps_maps()
2730 dev_maps = xmap_dereference(dev->xps_maps[type]); in clean_xps_maps()
2734 for (j = 0; j < dev_maps->nr_ids; j++) in clean_xps_maps()
2740 for (i = offset + (count - 1); count--; i--) in clean_xps_maps()
2766 netif_reset_xps_queues(dev, index, dev->num_tx_queues - index); in netif_reset_xps_queues_gt()
2776 for (pos = 0; map && pos < map->len; pos++) { in expand_xps_map()
2777 if (map->queues[pos] != index) in expand_xps_map()
2782 /* Need to add tx-queue to this CPU's/rx-queue's existing map */ in expand_xps_map()
2784 if (pos < map->alloc_len) in expand_xps_map()
2787 alloc_len = map->alloc_len * 2; in expand_xps_map()
2790 /* Need to allocate new map to store tx-queue on this CPU's/rx-queue's in expand_xps_map()
2802 new_map->queues[i] = map->queues[i]; in expand_xps_map()
2803 new_map->alloc_len = alloc_len; in expand_xps_map()
2804 new_map->len = pos; in expand_xps_map()
2814 int i, tci = index * dev_maps->num_tc; in xps_copy_dev_maps()
2818 for (i = 0; i < dev_maps->num_tc; i++, tci++) { in xps_copy_dev_maps()
2823 map = xmap_dereference(dev_maps->attr_map[tci]); in xps_copy_dev_maps()
2824 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map); in xps_copy_dev_maps()
2835 int i, j, tci, numa_node_id = -2; in __netif_set_xps_queue()
2840 WARN_ON_ONCE(index >= dev->num_tx_queues); in __netif_set_xps_queue()
2842 if (dev->num_tc) { in __netif_set_xps_queue()
2844 num_tc = dev->num_tc; in __netif_set_xps_queue()
2846 return -EINVAL; in __netif_set_xps_queue()
2849 dev = netdev_get_tx_queue(dev, index)->sb_dev ? : dev; in __netif_set_xps_queue()
2853 return -EINVAL; in __netif_set_xps_queue()
2858 dev_maps = xmap_dereference(dev->xps_maps[type]); in __netif_set_xps_queue()
2860 maps_sz = XPS_RXQ_DEV_MAPS_SIZE(num_tc, dev->num_rx_queues); in __netif_set_xps_queue()
2861 nr_ids = dev->num_rx_queues; in __netif_set_xps_queue()
2873 * setting up now, as dev->num_tc or nr_ids could have been updated in in __netif_set_xps_queue()
2878 dev_maps->num_tc == num_tc && dev_maps->nr_ids == nr_ids) in __netif_set_xps_queue()
2882 for (j = -1; j = netif_attrmask_next_and(j, online_mask, mask, nr_ids), in __netif_set_xps_queue()
2888 return -ENOMEM; in __netif_set_xps_queue()
2891 new_dev_maps->nr_ids = nr_ids; in __netif_set_xps_queue()
2892 new_dev_maps->num_tc = num_tc; in __netif_set_xps_queue()
2896 map = copy ? xmap_dereference(dev_maps->attr_map[tci]) : NULL; in __netif_set_xps_queue()
2902 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map); in __netif_set_xps_queue()
2921 /* add tx-queue to CPU/rx-queue maps */ in __netif_set_xps_queue()
2926 map = xmap_dereference(new_dev_maps->attr_map[tci]); in __netif_set_xps_queue()
2927 while ((pos < map->len) && (map->queues[pos] != index)) in __netif_set_xps_queue()
2930 if (pos == map->len) in __netif_set_xps_queue()
2931 map->queues[map->len++] = index; in __netif_set_xps_queue()
2934 if (numa_node_id == -2) in __netif_set_xps_queue()
2937 numa_node_id = -1; in __netif_set_xps_queue()
2947 rcu_assign_pointer(dev->xps_maps[type], new_dev_maps); in __netif_set_xps_queue()
2953 for (j = 0; j < dev_maps->nr_ids; j++) { in __netif_set_xps_queue()
2954 for (i = num_tc, tci = j * dev_maps->num_tc; i--; tci++) { in __netif_set_xps_queue()
2955 map = xmap_dereference(dev_maps->attr_map[tci]); in __netif_set_xps_queue()
2960 new_map = xmap_dereference(new_dev_maps->attr_map[tci]); in __netif_set_xps_queue()
2965 RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL); in __netif_set_xps_queue()
2986 /* removes tx-queue from unused CPUs/rx-queues */ in __netif_set_xps_queue()
2987 for (j = 0; j < dev_maps->nr_ids; j++) { in __netif_set_xps_queue()
2988 tci = j * dev_maps->num_tc; in __netif_set_xps_queue()
2990 for (i = 0; i < dev_maps->num_tc; i++, tci++) { in __netif_set_xps_queue()
2992 netif_attr_test_mask(j, mask, dev_maps->nr_ids) && in __netif_set_xps_queue()
2993 netif_attr_test_online(j, online_mask, dev_maps->nr_ids)) in __netif_set_xps_queue()
3016 for (i = num_tc, tci = j * num_tc; i--; tci++) { in __netif_set_xps_queue()
3017 new_map = xmap_dereference(new_dev_maps->attr_map[tci]); in __netif_set_xps_queue()
3019 xmap_dereference(dev_maps->attr_map[tci]) : in __netif_set_xps_queue()
3029 return -ENOMEM; in __netif_set_xps_queue()
3049 struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues]; in netdev_unbind_all_sb_channels()
3052 while (txq-- != &dev->_tx[0]) { in netdev_unbind_all_sb_channels()
3053 if (txq->sb_dev) in netdev_unbind_all_sb_channels()
3054 netdev_unbind_sb_channel(dev, txq->sb_dev); in netdev_unbind_all_sb_channels()
3066 dev->num_tc = 0; in netdev_reset_tc()
3067 memset(dev->tc_to_txq, 0, sizeof(dev->tc_to_txq)); in netdev_reset_tc()
3068 memset(dev->prio_tc_map, 0, sizeof(dev->prio_tc_map)); in netdev_reset_tc()
3074 if (tc >= dev->num_tc) in netdev_set_tc_queue()
3075 return -EINVAL; in netdev_set_tc_queue()
3080 dev->tc_to_txq[tc].count = count; in netdev_set_tc_queue()
3081 dev->tc_to_txq[tc].offset = offset; in netdev_set_tc_queue()
3089 return -EINVAL; in netdev_set_num_tc()
3096 dev->num_tc = num_tc; in netdev_set_num_tc()
3104 struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues]; in netdev_unbind_sb_channel()
3109 memset(sb_dev->tc_to_txq, 0, sizeof(sb_dev->tc_to_txq)); in netdev_unbind_sb_channel()
3110 memset(sb_dev->prio_tc_map, 0, sizeof(sb_dev->prio_tc_map)); in netdev_unbind_sb_channel()
3112 while (txq-- != &dev->_tx[0]) { in netdev_unbind_sb_channel()
3113 if (txq->sb_dev == sb_dev) in netdev_unbind_sb_channel()
3114 txq->sb_dev = NULL; in netdev_unbind_sb_channel()
3124 if (sb_dev->num_tc >= 0 || tc >= dev->num_tc) in netdev_bind_sb_channel_queue()
3125 return -EINVAL; in netdev_bind_sb_channel_queue()
3128 if ((offset + count) > dev->real_num_tx_queues) in netdev_bind_sb_channel_queue()
3129 return -EINVAL; in netdev_bind_sb_channel_queue()
3132 sb_dev->tc_to_txq[tc].count = count; in netdev_bind_sb_channel_queue()
3133 sb_dev->tc_to_txq[tc].offset = offset; in netdev_bind_sb_channel_queue()
3138 while (count--) in netdev_bind_sb_channel_queue()
3139 netdev_get_tx_queue(dev, count + offset)->sb_dev = sb_dev; in netdev_bind_sb_channel_queue()
3149 return -ENODEV; in netdev_set_sb_channel()
3151 /* We allow channels 1 - 32767 to be used for subordinate channels. in netdev_set_sb_channel()
3157 return -EINVAL; in netdev_set_sb_channel()
3159 dev->num_tc = -channel; in netdev_set_sb_channel()
3174 disabling = txq < dev->real_num_tx_queues; in netif_set_real_num_tx_queues()
3176 if (txq < 1 || txq > dev->num_tx_queues) in netif_set_real_num_tx_queues()
3177 return -EINVAL; in netif_set_real_num_tx_queues()
3179 if (dev->reg_state == NETREG_REGISTERED || in netif_set_real_num_tx_queues()
3180 dev->reg_state == NETREG_UNREGISTERING) { in netif_set_real_num_tx_queues()
3183 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues, in netif_set_real_num_tx_queues()
3188 if (dev->num_tc) in netif_set_real_num_tx_queues()
3195 dev->real_num_tx_queues = txq; in netif_set_real_num_tx_queues()
3205 dev->real_num_tx_queues = txq; in netif_set_real_num_tx_queues()
3213 * netif_set_real_num_rx_queues - set actual number of RX queues used
3226 if (rxq < 1 || rxq > dev->num_rx_queues) in netif_set_real_num_rx_queues()
3227 return -EINVAL; in netif_set_real_num_rx_queues()
3229 if (dev->reg_state == NETREG_REGISTERED) { in netif_set_real_num_rx_queues()
3232 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues, in netif_set_real_num_rx_queues()
3238 dev->real_num_rx_queues = rxq; in netif_set_real_num_rx_queues()
3244 * netif_set_real_num_queues - set actual number of RX and TX queues used
3255 unsigned int old_rxq = dev->real_num_rx_queues; in netif_set_real_num_queues()
3258 if (txq < 1 || txq > dev->num_tx_queues || in netif_set_real_num_queues()
3259 rxq < 1 || rxq > dev->num_rx_queues) in netif_set_real_num_queues()
3260 return -EINVAL; in netif_set_real_num_queues()
3262 /* Start from increases, so the error path only does decreases - in netif_set_real_num_queues()
3265 if (rxq > dev->real_num_rx_queues) { in netif_set_real_num_queues()
3270 if (txq > dev->real_num_tx_queues) { in netif_set_real_num_queues()
3275 if (rxq < dev->real_num_rx_queues) in netif_set_real_num_queues()
3277 if (txq < dev->real_num_tx_queues) in netif_set_real_num_queues()
3288 * netif_set_tso_max_size() - set the max size of TSO frames supported
3290 * @size: max skb->len of a TSO frame
3292 * Set the limit on the size of TSO super-frames the device can handle.
3298 dev->tso_max_size = min(GSO_MAX_SIZE, size); in netif_set_tso_max_size()
3299 if (size < READ_ONCE(dev->gso_max_size)) in netif_set_tso_max_size()
3301 if (size < READ_ONCE(dev->gso_ipv4_max_size)) in netif_set_tso_max_size()
3307 * netif_set_tso_max_segs() - set the max number of segs supported for TSO
3312 * a single TSO super-frame.
3317 dev->tso_max_segs = segs; in netif_set_tso_max_segs()
3318 if (segs < READ_ONCE(dev->gso_max_segs)) in netif_set_tso_max_segs()
3324 * netif_inherit_tso_max() - copy all TSO limits from a lower device to an upper
3330 netif_set_tso_max_size(to, from->tso_max_size); in netif_inherit_tso_max()
3331 netif_set_tso_max_segs(to, from->tso_max_segs); in netif_inherit_tso_max()
3336 * netif_get_num_default_rss_queues - default number of RSS queues
3367 q->next_sched = NULL; in __netif_reschedule()
3368 *sd->output_queue_tailp = q; in __netif_reschedule()
3369 sd->output_queue_tailp = &q->next_sched; in __netif_reschedule()
3376 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state)) in __netif_schedule()
3387 return (struct dev_kfree_skb_cb *)skb->cb; in get_kfree_skb_cb()
3394 struct Qdisc *q = rcu_dereference(txq->qdisc); in netif_schedule_queue()
3404 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) { in netif_tx_wake_queue()
3408 q = rcu_dereference(dev_queue->qdisc); in netif_tx_wake_queue()
3422 if (likely(refcount_read(&skb->users) == 1)) { in dev_kfree_skb_irq_reason()
3424 refcount_set(&skb->users, 0); in dev_kfree_skb_irq_reason()
3425 } else if (likely(!refcount_dec_and_test(&skb->users))) { in dev_kfree_skb_irq_reason()
3428 get_kfree_skb_cb(skb)->reason = reason; in dev_kfree_skb_irq_reason()
3430 skb->next = __this_cpu_read(softnet_data.completion_queue); in dev_kfree_skb_irq_reason()
3448 * netif_device_detach - mark device as removed
3455 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) && in netif_device_detach()
3463 * netif_device_attach - mark device as attached
3470 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) && in netif_device_attach()
3488 u16 qcount = dev->real_num_tx_queues; in skb_tx_hash()
3490 if (dev->num_tc) { in skb_tx_hash()
3491 u8 tc = netdev_get_prio_tc_map(dev, skb->priority); in skb_tx_hash()
3493 qoffset = sb_dev->tc_to_txq[tc].offset; in skb_tx_hash()
3494 qcount = sb_dev->tc_to_txq[tc].count; in skb_tx_hash()
3497 sb_dev->name, qoffset, tc); in skb_tx_hash()
3499 qcount = dev->real_num_tx_queues; in skb_tx_hash()
3507 hash -= qoffset; in skb_tx_hash()
3509 hash -= qcount; in skb_tx_hash()
3519 struct net_device *dev = skb->dev; in skb_warn_bad_offload()
3526 if (dev->dev.parent) in skb_warn_bad_offload()
3527 name = dev_driver_string(dev->dev.parent); in skb_warn_bad_offload()
3533 name, dev ? &dev->features : &null_features, in skb_warn_bad_offload()
3534 skb->sk ? &skb->sk->sk_route_caps : &null_features); in skb_warn_bad_offload()
3546 if (skb->ip_summed == CHECKSUM_COMPLETE) in skb_checksum_help()
3551 return -EINVAL; in skb_checksum_help()
3555 return -EFAULT; in skb_checksum_help()
3568 ret = -EINVAL; in skb_checksum_help()
3575 csum = skb_checksum(skb, offset, skb->len - offset, 0); in skb_checksum_help()
3577 offset += skb->csum_offset; in skb_checksum_help()
3588 *(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0; in skb_checksum_help()
3590 skb->ip_summed = CHECKSUM_NONE; in skb_checksum_help()
3602 if (skb->ip_summed != CHECKSUM_PARTIAL) in skb_crc32c_csum_help()
3619 ret = -EINVAL; in skb_crc32c_csum_help()
3627 crc = ~skb_crc32c(skb, start, skb->len - start, ~0); in skb_crc32c_csum_help()
3628 *(__le32 *)(skb->data + offset) = cpu_to_le32(crc); in skb_crc32c_csum_help()
3638 __be16 type = skb->protocol; in skb_network_protocol()
3647 eth = (struct ethhdr *)skb->data; in skb_network_protocol()
3648 type = eth->h_proto; in skb_network_protocol()
3677 if (!(dev->features & NETIF_F_HIGHDMA)) { in illegal_highdma()
3678 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { in illegal_highdma()
3679 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; in illegal_highdma()
3699 features &= skb->dev->mpls_features; in net_mpls_features()
3720 if (skb->ip_summed != CHECKSUM_NONE && in harmonize_features()
3724 if (illegal_highdma(skb->dev, skb)) in harmonize_features()
3749 u16 gso_segs = skb_shinfo(skb)->gso_segs; in gso_features_check()
3751 if (gso_segs > READ_ONCE(dev->gso_max_segs)) in gso_features_check()
3754 if (unlikely(skb->len >= netif_get_gso_max_size(dev, skb))) in gso_features_check()
3757 if (!skb_shinfo(skb)->gso_type) { in gso_features_check()
3768 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL)) in gso_features_check()
3769 features &= ~dev->gso_partial_features; in gso_features_check()
3778 * segmentation-offloads.rst). in gso_features_check()
3780 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) { in gso_features_check()
3781 struct iphdr *iph = skb->encapsulation ? in gso_features_check()
3784 if (!(iph->frag_off & htons(IP_DF))) in gso_features_check()
3792 (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6 || in gso_features_check()
3793 (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4 && in gso_features_check()
3805 struct net_device *dev = skb->dev; in netif_skb_features()
3806 netdev_features_t features = dev->features; in netif_skb_features()
3815 if (skb->encapsulation) in netif_skb_features()
3816 features &= dev->hw_enc_features; in netif_skb_features()
3820 dev->vlan_features | in netif_skb_features()
3824 if (dev->netdev_ops->ndo_features_check) in netif_skb_features()
3825 features &= dev->netdev_ops->ndo_features_check(skb, dev, in netif_skb_features()
3843 len = skb->len; in xmit_one()
3858 struct sk_buff *next = skb->next; in dev_hard_start_xmit()
3863 skb->next = next; in dev_hard_start_xmit()
3883 !vlan_hw_offload_capable(features, skb->vlan_proto)) in validate_xmit_vlan()
3904 switch (skb->csum_offset) { in skb_csum_hwoffload_help()
3926 struct sock *sk = skb->sk; in sk_validate_xmit_skb()
3931 sk_validate = sk->sk_validate_xmit_skb; in sk_validate_xmit_skb()
3932 else if (sk_is_inet(sk) && sk->sk_state == TCP_TIME_WAIT) in sk_validate_xmit_skb()
3933 sk_validate = inet_twsk(sk)->tw_validate_xmit_skb; in sk_validate_xmit_skb()
3939 pr_warn_ratelimited("unencrypted skb with no associated socket - dropping\n"); in sk_validate_xmit_skb()
3957 if (!dev->netmem_tx) in validate_xmit_unreadable_skb()
3962 if (shinfo->nr_frags > 0) { in validate_xmit_unreadable_skb()
3963 niov = netmem_to_net_iov(skb_frag_netmem(&shinfo->frags[0])); in validate_xmit_unreadable_skb()
3965 net_devmem_iov_binding(niov)->dev != dev) in validate_xmit_unreadable_skb()
4013 if (skb->ip_summed == CHECKSUM_PARTIAL) { in validate_xmit_skb()
4014 if (skb->encapsulation) in validate_xmit_skb()
4041 next = skb->next; in validate_xmit_skb_list()
4045 skb->prev = skb; in validate_xmit_skb_list()
4054 tail->next = skb; in validate_xmit_skb_list()
4055 /* If skb was segmented, skb->prev points to in validate_xmit_skb_list()
4058 tail = skb->prev; in validate_xmit_skb_list()
4068 qdisc_skb_cb(skb)->pkt_len = skb->len; in qdisc_pkt_len_init()
4073 if (shinfo->gso_size && skb_transport_header_was_set(skb)) { in qdisc_pkt_len_init()
4074 u16 gso_segs = shinfo->gso_segs; in qdisc_pkt_len_init()
4078 if (!skb->encapsulation) in qdisc_pkt_len_init()
4084 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) { in qdisc_pkt_len_init()
4092 } else if (shinfo->gso_type & SKB_GSO_UDP_L4) { in qdisc_pkt_len_init()
4100 if (unlikely(shinfo->gso_type & SKB_GSO_DODGY)) { in qdisc_pkt_len_init()
4101 int payload = skb->len - hdr_len; in qdisc_pkt_len_init()
4106 gso_segs = DIV_ROUND_UP(payload, shinfo->gso_size); in qdisc_pkt_len_init()
4108 qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len; in qdisc_pkt_len_init()
4118 rc = q->enqueue(skb, q, to_free) & NET_XMIT_MASK; in dev_qdisc_enqueue()
4137 if (q->flags & TCQ_F_NOLOCK) { in __dev_xmit_skb()
4138 if (q->flags & TCQ_F_CAN_BYPASS && nolock_qdisc_is_empty(q) && in __dev_xmit_skb()
4141 * of q->seqlock to protect from racing with requeuing. in __dev_xmit_skb()
4170 if (unlikely(READ_ONCE(q->owner) == smp_processor_id())) { in __dev_xmit_skb()
4177 * This permits qdisc->running owner to get the lock more in __dev_xmit_skb()
4186 spin_lock(&q->busylock); in __dev_xmit_skb()
4189 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) { in __dev_xmit_skb()
4192 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) && in __dev_xmit_skb()
4195 * This is a work-conserving queue; there are no old skbs in __dev_xmit_skb()
4196 * waiting to be sent out; and the qdisc is not running - in __dev_xmit_skb()
4204 spin_unlock(&q->busylock); in __dev_xmit_skb()
4213 WRITE_ONCE(q->owner, smp_processor_id()); in __dev_xmit_skb()
4215 WRITE_ONCE(q->owner, -1); in __dev_xmit_skb()
4218 spin_unlock(&q->busylock); in __dev_xmit_skb()
4230 spin_unlock(&q->busylock); in __dev_xmit_skb()
4241 if (skb->priority) in skb_update_prio()
4243 map = rcu_dereference_bh(skb->dev->priomap); in skb_update_prio()
4250 prioidx = sock_cgroup_prioidx(&sk->sk_cgrp_data); in skb_update_prio()
4252 if (prioidx < map->priomap_len) in skb_update_prio()
4253 skb->priority = map->priomap[prioidx]; in skb_update_prio()
4260 * dev_loopback_xmit - loop back @skb
4269 skb->pkt_type = PACKET_LOOPBACK; in dev_loopback_xmit()
4270 if (skb->ip_summed == CHECKSUM_NONE) in dev_loopback_xmit()
4271 skb->ip_summed = CHECKSUM_UNNECESSARY; in dev_loopback_xmit()
4303 return current->net_xmit.skip_txqueue; in netdev_xmit_txqueue_skipped()
4308 current->net_xmit.skip_txqueue = skip; in netdev_xmit_skip_txqueue()
4320 struct mini_Qdisc *miniq = rcu_dereference_bh(entry->miniq); in tc_run()
4330 /* Block-wise bypass */ in tc_run()
4331 if (tcf_block_bypass_sw(miniq->block)) in tc_run()
4334 tc_skb_cb(skb)->mru = 0; in tc_run()
4335 tc_skb_cb(skb)->post_ct = false; in tc_run()
4339 ret = tcf_classify(skb, miniq->block, miniq->filter_list, &res, false); in tc_run()
4348 skb->tc_index = TC_H_MIN(res.classid); in tc_run()
4376 __skb_push(skb, skb->mac_len); in tcx_run()
4384 __skb_pull(skb, skb->mac_len); in tcx_run()
4392 struct bpf_mprog_entry *entry = rcu_dereference_bh(skb->dev->tcx_ingress); in sch_handle_ingress()
4406 qdisc_skb_cb(skb)->pkt_len = skb->len; in sch_handle_ingress()
4422 __skb_push(skb, skb->mac_len); in sch_handle_ingress()
4423 if (skb_do_redirect(skb) == -EAGAIN) { in sch_handle_ingress()
4424 __skb_pull(skb, skb->mac_len); in sch_handle_ingress()
4455 struct bpf_mprog_entry *entry = rcu_dereference_bh(dev->tcx_egress); in sch_handle_egress()
4465 /* qdisc_skb_cb(skb)->pkt_len & tcx_set_ingress() was in sch_handle_egress()
4521 int tc = netdev_get_prio_tc_map(dev, skb->priority); in __get_xps_queue_idx()
4523 int queue_index = -1; in __get_xps_queue_idx()
4525 if (tc >= dev_maps->num_tc || tci >= dev_maps->nr_ids) in __get_xps_queue_idx()
4528 tci *= dev_maps->num_tc; in __get_xps_queue_idx()
4531 map = rcu_dereference(dev_maps->attr_map[tci]); in __get_xps_queue_idx()
4533 if (map->len == 1) in __get_xps_queue_idx()
4534 queue_index = map->queues[0]; in __get_xps_queue_idx()
4536 queue_index = map->queues[reciprocal_scale( in __get_xps_queue_idx()
4537 skb_get_hash(skb), map->len)]; in __get_xps_queue_idx()
4538 if (unlikely(queue_index >= dev->real_num_tx_queues)) in __get_xps_queue_idx()
4539 queue_index = -1; in __get_xps_queue_idx()
4550 struct sock *sk = skb->sk; in get_xps_queue()
4551 int queue_index = -1; in get_xps_queue()
4554 return -1; in get_xps_queue()
4560 dev_maps = rcu_dereference(sb_dev->xps_maps[XPS_RXQS]); in get_xps_queue()
4571 dev_maps = rcu_dereference(sb_dev->xps_maps[XPS_CPUS]); in get_xps_queue()
4573 unsigned int tci = skb->sender_cpu - 1; in get_xps_queue()
4583 return -1; in get_xps_queue()
4597 struct sock *sk = skb->sk; in netdev_pick_tx()
4602 if (queue_index < 0 || skb->ooo_okay || in netdev_pick_tx()
4603 queue_index >= dev->real_num_tx_queues) { in netdev_pick_tx()
4611 rcu_access_pointer(sk->sk_dst_cache)) in netdev_pick_tx()
4628 u32 sender_cpu = skb->sender_cpu - 1; in netdev_core_pick_tx()
4631 skb->sender_cpu = raw_smp_processor_id() + 1; in netdev_core_pick_tx()
4634 if (dev->real_num_tx_queues != 1) { in netdev_core_pick_tx()
4635 const struct net_device_ops *ops = dev->netdev_ops; in netdev_core_pick_tx()
4637 if (ops->ndo_select_queue) in netdev_core_pick_tx()
4638 queue_index = ops->ndo_select_queue(dev, skb, sb_dev); in netdev_core_pick_tx()
4650 * __dev_queue_xmit() - transmit a buffer
4666 * * 0 - buffer successfully transmitted
4667 * * positive qdisc return code - NET_XMIT_DROP etc.
4668 * * negative errno - other errors
4672 struct net_device *dev = skb->dev; in __dev_queue_xmit()
4675 int rc = -ENOMEM; in __dev_queue_xmit()
4681 if (unlikely(skb_shinfo(skb)->tx_flags & in __dev_queue_xmit()
4683 __skb_tstamp_tx(skb, NULL, NULL, skb->sk, SCM_TSTAMP_SCHED); in __dev_queue_xmit()
4714 /* If device/qdisc don't need skb->dst, release it right now while in __dev_queue_xmit()
4717 if (dev->priv_flags & IFF_XMIT_DST_RELEASE) in __dev_queue_xmit()
4725 q = rcu_dereference_bh(txq->qdisc); in __dev_queue_xmit()
4728 if (q->enqueue) { in __dev_queue_xmit()
4745 if (dev->flags & IFF_UP) { in __dev_queue_xmit()
4746 int cpu = smp_processor_id(); /* ok because BHs are off */ in __dev_queue_xmit()
4748 /* Other cpus might concurrently change txq->xmit_lock_owner in __dev_queue_xmit()
4749 * to -1 or to their cpu id, but not to our id. in __dev_queue_xmit()
4751 if (READ_ONCE(txq->xmit_lock_owner) != cpu) { in __dev_queue_xmit()
4772 dev->name); in __dev_queue_xmit()
4779 dev->name); in __dev_queue_xmit()
4783 rc = -ENETDOWN; in __dev_queue_xmit()
4797 struct net_device *dev = skb->dev; in __dev_direct_xmit()
4849 if (test_bit(NAPI_STATE_THREADED, &napi->state)) { in ____napi_schedule()
4853 * read on napi->thread. Only call in ____napi_schedule()
4856 thread = READ_ONCE(napi->thread); in ____napi_schedule()
4861 set_bit(NAPI_STATE_SCHED_THREADED, &napi->state); in ____napi_schedule()
4868 DEBUG_NET_WARN_ON_ONCE(!list_empty(&napi->poll_list)); in ____napi_schedule()
4869 list_add_tail(&napi->poll_list, &sd->poll_list); in ____napi_schedule()
4870 WRITE_ONCE(napi->list_owner, smp_processor_id()); in ____napi_schedule()
4874 if (!sd->in_net_rx_action) in ____napi_schedule()
4887 return hash_32(hash, flow_table->log); in rfs_slot()
4892 * rps_flow_is_active - check whether the flow is recently active.
4894 * @flow_table: per-queue flowtable that @rflow belongs to.
4913 flow_last_active = READ_ONCE(rflow->last_qtail); in rps_flow_is_active()
4915 return (int)(sd_input_head - flow_last_active) < in rps_flow_is_active()
4916 (int)(10 << flow_table->log); in rps_flow_is_active()
4937 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap || in set_rps_cpu()
4938 !(dev->features & NETIF_F_NTUPLE)) in set_rps_cpu()
4940 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu); in set_rps_cpu()
4944 rxqueue = dev->_rx + rxq_index; in set_rps_cpu()
4945 flow_table = rcu_dereference(rxqueue->rps_flow_table); in set_rps_cpu()
4949 tmp_rflow = &flow_table->flows[flow_id]; in set_rps_cpu()
4950 tmp_cpu = READ_ONCE(tmp_rflow->cpu); in set_rps_cpu()
4952 if (READ_ONCE(tmp_rflow->filter) != RPS_NO_FILTER) { in set_rps_cpu()
4955 if (hash != READ_ONCE(tmp_rflow->hash) || in set_rps_cpu()
4961 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb, in set_rps_cpu()
4968 WRITE_ONCE(rflow->filter, rc); in set_rps_cpu()
4969 WRITE_ONCE(rflow->hash, hash); in set_rps_cpu()
4971 if (old_rflow->filter == rc) in set_rps_cpu()
4972 WRITE_ONCE(old_rflow->filter, RPS_NO_FILTER); in set_rps_cpu()
4976 rps_input_queue_tail_save(&rflow->last_qtail, head); in set_rps_cpu()
4979 WRITE_ONCE(rflow->cpu, next_cpu); in set_rps_cpu()
4992 struct netdev_rx_queue *rxqueue = dev->_rx; in get_rps_cpu()
4995 int cpu = -1; in get_rps_cpu()
5003 if (unlikely(index >= dev->real_num_rx_queues)) { in get_rps_cpu()
5004 WARN_ONCE(dev->real_num_rx_queues > 1, in get_rps_cpu()
5007 dev->name, index, dev->real_num_rx_queues); in get_rps_cpu()
5015 flow_table = rcu_dereference(rxqueue->rps_flow_table); in get_rps_cpu()
5016 map = rcu_dereference(rxqueue->rps_map); in get_rps_cpu()
5034 ident = READ_ONCE(sock_flow_table->ents[hash & sock_flow_table->mask]); in get_rps_cpu()
5044 rflow = &flow_table->flows[flow_id]; in get_rps_cpu()
5045 tcpu = rflow->cpu; in get_rps_cpu()
5049 * different from current CPU (one in the rx-queue flow in get_rps_cpu()
5051 * - Current CPU is unset (>= nr_cpu_ids). in get_rps_cpu()
5052 * - Current CPU is offline. in get_rps_cpu()
5053 * - The current CPU's queue tail has advanced beyond the in get_rps_cpu()
5060 ((int)(READ_ONCE(per_cpu(softnet_data, tcpu).input_queue_head) - in get_rps_cpu()
5061 rflow->last_qtail)) >= 0)) { in get_rps_cpu()
5077 tcpu = map->cpus[reciprocal_scale(hash, map->len)]; in get_rps_cpu()
5091 * rps_may_expire_flow - check whether an RFS hardware filter may be removed
5104 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index; in rps_may_expire_flow()
5110 flow_table = rcu_dereference(rxqueue->rps_flow_table); in rps_may_expire_flow()
5111 if (flow_table && flow_id < (1UL << flow_table->log)) { in rps_may_expire_flow()
5114 rflow = &flow_table->flows[flow_id]; in rps_may_expire_flow()
5115 cpu = READ_ONCE(rflow->cpu); in rps_may_expire_flow()
5116 if (READ_ONCE(rflow->filter) == filter_id && in rps_may_expire_flow()
5132 ____napi_schedule(sd, &sd->backlog); in rps_trigger_softirq()
5134 WRITE_ONCE(sd->received_rps, sd->received_rps + 1); in rps_trigger_softirq()
5145 smp_store_release(&sd->defer_ipi_scheduled, 0); in trigger_rx_softirq()
5149 * After we queued a packet into sd->input_pkt_queue,
5152 * - If this is another cpu queue, link it to our rps_ipi_list,
5155 * - If this is our own queue, NAPI schedule our backlog.
5165 __napi_schedule_irqoff(&sd->backlog); in napi_schedule_rps()
5169 sd->rps_ipi_next = mysd->rps_ipi_list; in napi_schedule_rps()
5170 mysd->rps_ipi_list = sd; in napi_schedule_rps()
5175 if (!mysd->in_net_rx_action && !mysd->in_napi_threaded_poll) in napi_schedule_rps()
5180 __napi_schedule_irqoff(&mysd->backlog); in napi_schedule_rps()
5191 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) in kick_defer_list_purge()
5192 __napi_schedule_irqoff(&sd->backlog); in kick_defer_list_purge()
5196 } else if (!cmpxchg(&sd->defer_ipi_scheduled, 0, 1)) { in kick_defer_list_purge()
5197 smp_call_function_single_async(cpu, &sd->defer_csd); in kick_defer_list_purge()
5218 fl = rcu_dereference(sd->flow_limit); in skb_flow_limit()
5220 new_flow = hash_32(skb_get_hash(skb), fl->log_buckets); in skb_flow_limit()
5221 old_flow = fl->history[fl->history_head]; in skb_flow_limit()
5222 fl->history[fl->history_head] = new_flow; in skb_flow_limit()
5224 fl->history_head++; in skb_flow_limit()
5225 fl->history_head &= FLOW_LIMIT_HISTORY - 1; in skb_flow_limit()
5227 if (likely(fl->buckets[old_flow])) in skb_flow_limit()
5228 fl->buckets[old_flow]--; in skb_flow_limit()
5230 if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) { in skb_flow_limit()
5232 WRITE_ONCE(fl->count, fl->count + 1); in skb_flow_limit()
5257 if (!netif_running(skb->dev)) in enqueue_to_backlog()
5263 qlen = skb_queue_len_lockless(&sd->input_pkt_queue); in enqueue_to_backlog()
5268 qlen = skb_queue_len(&sd->input_pkt_queue); in enqueue_to_backlog()
5275 &sd->backlog.state)) in enqueue_to_backlog()
5278 __skb_queue_tail(&sd->input_pkt_queue, skb); in enqueue_to_backlog()
5290 numa_drop_add(&sd->drop_counters, 1); in enqueue_to_backlog()
5292 dev_core_stats_rx_dropped_inc(skb->dev); in enqueue_to_backlog()
5299 struct net_device *dev = skb->dev; in netif_get_rxqueue()
5302 rxqueue = dev->_rx; in netif_get_rxqueue()
5307 if (unlikely(index >= dev->real_num_rx_queues)) { in netif_get_rxqueue()
5308 WARN_ONCE(dev->real_num_rx_queues > 1, in netif_get_rxqueue()
5311 dev->name, index, dev->real_num_rx_queues); in netif_get_rxqueue()
5335 mac_len = skb->data - skb_mac_header(skb); in bpf_prog_run_generic_xdp()
5336 hard_start = skb->data - skb_headroom(skb); in bpf_prog_run_generic_xdp()
5339 frame_sz = (void *)skb_end_pointer(skb) - hard_start; in bpf_prog_run_generic_xdp()
5343 xdp_init_buff(xdp, frame_sz, &rxqueue->xdp_rxq); in bpf_prog_run_generic_xdp()
5344 xdp_prepare_buff(xdp, hard_start, skb_headroom(skb) - mac_len, in bpf_prog_run_generic_xdp()
5347 skb_shinfo(skb)->xdp_frags_size = skb->data_len; in bpf_prog_run_generic_xdp()
5353 orig_data_end = xdp->data_end; in bpf_prog_run_generic_xdp()
5354 orig_data = xdp->data; in bpf_prog_run_generic_xdp()
5355 eth = (struct ethhdr *)xdp->data; in bpf_prog_run_generic_xdp()
5356 orig_host = ether_addr_equal_64bits(eth->h_dest, skb->dev->dev_addr); in bpf_prog_run_generic_xdp()
5357 orig_bcast = is_multicast_ether_addr_64bits(eth->h_dest); in bpf_prog_run_generic_xdp()
5358 orig_eth_type = eth->h_proto; in bpf_prog_run_generic_xdp()
5363 off = xdp->data - orig_data; in bpf_prog_run_generic_xdp()
5368 __skb_push(skb, -off); in bpf_prog_run_generic_xdp()
5370 skb->mac_header += off; in bpf_prog_run_generic_xdp()
5375 off = xdp->data_end - orig_data_end; in bpf_prog_run_generic_xdp()
5377 skb_set_tail_pointer(skb, xdp->data_end - xdp->data); in bpf_prog_run_generic_xdp()
5378 skb->len += off; /* positive on grow, negative on shrink */ in bpf_prog_run_generic_xdp()
5385 skb->data_len = skb_shinfo(skb)->xdp_frags_size; in bpf_prog_run_generic_xdp()
5387 skb->data_len = 0; in bpf_prog_run_generic_xdp()
5390 eth = (struct ethhdr *)xdp->data; in bpf_prog_run_generic_xdp()
5391 if ((orig_eth_type != eth->h_proto) || in bpf_prog_run_generic_xdp()
5392 (orig_host != ether_addr_equal_64bits(eth->h_dest, in bpf_prog_run_generic_xdp()
5393 skb->dev->dev_addr)) || in bpf_prog_run_generic_xdp()
5394 (orig_bcast != is_multicast_ether_addr_64bits(eth->h_dest))) { in bpf_prog_run_generic_xdp()
5396 skb->pkt_type = PACKET_HOST; in bpf_prog_run_generic_xdp()
5397 skb->protocol = eth_type_trans(skb, skb->dev); in bpf_prog_run_generic_xdp()
5413 metalen = xdp->data - xdp->data_meta; in bpf_prog_run_generic_xdp()
5437 hroom = XDP_PACKET_HEADROOM - skb_headroom(skb); in netif_skb_check_for_xdp()
5438 troom = skb->tail + skb->data_len - skb->end; in netif_skb_check_for_xdp()
5465 mac_len = skb->data - skb_mac_header(skb); in netif_receive_generic_xdp()
5483 bpf_warn_invalid_xdp_action((*pskb)->dev, xdp_prog, act); in netif_receive_generic_xdp()
5486 trace_xdp_exception((*pskb)->dev, xdp_prog, act); in netif_receive_generic_xdp()
5498 * network taps in order to match in-driver-XDP behavior. This also means
5500 * and DDOS attacks will be more effective. In-driver-XDP use dedicated TX
5505 struct net_device *dev = skb->dev; in generic_xdp_tx()
5542 err = xdp_do_generic_redirect((*pskb)->dev, *pskb, in do_xdp_generic()
5579 cpu = get_rps_cpu(skb->dev, skb, &rflow); in netif_rx_internal()
5583 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail); in netif_rx_internal()
5597 * __netif_rx - Slightly optimized version of netif_rx
5618 * netif_rx - post buffer to the network code
5656 if (sd->completion_queue) { in net_tx_action()
5660 clist = sd->completion_queue; in net_tx_action()
5661 sd->completion_queue = NULL; in net_tx_action()
5667 clist = clist->next; in net_tx_action()
5669 WARN_ON(refcount_read(&skb->users)); in net_tx_action()
5670 if (likely(get_kfree_skb_cb(skb)->reason == SKB_CONSUMED)) in net_tx_action()
5674 get_kfree_skb_cb(skb)->reason, NULL); in net_tx_action()
5676 if (skb->fclone != SKB_FCLONE_UNAVAILABLE) in net_tx_action()
5680 get_kfree_skb_cb(skb)->reason); in net_tx_action()
5684 if (sd->output_queue) { in net_tx_action()
5688 head = sd->output_queue; in net_tx_action()
5689 sd->output_queue = NULL; in net_tx_action()
5690 sd->output_queue_tailp = &sd->output_queue; in net_tx_action()
5699 head = head->next_sched; in net_tx_action()
5701 /* We need to make sure head->next_sched is read in net_tx_action()
5706 if (!(q->flags & TCQ_F_NOLOCK)) { in net_tx_action()
5710 &q->state))) { in net_tx_action()
5719 clear_bit(__QDISC_STATE_SCHED, &q->state); in net_tx_action()
5723 clear_bit(__QDISC_STATE_SCHED, &q->state); in net_tx_action()
5743 * netdev_is_rx_handler_busy - check if receive handler is registered
5754 return dev && rtnl_dereference(dev->rx_handler); in netdev_is_rx_handler_busy()
5759 * netdev_rx_handler_register - register receive handler
5777 return -EBUSY; in netdev_rx_handler_register()
5779 if (dev->priv_flags & IFF_NO_RX_HANDLER) in netdev_rx_handler_register()
5780 return -EINVAL; in netdev_rx_handler_register()
5783 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data); in netdev_rx_handler_register()
5784 rcu_assign_pointer(dev->rx_handler, rx_handler); in netdev_rx_handler_register()
5791 * netdev_rx_handler_unregister - unregister receive handler
5802 RCU_INIT_POINTER(dev->rx_handler, NULL); in netdev_rx_handler_unregister()
5808 RCU_INIT_POINTER(dev->rx_handler_data, NULL); in netdev_rx_handler_unregister()
5818 switch (skb->protocol) { in skb_pfmemalloc_protocol()
5865 orig_dev = skb->dev; in __netif_receive_skb_core()
5881 skb->skb_iif = skb->dev->ifindex; in __netif_receive_skb_core()
5889 ret2 = do_xdp_generic(rcu_dereference(skb->dev->xdp_prog), in __netif_receive_skb_core()
5899 if (eth_type_vlan(skb->protocol)) { in __netif_receive_skb_core()
5911 list_for_each_entry_rcu(ptype, &dev_net_rcu(skb->dev)->ptype_all, in __netif_receive_skb_core()
5918 list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) { in __netif_receive_skb_core()
5960 rx_handler = rcu_dereference(skb->dev->rx_handler); in __netif_receive_skb_core()
5982 if (unlikely(skb_vlan_tag_present(skb)) && !netdev_uses_dsa(skb->dev)) { in __netif_receive_skb_core()
5988 skb->pkt_type = PACKET_OTHERHOST; in __netif_receive_skb_core()
5989 } else if (eth_type_vlan(skb->protocol)) { in __netif_receive_skb_core()
6013 * and set skb->priority like in vlan_do_receive() in __netif_receive_skb_core()
6019 type = skb->protocol; in __netif_receive_skb_core()
6027 /* orig_dev and skb->dev could belong to different netns; in __netif_receive_skb_core()
6029 * coming from skb->dev, as the ptype owner (packet socket) in __netif_receive_skb_core()
6030 * will use dev_net(skb->dev) to do namespace filtering. in __netif_receive_skb_core()
6033 &dev_net_rcu(skb->dev)->ptype_specific); in __netif_receive_skb_core()
6037 &orig_dev->ptype_specific); in __netif_receive_skb_core()
6039 if (unlikely(skb->dev != orig_dev)) { in __netif_receive_skb_core()
6041 &skb->dev->ptype_specific); in __netif_receive_skb_core()
6049 dev_core_stats_rx_dropped_inc(skb->dev); in __netif_receive_skb_core()
6051 dev_core_stats_rx_nohandler_inc(skb->dev); in __netif_receive_skb_core()
6055 * me how you were going to use this. :-) in __netif_receive_skb_core()
6062 * then skb should also be non-NULL. in __netif_receive_skb_core()
6073 struct net_device *orig_dev = skb->dev; in __netif_receive_skb_one_core()
6079 ret = INDIRECT_CALL_INET(pt_prev->func, ipv6_rcv, ip_rcv, skb, in __netif_receive_skb_one_core()
6080 skb->dev, pt_prev, orig_dev); in __netif_receive_skb_one_core()
6085 * netif_receive_skb_core - special purpose version of netif_receive_skb
6121 if (pt_prev->list_func != NULL) in __netif_receive_skb_list_ptype()
6122 INDIRECT_CALL_INET(pt_prev->list_func, ipv6_list_rcv, in __netif_receive_skb_list_ptype()
6127 pt_prev->func(skb, skb->dev, pt_prev, orig_dev); in __netif_receive_skb_list_ptype()
6133 /* Fast-path assumptions: in __netif_receive_skb_list_core()
6134 * - There is no RX handler. in __netif_receive_skb_list_core()
6135 * - Only one packet_type matches. in __netif_receive_skb_list_core()
6136 * If either of these fails, we will end up doing some per-packet in __netif_receive_skb_list_core()
6137 * processing in-line, then handling the 'last ptype' for the whole in __netif_receive_skb_list_core()
6138 * sublist. This can't cause out-of-order delivery to any single ptype, in __netif_receive_skb_list_core()
6140 * other ptypes are handled per-packet. in __netif_receive_skb_list_core()
6150 struct net_device *orig_dev = skb->dev; in __netif_receive_skb_list_core()
6165 list_add_tail(&skb->list, &sublist); in __netif_receive_skb_list_core()
6181 * - be delivered to SOCK_MEMALLOC sockets only in __netif_receive_skb()
6182 * - stay away from userspace in __netif_receive_skb()
6183 * - have bounded memory usage in __netif_receive_skb()
6208 list_cut_before(&sublist, head, &skb->list); in __netif_receive_skb_list()
6229 struct bpf_prog *old = rtnl_dereference(dev->xdp_prog); in generic_xdp_install()
6230 struct bpf_prog *new = xdp->prog; in generic_xdp_install()
6233 switch (xdp->command) { in generic_xdp_install()
6235 rcu_assign_pointer(dev->xdp_prog, new); in generic_xdp_install()
6249 ret = -EINVAL; in generic_xdp_install()
6269 int cpu = get_rps_cpu(skb->dev, skb, &rflow); in netif_receive_skb_internal()
6272 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail); in netif_receive_skb_internal()
6293 list_add_tail(&skb->list, &sublist); in netif_receive_skb_list_internal()
6302 int cpu = get_rps_cpu(skb->dev, skb, &rflow); in netif_receive_skb_list_internal()
6307 enqueue_to_backlog(skb, cpu, &rflow->last_qtail); in netif_receive_skb_list_internal()
6317 * netif_receive_skb - process receive buffer from network
6345 * netif_receive_skb_list - process many receive buffers from network
6381 skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) { in flush_backlog()
6382 if (READ_ONCE(skb->dev->reg_state) == NETREG_UNREGISTERING) { in flush_backlog()
6383 __skb_unlink(skb, &sd->input_pkt_queue); in flush_backlog()
6391 skb_queue_walk_safe(&sd->process_queue, skb, tmp) { in flush_backlog()
6392 if (READ_ONCE(skb->dev->reg_state) == NETREG_UNREGISTERING) { in flush_backlog()
6393 __skb_unlink(skb, &sd->process_queue); in flush_backlog()
6415 do_flush = !skb_queue_empty(&sd->input_pkt_queue) || in flush_required()
6416 !skb_queue_empty_lockless(&sd->process_queue); in flush_required()
6423 * input_pkt_queue and process_queue even if the latter could end-up in flush_required()
6452 cpumask_clear(&ptr->flush_cpus); in flush_all_backlogs()
6458 INIT_WORK(&ptr->w[cpu], flush_backlog); in flush_all_backlogs()
6459 queue_work_on(cpu, system_highpri_wq, &ptr->w[cpu]); in flush_all_backlogs()
6460 __cpumask_set_cpu(cpu, &ptr->flush_cpus); in flush_all_backlogs()
6468 for_each_cpu(cpu, &ptr->flush_cpus) in flush_all_backlogs()
6469 flush_work(&ptr->w[cpu]); in flush_all_backlogs()
6483 struct softnet_data *next = remsd->rps_ipi_next; in net_rps_send_ipi()
6485 if (cpu_online(remsd->cpu)) in net_rps_send_ipi()
6486 smp_call_function_single_async(remsd->cpu, &remsd->csd); in net_rps_send_ipi()
6499 struct softnet_data *remsd = sd->rps_ipi_list; in net_rps_action_and_irq_enable()
6502 sd->rps_ipi_list = NULL; in net_rps_action_and_irq_enable()
6516 return !use_backlog_threads() && sd->rps_ipi_list; in sd_has_rps_ipi_waiting()
6536 napi->weight = READ_ONCE(net_hotdata.dev_rx_weight); in process_backlog()
6541 while ((skb = __skb_dequeue(&sd->process_queue))) { in process_backlog()
6556 if (skb_queue_empty(&sd->input_pkt_queue)) { in process_backlog()
6565 napi->state &= NAPIF_STATE_THREADED; in process_backlog()
6569 skb_queue_splice_tail_init(&sd->input_pkt_queue, in process_backlog()
6570 &sd->process_queue); in process_backlog()
6582 * __napi_schedule - schedule for receive
6599 * napi_schedule_prep - check if napi can be scheduled
6609 unsigned long new, val = READ_ONCE(n->state); in napi_schedule_prep()
6624 } while (!try_cmpxchg(&n->state, &val, new)); in napi_schedule_prep()
6631 * __napi_schedule_irqoff - schedule for receive
6638 * due to force-threaded interrupts and spinlock substitution.
6660 if (unlikely(n->state & (NAPIF_STATE_NPSVC | in napi_complete_done()
6665 if (n->gro.bitmask) in napi_complete_done()
6667 n->defer_hard_irqs_count = napi_get_defer_hard_irqs(n); in napi_complete_done()
6669 if (n->defer_hard_irqs_count > 0) { in napi_complete_done()
6670 n->defer_hard_irqs_count--; in napi_complete_done()
6681 gro_flush_normal(&n->gro, !!timeout); in napi_complete_done()
6683 if (unlikely(!list_empty(&n->poll_list))) { in napi_complete_done()
6684 /* If n->poll_list is not empty, we need to mask irqs */ in napi_complete_done()
6686 list_del_init(&n->poll_list); in napi_complete_done()
6689 WRITE_ONCE(n->list_owner, -1); in napi_complete_done()
6691 val = READ_ONCE(n->state); in napi_complete_done()
6700 * because we will call napi->poll() one more time. in napi_complete_done()
6705 } while (!try_cmpxchg(&n->state, &val, new)); in napi_complete_done()
6713 hrtimer_start(&n->timer, ns_to_ktime(timeout), in napi_complete_done()
6729 if (llist_empty(&sdn->defer_list)) in skb_defer_free_flush()
6731 atomic_long_set(&sdn->defer_count, 0); in skb_defer_free_flush()
6732 free_list = llist_del_all(&sdn->defer_list); in skb_defer_free_flush()
6745 gro_normal_list(&napi->gro); in __busy_poll_stop()
6751 gro_flush_normal(&napi->gro, HZ >= 1000); in __busy_poll_stop()
6753 clear_bit(NAPI_STATE_SCHED, &napi->state); in __busy_poll_stop()
6772 * Since we are about to call napi->poll() once more, we can safely in busy_poll_stop()
6778 clear_bit(NAPI_STATE_MISSED, &napi->state); in busy_poll_stop()
6779 clear_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state); in busy_poll_stop()
6785 napi->defer_hard_irqs_count = napi_get_defer_hard_irqs(napi); in busy_poll_stop()
6787 if (napi->defer_hard_irqs_count && timeout) { in busy_poll_stop()
6788 hrtimer_start(&napi->timer, ns_to_ktime(timeout), HRTIMER_MODE_REL_PINNED); in busy_poll_stop()
6793 /* All we really want here is to re-enable device interrupts. in busy_poll_stop()
6796 rc = napi->poll(napi, budget); in busy_poll_stop()
6797 /* We can't gro_normal_list() here, because napi->poll() might have in busy_poll_stop()
6836 unsigned long val = READ_ONCE(napi->state); in __napi_busy_loop()
6839 * we avoid dirtying napi->state as much as we can. in __napi_busy_loop()
6844 set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state); in __napi_busy_loop()
6847 if (cmpxchg(&napi->state, val, in __napi_busy_loop()
6851 set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state); in __napi_busy_loop()
6855 napi_poll = napi->poll; in __napi_busy_loop()
6859 gro_normal_list(&napi->gro); in __napi_busy_loop()
6862 __NET_ADD_STATS(dev_net(napi->dev), in __napi_busy_loop()
6927 hrtimer_start(&napi->timer, ns_to_ktime(timeout), in napi_suspend_irqs()
6959 napi->gro.cached_napi_id = napi_id; in __napi_hash_add_with_id()
6961 WRITE_ONCE(napi->napi_id, napi_id); in __napi_hash_add_with_id()
6962 hlist_add_head_rcu(&napi->napi_hash_node, in __napi_hash_add_with_id()
6963 &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]); in __napi_hash_add_with_id()
6981 if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state)) in napi_hash_add()
7006 hlist_del_init_rcu(&napi->napi_hash_node); in napi_hash_del()
7021 !test_and_set_bit(NAPI_STATE_SCHED, &napi->state)) { in napi_watchdog()
7022 clear_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state); in napi_watchdog()
7035 val = READ_ONCE(napi->state); in napi_stop_kthread()
7048 if (try_cmpxchg(&napi->state, &val, new)) in napi_stop_kthread()
7056 if (!test_bit(NAPI_STATE_SCHED_THREADED, &napi->state)) in napi_stop_kthread()
7062 kthread_stop(napi->thread); in napi_stop_kthread()
7063 napi->thread = NULL; in napi_stop_kthread()
7070 if (!napi->thread) { in napi_set_threaded()
7078 if (napi->config) in napi_set_threaded()
7079 napi->config->threaded = threaded; in napi_set_threaded()
7087 if (!threaded && napi->thread) { in napi_set_threaded()
7092 assign_bit(NAPI_STATE_THREADED, &napi->state, threaded); in napi_set_threaded()
7107 list_for_each_entry(napi, &dev->napi_list, dev_list) { in netif_set_threaded()
7108 if (!napi->thread) { in netif_set_threaded()
7118 WRITE_ONCE(dev->threaded, threaded); in netif_set_threaded()
7121 list_for_each_entry(napi, &dev->napi_list, dev_list) in netif_set_threaded()
7125 for (i = 0; i < dev->num_napi_configs; i++) in netif_set_threaded()
7126 dev->napi_config[i].threaded = threaded; in netif_set_threaded()
7132 * netif_threaded_enable() - enable threaded NAPIs
7149 * netif_queue_set_napi - Associate queue with the napi
7156 * registering the NAPI handler for the queue-vector and the queues have been
7165 if (WARN_ON_ONCE(napi && !napi->dev)) in netif_queue_set_napi()
7172 rxq->napi = napi; in netif_queue_set_napi()
7176 txq->napi = napi; in netif_queue_set_napi()
7191 struct cpu_rmap *rmap = napi->dev->rx_cpu_rmap; in netif_napi_irq_notify()
7195 if (napi->config && napi->dev->irq_affinity_auto) in netif_napi_irq_notify()
7196 cpumask_copy(&napi->config->affinity_mask, mask); in netif_napi_irq_notify()
7199 if (napi->dev->rx_cpu_rmap_auto) { in netif_napi_irq_notify()
7200 err = cpu_rmap_update(rmap, napi->napi_rmap_idx, mask); in netif_napi_irq_notify()
7202 netdev_warn(napi->dev, "RMAP update failed (%d)\n", in netif_napi_irq_notify()
7213 struct cpu_rmap *rmap = napi->dev->rx_cpu_rmap; in netif_napi_affinity_release()
7215 netdev_assert_locked(napi->dev); in netif_napi_affinity_release()
7217 &napi->state)); in netif_napi_affinity_release()
7219 if (!napi->dev->rx_cpu_rmap_auto) in netif_napi_affinity_release()
7221 rmap->obj[napi->napi_rmap_idx] = NULL; in netif_napi_affinity_release()
7222 napi->napi_rmap_idx = -1; in netif_napi_affinity_release()
7228 if (dev->rx_cpu_rmap_auto) in netif_enable_cpu_rmap()
7231 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(num_irqs); in netif_enable_cpu_rmap()
7232 if (!dev->rx_cpu_rmap) in netif_enable_cpu_rmap()
7233 return -ENOMEM; in netif_enable_cpu_rmap()
7235 dev->rx_cpu_rmap_auto = true; in netif_enable_cpu_rmap()
7242 struct cpu_rmap *rmap = dev->rx_cpu_rmap; in netif_del_cpu_rmap()
7244 if (!dev->rx_cpu_rmap_auto) in netif_del_cpu_rmap()
7249 dev->rx_cpu_rmap = NULL; in netif_del_cpu_rmap()
7250 dev->rx_cpu_rmap_auto = false; in netif_del_cpu_rmap()
7273 maxqs = max(dev->num_tx_queues, dev->num_rx_queues); in netif_set_affinity_auto()
7274 numa = dev_to_node(&dev->dev); in netif_set_affinity_auto()
7278 &dev->napi_config[i].affinity_mask); in netif_set_affinity_auto()
7280 dev->irq_affinity_auto = true; in netif_set_affinity_auto()
7288 netdev_assert_locked_or_invisible(napi->dev); in netif_napi_set_irq_locked()
7290 if (napi->irq == irq) in netif_napi_set_irq_locked()
7294 if (test_and_clear_bit(NAPI_STATE_HAS_NOTIFIER, &napi->state)) in netif_napi_set_irq_locked()
7295 irq_set_affinity_notifier(napi->irq, NULL); in netif_napi_set_irq_locked()
7297 napi->irq = irq; in netif_napi_set_irq_locked()
7299 (!napi->dev->rx_cpu_rmap_auto && !napi->dev->irq_affinity_auto)) in netif_napi_set_irq_locked()
7303 if (napi->dev->irq_affinity_auto && WARN_ON_ONCE(!napi->config)) in netif_napi_set_irq_locked()
7307 if (napi->dev->rx_cpu_rmap_auto) { in netif_napi_set_irq_locked()
7308 rc = cpu_rmap_add(napi->dev->rx_cpu_rmap, napi); in netif_napi_set_irq_locked()
7312 cpu_rmap_get(napi->dev->rx_cpu_rmap); in netif_napi_set_irq_locked()
7313 napi->napi_rmap_idx = rc; in netif_napi_set_irq_locked()
7318 napi->notify.notify = netif_napi_irq_notify; in netif_napi_set_irq_locked()
7319 napi->notify.release = netif_napi_affinity_release; in netif_napi_set_irq_locked()
7320 rc = irq_set_affinity_notifier(irq, &napi->notify); in netif_napi_set_irq_locked()
7322 netdev_warn(napi->dev, "Unable to set IRQ notifier (%d)\n", in netif_napi_set_irq_locked()
7327 set_bit(NAPI_STATE_HAS_NOTIFIER, &napi->state); in netif_napi_set_irq_locked()
7332 if (napi->dev->rx_cpu_rmap_auto) { in netif_napi_set_irq_locked()
7333 napi->dev->rx_cpu_rmap->obj[napi->napi_rmap_idx] = NULL; in netif_napi_set_irq_locked()
7334 cpu_rmap_put(napi->dev->rx_cpu_rmap); in netif_napi_set_irq_locked()
7335 napi->napi_rmap_idx = -1; in netif_napi_set_irq_locked()
7338 napi->notify.notify = NULL; in netif_napi_set_irq_locked()
7339 napi->notify.release = NULL; in netif_napi_set_irq_locked()
7345 n->defer_hard_irqs = n->config->defer_hard_irqs; in napi_restore_config()
7346 n->gro_flush_timeout = n->config->gro_flush_timeout; in napi_restore_config()
7347 n->irq_suspend_timeout = n->config->irq_suspend_timeout; in napi_restore_config()
7349 if (n->dev->irq_affinity_auto && in napi_restore_config()
7350 test_bit(NAPI_STATE_HAS_NOTIFIER, &n->state)) in napi_restore_config()
7351 irq_set_affinity(n->irq, &n->config->affinity_mask); in napi_restore_config()
7356 if (n->config->napi_id) { in napi_restore_config()
7357 napi_hash_add_with_id(n, n->config->napi_id); in napi_restore_config()
7360 n->config->napi_id = n->napi_id; in napi_restore_config()
7363 WARN_ON_ONCE(napi_set_threaded(n, n->config->threaded)); in napi_restore_config()
7368 n->config->defer_hard_irqs = n->defer_hard_irqs; in napi_save_config()
7369 n->config->gro_flush_timeout = n->gro_flush_timeout; in napi_save_config()
7370 n->config->irq_suspend_timeout = n->irq_suspend_timeout; in napi_save_config()
7385 if (napi->config && napi->config->napi_id) in netif_napi_dev_list_add()
7386 new_id = napi->config->napi_id; in netif_napi_dev_list_add()
7388 higher = &dev->napi_list; in netif_napi_dev_list_add()
7389 list_for_each_entry(pos, &dev->napi_list, dev_list) { in netif_napi_dev_list_add()
7390 if (napi_id_valid(pos->napi_id)) in netif_napi_dev_list_add()
7391 pos_id = pos->napi_id; in netif_napi_dev_list_add()
7392 else if (pos->config) in netif_napi_dev_list_add()
7393 pos_id = pos->config->napi_id; in netif_napi_dev_list_add()
7399 higher = &pos->dev_list; in netif_napi_dev_list_add()
7401 list_add_rcu(&napi->dev_list, higher); /* adds after higher */ in netif_napi_dev_list_add()
7405 * skb->head being backed by slab, not a page fragment.
7407 * ("net: avoid 32 x truesize under-estimation for tiny skbs")
7416 WARN_ON_ONCE(skb && skb->head_frag); in napi_get_frags_check()
7427 if (WARN_ON(test_and_set_bit(NAPI_STATE_LISTED, &napi->state))) in netif_napi_add_weight_locked()
7430 INIT_LIST_HEAD(&napi->poll_list); in netif_napi_add_weight_locked()
7431 INIT_HLIST_NODE(&napi->napi_hash_node); in netif_napi_add_weight_locked()
7432 hrtimer_setup(&napi->timer, napi_watchdog, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); in netif_napi_add_weight_locked()
7433 gro_init(&napi->gro); in netif_napi_add_weight_locked()
7434 napi->skb = NULL; in netif_napi_add_weight_locked()
7435 napi->poll = poll; in netif_napi_add_weight_locked()
7439 napi->weight = weight; in netif_napi_add_weight_locked()
7440 napi->dev = dev; in netif_napi_add_weight_locked()
7442 napi->poll_owner = -1; in netif_napi_add_weight_locked()
7444 napi->list_owner = -1; in netif_napi_add_weight_locked()
7445 set_bit(NAPI_STATE_SCHED, &napi->state); in netif_napi_add_weight_locked()
7446 set_bit(NAPI_STATE_NPSVC, &napi->state); in netif_napi_add_weight_locked()
7449 /* default settings from sysfs are applied to all NAPIs. any per-NAPI in netif_napi_add_weight_locked()
7452 napi_set_defer_hard_irqs(napi, READ_ONCE(dev->napi_defer_hard_irqs)); in netif_napi_add_weight_locked()
7453 napi_set_gro_flush_timeout(napi, READ_ONCE(dev->gro_flush_timeout)); in netif_napi_add_weight_locked()
7456 /* Create kthread for this napi if dev->threaded is set. in netif_napi_add_weight_locked()
7457 * Clear dev->threaded if kthread creation failed so that in netif_napi_add_weight_locked()
7462 dev->threaded = NETDEV_NAPI_THREADED_DISABLED; in netif_napi_add_weight_locked()
7463 netif_napi_set_irq_locked(napi, -1); in netif_napi_add_weight_locked()
7472 netdev_assert_locked(n->dev); in napi_disable_locked()
7474 set_bit(NAPI_STATE_DISABLE, &n->state); in napi_disable_locked()
7476 val = READ_ONCE(n->state); in napi_disable_locked()
7480 val = READ_ONCE(n->state); in napi_disable_locked()
7485 } while (!try_cmpxchg(&n->state, &val, new)); in napi_disable_locked()
7487 hrtimer_cancel(&n->timer); in napi_disable_locked()
7489 if (n->config) in napi_disable_locked()
7494 clear_bit(NAPI_STATE_DISABLE, &n->state); in napi_disable_locked()
7499 * napi_disable() - prevent NAPI from scheduling
7508 netdev_lock(n->dev); in napi_disable()
7510 netdev_unlock(n->dev); in napi_disable()
7516 unsigned long new, val = READ_ONCE(n->state); in napi_enable_locked()
7518 if (n->config) in napi_enable_locked()
7527 if (n->dev->threaded && n->thread) in napi_enable_locked()
7529 } while (!try_cmpxchg(&n->state, &val, new)); in napi_enable_locked()
7534 * napi_enable() - enable NAPI scheduling
7543 netdev_lock(n->dev); in napi_enable()
7545 netdev_unlock(n->dev); in napi_enable()
7552 netdev_assert_locked(napi->dev); in __netif_napi_del_locked()
7554 if (!test_and_clear_bit(NAPI_STATE_LISTED, &napi->state)) in __netif_napi_del_locked()
7558 WARN_ON(!test_bit(NAPI_STATE_SCHED, &napi->state)); in __netif_napi_del_locked()
7560 if (test_and_clear_bit(NAPI_STATE_HAS_NOTIFIER, &napi->state)) in __netif_napi_del_locked()
7561 irq_set_affinity_notifier(napi->irq, NULL); in __netif_napi_del_locked()
7563 if (napi->config) { in __netif_napi_del_locked()
7564 napi->index = -1; in __netif_napi_del_locked()
7565 napi->config = NULL; in __netif_napi_del_locked()
7568 list_del_rcu(&napi->dev_list); in __netif_napi_del_locked()
7571 gro_cleanup(&napi->gro); in __netif_napi_del_locked()
7573 if (napi->thread) { in __netif_napi_del_locked()
7574 kthread_stop(napi->thread); in __netif_napi_del_locked()
7575 napi->thread = NULL; in __netif_napi_del_locked()
7584 weight = n->weight; in __napi_poll()
7589 * actually make the ->poll() call. Therefore we avoid in __napi_poll()
7590 * accidentally calling ->poll() when NAPI is not scheduled. in __napi_poll()
7594 work = n->poll(n, weight); in __napi_poll()
7601 netdev_err_once(n->dev, "NAPI poll function %pS returned %d, exceeding its budget of %d.\n", in __napi_poll()
7602 n->poll, work, weight); in __napi_poll()
7610 * move the instance around on the list at-will. in __napi_poll()
7617 /* The NAPI context has more processing work, but busy-polling in __napi_poll()
7623 * that the NAPI is re-scheduled. in __napi_poll()
7631 gro_flush_normal(&n->gro, HZ >= 1000); in __napi_poll()
7636 if (unlikely(!list_empty(&n->poll_list))) { in __napi_poll()
7638 n->dev ? n->dev->name : "backlog"); in __napi_poll()
7653 list_del_init(&n->poll_list); in napi_poll()
7663 n->dev->name, n->poll); in napi_poll()
7665 list_add_tail(&n->poll_list, repoll); in napi_poll()
7682 if (test_bit(NAPI_STATE_SCHED_THREADED, &napi->state)) { in napi_thread_wait()
7683 WARN_ON(!list_empty(&napi->poll_list)); in napi_thread_wait()
7693 return -1; in napi_thread_wait()
7710 sd->in_napi_threaded_poll = true; in napi_threaded_poll_loop()
7716 sd->in_napi_threaded_poll = false; in napi_threaded_poll_loop()
7757 sd->in_net_rx_action = true; in net_rx_action()
7759 list_splice_init(&sd->poll_list, &list); in net_rx_action()
7769 sd->in_net_rx_action = false; in net_rx_action()
7773 * sd->in_net_rx_action was true. in net_rx_action()
7775 if (!list_empty(&sd->poll_list)) in net_rx_action()
7784 budget -= napi_poll(n, &repoll); in net_rx_action()
7793 WRITE_ONCE(sd->time_squeeze, sd->time_squeeze + 1); in net_rx_action()
7800 list_splice_tail_init(&sd->poll_list, &list); in net_rx_action()
7802 list_splice(&list, &sd->poll_list); in net_rx_action()
7803 if (!list_empty(&sd->poll_list)) in net_rx_action()
7806 sd->in_net_rx_action = false; in net_rx_action()
7839 if (adj->dev == adj_dev) in __netdev_find_adj()
7848 struct net_device *dev = (struct net_device *)priv->data; in ____netdev_has_upper_dev()
7854 * netdev_has_upper_dev - Check if device is linked to an upper device
7877 * netdev_has_upper_dev_all_rcu - Check if device is linked to an upper device
7899 * netdev_has_any_upper_dev - Check if device is linked to some device
7909 return !list_empty(&dev->adj_list.upper); in netdev_has_any_upper_dev()
7914 * netdev_master_upper_dev_get - Get master upper device
7926 if (list_empty(&dev->adj_list.upper)) in netdev_master_upper_dev_get()
7929 upper = list_first_entry(&dev->adj_list.upper, in netdev_master_upper_dev_get()
7931 if (likely(upper->master)) in netdev_master_upper_dev_get()
7932 return upper->dev; in netdev_master_upper_dev_get()
7943 if (list_empty(&dev->adj_list.upper)) in __netdev_master_upper_dev_get()
7946 upper = list_first_entry(&dev->adj_list.upper, in __netdev_master_upper_dev_get()
7948 if (likely(upper->master) && !upper->ignore) in __netdev_master_upper_dev_get()
7949 return upper->dev; in __netdev_master_upper_dev_get()
7954 * netdev_has_any_lower_dev - Check if device is linked to some device
7964 return !list_empty(&dev->adj_list.lower); in netdev_has_any_lower_dev()
7973 return adj->private; in netdev_adjacent_get_private()
7978 * netdev_upper_get_next_dev_rcu - Get the next dev from upper list
7992 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); in netdev_upper_get_next_dev_rcu()
7994 if (&upper->list == &dev->adj_list.upper) in netdev_upper_get_next_dev_rcu()
7997 *iter = &upper->list; in netdev_upper_get_next_dev_rcu()
7999 return upper->dev; in netdev_upper_get_next_dev_rcu()
8009 upper = list_entry((*iter)->next, struct netdev_adjacent, list); in __netdev_next_upper_dev()
8011 if (&upper->list == &dev->adj_list.upper) in __netdev_next_upper_dev()
8014 *iter = &upper->list; in __netdev_next_upper_dev()
8015 *ignore = upper->ignore; in __netdev_next_upper_dev()
8017 return upper->dev; in __netdev_next_upper_dev()
8027 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); in netdev_next_upper_dev_rcu()
8029 if (&upper->list == &dev->adj_list.upper) in netdev_next_upper_dev_rcu()
8032 *iter = &upper->list; in netdev_next_upper_dev_rcu()
8034 return upper->dev; in netdev_next_upper_dev_rcu()
8048 iter = &dev->adj_list.upper; in __netdev_walk_all_upper_dev()
8066 niter = &udev->adj_list.upper; in __netdev_walk_all_upper_dev()
8075 next = dev_stack[--cur]; in __netdev_walk_all_upper_dev()
8096 iter = &dev->adj_list.upper; in netdev_walk_all_upper_dev_rcu()
8112 niter = &udev->adj_list.upper; in netdev_walk_all_upper_dev_rcu()
8121 next = dev_stack[--cur]; in netdev_walk_all_upper_dev_rcu()
8148 * netdev_lower_get_next_private - Get the next ->private from the
8153 * Gets the next netdev_adjacent->private from the dev's lower neighbour
8165 if (&lower->list == &dev->adj_list.lower) in netdev_lower_get_next_private()
8168 *iter = lower->list.next; in netdev_lower_get_next_private()
8170 return lower->private; in netdev_lower_get_next_private()
8175 * netdev_lower_get_next_private_rcu - Get the next ->private from the
8181 * Gets the next netdev_adjacent->private from the dev's lower neighbour
8191 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); in netdev_lower_get_next_private_rcu()
8193 if (&lower->list == &dev->adj_list.lower) in netdev_lower_get_next_private_rcu()
8196 *iter = &lower->list; in netdev_lower_get_next_private_rcu()
8198 return lower->private; in netdev_lower_get_next_private_rcu()
8203 * netdev_lower_get_next - Get the next device from the lower neighbour
8219 if (&lower->list == &dev->adj_list.lower) in netdev_lower_get_next()
8222 *iter = lower->list.next; in netdev_lower_get_next()
8224 return lower->dev; in netdev_lower_get_next()
8233 lower = list_entry((*iter)->next, struct netdev_adjacent, list); in netdev_next_lower_dev()
8235 if (&lower->list == &dev->adj_list.lower) in netdev_next_lower_dev()
8238 *iter = &lower->list; in netdev_next_lower_dev()
8240 return lower->dev; in netdev_next_lower_dev()
8249 lower = list_entry((*iter)->next, struct netdev_adjacent, list); in __netdev_next_lower_dev()
8251 if (&lower->list == &dev->adj_list.lower) in __netdev_next_lower_dev()
8254 *iter = &lower->list; in __netdev_next_lower_dev()
8255 *ignore = lower->ignore; in __netdev_next_lower_dev()
8257 return lower->dev; in __netdev_next_lower_dev()
8270 iter = &dev->adj_list.lower; in netdev_walk_all_lower_dev()
8286 niter = &ldev->adj_list.lower; in netdev_walk_all_lower_dev()
8295 next = dev_stack[--cur]; in netdev_walk_all_lower_dev()
8318 iter = &dev->adj_list.lower; in __netdev_walk_all_lower_dev()
8336 niter = &ldev->adj_list.lower; in __netdev_walk_all_lower_dev()
8345 next = dev_stack[--cur]; in __netdev_walk_all_lower_dev()
8361 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); in netdev_next_lower_dev_rcu()
8362 if (&lower->list == &dev->adj_list.lower) in netdev_next_lower_dev_rcu()
8365 *iter = &lower->list; in netdev_next_lower_dev_rcu()
8367 return lower->dev; in netdev_next_lower_dev_rcu()
8378 for (iter = &dev->adj_list.upper, in __netdev_upper_depth()
8384 if (max_depth < udev->upper_level) in __netdev_upper_depth()
8385 max_depth = udev->upper_level; in __netdev_upper_depth()
8398 for (iter = &dev->adj_list.lower, in __netdev_lower_depth()
8404 if (max_depth < ldev->lower_level) in __netdev_lower_depth()
8405 max_depth = ldev->lower_level; in __netdev_lower_depth()
8414 dev->upper_level = __netdev_upper_depth(dev) + 1; in __netdev_update_upper_level()
8423 if (list_empty(&dev->unlink_list)) in net_unlink_todo()
8424 list_add_tail(&dev->unlink_list, &net_unlink_list); in net_unlink_todo()
8431 dev->lower_level = __netdev_lower_depth(dev) + 1; in __netdev_update_lower_level()
8437 if (priv->flags & NESTED_SYNC_IMM) in __netdev_update_lower_level()
8438 dev->nested_level = dev->lower_level - 1; in __netdev_update_lower_level()
8439 if (priv->flags & NESTED_SYNC_TODO) in __netdev_update_lower_level()
8455 iter = &dev->adj_list.lower; in netdev_walk_all_lower_dev_rcu()
8471 niter = &ldev->adj_list.lower; in netdev_walk_all_lower_dev_rcu()
8480 next = dev_stack[--cur]; in netdev_walk_all_lower_dev_rcu()
8493 * netdev_lower_get_first_private_rcu - Get the first ->private from the
8498 * Gets the first netdev_adjacent->private from the dev's lower neighbour
8505 lower = list_first_or_null_rcu(&dev->adj_list.lower, in netdev_lower_get_first_private_rcu()
8508 return lower->private; in netdev_lower_get_first_private_rcu()
8514 * netdev_master_upper_dev_get_rcu - Get master upper device
8524 upper = list_first_or_null_rcu(&dev->adj_list.upper, in netdev_master_upper_dev_get_rcu()
8526 if (upper && likely(upper->master)) in netdev_master_upper_dev_get_rcu()
8527 return upper->dev; in netdev_master_upper_dev_get_rcu()
8538 sprintf(linkname, dev_list == &dev->adj_list.upper ? in netdev_adjacent_sysfs_add()
8539 "upper_%s" : "lower_%s", adj_dev->name); in netdev_adjacent_sysfs_add()
8540 return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj), in netdev_adjacent_sysfs_add()
8549 sprintf(linkname, dev_list == &dev->adj_list.upper ? in netdev_adjacent_sysfs_del()
8551 sysfs_remove_link(&(dev->dev.kobj), linkname); in netdev_adjacent_sysfs_del()
8558 return (dev_list == &dev->adj_list.upper || in netdev_adjacent_is_neigh_list()
8559 dev_list == &dev->adj_list.lower) && in netdev_adjacent_is_neigh_list()
8574 adj->ref_nr += 1; in __netdev_adjacent_dev_insert()
8575 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d\n", in __netdev_adjacent_dev_insert()
8576 dev->name, adj_dev->name, adj->ref_nr); in __netdev_adjacent_dev_insert()
8583 return -ENOMEM; in __netdev_adjacent_dev_insert()
8585 adj->dev = adj_dev; in __netdev_adjacent_dev_insert()
8586 adj->master = master; in __netdev_adjacent_dev_insert()
8587 adj->ref_nr = 1; in __netdev_adjacent_dev_insert()
8588 adj->private = private; in __netdev_adjacent_dev_insert()
8589 adj->ignore = false; in __netdev_adjacent_dev_insert()
8590 netdev_hold(adj_dev, &adj->dev_tracker, GFP_KERNEL); in __netdev_adjacent_dev_insert()
8592 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d; dev_hold on %s\n", in __netdev_adjacent_dev_insert()
8593 dev->name, adj_dev->name, adj->ref_nr, adj_dev->name); in __netdev_adjacent_dev_insert()
8603 ret = sysfs_create_link(&(dev->dev.kobj), in __netdev_adjacent_dev_insert()
8604 &(adj_dev->dev.kobj), "master"); in __netdev_adjacent_dev_insert()
8608 list_add_rcu(&adj->list, dev_list); in __netdev_adjacent_dev_insert()
8610 list_add_tail_rcu(&adj->list, dev_list); in __netdev_adjacent_dev_insert()
8617 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list); in __netdev_adjacent_dev_insert()
8619 netdev_put(adj_dev, &adj->dev_tracker); in __netdev_adjacent_dev_insert()
8633 dev->name, adj_dev->name, ref_nr); in __netdev_adjacent_dev_remove()
8639 dev->name, adj_dev->name); in __netdev_adjacent_dev_remove()
8644 if (adj->ref_nr > ref_nr) { in __netdev_adjacent_dev_remove()
8645 pr_debug("adjacency: %s to %s ref_nr - %d = %d\n", in __netdev_adjacent_dev_remove()
8646 dev->name, adj_dev->name, ref_nr, in __netdev_adjacent_dev_remove()
8647 adj->ref_nr - ref_nr); in __netdev_adjacent_dev_remove()
8648 adj->ref_nr -= ref_nr; in __netdev_adjacent_dev_remove()
8652 if (adj->master) in __netdev_adjacent_dev_remove()
8653 sysfs_remove_link(&(dev->dev.kobj), "master"); in __netdev_adjacent_dev_remove()
8656 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list); in __netdev_adjacent_dev_remove()
8658 list_del_rcu(&adj->list); in __netdev_adjacent_dev_remove()
8660 adj_dev->name, dev->name, adj_dev->name); in __netdev_adjacent_dev_remove()
8661 netdev_put(adj_dev, &adj->dev_tracker); in __netdev_adjacent_dev_remove()
8703 &dev->adj_list.upper, in __netdev_adjacent_dev_link_neighbour()
8704 &upper_dev->adj_list.lower, in __netdev_adjacent_dev_link_neighbour()
8712 &dev->adj_list.upper, in __netdev_adjacent_dev_unlink_neighbour()
8713 &upper_dev->adj_list.lower); in __netdev_adjacent_dev_unlink_neighbour()
8738 return -EBUSY; in __netdev_upper_dev_link()
8742 return -EBUSY; in __netdev_upper_dev_link()
8744 if ((dev->lower_level + upper_dev->upper_level) > MAX_NEST_DEV) in __netdev_upper_dev_link()
8745 return -EMLINK; in __netdev_upper_dev_link()
8749 return -EEXIST; in __netdev_upper_dev_link()
8753 return master_dev == upper_dev ? -EEXIST : -EBUSY; in __netdev_upper_dev_link()
8789 * netdev_upper_dev_link - Add a link to the upper device
8814 * netdev_master_upper_dev_link - Add a master link to the upper device
8822 * one master upper device can be linked, although other non-master devices
8875 * netdev_upper_dev_unlink - Removes a link to upper device
8900 adj = __netdev_find_adj(lower_dev, &upper_dev->adj_list.lower); in __netdev_adjacent_dev_set()
8902 adj->ignore = val; in __netdev_adjacent_dev_set()
8904 adj = __netdev_find_adj(upper_dev, &lower_dev->adj_list.upper); in __netdev_adjacent_dev_set()
8906 adj->ignore = val; in __netdev_adjacent_dev_set()
8989 * netdev_bonding_info_change - Dispatch event about slave change
9021 dev->offload_xstats_l3 = kzalloc(sizeof(*dev->offload_xstats_l3), in netdev_offload_xstats_enable_l3()
9023 if (!dev->offload_xstats_l3) in netdev_offload_xstats_enable_l3()
9024 return -ENOMEM; in netdev_offload_xstats_enable_l3()
9036 kfree(dev->offload_xstats_l3); in netdev_offload_xstats_enable_l3()
9037 dev->offload_xstats_l3 = NULL; in netdev_offload_xstats_enable_l3()
9048 return -EALREADY; in netdev_offload_xstats_enable()
9056 return -EINVAL; in netdev_offload_xstats_enable()
9069 kfree(dev->offload_xstats_l3); in netdev_offload_xstats_disable_l3()
9070 dev->offload_xstats_l3 = NULL; in netdev_offload_xstats_disable_l3()
9079 return -EALREADY; in netdev_offload_xstats_disable()
9088 return -EINVAL; in netdev_offload_xstats_disable()
9103 return dev->offload_xstats_l3; in netdev_offload_xstats_get_ptr()
9131 dest->rx_packets += src->rx_packets; in netdev_hw_stats64_add()
9132 dest->tx_packets += src->tx_packets; in netdev_hw_stats64_add()
9133 dest->rx_bytes += src->rx_bytes; in netdev_hw_stats64_add()
9134 dest->tx_bytes += src->tx_bytes; in netdev_hw_stats64_add()
9135 dest->rx_errors += src->rx_errors; in netdev_hw_stats64_add()
9136 dest->tx_errors += src->tx_errors; in netdev_hw_stats64_add()
9137 dest->rx_dropped += src->rx_dropped; in netdev_hw_stats64_add()
9138 dest->tx_dropped += src->tx_dropped; in netdev_hw_stats64_add()
9139 dest->multicast += src->multicast; in netdev_hw_stats64_add()
9181 return -EINVAL; in netdev_offload_xstats_get_stats()
9218 report_delta->used = true; in netdev_offload_xstats_report_delta()
9219 netdev_hw_stats64_add(&report_delta->stats, stats); in netdev_offload_xstats_report_delta()
9226 report_used->used = true; in netdev_offload_xstats_report_used()
9247 * netdev_get_xmit_slave - Get the xmit slave of master device
9261 const struct net_device_ops *ops = dev->netdev_ops; in netdev_get_xmit_slave()
9263 if (!ops->ndo_get_xmit_slave) in netdev_get_xmit_slave()
9265 return ops->ndo_get_xmit_slave(dev, skb, all_slaves); in netdev_get_xmit_slave()
9272 const struct net_device_ops *ops = dev->netdev_ops; in netdev_sk_get_lower_dev()
9274 if (!ops->ndo_sk_get_lower_dev) in netdev_sk_get_lower_dev()
9276 return ops->ndo_sk_get_lower_dev(dev, sk); in netdev_sk_get_lower_dev()
9280 * netdev_sk_get_lowest_dev - Get the lowest device in chain given device and socket
9308 list_for_each_entry(iter, &dev->adj_list.upper, list) { in netdev_adjacent_add_links()
9309 if (!net_eq(net, dev_net(iter->dev))) in netdev_adjacent_add_links()
9311 netdev_adjacent_sysfs_add(iter->dev, dev, in netdev_adjacent_add_links()
9312 &iter->dev->adj_list.lower); in netdev_adjacent_add_links()
9313 netdev_adjacent_sysfs_add(dev, iter->dev, in netdev_adjacent_add_links()
9314 &dev->adj_list.upper); in netdev_adjacent_add_links()
9317 list_for_each_entry(iter, &dev->adj_list.lower, list) { in netdev_adjacent_add_links()
9318 if (!net_eq(net, dev_net(iter->dev))) in netdev_adjacent_add_links()
9320 netdev_adjacent_sysfs_add(iter->dev, dev, in netdev_adjacent_add_links()
9321 &iter->dev->adj_list.upper); in netdev_adjacent_add_links()
9322 netdev_adjacent_sysfs_add(dev, iter->dev, in netdev_adjacent_add_links()
9323 &dev->adj_list.lower); in netdev_adjacent_add_links()
9333 list_for_each_entry(iter, &dev->adj_list.upper, list) { in netdev_adjacent_del_links()
9334 if (!net_eq(net, dev_net(iter->dev))) in netdev_adjacent_del_links()
9336 netdev_adjacent_sysfs_del(iter->dev, dev->name, in netdev_adjacent_del_links()
9337 &iter->dev->adj_list.lower); in netdev_adjacent_del_links()
9338 netdev_adjacent_sysfs_del(dev, iter->dev->name, in netdev_adjacent_del_links()
9339 &dev->adj_list.upper); in netdev_adjacent_del_links()
9342 list_for_each_entry(iter, &dev->adj_list.lower, list) { in netdev_adjacent_del_links()
9343 if (!net_eq(net, dev_net(iter->dev))) in netdev_adjacent_del_links()
9345 netdev_adjacent_sysfs_del(iter->dev, dev->name, in netdev_adjacent_del_links()
9346 &iter->dev->adj_list.upper); in netdev_adjacent_del_links()
9347 netdev_adjacent_sysfs_del(dev, iter->dev->name, in netdev_adjacent_del_links()
9348 &dev->adj_list.lower); in netdev_adjacent_del_links()
9358 list_for_each_entry(iter, &dev->adj_list.upper, list) { in netdev_adjacent_rename_links()
9359 if (!net_eq(net, dev_net(iter->dev))) in netdev_adjacent_rename_links()
9361 netdev_adjacent_sysfs_del(iter->dev, oldname, in netdev_adjacent_rename_links()
9362 &iter->dev->adj_list.lower); in netdev_adjacent_rename_links()
9363 netdev_adjacent_sysfs_add(iter->dev, dev, in netdev_adjacent_rename_links()
9364 &iter->dev->adj_list.lower); in netdev_adjacent_rename_links()
9367 list_for_each_entry(iter, &dev->adj_list.lower, list) { in netdev_adjacent_rename_links()
9368 if (!net_eq(net, dev_net(iter->dev))) in netdev_adjacent_rename_links()
9370 netdev_adjacent_sysfs_del(iter->dev, oldname, in netdev_adjacent_rename_links()
9371 &iter->dev->adj_list.upper); in netdev_adjacent_rename_links()
9372 netdev_adjacent_sysfs_add(iter->dev, dev, in netdev_adjacent_rename_links()
9373 &iter->dev->adj_list.upper); in netdev_adjacent_rename_links()
9384 lower = __netdev_find_adj(lower_dev, &dev->adj_list.lower); in netdev_lower_dev_get_private()
9388 return lower->private; in netdev_lower_dev_get_private()
9394 * netdev_lower_state_changed - Dispatch event about lower device state change
9417 const struct net_device_ops *ops = dev->netdev_ops; in dev_change_rx_flags()
9419 if (ops->ndo_change_rx_flags) in dev_change_rx_flags()
9420 ops->ndo_change_rx_flags(dev, flags); in dev_change_rx_flags()
9425 unsigned int old_flags = dev->flags; in __dev_set_promiscuity()
9432 promiscuity = dev->promiscuity + inc; in __dev_set_promiscuity()
9440 return -EOVERFLOW; in __dev_set_promiscuity()
9446 WRITE_ONCE(dev->promiscuity, promiscuity); in __dev_set_promiscuity()
9448 WRITE_ONCE(dev->flags, flags); in __dev_set_promiscuity()
9450 dev->flags & IFF_PROMISC ? "entered" : "left"); in __dev_set_promiscuity()
9456 dev->name, (dev->flags & IFF_PROMISC), in __dev_set_promiscuity()
9481 unsigned int old_flags = dev->flags; in netif_set_promiscuity()
9487 if (dev->flags != old_flags) in netif_set_promiscuity()
9494 unsigned int old_flags = dev->flags, old_gflags = dev->gflags; in netif_set_allmulti()
9499 allmulti = dev->allmulti + inc; in netif_set_allmulti()
9507 return -EOVERFLOW; in netif_set_allmulti()
9513 WRITE_ONCE(dev->allmulti, allmulti); in netif_set_allmulti()
9515 WRITE_ONCE(dev->flags, flags); in netif_set_allmulti()
9517 dev->flags & IFF_ALLMULTI ? "entered" : "left"); in netif_set_allmulti()
9522 dev->gflags ^ old_gflags, 0, NULL); in netif_set_allmulti()
9535 const struct net_device_ops *ops = dev->netdev_ops; in __dev_set_rx_mode()
9538 if (!(dev->flags&IFF_UP)) in __dev_set_rx_mode()
9544 if (!(dev->priv_flags & IFF_UNICAST_FLT)) { in __dev_set_rx_mode()
9548 if (!netdev_uc_empty(dev) && !dev->uc_promisc) { in __dev_set_rx_mode()
9550 dev->uc_promisc = true; in __dev_set_rx_mode()
9551 } else if (netdev_uc_empty(dev) && dev->uc_promisc) { in __dev_set_rx_mode()
9552 __dev_set_promiscuity(dev, -1, false); in __dev_set_rx_mode()
9553 dev->uc_promisc = false; in __dev_set_rx_mode()
9557 if (ops->ndo_set_rx_mode) in __dev_set_rx_mode()
9558 ops->ndo_set_rx_mode(dev); in __dev_set_rx_mode()
9569 * netif_get_flags() - get flags reported to userspace
9578 flags = (READ_ONCE(dev->flags) & ~(IFF_PROMISC | in netif_get_flags()
9583 (READ_ONCE(dev->gflags) & (IFF_PROMISC | in netif_get_flags()
9602 unsigned int old_flags = dev->flags; in __dev_change_flags()
9611 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP | in __dev_change_flags()
9614 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC | in __dev_change_flags()
9640 if ((flags ^ dev->gflags) & IFF_PROMISC) { in __dev_change_flags()
9641 int inc = (flags & IFF_PROMISC) ? 1 : -1; in __dev_change_flags()
9642 old_flags = dev->flags; in __dev_change_flags()
9644 dev->gflags ^= IFF_PROMISC; in __dev_change_flags()
9647 if (dev->flags != old_flags) in __dev_change_flags()
9655 if ((flags ^ dev->gflags) & IFF_ALLMULTI) { in __dev_change_flags()
9656 int inc = (flags & IFF_ALLMULTI) ? 1 : -1; in __dev_change_flags()
9658 dev->gflags ^= IFF_ALLMULTI; in __dev_change_flags()
9669 unsigned int changes = dev->flags ^ old_flags; in __dev_notify_flags()
9675 if (dev->flags & IFF_UP) in __dev_notify_flags()
9681 if (dev->flags & IFF_UP && in __dev_notify_flags()
9698 unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags; in netif_change_flags()
9704 changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags); in netif_change_flags()
9711 const struct net_device_ops *ops = dev->netdev_ops; in __netif_set_mtu()
9713 if (ops->ndo_change_mtu) in __netif_set_mtu()
9714 return ops->ndo_change_mtu(dev, new_mtu); in __netif_set_mtu()
9716 /* Pairs with all the lockless reads of dev->mtu in the stack */ in __netif_set_mtu()
9717 WRITE_ONCE(dev->mtu, new_mtu); in __netif_set_mtu()
9726 if (new_mtu < 0 || new_mtu < dev->min_mtu) { in dev_validate_mtu()
9728 return -EINVAL; in dev_validate_mtu()
9731 if (dev->max_mtu > 0 && new_mtu > dev->max_mtu) { in dev_validate_mtu()
9733 return -EINVAL; in dev_validate_mtu()
9739 * netif_set_mtu_ext() - Change maximum transfer unit
9746 * Return: 0 on success, -errno on failure.
9755 if (new_mtu == dev->mtu) in netif_set_mtu_ext()
9763 return -ENODEV; in netif_set_mtu_ext()
9770 orig_mtu = dev->mtu; in netif_set_mtu_ext()
9797 net_err_ratelimited("%s: %s\n", dev->name, extack._msg); in netif_set_mtu()
9804 unsigned int orig_len = dev->tx_queue_len; in netif_change_tx_queue_len()
9808 return -ERANGE; in netif_change_tx_queue_len()
9811 WRITE_ONCE(dev->tx_queue_len, new_len); in netif_change_tx_queue_len()
9825 WRITE_ONCE(dev->tx_queue_len, orig_len); in netif_change_tx_queue_len()
9831 dev->group = new_group; in netif_set_group()
9835 * netif_pre_changeaddr_notify() - Call NETDEV_PRE_CHANGEADDR.
9840 * Return: 0 on success, -errno on failure.
9860 const struct net_device_ops *ops = dev->netdev_ops; in netif_set_mac_address()
9863 if (!ops->ndo_set_mac_address) in netif_set_mac_address()
9864 return -EOPNOTSUPP; in netif_set_mac_address()
9865 if (ss->ss_family != dev->type) in netif_set_mac_address()
9866 return -EINVAL; in netif_set_mac_address()
9868 return -ENODEV; in netif_set_mac_address()
9869 err = netif_pre_changeaddr_notify(dev, ss->__data, extack); in netif_set_mac_address()
9872 if (memcmp(dev->dev_addr, ss->__data, dev->addr_len)) { in netif_set_mac_address()
9873 err = ops->ndo_set_mac_address(dev, ss); in netif_set_mac_address()
9877 dev->addr_assign_type = NET_ADDR_SET; in netif_set_mac_address()
9879 add_device_randomness(dev->dev_addr, dev->addr_len); in netif_set_mac_address()
9888 size_t size = sizeof(sa->sa_data_min); in netif_get_mac_address()
9897 ret = -ENODEV; in netif_get_mac_address()
9900 if (!dev->addr_len) in netif_get_mac_address()
9901 memset(sa->sa_data, 0, size); in netif_get_mac_address()
9903 memcpy(sa->sa_data, dev->dev_addr, in netif_get_mac_address()
9904 min_t(size_t, size, dev->addr_len)); in netif_get_mac_address()
9905 sa->sa_family = dev->type; in netif_get_mac_address()
9916 const struct net_device_ops *ops = dev->netdev_ops; in netif_change_carrier()
9918 if (!ops->ndo_change_carrier) in netif_change_carrier()
9919 return -EOPNOTSUPP; in netif_change_carrier()
9921 return -ENODEV; in netif_change_carrier()
9922 return ops->ndo_change_carrier(dev, new_carrier); in netif_change_carrier()
9926 * dev_get_phys_port_id - Get device physical port ID
9935 const struct net_device_ops *ops = dev->netdev_ops; in dev_get_phys_port_id()
9937 if (!ops->ndo_get_phys_port_id) in dev_get_phys_port_id()
9938 return -EOPNOTSUPP; in dev_get_phys_port_id()
9939 return ops->ndo_get_phys_port_id(dev, ppid); in dev_get_phys_port_id()
9943 * dev_get_phys_port_name - Get device physical port name
9953 const struct net_device_ops *ops = dev->netdev_ops; in dev_get_phys_port_name()
9956 if (ops->ndo_get_phys_port_name) { in dev_get_phys_port_name()
9957 err = ops->ndo_get_phys_port_name(dev, name, len); in dev_get_phys_port_name()
9958 if (err != -EOPNOTSUPP) in dev_get_phys_port_name()
9965 * netif_get_port_parent_id() - Get the device's port parent identifier
9972 * Return: 0 on success, -errno on failure.
9977 const struct net_device_ops *ops = dev->netdev_ops; in netif_get_port_parent_id()
9983 if (ops->ndo_get_port_parent_id) { in netif_get_port_parent_id()
9984 err = ops->ndo_get_port_parent_id(dev, ppid); in netif_get_port_parent_id()
9985 if (err != -EOPNOTSUPP) in netif_get_port_parent_id()
9990 if (!recurse || err != -EOPNOTSUPP) in netif_get_port_parent_id()
10000 return -EOPNOTSUPP; in netif_get_port_parent_id()
10008 * netdev_port_same_parent_id - Indicate if two network devices have
10028 if (!dev->change_proto_down) in netif_change_proto_down()
10029 return -EOPNOTSUPP; in netif_change_proto_down()
10031 return -ENODEV; in netif_change_proto_down()
10036 WRITE_ONCE(dev->proto_down, proto_down); in netif_change_proto_down()
10041 * netdev_change_proto_down_reason_locked - proto down reason
10056 proto_down_reason = dev->proto_down_reason; in netdev_change_proto_down_reason_locked()
10064 WRITE_ONCE(dev->proto_down_reason, proto_down_reason); in netdev_change_proto_down_reason_locked()
10081 return dev->netdev_ops->ndo_bpf ? XDP_MODE_DRV : XDP_MODE_SKB; in dev_xdp_mode()
10091 return dev->netdev_ops->ndo_bpf; in dev_xdp_bpf_op()
10100 return dev->xdp_state[mode].link; in dev_xdp_link()
10109 return link->link.prog; in dev_xdp_prog()
10110 return dev->xdp_state[mode].prog; in dev_xdp_prog()
10119 if (dev->xdp_state[i].prog || dev->xdp_state[i].link) in dev_xdp_prog_count()
10131 if (dev->xdp_state[i].prog && in dev_xdp_sb_prog_count()
10132 !dev->xdp_state[i].prog->aux->xdp_has_frags) in dev_xdp_sb_prog_count()
10139 if (!dev->netdev_ops->ndo_bpf) in netif_xdp_propagate()
10140 return -EOPNOTSUPP; in netif_xdp_propagate()
10142 if (dev->cfg->hds_config == ETHTOOL_TCP_DATA_SPLIT_ENABLED && in netif_xdp_propagate()
10143 bpf->command == XDP_SETUP_PROG && in netif_xdp_propagate()
10144 bpf->prog && !bpf->prog->aux->xdp_has_frags) { in netif_xdp_propagate()
10145 NL_SET_ERR_MSG(bpf->extack, in netif_xdp_propagate()
10146 "unable to propagate XDP to device using tcp-data-split"); in netif_xdp_propagate()
10147 return -EBUSY; in netif_xdp_propagate()
10151 NL_SET_ERR_MSG(bpf->extack, "unable to propagate XDP to device using memory provider"); in netif_xdp_propagate()
10152 return -EBUSY; in netif_xdp_propagate()
10155 return dev->netdev_ops->ndo_bpf(dev, bpf); in netif_xdp_propagate()
10163 return prog ? prog->aux->id : 0; in dev_xdp_prog_id()
10169 dev->xdp_state[mode].link = link; in dev_xdp_set_link()
10170 dev->xdp_state[mode].prog = NULL; in dev_xdp_set_link()
10176 dev->xdp_state[mode].link = NULL; in dev_xdp_set_prog()
10177 dev->xdp_state[mode].prog = prog; in dev_xdp_set_prog()
10189 if (dev->cfg->hds_config == ETHTOOL_TCP_DATA_SPLIT_ENABLED && in dev_xdp_install()
10190 prog && !prog->aux->xdp_has_frags) { in dev_xdp_install()
10191 NL_SET_ERR_MSG(extack, "unable to install XDP to device using tcp-data-split"); in dev_xdp_install()
10192 return -EBUSY; in dev_xdp_install()
10197 return -EBUSY; in dev_xdp_install()
10247 /* auto-detach link from net device */ in dev_xdp_uninstall()
10250 link->dev = NULL; in dev_xdp_uninstall()
10274 return -EINVAL; in dev_xdp_attach()
10278 return -EINVAL; in dev_xdp_attach()
10283 return -EINVAL; in dev_xdp_attach()
10289 return -EINVAL; in dev_xdp_attach()
10294 return -EINVAL; in dev_xdp_attach()
10301 return -EBUSY; in dev_xdp_attach()
10308 return -EEXIST; in dev_xdp_attach()
10316 return -EBUSY; in dev_xdp_attach()
10320 return -EEXIST; in dev_xdp_attach()
10325 new_prog = link->link.prog; in dev_xdp_attach()
10334 return -EBUSY; in dev_xdp_attach()
10338 return -EEXIST; in dev_xdp_attach()
10340 if (!offload && bpf_prog_is_offloaded(new_prog->aux)) { in dev_xdp_attach()
10342 return -EINVAL; in dev_xdp_attach()
10344 if (bpf_prog_is_dev_bound(new_prog->aux) && !bpf_offload_dev_match(new_prog, dev)) { in dev_xdp_attach()
10346 return -EINVAL; in dev_xdp_attach()
10348 if (bpf_prog_is_dev_bound(new_prog->aux) && mode == XDP_MODE_SKB) { in dev_xdp_attach()
10349 NL_SET_ERR_MSG(extack, "Can't attach device-bound programs in generic mode"); in dev_xdp_attach()
10350 return -EINVAL; in dev_xdp_attach()
10352 if (new_prog->expected_attach_type == BPF_XDP_DEVMAP) { in dev_xdp_attach()
10354 return -EINVAL; in dev_xdp_attach()
10356 if (new_prog->expected_attach_type == BPF_XDP_CPUMAP) { in dev_xdp_attach()
10358 return -EINVAL; in dev_xdp_attach()
10367 return -EOPNOTSUPP; in dev_xdp_attach()
10389 return dev_xdp_attach(dev, extack, link, NULL, NULL, link->flags); in dev_xdp_attach_link()
10401 mode = dev_xdp_mode(dev, link->flags); in dev_xdp_detach_link()
10403 return -EINVAL; in dev_xdp_detach_link()
10417 /* if racing with net_device's tear down, xdp_link->dev might be in bpf_xdp_link_release()
10418 * already NULL, in which case link was already auto-detached in bpf_xdp_link_release()
10420 if (xdp_link->dev) { in bpf_xdp_link_release()
10421 netdev_lock_ops(xdp_link->dev); in bpf_xdp_link_release()
10422 WARN_ON(dev_xdp_detach_link(xdp_link->dev, NULL, xdp_link)); in bpf_xdp_link_release()
10423 netdev_unlock_ops(xdp_link->dev); in bpf_xdp_link_release()
10424 xdp_link->dev = NULL; in bpf_xdp_link_release()
10450 if (xdp_link->dev) in bpf_xdp_link_show_fdinfo()
10451 ifindex = xdp_link->dev->ifindex; in bpf_xdp_link_show_fdinfo()
10464 if (xdp_link->dev) in bpf_xdp_link_fill_link_info()
10465 ifindex = xdp_link->dev->ifindex; in bpf_xdp_link_fill_link_info()
10468 info->xdp.ifindex = ifindex; in bpf_xdp_link_fill_link_info()
10482 /* link might have been auto-released already, so fail */ in bpf_xdp_link_update()
10483 if (!xdp_link->dev) { in bpf_xdp_link_update()
10484 err = -ENOLINK; in bpf_xdp_link_update()
10488 if (old_prog && link->prog != old_prog) { in bpf_xdp_link_update()
10489 err = -EPERM; in bpf_xdp_link_update()
10492 old_prog = link->prog; in bpf_xdp_link_update()
10493 if (old_prog->type != new_prog->type || in bpf_xdp_link_update()
10494 old_prog->expected_attach_type != new_prog->expected_attach_type) { in bpf_xdp_link_update()
10495 err = -EINVAL; in bpf_xdp_link_update()
10500 /* no-op, don't disturb drivers */ in bpf_xdp_link_update()
10505 netdev_lock_ops(xdp_link->dev); in bpf_xdp_link_update()
10506 mode = dev_xdp_mode(xdp_link->dev, xdp_link->flags); in bpf_xdp_link_update()
10507 bpf_op = dev_xdp_bpf_op(xdp_link->dev, mode); in bpf_xdp_link_update()
10508 err = dev_xdp_install(xdp_link->dev, mode, bpf_op, NULL, in bpf_xdp_link_update()
10509 xdp_link->flags, new_prog); in bpf_xdp_link_update()
10510 netdev_unlock_ops(xdp_link->dev); in bpf_xdp_link_update()
10514 old_prog = xchg(&link->prog, new_prog); in bpf_xdp_link_update()
10533 struct net *net = current->nsproxy->net_ns; in bpf_xdp_link_attach()
10541 dev = dev_get_by_index(net, attr->link_create.target_ifindex); in bpf_xdp_link_attach()
10544 return -EINVAL; in bpf_xdp_link_attach()
10549 err = -ENOMEM; in bpf_xdp_link_attach()
10553 bpf_link_init(&link->link, BPF_LINK_TYPE_XDP, &bpf_xdp_link_lops, prog, in bpf_xdp_link_attach()
10554 attr->link_create.attach_type); in bpf_xdp_link_attach()
10555 link->dev = dev; in bpf_xdp_link_attach()
10556 link->flags = attr->link_create.flags; in bpf_xdp_link_attach()
10558 err = bpf_link_prime(&link->link, &link_primer); in bpf_xdp_link_attach()
10570 link->dev = NULL; in bpf_xdp_link_attach()
10590 * dev_change_xdp_fd - set or clear a bpf program for a device rx path
10595 * @flags: xdp-related flags
10641 for (i = dev->real_num_rx_queues - 1; i >= 0; i--) in dev_get_min_mp_channel_count()
10642 if (dev->_rx[i].mp_params.mp_priv) in dev_get_min_mp_channel_count()
10650 * dev_index_reserve() - allocate an ifindex in a namespace
10658 * Return: a suitable unique value for a new device interface number or -errno.
10666 return -EINVAL; in dev_index_reserve()
10670 err = xa_alloc_cyclic(&net->dev_by_index, &ifindex, NULL, in dev_index_reserve()
10671 xa_limit_31b, &net->ifindex, GFP_KERNEL); in dev_index_reserve()
10673 err = xa_insert(&net->dev_by_index, ifindex, NULL, GFP_KERNEL); in dev_index_reserve()
10683 WARN_ON(xa_erase(&net->dev_by_index, ifindex)); in dev_index_release()
10702 list_add_tail(&dev->todo_list, &net_todo_list); in net_set_todo()
10714 if (!(upper->wanted_features & feature) in netdev_sync_upper_features()
10717 &feature, upper->name); in netdev_sync_upper_features()
10734 if (!(features & feature) && (lower->features & feature)) { in netdev_sync_lower_features()
10736 &feature, lower->name); in netdev_sync_lower_features()
10738 lower->wanted_features &= ~feature; in netdev_sync_lower_features()
10741 if (unlikely(lower->features & feature)) in netdev_sync_lower_features()
10743 &feature, lower->name); in netdev_sync_lower_features()
10804 if ((features & dev->gso_partial_features) && in netdev_fix_features()
10808 features &= ~dev->gso_partial_features; in netdev_fix_features()
10823 /* LRO/HW-GRO features cannot be combined with RX-FCS */ in netdev_fix_features()
10826 netdev_dbg(dev, "Dropping LRO feature since RX-FCS is requested.\n"); in netdev_fix_features()
10831 netdev_dbg(dev, "Dropping HW-GRO feature since RX-FCS is requested.\n"); in netdev_fix_features()
10837 netdev_dbg(dev, "Dropping LRO feature since HW-GRO is requested.\n"); in netdev_fix_features()
10864 int err = -1; in __netdev_update_features()
10871 if (dev->netdev_ops->ndo_fix_features) in __netdev_update_features()
10872 features = dev->netdev_ops->ndo_fix_features(dev, features); in __netdev_update_features()
10881 if (dev->features == features) in __netdev_update_features()
10884 netdev_dbg(dev, "Features changed: %pNF -> %pNF\n", in __netdev_update_features()
10885 &dev->features, &features); in __netdev_update_features()
10887 if (dev->netdev_ops->ndo_set_features) in __netdev_update_features()
10888 err = dev->netdev_ops->ndo_set_features(dev, features); in __netdev_update_features()
10895 err, &features, &dev->features); in __netdev_update_features()
10896 /* return non-0 since some features might have changed and in __netdev_update_features()
10899 return -1; in __netdev_update_features()
10910 netdev_features_t diff = features ^ dev->features; in __netdev_update_features()
10916 * Thus we need to update dev->features in __netdev_update_features()
10922 dev->features = features; in __netdev_update_features()
10932 dev->features = features; in __netdev_update_features()
10941 dev->features = features; in __netdev_update_features()
10948 dev->features = features; in __netdev_update_features()
10955 * netdev_update_features - recalculate device features
10958 * Recalculate dev->features set and send notifications if it
10970 * netdev_change_features - recalculate device features
10973 * Recalculate dev->features set and send notifications even
10975 * netdev_update_features() if also dev->vlan_features might
10987 * netif_stacked_transfer_operstate - transfer operstate
10998 if (rootdev->operstate == IF_OPER_DORMANT) in netif_stacked_transfer_operstate()
11003 if (rootdev->operstate == IF_OPER_TESTING) in netif_stacked_transfer_operstate()
11017 unsigned int i, count = dev->num_rx_queues; in netif_alloc_rx_queues()
11026 return -ENOMEM; in netif_alloc_rx_queues()
11028 dev->_rx = rx; in netif_alloc_rx_queues()
11033 /* XDP RX-queue setup */ in netif_alloc_rx_queues()
11041 /* Rollback successful reg's and free other resources */ in netif_alloc_rx_queues()
11042 while (i--) in netif_alloc_rx_queues()
11044 kvfree(dev->_rx); in netif_alloc_rx_queues()
11045 dev->_rx = NULL; in netif_alloc_rx_queues()
11051 unsigned int i, count = dev->num_rx_queues; in netif_free_rx_queues()
11054 if (!dev->_rx) in netif_free_rx_queues()
11058 xdp_rxq_info_unreg(&dev->_rx[i].xdp_rxq); in netif_free_rx_queues()
11060 kvfree(dev->_rx); in netif_free_rx_queues()
11067 spin_lock_init(&queue->_xmit_lock); in netdev_init_one_queue()
11068 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type); in netdev_init_one_queue()
11069 queue->xmit_lock_owner = -1; in netdev_init_one_queue()
11071 queue->dev = dev; in netdev_init_one_queue()
11073 dql_init(&queue->dql, HZ); in netdev_init_one_queue()
11079 kvfree(dev->_tx); in netif_free_tx_queues()
11084 unsigned int count = dev->num_tx_queues; in netif_alloc_netdev_queues()
11089 return -EINVAL; in netif_alloc_netdev_queues()
11093 return -ENOMEM; in netif_alloc_netdev_queues()
11095 dev->_tx = tx; in netif_alloc_netdev_queues()
11098 spin_lock_init(&dev->tx_global_lock); in netif_alloc_netdev_queues()
11107 for (i = 0; i < dev->num_tx_queues; i++) { in netif_tx_stop_all_queues()
11123 if (dev->netdev_ops->ndo_get_peer_dev && in netdev_do_alloc_pcpu_stats()
11124 dev->pcpu_stat_type != NETDEV_PCPU_STAT_TSTATS) in netdev_do_alloc_pcpu_stats()
11125 return -EOPNOTSUPP; in netdev_do_alloc_pcpu_stats()
11127 switch (dev->pcpu_stat_type) { in netdev_do_alloc_pcpu_stats()
11131 v = dev->lstats = netdev_alloc_pcpu_stats(struct pcpu_lstats); in netdev_do_alloc_pcpu_stats()
11134 v = dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats); in netdev_do_alloc_pcpu_stats()
11137 v = dev->dstats = netdev_alloc_pcpu_stats(struct pcpu_dstats); in netdev_do_alloc_pcpu_stats()
11140 return -EINVAL; in netdev_do_alloc_pcpu_stats()
11143 return v ? 0 : -ENOMEM; in netdev_do_alloc_pcpu_stats()
11148 switch (dev->pcpu_stat_type) { in netdev_do_free_pcpu_stats()
11152 free_percpu(dev->lstats); in netdev_do_free_pcpu_stats()
11155 free_percpu(dev->tstats); in netdev_do_free_pcpu_stats()
11158 free_percpu(dev->dstats); in netdev_do_free_pcpu_stats()
11165 struct phy_link_topology *topo = dev->link_topo; in netdev_free_phy_link_topology()
11168 xa_destroy(&topo->phys); in netdev_free_phy_link_topology()
11170 dev->link_topo = NULL; in netdev_free_phy_link_topology()
11175 * register_netdevice() - register a network device
11180 * Callers must hold the rtnl lock - you may want register_netdev()
11196 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED); in register_netdevice()
11199 ret = ethtool_check_ops(dev->ethtool_ops); in register_netdevice()
11204 xa_init_flags(&dev->ethtool->rss_ctx, XA_FLAGS_ALLOC1); in register_netdevice()
11205 mutex_init(&dev->ethtool->rss_lock); in register_netdevice()
11207 spin_lock_init(&dev->addr_list_lock); in register_netdevice()
11210 ret = dev_get_valid_name(net, dev, dev->name); in register_netdevice()
11214 ret = -ENOMEM; in register_netdevice()
11215 dev->name_node = netdev_name_node_head_alloc(dev); in register_netdevice()
11216 if (!dev->name_node) in register_netdevice()
11220 if (dev->netdev_ops->ndo_init) { in register_netdevice()
11221 ret = dev->netdev_ops->ndo_init(dev); in register_netdevice()
11224 ret = -EIO; in register_netdevice()
11229 if (((dev->hw_features | dev->features) & in register_netdevice()
11231 (!dev->netdev_ops->ndo_vlan_rx_add_vid || in register_netdevice()
11232 !dev->netdev_ops->ndo_vlan_rx_kill_vid)) { in register_netdevice()
11234 ret = -EINVAL; in register_netdevice()
11242 ret = dev_index_reserve(net, dev->ifindex); in register_netdevice()
11245 dev->ifindex = ret; in register_netdevice()
11250 dev->hw_features |= (NETIF_F_SOFT_FEATURES | NETIF_F_SOFT_FEATURES_OFF); in register_netdevice()
11251 dev->features |= NETIF_F_SOFT_FEATURES; in register_netdevice()
11253 if (dev->udp_tunnel_nic_info) { in register_netdevice()
11254 dev->features |= NETIF_F_RX_UDP_TUNNEL_PORT; in register_netdevice()
11255 dev->hw_features |= NETIF_F_RX_UDP_TUNNEL_PORT; in register_netdevice()
11258 dev->wanted_features = dev->features & dev->hw_features; in register_netdevice()
11260 if (!(dev->flags & IFF_LOOPBACK)) in register_netdevice()
11261 dev->hw_features |= NETIF_F_NOCACHE_COPY; in register_netdevice()
11268 if (dev->hw_features & NETIF_F_TSO) in register_netdevice()
11269 dev->hw_features |= NETIF_F_TSO_MANGLEID; in register_netdevice()
11270 if (dev->vlan_features & NETIF_F_TSO) in register_netdevice()
11271 dev->vlan_features |= NETIF_F_TSO_MANGLEID; in register_netdevice()
11272 if (dev->mpls_features & NETIF_F_TSO) in register_netdevice()
11273 dev->mpls_features |= NETIF_F_TSO_MANGLEID; in register_netdevice()
11274 if (dev->hw_enc_features & NETIF_F_TSO) in register_netdevice()
11275 dev->hw_enc_features |= NETIF_F_TSO_MANGLEID; in register_netdevice()
11279 dev->vlan_features |= NETIF_F_HIGHDMA; in register_netdevice()
11283 dev->hw_enc_features |= NETIF_F_SG | NETIF_F_GSO_PARTIAL; in register_netdevice()
11287 dev->mpls_features |= NETIF_F_SG; in register_netdevice()
11297 WRITE_ONCE(dev->reg_state, ret ? NETREG_UNREGISTERED : NETREG_REGISTERED); in register_netdevice()
11312 set_bit(__LINK_STATE_PRESENT, &dev->state); in register_netdevice()
11318 netdev_hold(dev, &dev->dev_registered_tracker, GFP_KERNEL); in register_netdevice()
11321 add_device_randomness(dev->dev_addr, dev->addr_len); in register_netdevice()
11327 if (dev->addr_assign_type == NET_ADDR_PERM) in register_netdevice()
11328 memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len); in register_netdevice()
11337 dev->needs_free_netdev = false; in register_netdevice()
11345 if (!(dev->rtnl_link_ops && dev->rtnl_link_initializing)) in register_netdevice()
11354 dev_index_release(net, dev->ifindex); in register_netdevice()
11358 if (dev->netdev_ops->ndo_uninit) in register_netdevice()
11359 dev->netdev_ops->ndo_uninit(dev); in register_netdevice()
11360 if (dev->priv_destructor) in register_netdevice()
11361 dev->priv_destructor(dev); in register_netdevice()
11363 netdev_name_node_free(dev->name_node); in register_netdevice()
11377 dev->reg_state = NETREG_DUMMY; in init_dummy_netdev()
11380 set_bit(__LINK_STATE_PRESENT, &dev->state); in init_dummy_netdev()
11381 set_bit(__LINK_STATE_START, &dev->state); in init_dummy_netdev()
11390 * register_netdev - register a network device
11408 return -EINTR; in register_netdev()
11424 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i); in netdev_refcnt_read()
11427 return refcount_read(&dev->dev_refcnt); in netdev_refcnt_read()
11437 * netdev_wait_allrefs_any - wait until all references are gone.
11474 &dev->state)) { in netdev_wait_allrefs_any()
11507 dev->name, netdev_refcnt_read(dev)); in netdev_wait_allrefs_any()
11508 ref_tracker_dir_print(&dev->refcnt_tracker, 10); in netdev_wait_allrefs_any()
11553 list_del_init(&dev->unlink_list); in netdev_run_todo()
11554 dev->nested_level = dev->lower_level - 1; in netdev_run_todo()
11568 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) { in netdev_run_todo()
11570 list_del(&dev->todo_list); in netdev_run_todo()
11575 WRITE_ONCE(dev->reg_state, NETREG_UNREGISTERED); in netdev_run_todo()
11583 list_del(&dev->todo_list); in netdev_run_todo()
11587 BUG_ON(!list_empty(&dev->ptype_all)); in netdev_run_todo()
11588 BUG_ON(!list_empty(&dev->ptype_specific)); in netdev_run_todo()
11589 WARN_ON(rcu_access_pointer(dev->ip_ptr)); in netdev_run_todo()
11590 WARN_ON(rcu_access_pointer(dev->ip6_ptr)); in netdev_run_todo()
11593 if (dev->priv_destructor) in netdev_run_todo()
11594 dev->priv_destructor(dev); in netdev_run_todo()
11595 if (dev->needs_free_netdev) in netdev_run_todo()
11601 kobject_put(&dev->dev.kobj); in netdev_run_todo()
11607 /* Collate per-cpu network dstats statistics
11609 * Read per-cpu network statistics from dev->dstats and populate the related
11625 start = u64_stats_fetch_begin(&stats->syncp); in dev_fetch_dstats()
11626 rx_packets = u64_stats_read(&stats->rx_packets); in dev_fetch_dstats()
11627 rx_bytes = u64_stats_read(&stats->rx_bytes); in dev_fetch_dstats()
11628 rx_drops = u64_stats_read(&stats->rx_drops); in dev_fetch_dstats()
11629 tx_packets = u64_stats_read(&stats->tx_packets); in dev_fetch_dstats()
11630 tx_bytes = u64_stats_read(&stats->tx_bytes); in dev_fetch_dstats()
11631 tx_drops = u64_stats_read(&stats->tx_drops); in dev_fetch_dstats()
11632 } while (u64_stats_fetch_retry(&stats->syncp, start)); in dev_fetch_dstats()
11634 s->rx_packets += rx_packets; in dev_fetch_dstats()
11635 s->rx_bytes += rx_bytes; in dev_fetch_dstats()
11636 s->rx_dropped += rx_drops; in dev_fetch_dstats()
11637 s->tx_packets += tx_packets; in dev_fetch_dstats()
11638 s->tx_bytes += tx_bytes; in dev_fetch_dstats()
11639 s->tx_dropped += tx_drops; in dev_fetch_dstats()
11643 /* ndo_get_stats64 implementation for dtstats-based accounting.
11645 * Populate @s from dev->stats and dev->dstats. This is used internally by the
11646 * core for NETDEV_PCPU_STAT_DSTAT-type stats collection.
11651 netdev_stats_to_stats64(s, &dev->stats); in dev_get_dstats64()
11652 dev_fetch_dstats(s, dev->dstats); in dev_get_dstats64()
11672 sizeof(*stats64) - n * sizeof(u64)); in netdev_stats_to_stats64()
11684 if (p && cmpxchg(&dev->core_stats, NULL, p)) in netdev_core_stats_alloc()
11688 return READ_ONCE(dev->core_stats); in netdev_core_stats_alloc()
11694 struct net_device_core_stats __percpu *p = READ_ONCE(dev->core_stats); in netdev_core_stats_inc()
11709 * dev_get_stats - get network device statistics
11715 * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
11721 const struct net_device_ops *ops = dev->netdev_ops; in dev_get_stats()
11738 if (ops->ndo_get_stats64) { in dev_get_stats()
11740 ops->ndo_get_stats64(dev, storage); in dev_get_stats()
11741 } else if (ops->ndo_get_stats) { in dev_get_stats()
11742 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev)); in dev_get_stats()
11743 } else if (dev->pcpu_stat_type == NETDEV_PCPU_STAT_TSTATS) { in dev_get_stats()
11745 } else if (dev->pcpu_stat_type == NETDEV_PCPU_STAT_DSTATS) { in dev_get_stats()
11748 netdev_stats_to_stats64(storage, &dev->stats); in dev_get_stats()
11752 p = READ_ONCE(dev->core_stats); in dev_get_stats()
11759 storage->rx_dropped += READ_ONCE(core_stats->rx_dropped); in dev_get_stats()
11760 storage->tx_dropped += READ_ONCE(core_stats->tx_dropped); in dev_get_stats()
11761 storage->rx_nohandler += READ_ONCE(core_stats->rx_nohandler); in dev_get_stats()
11762 storage->rx_otherhost_dropped += READ_ONCE(core_stats->rx_otherhost_dropped); in dev_get_stats()
11770 * dev_fetch_sw_netstats - get per-cpu network device statistics
11772 * @netstats: per-cpu network stats to read from
11774 * Read per-cpu network statistics and populate the related fields in @s.
11788 start = u64_stats_fetch_begin(&stats->syncp); in dev_fetch_sw_netstats()
11789 rx_packets = u64_stats_read(&stats->rx_packets); in dev_fetch_sw_netstats()
11790 rx_bytes = u64_stats_read(&stats->rx_bytes); in dev_fetch_sw_netstats()
11791 tx_packets = u64_stats_read(&stats->tx_packets); in dev_fetch_sw_netstats()
11792 tx_bytes = u64_stats_read(&stats->tx_bytes); in dev_fetch_sw_netstats()
11793 } while (u64_stats_fetch_retry(&stats->syncp, start)); in dev_fetch_sw_netstats()
11795 s->rx_packets += rx_packets; in dev_fetch_sw_netstats()
11796 s->rx_bytes += rx_bytes; in dev_fetch_sw_netstats()
11797 s->tx_packets += tx_packets; in dev_fetch_sw_netstats()
11798 s->tx_bytes += tx_bytes; in dev_fetch_sw_netstats()
11804 * dev_get_tstats64 - ndo_get_stats64 implementation
11808 * Populate @s from dev->stats and dev->tstats. Can be used as
11813 netdev_stats_to_stats64(s, &dev->stats); in dev_get_tstats64()
11814 dev_fetch_sw_netstats(s, dev->tstats); in dev_get_tstats64()
11829 RCU_INIT_POINTER(queue->qdisc, &noop_qdisc); in dev_ingress_queue_create()
11830 RCU_INIT_POINTER(queue->qdisc_sleeping, &noop_qdisc); in dev_ingress_queue_create()
11831 rcu_assign_pointer(dev->ingress_queue, queue); in dev_ingress_queue_create()
11841 if (dev->ethtool_ops == &default_ethtool_ops) in netdev_set_default_ethtool_ops()
11842 dev->ethtool_ops = ops; in netdev_set_default_ethtool_ops()
11847 * netdev_sw_irq_coalesce_default_on() - enable SW IRQ coalescing by default
11855 WARN_ON(dev->reg_state == NETREG_REGISTERED); in netdev_sw_irq_coalesce_default_on()
11865 * alloc_netdev_mqs - allocate network device
11886 BUG_ON(strlen(name) >= sizeof(dev->name)); in alloc_netdev_mqs()
11905 dev->priv_len = sizeof_priv; in alloc_netdev_mqs()
11907 ref_tracker_dir_init(&dev->refcnt_tracker, 128, "netdev"); in alloc_netdev_mqs()
11909 dev->pcpu_refcnt = alloc_percpu(int); in alloc_netdev_mqs()
11910 if (!dev->pcpu_refcnt) in alloc_netdev_mqs()
11914 refcount_set(&dev->dev_refcnt, 1); in alloc_netdev_mqs()
11925 dev->gso_max_size = GSO_LEGACY_MAX_SIZE; in alloc_netdev_mqs()
11926 dev->xdp_zc_max_segs = 1; in alloc_netdev_mqs()
11927 dev->gso_max_segs = GSO_MAX_SEGS; in alloc_netdev_mqs()
11928 dev->gro_max_size = GRO_LEGACY_MAX_SIZE; in alloc_netdev_mqs()
11929 dev->gso_ipv4_max_size = GSO_LEGACY_MAX_SIZE; in alloc_netdev_mqs()
11930 dev->gro_ipv4_max_size = GRO_LEGACY_MAX_SIZE; in alloc_netdev_mqs()
11931 dev->tso_max_size = TSO_LEGACY_MAX_SIZE; in alloc_netdev_mqs()
11932 dev->tso_max_segs = TSO_MAX_SEGS; in alloc_netdev_mqs()
11933 dev->upper_level = 1; in alloc_netdev_mqs()
11934 dev->lower_level = 1; in alloc_netdev_mqs()
11936 dev->nested_level = 0; in alloc_netdev_mqs()
11937 INIT_LIST_HEAD(&dev->unlink_list); in alloc_netdev_mqs()
11940 INIT_LIST_HEAD(&dev->napi_list); in alloc_netdev_mqs()
11941 INIT_LIST_HEAD(&dev->unreg_list); in alloc_netdev_mqs()
11942 INIT_LIST_HEAD(&dev->close_list); in alloc_netdev_mqs()
11943 INIT_LIST_HEAD(&dev->link_watch_list); in alloc_netdev_mqs()
11944 INIT_LIST_HEAD(&dev->adj_list.upper); in alloc_netdev_mqs()
11945 INIT_LIST_HEAD(&dev->adj_list.lower); in alloc_netdev_mqs()
11946 INIT_LIST_HEAD(&dev->ptype_all); in alloc_netdev_mqs()
11947 INIT_LIST_HEAD(&dev->ptype_specific); in alloc_netdev_mqs()
11948 INIT_LIST_HEAD(&dev->net_notifier_list); in alloc_netdev_mqs()
11950 hash_init(dev->qdisc_hash); in alloc_netdev_mqs()
11953 mutex_init(&dev->lock); in alloc_netdev_mqs()
11955 dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM; in alloc_netdev_mqs()
11958 if (!dev->tx_queue_len) { in alloc_netdev_mqs()
11959 dev->priv_flags |= IFF_NO_QUEUE; in alloc_netdev_mqs()
11960 dev->tx_queue_len = DEFAULT_TX_QUEUE_LEN; in alloc_netdev_mqs()
11963 dev->num_tx_queues = txqs; in alloc_netdev_mqs()
11964 dev->real_num_tx_queues = txqs; in alloc_netdev_mqs()
11968 dev->num_rx_queues = rxqs; in alloc_netdev_mqs()
11969 dev->real_num_rx_queues = rxqs; in alloc_netdev_mqs()
11972 dev->ethtool = kzalloc(sizeof(*dev->ethtool), GFP_KERNEL_ACCOUNT); in alloc_netdev_mqs()
11973 if (!dev->ethtool) in alloc_netdev_mqs()
11976 dev->cfg = kzalloc(sizeof(*dev->cfg), GFP_KERNEL_ACCOUNT); in alloc_netdev_mqs()
11977 if (!dev->cfg) in alloc_netdev_mqs()
11979 dev->cfg_pending = dev->cfg; in alloc_netdev_mqs()
11981 dev->num_napi_configs = maxqs; in alloc_netdev_mqs()
11982 napi_config_sz = array_size(maxqs, sizeof(*dev->napi_config)); in alloc_netdev_mqs()
11983 dev->napi_config = kvzalloc(napi_config_sz, GFP_KERNEL_ACCOUNT); in alloc_netdev_mqs()
11984 if (!dev->napi_config) in alloc_netdev_mqs()
11987 strscpy(dev->name, name); in alloc_netdev_mqs()
11988 dev->name_assign_type = name_assign_type; in alloc_netdev_mqs()
11989 dev->group = INIT_NETDEV_GROUP; in alloc_netdev_mqs()
11990 if (!dev->ethtool_ops) in alloc_netdev_mqs()
11991 dev->ethtool_ops = &default_ethtool_ops; in alloc_netdev_mqs()
12003 free_percpu(dev->pcpu_refcnt); in alloc_netdev_mqs()
12013 if (!list_empty(&dev->napi_list)) { in netdev_napi_exit()
12017 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list) in netdev_napi_exit()
12024 kvfree(dev->napi_config); in netdev_napi_exit()
12028 * free_netdev - free network device
12044 if (dev->reg_state == NETREG_UNREGISTERING) { in free_netdev()
12046 dev->needs_free_netdev = true; in free_netdev()
12050 WARN_ON(dev->cfg != dev->cfg_pending); in free_netdev()
12051 kfree(dev->cfg); in free_netdev()
12052 kfree(dev->ethtool); in free_netdev()
12056 kfree(rcu_dereference_protected(dev->ingress_queue, 1)); in free_netdev()
12065 ref_tracker_dir_exit(&dev->refcnt_tracker); in free_netdev()
12067 free_percpu(dev->pcpu_refcnt); in free_netdev()
12068 dev->pcpu_refcnt = NULL; in free_netdev()
12070 free_percpu(dev->core_stats); in free_netdev()
12071 dev->core_stats = NULL; in free_netdev()
12072 free_percpu(dev->xdp_bulkq); in free_netdev()
12073 dev->xdp_bulkq = NULL; in free_netdev()
12077 mutex_destroy(&dev->lock); in free_netdev()
12080 if (dev->reg_state == NETREG_UNINITIALIZED || in free_netdev()
12081 dev->reg_state == NETREG_DUMMY) { in free_netdev()
12086 BUG_ON(dev->reg_state != NETREG_UNREGISTERED); in free_netdev()
12087 WRITE_ONCE(dev->reg_state, NETREG_RELEASED); in free_netdev()
12090 put_device(&dev->dev); in free_netdev()
12095 * alloc_netdev_dummy - Allocate and initialize a dummy net device.
12108 * synchronize_net - Synchronize with packet receive processing
12128 mutex_lock(&dev->ethtool->rss_lock); in netdev_rss_contexts_free()
12129 xa_for_each(&dev->ethtool->rss_ctx, context, ctx) { in netdev_rss_contexts_free()
12130 xa_erase(&dev->ethtool->rss_ctx, context); in netdev_rss_contexts_free()
12131 dev->ethtool_ops->remove_rxfh_context(dev, ctx, context, NULL); in netdev_rss_contexts_free()
12134 xa_destroy(&dev->ethtool->rss_ctx); in netdev_rss_contexts_free()
12135 mutex_unlock(&dev->ethtool->rss_lock); in netdev_rss_contexts_free()
12139 * unregister_netdevice_queue - remove device from the kernel
12156 list_move_tail(&dev->unreg_list, head); in unregister_netdevice_queue()
12160 list_add(&dev->unreg_list, &single); in unregister_netdevice_queue()
12170 for (i = 0; i < dev->real_num_rx_queues; i++) { in dev_memory_provider_uninstall()
12171 struct netdev_rx_queue *rxq = &dev->_rx[i]; in dev_memory_provider_uninstall()
12172 struct pp_memory_provider_params *p = &rxq->mp_params; in dev_memory_provider_uninstall()
12174 if (p->mp_ops && p->mp_ops->uninstall) in dev_memory_provider_uninstall()
12175 p->mp_ops->uninstall(rxq->mp_params.mp_priv, rxq); in dev_memory_provider_uninstall()
12189 list_del_init(&dev->close_list); in netif_close_many_and_unlock()
12226 if (dev->reg_state == NETREG_UNINITIALIZED) { in unregister_netdevice_many_notify()
12228 dev->name, dev); in unregister_netdevice_many_notify()
12231 list_del(&dev->unreg_list); in unregister_netdevice_many_notify()
12234 dev->dismantle = true; in unregister_netdevice_many_notify()
12235 BUG_ON(dev->reg_state != NETREG_REGISTERED); in unregister_netdevice_many_notify()
12240 if (!(dev->flags & IFF_UP)) in unregister_netdevice_many_notify()
12243 list_add_tail(&dev->close_list, &close_head); in unregister_netdevice_many_notify()
12252 list_add_tail(&dev->close_list, &close_head); in unregister_netdevice_many_notify()
12260 WRITE_ONCE(dev->reg_state, NETREG_UNREGISTERING); in unregister_netdevice_many_notify()
12286 if (!(dev->rtnl_link_ops && dev->rtnl_link_initializing)) in unregister_netdevice_many_notify()
12298 netdev_name_node_free(dev->name_node); in unregister_netdevice_many_notify()
12304 if (dev->netdev_ops->ndo_uninit) in unregister_netdevice_many_notify()
12305 dev->netdev_ops->ndo_uninit(dev); in unregister_netdevice_many_notify()
12307 mutex_destroy(&dev->ethtool->rss_lock); in unregister_netdevice_many_notify()
12329 netdev_put(dev, &dev->dev_registered_tracker); in unregister_netdevice_many_notify()
12339 * unregister_netdevice_many - unregister many devices
12352 * unregister_netdev - remove device from the kernel
12382 err = -EINVAL; in __dev_change_net_namespace()
12383 if (dev->netns_immutable) { in __dev_change_net_namespace()
12389 if (dev->reg_state != NETREG_REGISTERED) { in __dev_change_net_namespace()
12402 err = -EEXIST; in __dev_change_net_namespace()
12403 if (netdev_name_in_use(net, dev->name)) { in __dev_change_net_namespace()
12419 err = -EEXIST; in __dev_change_net_namespace()
12421 if (netdev_name_in_use(net, name_node->name)) { in __dev_change_net_namespace()
12424 name_node->name); in __dev_change_net_namespace()
12440 err = dev_index_reserve(net, dev->ifindex); in __dev_change_net_namespace()
12441 if (err == -EBUSY) in __dev_change_net_namespace()
12463 dev->moving_ns = true; in __dev_change_net_namespace()
12476 * Note that dev->reg_state stays at NETREG_REGISTERED. in __dev_change_net_namespace()
12494 /* Send a netdev-removed uevent to the old namespace */ in __dev_change_net_namespace()
12495 kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE); in __dev_change_net_namespace()
12498 /* Move per-net netdevice notifiers that are following the netdevice */ in __dev_change_net_namespace()
12505 dev->ifindex = new_ifindex; in __dev_change_net_namespace()
12510 strscpy(dev->name, new_name, IFNAMSIZ); in __dev_change_net_namespace()
12515 dev_set_uevent_suppress(&dev->dev, 1); in __dev_change_net_namespace()
12516 err = device_rename(&dev->dev, dev->name); in __dev_change_net_namespace()
12517 dev_set_uevent_suppress(&dev->dev, 0); in __dev_change_net_namespace()
12520 /* Send a netdev-add uevent to the new namespace */ in __dev_change_net_namespace()
12521 kobject_uevent(&dev->dev.kobj, KOBJ_ADD); in __dev_change_net_namespace()
12531 dev->moving_ns = false; in __dev_change_net_namespace()
12566 list_skb = &sd->completion_queue; in dev_cpu_dead()
12568 list_skb = &(*list_skb)->next; in dev_cpu_dead()
12570 *list_skb = oldsd->completion_queue; in dev_cpu_dead()
12571 oldsd->completion_queue = NULL; in dev_cpu_dead()
12574 if (oldsd->output_queue) { in dev_cpu_dead()
12575 *sd->output_queue_tailp = oldsd->output_queue; in dev_cpu_dead()
12576 sd->output_queue_tailp = oldsd->output_queue_tailp; in dev_cpu_dead()
12577 oldsd->output_queue = NULL; in dev_cpu_dead()
12578 oldsd->output_queue_tailp = &oldsd->output_queue; in dev_cpu_dead()
12584 while (!list_empty(&oldsd->poll_list)) { in dev_cpu_dead()
12585 struct napi_struct *napi = list_first_entry(&oldsd->poll_list, in dev_cpu_dead()
12589 list_del_init(&napi->poll_list); in dev_cpu_dead()
12590 if (napi->poll == process_backlog) in dev_cpu_dead()
12591 napi->state &= NAPIF_STATE_THREADED; in dev_cpu_dead()
12601 remsd = oldsd->rps_ipi_list; in dev_cpu_dead()
12602 oldsd->rps_ipi_list = NULL; in dev_cpu_dead()
12609 while ((skb = __skb_dequeue(&oldsd->process_queue))) { in dev_cpu_dead()
12613 while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) { in dev_cpu_dead()
12622 * netdev_increment_features - increment feature set by one
12668 INIT_LIST_HEAD(&net->dev_base_head); in netdev_init()
12670 net->dev_name_head = netdev_create_hash(); in netdev_init()
12671 if (net->dev_name_head == NULL) in netdev_init()
12674 net->dev_index_head = netdev_create_hash(); in netdev_init()
12675 if (net->dev_index_head == NULL) in netdev_init()
12678 xa_init_flags(&net->dev_by_index, XA_FLAGS_ALLOC1); in netdev_init()
12680 RAW_INIT_NOTIFIER_HEAD(&net->netdev_chain); in netdev_init()
12685 kfree(net->dev_name_head); in netdev_init()
12687 return -ENOMEM; in netdev_init()
12691 * netdev_drivername - network driver for the device
12702 parent = dev->dev.parent; in netdev_drivername()
12706 driver = parent->driver; in netdev_drivername()
12707 if (driver && driver->name) in netdev_drivername()
12708 return driver->name; in netdev_drivername()
12715 if (dev && dev->dev.parent) { in __netdev_printk()
12716 dev_printk_emit(level[1] - '0', in __netdev_printk()
12717 dev->dev.parent, in __netdev_printk()
12719 dev_driver_string(dev->dev.parent), in __netdev_printk()
12720 dev_name(dev->dev.parent), in __netdev_printk()
12775 kfree(net->dev_name_head); in netdev_exit()
12776 kfree(net->dev_index_head); in netdev_exit()
12777 xa_destroy(&net->dev_by_index); in netdev_exit()
12779 WARN_ON_ONCE(!list_empty(&net->dev_base_head)); in netdev_exit()
12801 if (dev->netns_immutable) in default_device_exit_net()
12805 if (dev->rtnl_link_ops && !dev->rtnl_link_ops->netns_refund) in default_device_exit_net()
12809 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex); in default_device_exit_net()
12814 if (netdev_name_in_use(&init_net, name_node->name)) in default_device_exit_net()
12820 __func__, dev->name, err); in default_device_exit_net()
12845 if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink) in default_device_exit_batch()
12846 dev->rtnl_link_ops->dellink(dev, &dev_kill_list); in default_device_exit_batch()
12861 /* TX read-mostly hotpath */ in net_dev_struct_check()
12886 /* TXRX read-mostly hotpath */ in net_dev_struct_check()
12895 /* RX read-mostly hotpath */ in net_dev_struct_check()
12937 return -ENOMEM; in net_page_pool_create()
12953 struct napi_struct *napi = &sd->backlog; in backlog_napi_should_run()
12955 return test_bit(NAPI_STATE_SCHED_THREADED, &napi->state); in backlog_napi_should_run()
12962 napi_threaded_poll_loop(&sd->backlog); in run_backlog_napi()
12968 struct napi_struct *napi = &sd->backlog; in backlog_napi_setup()
12970 napi->thread = this_cpu_read(backlog_napi); in backlog_napi_setup()
12971 set_bit(NAPI_STATE_THREADED, &napi->state); in backlog_napi_setup()
12988 int i, rc = -ENOMEM; in net_dev_init()
13017 skb_queue_head_init(&sd->input_pkt_queue); in net_dev_init()
13018 skb_queue_head_init(&sd->process_queue); in net_dev_init()
13020 skb_queue_head_init(&sd->xfrm_backlog); in net_dev_init()
13022 INIT_LIST_HEAD(&sd->poll_list); in net_dev_init()
13023 sd->output_queue_tailp = &sd->output_queue; in net_dev_init()
13025 INIT_CSD(&sd->csd, rps_trigger_softirq, sd); in net_dev_init()
13026 sd->cpu = i; in net_dev_init()
13028 INIT_CSD(&sd->defer_csd, trigger_rx_softirq, sd); in net_dev_init()
13030 gro_init(&sd->backlog.gro); in net_dev_init()
13031 sd->backlog.poll = process_backlog; in net_dev_init()
13032 sd->backlog.weight = weight_p; in net_dev_init()
13033 INIT_LIST_HEAD(&sd->backlog.poll_list); in net_dev_init()