Lines Matching +full:lower +full:- +full:case
1 // SPDX-License-Identifier: GPL-2.0-or-later
11 * Florian la Roche <rzsfl@rz.uni-sb.de>
19 * D.J. Barrow : Fixed bug where dev->refcnt gets set
64 * Pekka Riikonen : Netdev boot-time settings code
66 * indefinitely on dev->refcnt
67 * J Hadi Salim : - Backlog queue sampling
68 * - netif_rx() feedback
167 #include "net-sysfs.h"
187 unsigned int val = net->dev_base_seq + 1; in dev_base_seq_inc()
189 WRITE_ONCE(net->dev_base_seq, val ?: 1); in dev_base_seq_inc()
196 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)]; in dev_name_hash()
201 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)]; in dev_index_hash()
233 spin_lock_irqsave(&sd->input_pkt_queue.lock, *flags); in backlog_lock_irq_save()
241 spin_lock_irq(&sd->input_pkt_queue.lock); in backlog_lock_irq_disable()
250 spin_unlock_irqrestore(&sd->input_pkt_queue.lock, *flags); in backlog_unlock_irq_restore()
258 spin_unlock_irq(&sd->input_pkt_queue.lock); in backlog_unlock_irq_enable()
271 INIT_HLIST_NODE(&name_node->hlist); in netdev_name_node_alloc()
272 name_node->dev = dev; in netdev_name_node_alloc()
273 name_node->name = name; in netdev_name_node_alloc()
282 name_node = netdev_name_node_alloc(dev, dev->name); in netdev_name_node_head_alloc()
285 INIT_LIST_HEAD(&name_node->list); in netdev_name_node_head_alloc()
297 hlist_add_head_rcu(&name_node->hlist, in netdev_name_node_add()
298 dev_name_hash(net, name_node->name)); in netdev_name_node_add()
303 hlist_del_rcu(&name_node->hlist); in netdev_name_node_del()
313 if (!strcmp(name_node->name, name)) in netdev_name_node_lookup()
325 if (!strcmp(name_node->name, name)) in netdev_name_node_lookup_rcu()
343 return -EEXIST; in netdev_name_node_alt_create()
346 return -ENOMEM; in netdev_name_node_alt_create()
348 /* The node that holds dev->name acts as a head of per-device list. */ in netdev_name_node_alt_create()
349 list_add_tail_rcu(&name_node->list, &dev->name_node->list); in netdev_name_node_alt_create()
359 kfree(name_node->name); in netdev_name_node_alt_free()
366 list_del(&name_node->list); in __netdev_name_node_alt_destroy()
367 call_rcu(&name_node->rcu, netdev_name_node_alt_free); in __netdev_name_node_alt_destroy()
377 return -ENOENT; in netdev_name_node_alt_destroy()
381 if (name_node == dev->name_node || name_node->dev != dev) in netdev_name_node_alt_destroy()
382 return -EINVAL; in netdev_name_node_alt_destroy()
392 list_for_each_entry_safe(name_node, tmp, &dev->name_node->list, list) { in netdev_name_node_alt_flush()
393 list_del(&name_node->list); in netdev_name_node_alt_flush()
394 netdev_name_node_alt_free(&name_node->rcu); in netdev_name_node_alt_flush()
406 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head); in list_netdevice()
407 netdev_name_node_add(net, dev->name_node); in list_netdevice()
408 hlist_add_head_rcu(&dev->index_hlist, in list_netdevice()
409 dev_index_hash(net, dev->ifindex)); in list_netdevice()
415 WARN_ON(xa_store(&net->dev_by_index, dev->ifindex, dev, GFP_KERNEL)); in list_netdevice()
430 xa_erase(&net->dev_by_index, dev->ifindex); in unlist_netdevice()
436 list_del_rcu(&dev->dev_list); in unlist_netdevice()
437 netdev_name_node_del(dev->name_node); in unlist_netdevice()
438 hlist_del_rcu(&dev->index_hlist); in unlist_netdevice()
467 * register_netdevice() inits txq->_xmit_lock and sets lockdep class
468 * according to dev->type
515 return ARRAY_SIZE(netdev_lock_type) - 1; in netdev_lock_pos()
532 i = netdev_lock_pos(dev->type); in netdev_set_addr_lockdep_class()
533 lockdep_set_class_and_name(&dev->addr_list_lock, in netdev_set_addr_lockdep_class()
566 * is cloned and should be copied-on-write, so that it will
568 * --ANK (980803)
573 if (pt->type == htons(ETH_P_ALL)) in ptype_head()
574 return pt->dev ? &pt->dev->ptype_all : &net_hotdata.ptype_all; in ptype_head()
576 return pt->dev ? &pt->dev->ptype_specific : in ptype_head()
577 &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK]; in ptype_head()
581 * dev_add_pack - add packet handler
598 list_add_rcu(&pt->list, head); in dev_add_pack()
604 * __dev_remove_pack - remove packet handler
625 list_del_rcu(&pt->list); in __dev_remove_pack()
637 * dev_remove_pack - remove packet handler
664 * dev_get_iflink - get 'iflink' value of a interface
673 if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink) in dev_get_iflink()
674 return dev->netdev_ops->ndo_get_iflink(dev); in dev_get_iflink()
676 return READ_ONCE(dev->ifindex); in dev_get_iflink()
681 * dev_fill_metadata_dst - Retrieve tunnel egress information.
693 if (!dev->netdev_ops || !dev->netdev_ops->ndo_fill_metadata_dst) in dev_fill_metadata_dst()
694 return -EINVAL; in dev_fill_metadata_dst()
698 return -ENOMEM; in dev_fill_metadata_dst()
699 if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX))) in dev_fill_metadata_dst()
700 return -EINVAL; in dev_fill_metadata_dst()
702 return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb); in dev_fill_metadata_dst()
708 int k = stack->num_paths++; in dev_fwd_path()
713 return &stack->path[k]; in dev_fwd_path()
727 stack->num_paths = 0; in dev_fill_forward_path()
728 while (ctx.dev && ctx.dev->netdev_ops->ndo_fill_forward_path) { in dev_fill_forward_path()
732 return -1; in dev_fill_forward_path()
735 ret = ctx.dev->netdev_ops->ndo_fill_forward_path(&ctx, path); in dev_fill_forward_path()
737 return -1; in dev_fill_forward_path()
740 return -1; in dev_fill_forward_path()
748 return -1; in dev_fill_forward_path()
749 path->type = DEV_PATH_ETHERNET; in dev_fill_forward_path()
750 path->dev = ctx.dev; in dev_fill_forward_path()
763 if (napi->napi_id == napi_id) in napi_by_id()
779 if (WARN_ON_ONCE(!napi->dev)) in netdev_napi_by_id()
781 if (!net_eq(net, dev_net(napi->dev))) in netdev_napi_by_id()
788 * netdev_napi_by_id_lock() - find a device by NAPI ID and lock it
806 if (!napi || READ_ONCE(napi->dev->reg_state) != NETREG_REGISTERED) { in netdev_napi_by_id_lock()
811 dev = napi->dev; in netdev_napi_by_id_lock()
821 if (napi && napi->dev != dev) in netdev_napi_by_id_lock()
831 * __dev_get_by_name - find a device by its name
847 return node_name ? node_name->dev : NULL; in __dev_get_by_name()
852 * dev_get_by_name_rcu - find a device by its name
868 return node_name ? node_name->dev : NULL; in dev_get_by_name_rcu()
886 * netdev_get_by_name() - find a device by its name
911 * __dev_get_by_index - find a device by its ifindex
927 if (dev->ifindex == ifindex) in __dev_get_by_index()
935 * dev_get_by_index_rcu - find a device by its ifindex
951 if (dev->ifindex == ifindex) in dev_get_by_index_rcu()
972 * netdev_get_by_index() - find a device by its ifindex
996 * dev_get_by_napi_id - find a device by napi_id
1015 return napi ? napi->dev : NULL; in dev_get_by_napi_id()
1029 if (dev->reg_state > NETREG_REGISTERED) { in __netdev_put_lock()
1039 * netdev_get_by_index_lock() - find a device by its ifindex
1044 * with @ifindex is found it will be returned with netdev->lock held.
1069 dev = xa_find(&net->dev_by_index, index, ULONG_MAX, XA_PRESENT); in netdev_xa_find_lock()
1093 strscpy(name, dev->name, IFNAMSIZ); in netdev_copy_name()
1098 * netdev_get_name - get a netdevice name, knowing its ifindex.
1112 ret = -ENODEV; in netdev_get_name()
1127 return dev->type == type && !memcmp(dev->dev_addr, ha, dev->addr_len); in dev_addr_cmp()
1131 * dev_getbyhwaddr_rcu - find a device by its hardware address
1158 * dev_getbyhwaddr() - find a device by its hardware address
1189 if (dev->type == type) { in dev_getfirstbyhwtype()
1200 * __dev_get_by_flags - find any device with given flags
1219 if (((dev->flags ^ if_flags) & mask) == 0) { in __dev_get_by_flags()
1229 * dev_valid_name - check if name is okay for network device
1255 * __dev_alloc_name - allocate a name for a device
1260 * Passed a format string - eg "lt%d" it will try and find a suitable
1283 return -EINVAL; in __dev_alloc_name()
1288 return -ENOMEM; in __dev_alloc_name()
1294 if (!sscanf(name_node->name, name, &i)) in __dev_alloc_name()
1301 if (!strncmp(buf, name_node->name, IFNAMSIZ)) in __dev_alloc_name()
1304 if (!sscanf(d->name, name, &i)) in __dev_alloc_name()
1311 if (!strncmp(buf, d->name, IFNAMSIZ)) in __dev_alloc_name()
1318 return -ENFILE; in __dev_alloc_name()
1332 return -EINVAL; in dev_prep_valid_name()
1338 return -dup_errno; in dev_prep_valid_name()
1345 * dev_alloc_name - allocate a name for a device
1349 * Passed a format string - eg "lt%d" it will try and find a suitable
1360 return dev_prep_valid_name(dev_net(dev), dev, name, dev->name, ENFILE); in dev_alloc_name()
1369 ret = dev_prep_valid_name(net, dev, name, dev->name, EEXIST); in dev_get_valid_name()
1374 * dev_change_name - change name of a device
1391 if (!strncmp(newname, dev->name, IFNAMSIZ)) in dev_change_name()
1394 memcpy(oldname, dev->name, IFNAMSIZ); in dev_change_name()
1405 dev->flags & IFF_UP ? " (while UP)" : ""); in dev_change_name()
1407 old_assign_type = dev->name_assign_type; in dev_change_name()
1408 WRITE_ONCE(dev->name_assign_type, NET_NAME_RENAMED); in dev_change_name()
1411 ret = device_rename(&dev->dev, dev->name); in dev_change_name()
1414 memcpy(dev->name, oldname, IFNAMSIZ); in dev_change_name()
1416 WRITE_ONCE(dev->name_assign_type, old_assign_type); in dev_change_name()
1422 netdev_name_node_del(dev->name_node); in dev_change_name()
1426 netdev_name_node_add(net, dev->name_node); in dev_change_name()
1436 memcpy(dev->name, oldname, IFNAMSIZ); in dev_change_name()
1439 WRITE_ONCE(dev->name_assign_type, old_assign_type); in dev_change_name()
1452 * dev_set_alias - change ifalias of a device
1464 return -EINVAL; in dev_set_alias()
1469 return -ENOMEM; in dev_set_alias()
1471 memcpy(new_alias->ifalias, alias, len); in dev_set_alias()
1472 new_alias->ifalias[len] = 0; in dev_set_alias()
1476 new_alias = rcu_replace_pointer(dev->ifalias, new_alias, in dev_set_alias()
1488 * dev_get_alias - get ifalias of a device
1502 alias = rcu_dereference(dev->ifalias); in dev_get_alias()
1504 ret = snprintf(name, len, "%s", alias->ifalias); in dev_get_alias()
1511 * netdev_features_change - device changes features
1523 * netdev_state_change - device changes state
1532 if (dev->flags & IFF_UP) { in netdev_state_change()
1545 * __netdev_notify_peers - notify network peers about existence of @dev,
1564 * netdev_notify_peers - notify network peers about existence of @dev
1591 n->thread = kthread_run(napi_threaded_poll, n, "napi/%s-%d", in napi_kthread_create()
1592 n->dev->name, n->napi_id); in napi_kthread_create()
1593 if (IS_ERR(n->thread)) { in napi_kthread_create()
1594 err = PTR_ERR(n->thread); in napi_kthread_create()
1596 n->thread = NULL; in napi_kthread_create()
1604 const struct net_device_ops *ops = dev->netdev_ops; in __dev_open()
1611 /* may be detached because parent is runtime-suspended */ in __dev_open()
1612 if (dev->dev.parent) in __dev_open()
1613 pm_runtime_resume(dev->dev.parent); in __dev_open()
1615 return -ENODEV; in __dev_open()
1629 set_bit(__LINK_STATE_START, &dev->state); in __dev_open()
1631 if (ops->ndo_validate_addr) in __dev_open()
1632 ret = ops->ndo_validate_addr(dev); in __dev_open()
1634 if (!ret && ops->ndo_open) in __dev_open()
1635 ret = ops->ndo_open(dev); in __dev_open()
1640 clear_bit(__LINK_STATE_START, &dev->state); in __dev_open()
1645 add_device_randomness(dev->dev_addr, dev->addr_len); in __dev_open()
1652 * dev_open - prepare an interface for use.
1668 if (dev->flags & IFF_UP) in dev_open()
1695 clear_bit(__LINK_STATE_START, &dev->state); in __dev_close_many()
1700 * dev->stop() will invoke napi_disable() on all of it's in __dev_close_many()
1709 const struct net_device_ops *ops = dev->netdev_ops; in __dev_close_many()
1715 * We allow it to be called even after a DETACH hot-plug in __dev_close_many()
1718 if (ops->ndo_stop) in __dev_close_many()
1719 ops->ndo_stop(dev); in __dev_close_many()
1730 list_add(&dev->close_list, &single); in __dev_close()
1741 if (!(dev->flags & IFF_UP)) in dev_close_many()
1742 list_del_init(&dev->close_list); in dev_close_many()
1750 list_del_init(&dev->close_list); in dev_close_many()
1756 * dev_close - shutdown an interface.
1766 if (dev->flags & IFF_UP) { in dev_close()
1769 list_add(&dev->close_list, &single); in dev_close()
1778 * dev_disable_lro - disable Large Receive Offload on a device
1790 dev->wanted_features &= ~NETIF_F_LRO; in dev_disable_lro()
1793 if (unlikely(dev->features & NETIF_F_LRO)) in dev_disable_lro()
1802 * dev_disable_gro_hw - disable HW Generic Receive Offload on a device
1811 dev->wanted_features &= ~NETIF_F_GRO_HW; in dev_disable_gro_hw()
1814 if (unlikely(dev->features & NETIF_F_GRO_HW)) in dev_disable_gro_hw()
1821 case NETDEV_##val: \ in netdev_cmd_to_name()
1849 return nb->notifier_call(nb, val, &info); in call_netdevice_notifier()
1862 if (!(dev->flags & IFF_UP)) in call_netdevice_register_notifiers()
1872 if (dev->flags & IFF_UP) { in call_netdevice_unregister_notifiers()
1911 * register_netdevice_notifier - register a network notifier block
1966 * unregister_netdevice_notifier - unregister a network notifier block
1976 * the need for special case cleanup code.
2010 err = raw_notifier_chain_register(&net->netdev_chain, nb); in __register_netdevice_notifier_net()
2023 raw_notifier_chain_unregister(&net->netdev_chain, nb); in __register_netdevice_notifier_net()
2032 err = raw_notifier_chain_unregister(&net->netdev_chain, nb); in __unregister_netdevice_notifier_net()
2041 * register_netdevice_notifier_net - register a per-netns network notifier block
2068 * unregister_netdevice_notifier_net - unregister a per-netns
2080 * the need for special case cleanup code.
2123 if (!net_eq(net, rcu_access_pointer(dev->nd_net.net))) { in rtnl_net_dev_lock()
2149 nn->nb = nb; in register_netdevice_notifier_dev_net()
2150 list_add(&nn->list, &dev->net_notifier_list); in register_netdevice_notifier_dev_net()
2165 list_del(&nn->list); in unregister_netdevice_notifier_dev_net()
2178 list_for_each_entry(nn, &dev->net_notifier_list, list) in move_netdevice_notifiers_dev_net()
2179 __move_netdevice_notifier_net(dev_net(dev), net, nn->nb); in move_netdevice_notifiers_dev_net()
2183 * call_netdevice_notifiers_info - call all network notifier blocks
2194 struct net *net = dev_net(info->dev); in call_netdevice_notifiers_info()
2199 /* Run per-netns notifier block chain first, then run the global one. in call_netdevice_notifiers_info()
2201 * all notifier block registrators get converted to be per-netns. in call_netdevice_notifiers_info()
2203 ret = raw_notifier_call_chain(&net->netdev_chain, val, info); in call_netdevice_notifiers_info()
2210 * call_netdevice_notifiers_info_robust - call per-netns notifier blocks
2217 * Call all per-netns network notifier blocks, but not notifier blocks on
2227 struct net *net = dev_net(info->dev); in call_netdevice_notifiers_info_robust()
2231 return raw_notifier_call_chain_robust(&net->netdev_chain, in call_netdevice_notifiers_info_robust()
2248 * call_netdevice_notifiers - call all network notifier blocks
2263 * call_netdevice_notifiers_mtu - call all network notifier blocks
2363 if (atomic_try_cmpxchg(&netstamp_wanted, &wanted, wanted - 1)) in net_disable_timestamp()
2376 skb->tstamp = 0; in net_timestamp_set()
2377 skb->tstamp_type = SKB_CLOCK_REALTIME; in net_timestamp_set()
2379 skb->tstamp = ktime_get_real(); in net_timestamp_set()
2384 if ((COND) && !(SKB)->tstamp) \
2385 (SKB)->tstamp = ktime_get_real(); \
2400 skb->protocol = eth_type_trans(skb, dev); in __dev_forward_skb2()
2414 * dev_forward_skb - loopback an skb to another netif
2447 return -ENOMEM; in deliver_skb()
2448 refcount_inc(&skb->users); in deliver_skb()
2449 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev); in deliver_skb()
2461 if (ptype->type != type) in deliver_ptype_list_skb()
2472 if (!ptype->af_packet_priv || !skb->sk) in skb_loop_sk()
2475 if (ptype->id_match) in skb_loop_sk()
2476 return ptype->id_match(ptype, skb->sk); in skb_loop_sk()
2477 else if ((struct sock *)ptype->af_packet_priv == skb->sk) in skb_loop_sk()
2484 * dev_nit_active - return true if any network interface taps are in use
2491 !list_empty(&dev->ptype_all); in dev_nit_active()
2509 if (READ_ONCE(ptype->ignore_outgoing)) in dev_queue_xmit_nit()
2513 * they originated from - MvS (miquels@drinkel.ow.org) in dev_queue_xmit_nit()
2519 deliver_skb(skb2, pt_prev, skb->dev); in dev_queue_xmit_nit()
2531 /* skb->nh should be correctly in dev_queue_xmit_nit()
2537 if (skb_network_header(skb2) < skb2->data || in dev_queue_xmit_nit()
2540 ntohs(skb2->protocol), in dev_queue_xmit_nit()
2541 dev->name); in dev_queue_xmit_nit()
2545 skb2->transport_header = skb2->network_header; in dev_queue_xmit_nit()
2546 skb2->pkt_type = PACKET_OUTGOING; in dev_queue_xmit_nit()
2551 ptype_list = &dev->ptype_all; in dev_queue_xmit_nit()
2557 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev); in dev_queue_xmit_nit()
2566 * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
2573 * offset/count pair it will no longer be used. In the worst case TC0
2581 struct netdev_tc_txq *tc = &dev->tc_to_txq[0]; in netif_setup_tc()
2584 if (tc->offset + tc->count > txq) { in netif_setup_tc()
2586 dev->num_tc = 0; in netif_setup_tc()
2594 tc = &dev->tc_to_txq[q]; in netif_setup_tc()
2595 if (tc->offset + tc->count > txq) { in netif_setup_tc()
2605 if (dev->num_tc) { in netdev_txq_to_tc()
2606 struct netdev_tc_txq *tc = &dev->tc_to_txq[0]; in netdev_txq_to_tc()
2611 if ((txq - tc->offset) < tc->count) in netdev_txq_to_tc()
2615 /* didn't find it, just return -1 to indicate no match */ in netdev_txq_to_tc()
2616 return -1; in netdev_txq_to_tc()
2636 map = xmap_dereference(dev_maps->attr_map[tci]); in remove_xps_queue()
2640 for (pos = map->len; pos--;) { in remove_xps_queue()
2641 if (map->queues[pos] != index) in remove_xps_queue()
2644 if (map->len > 1) { in remove_xps_queue()
2645 map->queues[pos] = map->queues[--map->len]; in remove_xps_queue()
2650 RCU_INIT_POINTER(old_maps->attr_map[tci], NULL); in remove_xps_queue()
2651 RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL); in remove_xps_queue()
2663 int num_tc = dev_maps->num_tc; in remove_xps_queue_cpu()
2667 for (tci = cpu * num_tc; num_tc--; tci++) { in remove_xps_queue_cpu()
2670 for (i = count, j = offset; i--; j++) { in remove_xps_queue_cpu()
2689 RCU_INIT_POINTER(dev->xps_maps[type], NULL); in reset_xps_maps()
2701 dev_maps = xmap_dereference(dev->xps_maps[type]); in clean_xps_maps()
2705 for (j = 0; j < dev_maps->nr_ids; j++) in clean_xps_maps()
2711 for (i = offset + (count - 1); count--; i--) in clean_xps_maps()
2737 netif_reset_xps_queues(dev, index, dev->num_tx_queues - index); in netif_reset_xps_queues_gt()
2747 for (pos = 0; map && pos < map->len; pos++) { in expand_xps_map()
2748 if (map->queues[pos] != index) in expand_xps_map()
2753 /* Need to add tx-queue to this CPU's/rx-queue's existing map */ in expand_xps_map()
2755 if (pos < map->alloc_len) in expand_xps_map()
2758 alloc_len = map->alloc_len * 2; in expand_xps_map()
2761 /* Need to allocate new map to store tx-queue on this CPU's/rx-queue's in expand_xps_map()
2773 new_map->queues[i] = map->queues[i]; in expand_xps_map()
2774 new_map->alloc_len = alloc_len; in expand_xps_map()
2775 new_map->len = pos; in expand_xps_map()
2785 int i, tci = index * dev_maps->num_tc; in xps_copy_dev_maps()
2789 for (i = 0; i < dev_maps->num_tc; i++, tci++) { in xps_copy_dev_maps()
2794 map = xmap_dereference(dev_maps->attr_map[tci]); in xps_copy_dev_maps()
2795 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map); in xps_copy_dev_maps()
2806 int i, j, tci, numa_node_id = -2; in __netif_set_xps_queue()
2811 WARN_ON_ONCE(index >= dev->num_tx_queues); in __netif_set_xps_queue()
2813 if (dev->num_tc) { in __netif_set_xps_queue()
2815 num_tc = dev->num_tc; in __netif_set_xps_queue()
2817 return -EINVAL; in __netif_set_xps_queue()
2820 dev = netdev_get_tx_queue(dev, index)->sb_dev ? : dev; in __netif_set_xps_queue()
2824 return -EINVAL; in __netif_set_xps_queue()
2829 dev_maps = xmap_dereference(dev->xps_maps[type]); in __netif_set_xps_queue()
2831 maps_sz = XPS_RXQ_DEV_MAPS_SIZE(num_tc, dev->num_rx_queues); in __netif_set_xps_queue()
2832 nr_ids = dev->num_rx_queues; in __netif_set_xps_queue()
2844 * setting up now, as dev->num_tc or nr_ids could have been updated in in __netif_set_xps_queue()
2849 dev_maps->num_tc == num_tc && dev_maps->nr_ids == nr_ids) in __netif_set_xps_queue()
2853 for (j = -1; j = netif_attrmask_next_and(j, online_mask, mask, nr_ids), in __netif_set_xps_queue()
2859 return -ENOMEM; in __netif_set_xps_queue()
2862 new_dev_maps->nr_ids = nr_ids; in __netif_set_xps_queue()
2863 new_dev_maps->num_tc = num_tc; in __netif_set_xps_queue()
2867 map = copy ? xmap_dereference(dev_maps->attr_map[tci]) : NULL; in __netif_set_xps_queue()
2873 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map); in __netif_set_xps_queue()
2892 /* add tx-queue to CPU/rx-queue maps */ in __netif_set_xps_queue()
2897 map = xmap_dereference(new_dev_maps->attr_map[tci]); in __netif_set_xps_queue()
2898 while ((pos < map->len) && (map->queues[pos] != index)) in __netif_set_xps_queue()
2901 if (pos == map->len) in __netif_set_xps_queue()
2902 map->queues[map->len++] = index; in __netif_set_xps_queue()
2905 if (numa_node_id == -2) in __netif_set_xps_queue()
2908 numa_node_id = -1; in __netif_set_xps_queue()
2918 rcu_assign_pointer(dev->xps_maps[type], new_dev_maps); in __netif_set_xps_queue()
2924 for (j = 0; j < dev_maps->nr_ids; j++) { in __netif_set_xps_queue()
2925 for (i = num_tc, tci = j * dev_maps->num_tc; i--; tci++) { in __netif_set_xps_queue()
2926 map = xmap_dereference(dev_maps->attr_map[tci]); in __netif_set_xps_queue()
2931 new_map = xmap_dereference(new_dev_maps->attr_map[tci]); in __netif_set_xps_queue()
2936 RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL); in __netif_set_xps_queue()
2957 /* removes tx-queue from unused CPUs/rx-queues */ in __netif_set_xps_queue()
2958 for (j = 0; j < dev_maps->nr_ids; j++) { in __netif_set_xps_queue()
2959 tci = j * dev_maps->num_tc; in __netif_set_xps_queue()
2961 for (i = 0; i < dev_maps->num_tc; i++, tci++) { in __netif_set_xps_queue()
2963 netif_attr_test_mask(j, mask, dev_maps->nr_ids) && in __netif_set_xps_queue()
2964 netif_attr_test_online(j, online_mask, dev_maps->nr_ids)) in __netif_set_xps_queue()
2987 for (i = num_tc, tci = j * num_tc; i--; tci++) { in __netif_set_xps_queue()
2988 new_map = xmap_dereference(new_dev_maps->attr_map[tci]); in __netif_set_xps_queue()
2990 xmap_dereference(dev_maps->attr_map[tci]) : in __netif_set_xps_queue()
3000 return -ENOMEM; in __netif_set_xps_queue()
3020 struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues]; in netdev_unbind_all_sb_channels()
3023 while (txq-- != &dev->_tx[0]) { in netdev_unbind_all_sb_channels()
3024 if (txq->sb_dev) in netdev_unbind_all_sb_channels()
3025 netdev_unbind_sb_channel(dev, txq->sb_dev); in netdev_unbind_all_sb_channels()
3037 dev->num_tc = 0; in netdev_reset_tc()
3038 memset(dev->tc_to_txq, 0, sizeof(dev->tc_to_txq)); in netdev_reset_tc()
3039 memset(dev->prio_tc_map, 0, sizeof(dev->prio_tc_map)); in netdev_reset_tc()
3045 if (tc >= dev->num_tc) in netdev_set_tc_queue()
3046 return -EINVAL; in netdev_set_tc_queue()
3051 dev->tc_to_txq[tc].count = count; in netdev_set_tc_queue()
3052 dev->tc_to_txq[tc].offset = offset; in netdev_set_tc_queue()
3060 return -EINVAL; in netdev_set_num_tc()
3067 dev->num_tc = num_tc; in netdev_set_num_tc()
3075 struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues]; in netdev_unbind_sb_channel()
3080 memset(sb_dev->tc_to_txq, 0, sizeof(sb_dev->tc_to_txq)); in netdev_unbind_sb_channel()
3081 memset(sb_dev->prio_tc_map, 0, sizeof(sb_dev->prio_tc_map)); in netdev_unbind_sb_channel()
3083 while (txq-- != &dev->_tx[0]) { in netdev_unbind_sb_channel()
3084 if (txq->sb_dev == sb_dev) in netdev_unbind_sb_channel()
3085 txq->sb_dev = NULL; in netdev_unbind_sb_channel()
3095 if (sb_dev->num_tc >= 0 || tc >= dev->num_tc) in netdev_bind_sb_channel_queue()
3096 return -EINVAL; in netdev_bind_sb_channel_queue()
3099 if ((offset + count) > dev->real_num_tx_queues) in netdev_bind_sb_channel_queue()
3100 return -EINVAL; in netdev_bind_sb_channel_queue()
3103 sb_dev->tc_to_txq[tc].count = count; in netdev_bind_sb_channel_queue()
3104 sb_dev->tc_to_txq[tc].offset = offset; in netdev_bind_sb_channel_queue()
3109 while (count--) in netdev_bind_sb_channel_queue()
3110 netdev_get_tx_queue(dev, count + offset)->sb_dev = sb_dev; in netdev_bind_sb_channel_queue()
3120 return -ENODEV; in netdev_set_sb_channel()
3122 /* We allow channels 1 - 32767 to be used for subordinate channels. in netdev_set_sb_channel()
3128 return -EINVAL; in netdev_set_sb_channel()
3130 dev->num_tc = -channel; in netdev_set_sb_channel()
3145 disabling = txq < dev->real_num_tx_queues; in netif_set_real_num_tx_queues()
3147 if (txq < 1 || txq > dev->num_tx_queues) in netif_set_real_num_tx_queues()
3148 return -EINVAL; in netif_set_real_num_tx_queues()
3150 if (dev->reg_state == NETREG_REGISTERED || in netif_set_real_num_tx_queues()
3151 dev->reg_state == NETREG_UNREGISTERING) { in netif_set_real_num_tx_queues()
3154 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues, in netif_set_real_num_tx_queues()
3159 if (dev->num_tc) in netif_set_real_num_tx_queues()
3166 dev->real_num_tx_queues = txq; in netif_set_real_num_tx_queues()
3176 dev->real_num_tx_queues = txq; in netif_set_real_num_tx_queues()
3185 * netif_set_real_num_rx_queues - set actual number of RX queues used
3198 if (rxq < 1 || rxq > dev->num_rx_queues) in netif_set_real_num_rx_queues()
3199 return -EINVAL; in netif_set_real_num_rx_queues()
3201 if (dev->reg_state == NETREG_REGISTERED) { in netif_set_real_num_rx_queues()
3204 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues, in netif_set_real_num_rx_queues()
3210 dev->real_num_rx_queues = rxq; in netif_set_real_num_rx_queues()
3217 * netif_set_real_num_queues - set actual number of RX and TX queues used
3228 unsigned int old_rxq = dev->real_num_rx_queues; in netif_set_real_num_queues()
3231 if (txq < 1 || txq > dev->num_tx_queues || in netif_set_real_num_queues()
3232 rxq < 1 || rxq > dev->num_rx_queues) in netif_set_real_num_queues()
3233 return -EINVAL; in netif_set_real_num_queues()
3235 /* Start from increases, so the error path only does decreases - in netif_set_real_num_queues()
3238 if (rxq > dev->real_num_rx_queues) { in netif_set_real_num_queues()
3243 if (txq > dev->real_num_tx_queues) { in netif_set_real_num_queues()
3248 if (rxq < dev->real_num_rx_queues) in netif_set_real_num_queues()
3250 if (txq < dev->real_num_tx_queues) in netif_set_real_num_queues()
3261 * netif_set_tso_max_size() - set the max size of TSO frames supported
3263 * @size: max skb->len of a TSO frame
3265 * Set the limit on the size of TSO super-frames the device can handle.
3271 dev->tso_max_size = min(GSO_MAX_SIZE, size); in netif_set_tso_max_size()
3272 if (size < READ_ONCE(dev->gso_max_size)) in netif_set_tso_max_size()
3274 if (size < READ_ONCE(dev->gso_ipv4_max_size)) in netif_set_tso_max_size()
3280 * netif_set_tso_max_segs() - set the max number of segs supported for TSO
3285 * a single TSO super-frame.
3290 dev->tso_max_segs = segs; in netif_set_tso_max_segs()
3291 if (segs < READ_ONCE(dev->gso_max_segs)) in netif_set_tso_max_segs()
3297 * netif_inherit_tso_max() - copy all TSO limits from a lower device to an upper
3303 netif_set_tso_max_size(to, from->tso_max_size); in netif_inherit_tso_max()
3304 netif_set_tso_max_segs(to, from->tso_max_segs); in netif_inherit_tso_max()
3309 * netif_get_num_default_rss_queues - default number of RSS queues
3340 q->next_sched = NULL; in __netif_reschedule()
3341 *sd->output_queue_tailp = q; in __netif_reschedule()
3342 sd->output_queue_tailp = &q->next_sched; in __netif_reschedule()
3349 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state)) in __netif_schedule()
3360 return (struct dev_kfree_skb_cb *)skb->cb; in get_kfree_skb_cb()
3367 struct Qdisc *q = rcu_dereference(txq->qdisc); in netif_schedule_queue()
3377 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) { in netif_tx_wake_queue()
3381 q = rcu_dereference(dev_queue->qdisc); in netif_tx_wake_queue()
3395 if (likely(refcount_read(&skb->users) == 1)) { in dev_kfree_skb_irq_reason()
3397 refcount_set(&skb->users, 0); in dev_kfree_skb_irq_reason()
3398 } else if (likely(!refcount_dec_and_test(&skb->users))) { in dev_kfree_skb_irq_reason()
3401 get_kfree_skb_cb(skb)->reason = reason; in dev_kfree_skb_irq_reason()
3403 skb->next = __this_cpu_read(softnet_data.completion_queue); in dev_kfree_skb_irq_reason()
3421 * netif_device_detach - mark device as removed
3428 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) && in netif_device_detach()
3436 * netif_device_attach - mark device as attached
3443 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) && in netif_device_attach()
3461 u16 qcount = dev->real_num_tx_queues; in skb_tx_hash()
3463 if (dev->num_tc) { in skb_tx_hash()
3464 u8 tc = netdev_get_prio_tc_map(dev, skb->priority); in skb_tx_hash()
3466 qoffset = sb_dev->tc_to_txq[tc].offset; in skb_tx_hash()
3467 qcount = sb_dev->tc_to_txq[tc].count; in skb_tx_hash()
3470 sb_dev->name, qoffset, tc); in skb_tx_hash()
3472 qcount = dev->real_num_tx_queues; in skb_tx_hash()
3480 hash -= qoffset; in skb_tx_hash()
3482 hash -= qcount; in skb_tx_hash()
3492 struct net_device *dev = skb->dev; in skb_warn_bad_offload()
3499 if (dev->dev.parent) in skb_warn_bad_offload()
3500 name = dev_driver_string(dev->dev.parent); in skb_warn_bad_offload()
3506 name, dev ? &dev->features : &null_features, in skb_warn_bad_offload()
3507 skb->sk ? &skb->sk->sk_route_caps : &null_features); in skb_warn_bad_offload()
3519 if (skb->ip_summed == CHECKSUM_COMPLETE) in skb_checksum_help()
3524 return -EINVAL; in skb_checksum_help()
3528 return -EFAULT; in skb_checksum_help()
3541 ret = -EINVAL; in skb_checksum_help()
3548 csum = skb_checksum(skb, offset, skb->len - offset, 0); in skb_checksum_help()
3550 offset += skb->csum_offset; in skb_checksum_help()
3561 *(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0; in skb_checksum_help()
3563 skb->ip_summed = CHECKSUM_NONE; in skb_checksum_help()
3574 if (skb->ip_summed != CHECKSUM_PARTIAL) in skb_crc32c_csum_help()
3591 ret = -EINVAL; in skb_crc32c_csum_help()
3600 skb->len - start, ~(__u32)0, in skb_crc32c_csum_help()
3602 *(__le32 *)(skb->data + offset) = crc32c_csum; in skb_crc32c_csum_help()
3611 __be16 type = skb->protocol; in skb_network_protocol()
3620 eth = (struct ethhdr *)skb->data; in skb_network_protocol()
3621 type = eth->h_proto; in skb_network_protocol()
3650 if (!(dev->features & NETIF_F_HIGHDMA)) { in illegal_highdma()
3651 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { in illegal_highdma()
3652 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; in illegal_highdma()
3672 features &= skb->dev->mpls_features; in net_mpls_features()
3693 if (skb->ip_summed != CHECKSUM_NONE && in harmonize_features()
3697 if (illegal_highdma(skb->dev, skb)) in harmonize_features()
3722 u16 gso_segs = skb_shinfo(skb)->gso_segs; in gso_features_check()
3724 if (gso_segs > READ_ONCE(dev->gso_max_segs)) in gso_features_check()
3727 if (unlikely(skb->len >= netif_get_gso_max_size(dev, skb))) in gso_features_check()
3730 if (!skb_shinfo(skb)->gso_type) { in gso_features_check()
3741 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL)) in gso_features_check()
3742 features &= ~dev->gso_partial_features; in gso_features_check()
3747 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) { in gso_features_check()
3748 struct iphdr *iph = skb->encapsulation ? in gso_features_check()
3751 if (!(iph->frag_off & htons(IP_DF))) in gso_features_check()
3760 struct net_device *dev = skb->dev; in netif_skb_features()
3761 netdev_features_t features = dev->features; in netif_skb_features()
3770 if (skb->encapsulation) in netif_skb_features()
3771 features &= dev->hw_enc_features; in netif_skb_features()
3775 dev->vlan_features | in netif_skb_features()
3779 if (dev->netdev_ops->ndo_features_check) in netif_skb_features()
3780 features &= dev->netdev_ops->ndo_features_check(skb, dev, in netif_skb_features()
3798 len = skb->len; in xmit_one()
3813 struct sk_buff *next = skb->next; in dev_hard_start_xmit()
3818 skb->next = next; in dev_hard_start_xmit()
3838 !vlan_hw_offload_capable(features, skb->vlan_proto)) in validate_xmit_vlan()
3859 switch (skb->csum_offset) { in skb_csum_hwoffload_help()
3860 case offsetof(struct tcphdr, check): in skb_csum_hwoffload_help()
3861 case offsetof(struct udphdr, check): in skb_csum_hwoffload_help()
3906 if (skb->ip_summed == CHECKSUM_PARTIAL) { in validate_xmit_skb()
3907 if (skb->encapsulation) in validate_xmit_skb()
3934 next = skb->next; in validate_xmit_skb_list()
3937 /* in case skb won't be segmented, point to itself */ in validate_xmit_skb_list()
3938 skb->prev = skb; in validate_xmit_skb_list()
3947 tail->next = skb; in validate_xmit_skb_list()
3948 /* If skb was segmented, skb->prev points to in validate_xmit_skb_list()
3951 tail = skb->prev; in validate_xmit_skb_list()
3961 qdisc_skb_cb(skb)->pkt_len = skb->len; in qdisc_pkt_len_init()
3966 if (shinfo->gso_size && skb_transport_header_was_set(skb)) { in qdisc_pkt_len_init()
3967 u16 gso_segs = shinfo->gso_segs; in qdisc_pkt_len_init()
3974 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) { in qdisc_pkt_len_init()
3982 } else if (shinfo->gso_type & SKB_GSO_UDP_L4) { in qdisc_pkt_len_init()
3990 if (unlikely(shinfo->gso_type & SKB_GSO_DODGY)) { in qdisc_pkt_len_init()
3991 int payload = skb->len - hdr_len; in qdisc_pkt_len_init()
3996 gso_segs = DIV_ROUND_UP(payload, shinfo->gso_size); in qdisc_pkt_len_init()
3998 qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len; in qdisc_pkt_len_init()
4008 rc = q->enqueue(skb, q, to_free) & NET_XMIT_MASK; in dev_qdisc_enqueue()
4027 if (q->flags & TCQ_F_NOLOCK) { in __dev_xmit_skb()
4028 if (q->flags & TCQ_F_CAN_BYPASS && nolock_qdisc_is_empty(q) && in __dev_xmit_skb()
4031 * of q->seqlock to protect from racing with requeuing. in __dev_xmit_skb()
4060 if (unlikely(READ_ONCE(q->owner) == smp_processor_id())) { in __dev_xmit_skb()
4067 * This permits qdisc->running owner to get the lock more in __dev_xmit_skb()
4076 spin_lock(&q->busylock); in __dev_xmit_skb()
4079 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) { in __dev_xmit_skb()
4082 } else if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) && in __dev_xmit_skb()
4085 * This is a work-conserving queue; there are no old skbs in __dev_xmit_skb()
4086 * waiting to be sent out; and the qdisc is not running - in __dev_xmit_skb()
4094 spin_unlock(&q->busylock); in __dev_xmit_skb()
4103 WRITE_ONCE(q->owner, smp_processor_id()); in __dev_xmit_skb()
4105 WRITE_ONCE(q->owner, -1); in __dev_xmit_skb()
4108 spin_unlock(&q->busylock); in __dev_xmit_skb()
4120 spin_unlock(&q->busylock); in __dev_xmit_skb()
4131 if (skb->priority) in skb_update_prio()
4133 map = rcu_dereference_bh(skb->dev->priomap); in skb_update_prio()
4140 prioidx = sock_cgroup_prioidx(&sk->sk_cgrp_data); in skb_update_prio()
4142 if (prioidx < map->priomap_len) in skb_update_prio()
4143 skb->priority = map->priomap[prioidx]; in skb_update_prio()
4150 * dev_loopback_xmit - loop back @skb
4159 skb->pkt_type = PACKET_LOOPBACK; in dev_loopback_xmit()
4160 if (skb->ip_summed == CHECKSUM_NONE) in dev_loopback_xmit()
4161 skb->ip_summed = CHECKSUM_UNNECESSARY; in dev_loopback_xmit()
4193 return current->net_xmit.skip_txqueue; in netdev_xmit_txqueue_skipped()
4198 current->net_xmit.skip_txqueue = skip; in netdev_xmit_skip_txqueue()
4210 struct mini_Qdisc *miniq = rcu_dereference_bh(entry->miniq); in tc_run()
4220 /* Block-wise bypass */ in tc_run()
4221 if (tcf_block_bypass_sw(miniq->block)) in tc_run()
4224 tc_skb_cb(skb)->mru = 0; in tc_run()
4225 tc_skb_cb(skb)->post_ct = false; in tc_run()
4229 ret = tcf_classify(skb, miniq->block, miniq->filter_list, &res, false); in tc_run()
4232 case TC_ACT_SHOT: in tc_run()
4236 case TC_ACT_OK: in tc_run()
4237 case TC_ACT_RECLASSIFY: in tc_run()
4238 skb->tc_index = TC_H_MIN(res.classid); in tc_run()
4266 __skb_push(skb, skb->mac_len); in tcx_run()
4274 __skb_pull(skb, skb->mac_len); in tcx_run()
4282 struct bpf_mprog_entry *entry = rcu_dereference_bh(skb->dev->tcx_ingress); in sch_handle_ingress()
4296 qdisc_skb_cb(skb)->pkt_len = skb->len; in sch_handle_ingress()
4307 case TC_ACT_REDIRECT: in sch_handle_ingress()
4312 __skb_push(skb, skb->mac_len); in sch_handle_ingress()
4313 if (skb_do_redirect(skb) == -EAGAIN) { in sch_handle_ingress()
4314 __skb_pull(skb, skb->mac_len); in sch_handle_ingress()
4321 case TC_ACT_SHOT: in sch_handle_ingress()
4327 case TC_ACT_STOLEN: in sch_handle_ingress()
4328 case TC_ACT_QUEUED: in sch_handle_ingress()
4329 case TC_ACT_TRAP: in sch_handle_ingress()
4332 case TC_ACT_CONSUMED: in sch_handle_ingress()
4345 struct bpf_mprog_entry *entry = rcu_dereference_bh(dev->tcx_egress); in sch_handle_egress()
4355 /* qdisc_skb_cb(skb)->pkt_len & tcx_set_ingress() was in sch_handle_egress()
4366 case TC_ACT_REDIRECT: in sch_handle_egress()
4372 case TC_ACT_SHOT: in sch_handle_egress()
4378 case TC_ACT_STOLEN: in sch_handle_egress()
4379 case TC_ACT_QUEUED: in sch_handle_egress()
4380 case TC_ACT_TRAP: in sch_handle_egress()
4383 case TC_ACT_CONSUMED: in sch_handle_egress()
4411 int tc = netdev_get_prio_tc_map(dev, skb->priority); in __get_xps_queue_idx()
4413 int queue_index = -1; in __get_xps_queue_idx()
4415 if (tc >= dev_maps->num_tc || tci >= dev_maps->nr_ids) in __get_xps_queue_idx()
4418 tci *= dev_maps->num_tc; in __get_xps_queue_idx()
4421 map = rcu_dereference(dev_maps->attr_map[tci]); in __get_xps_queue_idx()
4423 if (map->len == 1) in __get_xps_queue_idx()
4424 queue_index = map->queues[0]; in __get_xps_queue_idx()
4426 queue_index = map->queues[reciprocal_scale( in __get_xps_queue_idx()
4427 skb_get_hash(skb), map->len)]; in __get_xps_queue_idx()
4428 if (unlikely(queue_index >= dev->real_num_tx_queues)) in __get_xps_queue_idx()
4429 queue_index = -1; in __get_xps_queue_idx()
4440 struct sock *sk = skb->sk; in get_xps_queue()
4441 int queue_index = -1; in get_xps_queue()
4444 return -1; in get_xps_queue()
4450 dev_maps = rcu_dereference(sb_dev->xps_maps[XPS_RXQS]); in get_xps_queue()
4461 dev_maps = rcu_dereference(sb_dev->xps_maps[XPS_CPUS]); in get_xps_queue()
4463 unsigned int tci = skb->sender_cpu - 1; in get_xps_queue()
4473 return -1; in get_xps_queue()
4487 struct sock *sk = skb->sk; in netdev_pick_tx()
4492 if (queue_index < 0 || skb->ooo_okay || in netdev_pick_tx()
4493 queue_index >= dev->real_num_tx_queues) { in netdev_pick_tx()
4501 rcu_access_pointer(sk->sk_dst_cache)) in netdev_pick_tx()
4518 u32 sender_cpu = skb->sender_cpu - 1; in netdev_core_pick_tx()
4521 skb->sender_cpu = raw_smp_processor_id() + 1; in netdev_core_pick_tx()
4524 if (dev->real_num_tx_queues != 1) { in netdev_core_pick_tx()
4525 const struct net_device_ops *ops = dev->netdev_ops; in netdev_core_pick_tx()
4527 if (ops->ndo_select_queue) in netdev_core_pick_tx()
4528 queue_index = ops->ndo_select_queue(dev, skb, sb_dev); in netdev_core_pick_tx()
4540 * __dev_queue_xmit() - transmit a buffer
4556 * * 0 - buffer successfully transmitted
4557 * * positive qdisc return code - NET_XMIT_DROP etc.
4558 * * negative errno - other errors
4562 struct net_device *dev = skb->dev; in __dev_queue_xmit()
4565 int rc = -ENOMEM; in __dev_queue_xmit()
4571 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_SCHED_TSTAMP)) in __dev_queue_xmit()
4572 __skb_tstamp_tx(skb, NULL, NULL, skb->sk, SCM_TSTAMP_SCHED); in __dev_queue_xmit()
4603 /* If device/qdisc don't need skb->dst, release it right now while in __dev_queue_xmit()
4606 if (dev->priv_flags & IFF_XMIT_DST_RELEASE) in __dev_queue_xmit()
4614 q = rcu_dereference_bh(txq->qdisc); in __dev_queue_xmit()
4617 if (q->enqueue) { in __dev_queue_xmit()
4622 /* The device has no queue. Common case for software devices: in __dev_queue_xmit()
4634 if (dev->flags & IFF_UP) { in __dev_queue_xmit()
4637 /* Other cpus might concurrently change txq->xmit_lock_owner in __dev_queue_xmit()
4638 * to -1 or to their cpu id, but not to our id. in __dev_queue_xmit()
4640 if (READ_ONCE(txq->xmit_lock_owner) != cpu) { in __dev_queue_xmit()
4661 dev->name); in __dev_queue_xmit()
4668 dev->name); in __dev_queue_xmit()
4672 rc = -ENETDOWN; in __dev_queue_xmit()
4686 struct net_device *dev = skb->dev; in __dev_direct_xmit()
4738 if (test_bit(NAPI_STATE_THREADED, &napi->state)) { in ____napi_schedule()
4742 * read on napi->thread. Only call in ____napi_schedule()
4745 thread = READ_ONCE(napi->thread); in ____napi_schedule()
4750 set_bit(NAPI_STATE_SCHED_THREADED, &napi->state); in ____napi_schedule()
4757 list_add_tail(&napi->poll_list, &sd->poll_list); in ____napi_schedule()
4758 WRITE_ONCE(napi->list_owner, smp_processor_id()); in ____napi_schedule()
4762 if (!sd->in_net_rx_action) in ____napi_schedule()
4788 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap || in set_rps_cpu()
4789 !(dev->features & NETIF_F_NTUPLE)) in set_rps_cpu()
4791 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu); in set_rps_cpu()
4795 rxqueue = dev->_rx + rxq_index; in set_rps_cpu()
4796 flow_table = rcu_dereference(rxqueue->rps_flow_table); in set_rps_cpu()
4799 flow_id = skb_get_hash(skb) & flow_table->mask; in set_rps_cpu()
4800 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb, in set_rps_cpu()
4805 rflow = &flow_table->flows[flow_id]; in set_rps_cpu()
4806 WRITE_ONCE(rflow->filter, rc); in set_rps_cpu()
4807 if (old_rflow->filter == rc) in set_rps_cpu()
4808 WRITE_ONCE(old_rflow->filter, RPS_NO_FILTER); in set_rps_cpu()
4812 rps_input_queue_tail_save(&rflow->last_qtail, head); in set_rps_cpu()
4815 WRITE_ONCE(rflow->cpu, next_cpu); in set_rps_cpu()
4828 struct netdev_rx_queue *rxqueue = dev->_rx; in get_rps_cpu()
4831 int cpu = -1; in get_rps_cpu()
4838 if (unlikely(index >= dev->real_num_rx_queues)) { in get_rps_cpu()
4839 WARN_ONCE(dev->real_num_rx_queues > 1, in get_rps_cpu()
4842 dev->name, index, dev->real_num_rx_queues); in get_rps_cpu()
4850 flow_table = rcu_dereference(rxqueue->rps_flow_table); in get_rps_cpu()
4851 map = rcu_dereference(rxqueue->rps_map); in get_rps_cpu()
4869 ident = READ_ONCE(sock_flow_table->ents[hash & sock_flow_table->mask]); in get_rps_cpu()
4878 rflow = &flow_table->flows[hash & flow_table->mask]; in get_rps_cpu()
4879 tcpu = rflow->cpu; in get_rps_cpu()
4883 * different from current CPU (one in the rx-queue flow in get_rps_cpu()
4885 * - Current CPU is unset (>= nr_cpu_ids). in get_rps_cpu()
4886 * - Current CPU is offline. in get_rps_cpu()
4887 * - The current CPU's queue tail has advanced beyond the in get_rps_cpu()
4894 ((int)(READ_ONCE(per_cpu(softnet_data, tcpu).input_queue_head) - in get_rps_cpu()
4895 rflow->last_qtail)) >= 0)) { in get_rps_cpu()
4910 tcpu = map->cpus[reciprocal_scale(hash, map->len)]; in get_rps_cpu()
4924 * rps_may_expire_flow - check whether an RFS hardware filter may be removed
4937 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index; in rps_may_expire_flow()
4944 flow_table = rcu_dereference(rxqueue->rps_flow_table); in rps_may_expire_flow()
4945 if (flow_table && flow_id <= flow_table->mask) { in rps_may_expire_flow()
4946 rflow = &flow_table->flows[flow_id]; in rps_may_expire_flow()
4947 cpu = READ_ONCE(rflow->cpu); in rps_may_expire_flow()
4948 if (READ_ONCE(rflow->filter) == filter_id && cpu < nr_cpu_ids && in rps_may_expire_flow()
4949 ((int)(READ_ONCE(per_cpu(softnet_data, cpu).input_queue_head) - in rps_may_expire_flow()
4950 READ_ONCE(rflow->last_qtail)) < in rps_may_expire_flow()
4951 (int)(10 * flow_table->mask))) in rps_may_expire_flow()
4966 ____napi_schedule(sd, &sd->backlog); in rps_trigger_softirq()
4967 sd->received_rps++; in rps_trigger_softirq()
4978 smp_store_release(&sd->defer_ipi_scheduled, 0); in trigger_rx_softirq()
4982 * After we queued a packet into sd->input_pkt_queue,
4985 * - If this is another cpu queue, link it to our rps_ipi_list,
4988 * - If this is our own queue, NAPI schedule our backlog.
4998 __napi_schedule_irqoff(&sd->backlog); in napi_schedule_rps()
5002 sd->rps_ipi_next = mysd->rps_ipi_list; in napi_schedule_rps()
5003 mysd->rps_ipi_list = sd; in napi_schedule_rps()
5008 if (!mysd->in_net_rx_action && !mysd->in_napi_threaded_poll) in napi_schedule_rps()
5013 __napi_schedule_irqoff(&mysd->backlog); in napi_schedule_rps()
5023 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) in kick_defer_list_purge()
5024 __napi_schedule_irqoff(&sd->backlog); in kick_defer_list_purge()
5028 } else if (!cmpxchg(&sd->defer_ipi_scheduled, 0, 1)) { in kick_defer_list_purge()
5029 smp_call_function_single_async(cpu, &sd->defer_csd); in kick_defer_list_purge()
5050 fl = rcu_dereference(sd->flow_limit); in skb_flow_limit()
5052 new_flow = skb_get_hash(skb) & (fl->num_buckets - 1); in skb_flow_limit()
5053 old_flow = fl->history[fl->history_head]; in skb_flow_limit()
5054 fl->history[fl->history_head] = new_flow; in skb_flow_limit()
5056 fl->history_head++; in skb_flow_limit()
5057 fl->history_head &= FLOW_LIMIT_HISTORY - 1; in skb_flow_limit()
5059 if (likely(fl->buckets[old_flow])) in skb_flow_limit()
5060 fl->buckets[old_flow]--; in skb_flow_limit()
5062 if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) { in skb_flow_limit()
5063 fl->count++; in skb_flow_limit()
5088 if (!netif_running(skb->dev)) in enqueue_to_backlog()
5094 qlen = skb_queue_len_lockless(&sd->input_pkt_queue); in enqueue_to_backlog()
5099 qlen = skb_queue_len(&sd->input_pkt_queue); in enqueue_to_backlog()
5106 &sd->backlog.state)) in enqueue_to_backlog()
5109 __skb_queue_tail(&sd->input_pkt_queue, skb); in enqueue_to_backlog()
5121 atomic_inc(&sd->dropped); in enqueue_to_backlog()
5123 dev_core_stats_rx_dropped_inc(skb->dev); in enqueue_to_backlog()
5130 struct net_device *dev = skb->dev; in netif_get_rxqueue()
5133 rxqueue = dev->_rx; in netif_get_rxqueue()
5138 if (unlikely(index >= dev->real_num_rx_queues)) { in netif_get_rxqueue()
5139 WARN_ONCE(dev->real_num_rx_queues > 1, in netif_get_rxqueue()
5142 dev->name, index, dev->real_num_rx_queues); in netif_get_rxqueue()
5166 mac_len = skb->data - skb_mac_header(skb); in bpf_prog_run_generic_xdp()
5167 hard_start = skb->data - skb_headroom(skb); in bpf_prog_run_generic_xdp()
5170 frame_sz = (void *)skb_end_pointer(skb) - hard_start; in bpf_prog_run_generic_xdp()
5174 xdp_init_buff(xdp, frame_sz, &rxqueue->xdp_rxq); in bpf_prog_run_generic_xdp()
5175 xdp_prepare_buff(xdp, hard_start, skb_headroom(skb) - mac_len, in bpf_prog_run_generic_xdp()
5178 skb_shinfo(skb)->xdp_frags_size = skb->data_len; in bpf_prog_run_generic_xdp()
5184 orig_data_end = xdp->data_end; in bpf_prog_run_generic_xdp()
5185 orig_data = xdp->data; in bpf_prog_run_generic_xdp()
5186 eth = (struct ethhdr *)xdp->data; in bpf_prog_run_generic_xdp()
5187 orig_host = ether_addr_equal_64bits(eth->h_dest, skb->dev->dev_addr); in bpf_prog_run_generic_xdp()
5188 orig_bcast = is_multicast_ether_addr_64bits(eth->h_dest); in bpf_prog_run_generic_xdp()
5189 orig_eth_type = eth->h_proto; in bpf_prog_run_generic_xdp()
5194 off = xdp->data - orig_data; in bpf_prog_run_generic_xdp()
5199 __skb_push(skb, -off); in bpf_prog_run_generic_xdp()
5201 skb->mac_header += off; in bpf_prog_run_generic_xdp()
5206 off = xdp->data_end - orig_data_end; in bpf_prog_run_generic_xdp()
5208 skb_set_tail_pointer(skb, xdp->data_end - xdp->data); in bpf_prog_run_generic_xdp()
5209 skb->len += off; /* positive on grow, negative on shrink */ in bpf_prog_run_generic_xdp()
5216 skb->data_len = skb_shinfo(skb)->xdp_frags_size; in bpf_prog_run_generic_xdp()
5218 skb->data_len = 0; in bpf_prog_run_generic_xdp()
5221 eth = (struct ethhdr *)xdp->data; in bpf_prog_run_generic_xdp()
5222 if ((orig_eth_type != eth->h_proto) || in bpf_prog_run_generic_xdp()
5223 (orig_host != ether_addr_equal_64bits(eth->h_dest, in bpf_prog_run_generic_xdp()
5224 skb->dev->dev_addr)) || in bpf_prog_run_generic_xdp()
5225 (orig_bcast != is_multicast_ether_addr_64bits(eth->h_dest))) { in bpf_prog_run_generic_xdp()
5227 skb->pkt_type = PACKET_HOST; in bpf_prog_run_generic_xdp()
5228 skb->protocol = eth_type_trans(skb, skb->dev); in bpf_prog_run_generic_xdp()
5239 case XDP_REDIRECT: in bpf_prog_run_generic_xdp()
5240 case XDP_TX: in bpf_prog_run_generic_xdp()
5243 case XDP_PASS: in bpf_prog_run_generic_xdp()
5244 metalen = xdp->data - xdp->data_meta; in bpf_prog_run_generic_xdp()
5262 /* In case we have to go down the path and also linearize, in netif_skb_check_for_xdp()
5265 hroom = XDP_PACKET_HEADROOM - skb_headroom(skb); in netif_skb_check_for_xdp()
5266 troom = skb->tail + skb->data_len - skb->end; in netif_skb_check_for_xdp()
5293 mac_len = skb->data - skb_mac_header(skb); in netif_receive_generic_xdp()
5306 case XDP_REDIRECT: in netif_receive_generic_xdp()
5307 case XDP_TX: in netif_receive_generic_xdp()
5308 case XDP_PASS: in netif_receive_generic_xdp()
5311 bpf_warn_invalid_xdp_action((*pskb)->dev, xdp_prog, act); in netif_receive_generic_xdp()
5313 case XDP_ABORTED: in netif_receive_generic_xdp()
5314 trace_xdp_exception((*pskb)->dev, xdp_prog, act); in netif_receive_generic_xdp()
5316 case XDP_DROP: in netif_receive_generic_xdp()
5326 * network taps in order to match in-driver-XDP behavior. This also means
5328 * and DDOS attacks will be more effective. In-driver-XDP use dedicated TX
5333 struct net_device *dev = skb->dev; in generic_xdp_tx()
5369 case XDP_REDIRECT: in do_xdp_generic()
5370 err = xdp_do_generic_redirect((*pskb)->dev, *pskb, in do_xdp_generic()
5375 case XDP_TX: in do_xdp_generic()
5407 cpu = get_rps_cpu(skb->dev, skb, &rflow); in netif_rx_internal()
5411 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail); in netif_rx_internal()
5425 * __netif_rx - Slightly optimized version of netif_rx
5446 * netif_rx - post buffer to the network code
5484 if (sd->completion_queue) { in net_tx_action()
5488 clist = sd->completion_queue; in net_tx_action()
5489 sd->completion_queue = NULL; in net_tx_action()
5495 clist = clist->next; in net_tx_action()
5497 WARN_ON(refcount_read(&skb->users)); in net_tx_action()
5498 if (likely(get_kfree_skb_cb(skb)->reason == SKB_CONSUMED)) in net_tx_action()
5502 get_kfree_skb_cb(skb)->reason, NULL); in net_tx_action()
5504 if (skb->fclone != SKB_FCLONE_UNAVAILABLE) in net_tx_action()
5508 get_kfree_skb_cb(skb)->reason); in net_tx_action()
5512 if (sd->output_queue) { in net_tx_action()
5516 head = sd->output_queue; in net_tx_action()
5517 sd->output_queue = NULL; in net_tx_action()
5518 sd->output_queue_tailp = &sd->output_queue; in net_tx_action()
5527 head = head->next_sched; in net_tx_action()
5529 /* We need to make sure head->next_sched is read in net_tx_action()
5534 if (!(q->flags & TCQ_F_NOLOCK)) { in net_tx_action()
5538 &q->state))) { in net_tx_action()
5547 clear_bit(__QDISC_STATE_SCHED, &q->state); in net_tx_action()
5551 clear_bit(__QDISC_STATE_SCHED, &q->state); in net_tx_action()
5571 * netdev_is_rx_handler_busy - check if receive handler is registered
5582 return dev && rtnl_dereference(dev->rx_handler); in netdev_is_rx_handler_busy()
5587 * netdev_rx_handler_register - register receive handler
5605 return -EBUSY; in netdev_rx_handler_register()
5607 if (dev->priv_flags & IFF_NO_RX_HANDLER) in netdev_rx_handler_register()
5608 return -EINVAL; in netdev_rx_handler_register()
5611 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data); in netdev_rx_handler_register()
5612 rcu_assign_pointer(dev->rx_handler, rx_handler); in netdev_rx_handler_register()
5619 * netdev_rx_handler_unregister - unregister receive handler
5630 RCU_INIT_POINTER(dev->rx_handler, NULL); in netdev_rx_handler_unregister()
5636 RCU_INIT_POINTER(dev->rx_handler_data, NULL); in netdev_rx_handler_unregister()
5646 switch (skb->protocol) { in skb_pfmemalloc_protocol()
5647 case htons(ETH_P_ARP): in skb_pfmemalloc_protocol()
5648 case htons(ETH_P_IP): in skb_pfmemalloc_protocol()
5649 case htons(ETH_P_IPV6): in skb_pfmemalloc_protocol()
5650 case htons(ETH_P_8021Q): in skb_pfmemalloc_protocol()
5651 case htons(ETH_P_8021AD): in skb_pfmemalloc_protocol()
5692 orig_dev = skb->dev; in __netif_receive_skb_core()
5708 skb->skb_iif = skb->dev->ifindex; in __netif_receive_skb_core()
5716 ret2 = do_xdp_generic(rcu_dereference(skb->dev->xdp_prog), in __netif_receive_skb_core()
5726 if (eth_type_vlan(skb->protocol)) { in __netif_receive_skb_core()
5744 list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) { in __netif_receive_skb_core()
5784 rx_handler = rcu_dereference(skb->dev->rx_handler); in __netif_receive_skb_core()
5791 case RX_HANDLER_CONSUMED: in __netif_receive_skb_core()
5794 case RX_HANDLER_ANOTHER: in __netif_receive_skb_core()
5796 case RX_HANDLER_EXACT: in __netif_receive_skb_core()
5799 case RX_HANDLER_PASS: in __netif_receive_skb_core()
5806 if (unlikely(skb_vlan_tag_present(skb)) && !netdev_uses_dsa(skb->dev)) { in __netif_receive_skb_core()
5812 skb->pkt_type = PACKET_OTHERHOST; in __netif_receive_skb_core()
5813 } else if (eth_type_vlan(skb->protocol)) { in __netif_receive_skb_core()
5837 * and set skb->priority like in vlan_do_receive() in __netif_receive_skb_core()
5843 type = skb->protocol; in __netif_receive_skb_core()
5853 &orig_dev->ptype_specific); in __netif_receive_skb_core()
5855 if (unlikely(skb->dev != orig_dev)) { in __netif_receive_skb_core()
5857 &skb->dev->ptype_specific); in __netif_receive_skb_core()
5867 dev_core_stats_rx_dropped_inc(skb->dev); in __netif_receive_skb_core()
5869 dev_core_stats_rx_nohandler_inc(skb->dev); in __netif_receive_skb_core()
5872 * me how you were going to use this. :-) in __netif_receive_skb_core()
5879 * then skb should also be non-NULL. in __netif_receive_skb_core()
5890 struct net_device *orig_dev = skb->dev; in __netif_receive_skb_one_core()
5896 ret = INDIRECT_CALL_INET(pt_prev->func, ipv6_rcv, ip_rcv, skb, in __netif_receive_skb_one_core()
5897 skb->dev, pt_prev, orig_dev); in __netif_receive_skb_one_core()
5902 * netif_receive_skb_core - special purpose version of netif_receive_skb
5938 if (pt_prev->list_func != NULL) in __netif_receive_skb_list_ptype()
5939 INDIRECT_CALL_INET(pt_prev->list_func, ipv6_list_rcv, in __netif_receive_skb_list_ptype()
5944 pt_prev->func(skb, skb->dev, pt_prev, orig_dev); in __netif_receive_skb_list_ptype()
5950 /* Fast-path assumptions: in __netif_receive_skb_list_core()
5951 * - There is no RX handler. in __netif_receive_skb_list_core()
5952 * - Only one packet_type matches. in __netif_receive_skb_list_core()
5953 * If either of these fails, we will end up doing some per-packet in __netif_receive_skb_list_core()
5954 * processing in-line, then handling the 'last ptype' for the whole in __netif_receive_skb_list_core()
5955 * sublist. This can't cause out-of-order delivery to any single ptype, in __netif_receive_skb_list_core()
5957 * other ptypes are handled per-packet. in __netif_receive_skb_list_core()
5967 struct net_device *orig_dev = skb->dev; in __netif_receive_skb_list_core()
5982 list_add_tail(&skb->list, &sublist); in __netif_receive_skb_list_core()
5998 * - be delivered to SOCK_MEMALLOC sockets only in __netif_receive_skb()
5999 * - stay away from userspace in __netif_receive_skb()
6000 * - have bounded memory usage in __netif_receive_skb()
6025 list_cut_before(&sublist, head, &skb->list); in __netif_receive_skb_list()
6046 struct bpf_prog *old = rtnl_dereference(dev->xdp_prog); in generic_xdp_install()
6047 struct bpf_prog *new = xdp->prog; in generic_xdp_install()
6050 switch (xdp->command) { in generic_xdp_install()
6051 case XDP_SETUP_PROG: in generic_xdp_install()
6052 rcu_assign_pointer(dev->xdp_prog, new); in generic_xdp_install()
6066 ret = -EINVAL; in generic_xdp_install()
6086 int cpu = get_rps_cpu(skb->dev, skb, &rflow); in netif_receive_skb_internal()
6089 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail); in netif_receive_skb_internal()
6110 list_add_tail(&skb->list, &sublist); in netif_receive_skb_list_internal()
6119 int cpu = get_rps_cpu(skb->dev, skb, &rflow); in netif_receive_skb_list_internal()
6124 enqueue_to_backlog(skb, cpu, &rflow->last_qtail); in netif_receive_skb_list_internal()
6134 * netif_receive_skb - process receive buffer from network
6162 * netif_receive_skb_list - process many receive buffers from network
6196 skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) { in flush_backlog()
6197 if (skb->dev->reg_state == NETREG_UNREGISTERING) { in flush_backlog()
6198 __skb_unlink(skb, &sd->input_pkt_queue); in flush_backlog()
6206 skb_queue_walk_safe(&sd->process_queue, skb, tmp) { in flush_backlog()
6207 if (skb->dev->reg_state == NETREG_UNREGISTERING) { in flush_backlog()
6208 __skb_unlink(skb, &sd->process_queue); in flush_backlog()
6228 do_flush = !skb_queue_empty(&sd->input_pkt_queue) || in flush_required()
6229 !skb_queue_empty_lockless(&sd->process_queue); in flush_required()
6236 * input_pkt_queue and process_queue even if the latter could end-up in flush_required()
6265 cpumask_clear(&ptr->flush_cpus); in flush_all_backlogs()
6271 INIT_WORK(&ptr->w[cpu], flush_backlog); in flush_all_backlogs()
6272 queue_work_on(cpu, system_highpri_wq, &ptr->w[cpu]); in flush_all_backlogs()
6273 __cpumask_set_cpu(cpu, &ptr->flush_cpus); in flush_all_backlogs()
6281 for_each_cpu(cpu, &ptr->flush_cpus) in flush_all_backlogs()
6282 flush_work(&ptr->w[cpu]); in flush_all_backlogs()
6296 struct softnet_data *next = remsd->rps_ipi_next; in net_rps_send_ipi()
6298 if (cpu_online(remsd->cpu)) in net_rps_send_ipi()
6299 smp_call_function_single_async(remsd->cpu, &remsd->csd); in net_rps_send_ipi()
6312 struct softnet_data *remsd = sd->rps_ipi_list; in net_rps_action_and_irq_enable()
6315 sd->rps_ipi_list = NULL; in net_rps_action_and_irq_enable()
6329 return !use_backlog_threads() && sd->rps_ipi_list; in sd_has_rps_ipi_waiting()
6349 napi->weight = READ_ONCE(net_hotdata.dev_rx_weight); in process_backlog()
6354 while ((skb = __skb_dequeue(&sd->process_queue))) { in process_backlog()
6369 if (skb_queue_empty(&sd->input_pkt_queue)) { in process_backlog()
6378 napi->state &= NAPIF_STATE_THREADED; in process_backlog()
6382 skb_queue_splice_tail_init(&sd->input_pkt_queue, in process_backlog()
6383 &sd->process_queue); in process_backlog()
6395 * __napi_schedule - schedule for receive
6412 * napi_schedule_prep - check if napi can be scheduled
6422 unsigned long new, val = READ_ONCE(n->state); in napi_schedule_prep()
6437 } while (!try_cmpxchg(&n->state, &val, new)); in napi_schedule_prep()
6444 * __napi_schedule_irqoff - schedule for receive
6451 * due to force-threaded interrupts and spinlock substitution.
6469 * just in case its running on a different cpu. in napi_complete_done()
6473 if (unlikely(n->state & (NAPIF_STATE_NPSVC | in napi_complete_done()
6478 if (n->gro_bitmask) in napi_complete_done()
6480 n->defer_hard_irqs_count = napi_get_defer_hard_irqs(n); in napi_complete_done()
6482 if (n->defer_hard_irqs_count > 0) { in napi_complete_done()
6483 n->defer_hard_irqs_count--; in napi_complete_done()
6488 if (n->gro_bitmask) { in napi_complete_done()
6498 if (unlikely(!list_empty(&n->poll_list))) { in napi_complete_done()
6499 /* If n->poll_list is not empty, we need to mask irqs */ in napi_complete_done()
6501 list_del_init(&n->poll_list); in napi_complete_done()
6504 WRITE_ONCE(n->list_owner, -1); in napi_complete_done()
6506 val = READ_ONCE(n->state); in napi_complete_done()
6515 * because we will call napi->poll() one more time. in napi_complete_done()
6520 } while (!try_cmpxchg(&n->state, &val, new)); in napi_complete_done()
6528 hrtimer_start(&n->timer, ns_to_ktime(timeout), in napi_complete_done()
6539 if (!READ_ONCE(sd->defer_list)) in skb_defer_free_flush()
6542 spin_lock(&sd->defer_lock); in skb_defer_free_flush()
6543 skb = sd->defer_list; in skb_defer_free_flush()
6544 sd->defer_list = NULL; in skb_defer_free_flush()
6545 sd->defer_count = 0; in skb_defer_free_flush()
6546 spin_unlock(&sd->defer_lock); in skb_defer_free_flush()
6549 next = skb->next; in skb_defer_free_flush()
6565 if (napi->gro_bitmask) { in __busy_poll_stop()
6573 clear_bit(NAPI_STATE_SCHED, &napi->state); in __busy_poll_stop()
6592 * Since we are about to call napi->poll() once more, we can safely in busy_poll_stop()
6598 clear_bit(NAPI_STATE_MISSED, &napi->state); in busy_poll_stop()
6599 clear_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state); in busy_poll_stop()
6605 napi->defer_hard_irqs_count = napi_get_defer_hard_irqs(napi); in busy_poll_stop()
6607 if (napi->defer_hard_irqs_count && timeout) { in busy_poll_stop()
6608 hrtimer_start(&napi->timer, ns_to_ktime(timeout), HRTIMER_MODE_REL_PINNED); in busy_poll_stop()
6613 /* All we really want here is to re-enable device interrupts. in busy_poll_stop()
6616 rc = napi->poll(napi, budget); in busy_poll_stop()
6617 /* We can't gro_normal_list() here, because napi->poll() might have in busy_poll_stop()
6618 * rearmed the napi (napi_complete_done()) in which case it could in busy_poll_stop()
6656 unsigned long val = READ_ONCE(napi->state); in __napi_busy_loop()
6659 * we avoid dirtying napi->state as much as we can. in __napi_busy_loop()
6664 set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state); in __napi_busy_loop()
6667 if (cmpxchg(&napi->state, val, in __napi_busy_loop()
6671 set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state); in __napi_busy_loop()
6675 napi_poll = napi->poll; in __napi_busy_loop()
6682 __NET_ADD_STATS(dev_net(napi->dev), in __napi_busy_loop()
6747 hrtimer_start(&napi->timer, ns_to_ktime(timeout), in napi_suspend_irqs()
6779 WRITE_ONCE(napi->napi_id, napi_id); in __napi_hash_add_with_id()
6780 hlist_add_head_rcu(&napi->napi_hash_node, in __napi_hash_add_with_id()
6781 &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]); in __napi_hash_add_with_id()
6799 if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state)) in napi_hash_add()
6824 hlist_del_init_rcu(&napi->napi_hash_node); in napi_hash_del()
6839 !test_and_set_bit(NAPI_STATE_SCHED, &napi->state)) { in napi_watchdog()
6840 clear_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state); in napi_watchdog()
6852 INIT_LIST_HEAD(&napi->gro_hash[i].list); in init_gro_hash()
6853 napi->gro_hash[i].count = 0; in init_gro_hash()
6855 napi->gro_bitmask = 0; in init_gro_hash()
6865 if (dev->threaded == threaded) in dev_set_threaded()
6869 list_for_each_entry(napi, &dev->napi_list, dev_list) { in dev_set_threaded()
6870 if (!napi->thread) { in dev_set_threaded()
6880 WRITE_ONCE(dev->threaded, threaded); in dev_set_threaded()
6889 * polled. In this case, the switch between threaded mode and in dev_set_threaded()
6893 list_for_each_entry(napi, &dev->napi_list, dev_list) in dev_set_threaded()
6894 assign_bit(NAPI_STATE_THREADED, &napi->state, threaded); in dev_set_threaded()
6901 * netif_queue_set_napi - Associate queue with the napi
6908 * registering the NAPI handler for the queue-vector and the queues have been
6917 if (WARN_ON_ONCE(napi && !napi->dev)) in netif_queue_set_napi()
6919 if (dev->reg_state >= NETREG_REGISTERED) in netif_queue_set_napi()
6923 case NETDEV_QUEUE_TYPE_RX: in netif_queue_set_napi()
6925 rxq->napi = napi; in netif_queue_set_napi()
6927 case NETDEV_QUEUE_TYPE_TX: in netif_queue_set_napi()
6929 txq->napi = napi; in netif_queue_set_napi()
6939 n->defer_hard_irqs = n->config->defer_hard_irqs; in napi_restore_config()
6940 n->gro_flush_timeout = n->config->gro_flush_timeout; in napi_restore_config()
6941 n->irq_suspend_timeout = n->config->irq_suspend_timeout; in napi_restore_config()
6945 if (n->config->napi_id) { in napi_restore_config()
6946 napi_hash_add_with_id(n, n->config->napi_id); in napi_restore_config()
6949 n->config->napi_id = n->napi_id; in napi_restore_config()
6955 n->config->defer_hard_irqs = n->defer_hard_irqs; in napi_save_config()
6956 n->config->gro_flush_timeout = n->gro_flush_timeout; in napi_save_config()
6957 n->config->irq_suspend_timeout = n->irq_suspend_timeout; in napi_save_config()
6972 if (napi->config && napi->config->napi_id) in netif_napi_dev_list_add()
6973 new_id = napi->config->napi_id; in netif_napi_dev_list_add()
6975 higher = &dev->napi_list; in netif_napi_dev_list_add()
6976 list_for_each_entry(pos, &dev->napi_list, dev_list) { in netif_napi_dev_list_add()
6977 if (pos->napi_id >= MIN_NAPI_ID) in netif_napi_dev_list_add()
6978 pos_id = pos->napi_id; in netif_napi_dev_list_add()
6979 else if (pos->config) in netif_napi_dev_list_add()
6980 pos_id = pos->config->napi_id; in netif_napi_dev_list_add()
6986 higher = &pos->dev_list; in netif_napi_dev_list_add()
6988 list_add_rcu(&napi->dev_list, higher); /* adds after higher */ in netif_napi_dev_list_add()
6992 * skb->head being backed by slab, not a page fragment.
6994 * ("net: avoid 32 x truesize under-estimation for tiny skbs")
7003 WARN_ON_ONCE(skb && skb->head_frag); in napi_get_frags_check()
7014 if (WARN_ON(test_and_set_bit(NAPI_STATE_LISTED, &napi->state))) in netif_napi_add_weight_locked()
7017 INIT_LIST_HEAD(&napi->poll_list); in netif_napi_add_weight_locked()
7018 INIT_HLIST_NODE(&napi->napi_hash_node); in netif_napi_add_weight_locked()
7019 hrtimer_init(&napi->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); in netif_napi_add_weight_locked()
7020 napi->timer.function = napi_watchdog; in netif_napi_add_weight_locked()
7022 napi->skb = NULL; in netif_napi_add_weight_locked()
7023 INIT_LIST_HEAD(&napi->rx_list); in netif_napi_add_weight_locked()
7024 napi->rx_count = 0; in netif_napi_add_weight_locked()
7025 napi->poll = poll; in netif_napi_add_weight_locked()
7029 napi->weight = weight; in netif_napi_add_weight_locked()
7030 napi->dev = dev; in netif_napi_add_weight_locked()
7032 napi->poll_owner = -1; in netif_napi_add_weight_locked()
7034 napi->list_owner = -1; in netif_napi_add_weight_locked()
7035 set_bit(NAPI_STATE_SCHED, &napi->state); in netif_napi_add_weight_locked()
7036 set_bit(NAPI_STATE_NPSVC, &napi->state); in netif_napi_add_weight_locked()
7039 /* default settings from sysfs are applied to all NAPIs. any per-NAPI in netif_napi_add_weight_locked()
7042 napi_set_defer_hard_irqs(napi, READ_ONCE(dev->napi_defer_hard_irqs)); in netif_napi_add_weight_locked()
7043 napi_set_gro_flush_timeout(napi, READ_ONCE(dev->gro_flush_timeout)); in netif_napi_add_weight_locked()
7046 /* Create kthread for this napi if dev->threaded is set. in netif_napi_add_weight_locked()
7047 * Clear dev->threaded if kthread creation failed so that in netif_napi_add_weight_locked()
7050 if (dev->threaded && napi_kthread_create(napi)) in netif_napi_add_weight_locked()
7051 dev->threaded = false; in netif_napi_add_weight_locked()
7052 netif_napi_set_irq_locked(napi, -1); in netif_napi_add_weight_locked()
7061 netdev_assert_locked(n->dev); in napi_disable_locked()
7063 set_bit(NAPI_STATE_DISABLE, &n->state); in napi_disable_locked()
7065 val = READ_ONCE(n->state); in napi_disable_locked()
7069 val = READ_ONCE(n->state); in napi_disable_locked()
7074 } while (!try_cmpxchg(&n->state, &val, new)); in napi_disable_locked()
7076 hrtimer_cancel(&n->timer); in napi_disable_locked()
7078 if (n->config) in napi_disable_locked()
7083 clear_bit(NAPI_STATE_DISABLE, &n->state); in napi_disable_locked()
7088 * napi_disable() - prevent NAPI from scheduling
7097 netdev_lock(n->dev); in napi_disable()
7099 netdev_unlock(n->dev); in napi_disable()
7105 unsigned long new, val = READ_ONCE(n->state); in napi_enable_locked()
7107 if (n->config) in napi_enable_locked()
7116 if (n->dev->threaded && n->thread) in napi_enable_locked()
7118 } while (!try_cmpxchg(&n->state, &val, new)); in napi_enable_locked()
7123 * napi_enable() - enable NAPI scheduling
7132 netdev_lock(n->dev); in napi_enable()
7134 netdev_unlock(n->dev); in napi_enable()
7145 list_for_each_entry_safe(skb, n, &napi->gro_hash[i].list, list) in flush_gro_hash()
7147 napi->gro_hash[i].count = 0; in flush_gro_hash()
7154 netdev_assert_locked(napi->dev); in __netif_napi_del_locked()
7156 if (!test_and_clear_bit(NAPI_STATE_LISTED, &napi->state)) in __netif_napi_del_locked()
7159 if (napi->config) { in __netif_napi_del_locked()
7160 napi->index = -1; in __netif_napi_del_locked()
7161 napi->config = NULL; in __netif_napi_del_locked()
7164 list_del_rcu(&napi->dev_list); in __netif_napi_del_locked()
7168 napi->gro_bitmask = 0; in __netif_napi_del_locked()
7170 if (napi->thread) { in __netif_napi_del_locked()
7171 kthread_stop(napi->thread); in __netif_napi_del_locked()
7172 napi->thread = NULL; in __netif_napi_del_locked()
7181 weight = n->weight; in __napi_poll()
7186 * actually make the ->poll() call. Therefore we avoid in __napi_poll()
7187 * accidentally calling ->poll() when NAPI is not scheduled. in __napi_poll()
7191 work = n->poll(n, weight); in __napi_poll()
7198 netdev_err_once(n->dev, "NAPI poll function %pS returned %d, exceeding its budget of %d.\n", in __napi_poll()
7199 n->poll, work, weight); in __napi_poll()
7207 * move the instance around on the list at-will. in __napi_poll()
7214 /* The NAPI context has more processing work, but busy-polling in __napi_poll()
7220 * that the NAPI is re-scheduled. in __napi_poll()
7227 if (n->gro_bitmask) { in __napi_poll()
7239 if (unlikely(!list_empty(&n->poll_list))) { in __napi_poll()
7241 n->dev ? n->dev->name : "backlog"); in __napi_poll()
7256 list_del_init(&n->poll_list); in napi_poll()
7263 list_add_tail(&n->poll_list, repoll); in napi_poll()
7280 if (test_bit(NAPI_STATE_SCHED_THREADED, &napi->state)) { in napi_thread_wait()
7281 WARN_ON(!list_empty(&napi->poll_list)); in napi_thread_wait()
7291 return -1; in napi_thread_wait()
7308 sd->in_napi_threaded_poll = true; in napi_threaded_poll_loop()
7314 sd->in_napi_threaded_poll = false; in napi_threaded_poll_loop()
7355 sd->in_net_rx_action = true; in net_rx_action()
7357 list_splice_init(&sd->poll_list, &list); in net_rx_action()
7367 sd->in_net_rx_action = false; in net_rx_action()
7371 * sd->in_net_rx_action was true. in net_rx_action()
7373 if (!list_empty(&sd->poll_list)) in net_rx_action()
7382 budget -= napi_poll(n, &repoll); in net_rx_action()
7390 sd->time_squeeze++; in net_rx_action()
7397 list_splice_tail_init(&sd->poll_list, &list); in net_rx_action()
7399 list_splice(&list, &sd->poll_list); in net_rx_action()
7400 if (!list_empty(&sd->poll_list)) in net_rx_action()
7403 sd->in_net_rx_action = false; in net_rx_action()
7436 if (adj->dev == adj_dev) in __netdev_find_adj()
7445 struct net_device *dev = (struct net_device *)priv->data; in ____netdev_has_upper_dev()
7451 * netdev_has_upper_dev - Check if device is linked to an upper device
7456 * in case it is. Note that this checks only immediate upper device,
7474 * netdev_has_upper_dev_all_rcu - Check if device is linked to an upper device
7479 * in case it is. Note that this checks the entire upper device chain.
7496 * netdev_has_any_upper_dev - Check if device is linked to some device
7499 * Find out if a device is linked to an upper device and return true in case
7506 return !list_empty(&dev->adj_list.upper); in netdev_has_any_upper_dev()
7511 * netdev_master_upper_dev_get - Get master upper device
7514 * Find a master upper device and return pointer to it or NULL in case
7523 if (list_empty(&dev->adj_list.upper)) in netdev_master_upper_dev_get()
7526 upper = list_first_entry(&dev->adj_list.upper, in netdev_master_upper_dev_get()
7528 if (likely(upper->master)) in netdev_master_upper_dev_get()
7529 return upper->dev; in netdev_master_upper_dev_get()
7540 if (list_empty(&dev->adj_list.upper)) in __netdev_master_upper_dev_get()
7543 upper = list_first_entry(&dev->adj_list.upper, in __netdev_master_upper_dev_get()
7545 if (likely(upper->master) && !upper->ignore) in __netdev_master_upper_dev_get()
7546 return upper->dev; in __netdev_master_upper_dev_get()
7551 * netdev_has_any_lower_dev - Check if device is linked to some device
7554 * Find out if a device is linked to a lower device and return true in case
7561 return !list_empty(&dev->adj_list.lower); in netdev_has_any_lower_dev()
7570 return adj->private; in netdev_adjacent_get_private()
7575 * netdev_upper_get_next_dev_rcu - Get the next dev from upper list
7589 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); in netdev_upper_get_next_dev_rcu()
7591 if (&upper->list == &dev->adj_list.upper) in netdev_upper_get_next_dev_rcu()
7594 *iter = &upper->list; in netdev_upper_get_next_dev_rcu()
7596 return upper->dev; in netdev_upper_get_next_dev_rcu()
7606 upper = list_entry((*iter)->next, struct netdev_adjacent, list); in __netdev_next_upper_dev()
7608 if (&upper->list == &dev->adj_list.upper) in __netdev_next_upper_dev()
7611 *iter = &upper->list; in __netdev_next_upper_dev()
7612 *ignore = upper->ignore; in __netdev_next_upper_dev()
7614 return upper->dev; in __netdev_next_upper_dev()
7624 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); in netdev_next_upper_dev_rcu()
7626 if (&upper->list == &dev->adj_list.upper) in netdev_next_upper_dev_rcu()
7629 *iter = &upper->list; in netdev_next_upper_dev_rcu()
7631 return upper->dev; in netdev_next_upper_dev_rcu()
7645 iter = &dev->adj_list.upper; in __netdev_walk_all_upper_dev()
7663 niter = &udev->adj_list.upper; in __netdev_walk_all_upper_dev()
7672 next = dev_stack[--cur]; in __netdev_walk_all_upper_dev()
7693 iter = &dev->adj_list.upper; in netdev_walk_all_upper_dev_rcu()
7709 niter = &udev->adj_list.upper; in netdev_walk_all_upper_dev_rcu()
7718 next = dev_stack[--cur]; in netdev_walk_all_upper_dev_rcu()
7745 * netdev_lower_get_next_private - Get the next ->private from the
7746 * lower neighbour list
7750 * Gets the next netdev_adjacent->private from the dev's lower neighbour
7752 * RTNL lock or its own locking that guarantees that the neighbour lower
7758 struct netdev_adjacent *lower; in netdev_lower_get_next_private() local
7760 lower = list_entry(*iter, struct netdev_adjacent, list); in netdev_lower_get_next_private()
7762 if (&lower->list == &dev->adj_list.lower) in netdev_lower_get_next_private()
7765 *iter = lower->list.next; in netdev_lower_get_next_private()
7767 return lower->private; in netdev_lower_get_next_private()
7772 * netdev_lower_get_next_private_rcu - Get the next ->private from the
7773 * lower neighbour list, RCU
7778 * Gets the next netdev_adjacent->private from the dev's lower neighbour
7784 struct netdev_adjacent *lower; in netdev_lower_get_next_private_rcu() local
7788 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); in netdev_lower_get_next_private_rcu()
7790 if (&lower->list == &dev->adj_list.lower) in netdev_lower_get_next_private_rcu()
7793 *iter = &lower->list; in netdev_lower_get_next_private_rcu()
7795 return lower->private; in netdev_lower_get_next_private_rcu()
7800 * netdev_lower_get_next - Get the next device from the lower neighbour
7805 * Gets the next netdev_adjacent from the dev's lower neighbour
7807 * its own locking that guarantees that the neighbour lower
7812 struct netdev_adjacent *lower; in netdev_lower_get_next() local
7814 lower = list_entry(*iter, struct netdev_adjacent, list); in netdev_lower_get_next()
7816 if (&lower->list == &dev->adj_list.lower) in netdev_lower_get_next()
7819 *iter = lower->list.next; in netdev_lower_get_next()
7821 return lower->dev; in netdev_lower_get_next()
7828 struct netdev_adjacent *lower; in netdev_next_lower_dev() local
7830 lower = list_entry((*iter)->next, struct netdev_adjacent, list); in netdev_next_lower_dev()
7832 if (&lower->list == &dev->adj_list.lower) in netdev_next_lower_dev()
7835 *iter = &lower->list; in netdev_next_lower_dev()
7837 return lower->dev; in netdev_next_lower_dev()
7844 struct netdev_adjacent *lower; in __netdev_next_lower_dev() local
7846 lower = list_entry((*iter)->next, struct netdev_adjacent, list); in __netdev_next_lower_dev()
7848 if (&lower->list == &dev->adj_list.lower) in __netdev_next_lower_dev()
7851 *iter = &lower->list; in __netdev_next_lower_dev()
7852 *ignore = lower->ignore; in __netdev_next_lower_dev()
7854 return lower->dev; in __netdev_next_lower_dev()
7867 iter = &dev->adj_list.lower; in netdev_walk_all_lower_dev()
7883 niter = &ldev->adj_list.lower; in netdev_walk_all_lower_dev()
7892 next = dev_stack[--cur]; in netdev_walk_all_lower_dev()
7915 iter = &dev->adj_list.lower; in __netdev_walk_all_lower_dev()
7933 niter = &ldev->adj_list.lower; in __netdev_walk_all_lower_dev()
7942 next = dev_stack[--cur]; in __netdev_walk_all_lower_dev()
7956 struct netdev_adjacent *lower; in netdev_next_lower_dev_rcu() local
7958 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); in netdev_next_lower_dev_rcu()
7959 if (&lower->list == &dev->adj_list.lower) in netdev_next_lower_dev_rcu()
7962 *iter = &lower->list; in netdev_next_lower_dev_rcu()
7964 return lower->dev; in netdev_next_lower_dev_rcu()
7975 for (iter = &dev->adj_list.upper, in __netdev_upper_depth()
7981 if (max_depth < udev->upper_level) in __netdev_upper_depth()
7982 max_depth = udev->upper_level; in __netdev_upper_depth()
7995 for (iter = &dev->adj_list.lower, in __netdev_lower_depth()
8001 if (max_depth < ldev->lower_level) in __netdev_lower_depth()
8002 max_depth = ldev->lower_level; in __netdev_lower_depth()
8011 dev->upper_level = __netdev_upper_depth(dev) + 1; in __netdev_update_upper_level()
8020 if (list_empty(&dev->unlink_list)) in net_unlink_todo()
8021 list_add_tail(&dev->unlink_list, &net_unlink_list); in net_unlink_todo()
8028 dev->lower_level = __netdev_lower_depth(dev) + 1; in __netdev_update_lower_level()
8034 if (priv->flags & NESTED_SYNC_IMM) in __netdev_update_lower_level()
8035 dev->nested_level = dev->lower_level - 1; in __netdev_update_lower_level()
8036 if (priv->flags & NESTED_SYNC_TODO) in __netdev_update_lower_level()
8052 iter = &dev->adj_list.lower; in netdev_walk_all_lower_dev_rcu()
8068 niter = &ldev->adj_list.lower; in netdev_walk_all_lower_dev_rcu()
8077 next = dev_stack[--cur]; in netdev_walk_all_lower_dev_rcu()
8090 * netdev_lower_get_first_private_rcu - Get the first ->private from the
8091 * lower neighbour list, RCU
8095 * Gets the first netdev_adjacent->private from the dev's lower neighbour
8100 struct netdev_adjacent *lower; in netdev_lower_get_first_private_rcu() local
8102 lower = list_first_or_null_rcu(&dev->adj_list.lower, in netdev_lower_get_first_private_rcu()
8104 if (lower) in netdev_lower_get_first_private_rcu()
8105 return lower->private; in netdev_lower_get_first_private_rcu()
8111 * netdev_master_upper_dev_get_rcu - Get master upper device
8114 * Find a master upper device and return pointer to it or NULL in case
8121 upper = list_first_or_null_rcu(&dev->adj_list.upper, in netdev_master_upper_dev_get_rcu()
8123 if (upper && likely(upper->master)) in netdev_master_upper_dev_get_rcu()
8124 return upper->dev; in netdev_master_upper_dev_get_rcu()
8135 sprintf(linkname, dev_list == &dev->adj_list.upper ? in netdev_adjacent_sysfs_add()
8136 "upper_%s" : "lower_%s", adj_dev->name); in netdev_adjacent_sysfs_add()
8137 return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj), in netdev_adjacent_sysfs_add()
8146 sprintf(linkname, dev_list == &dev->adj_list.upper ? in netdev_adjacent_sysfs_del()
8148 sysfs_remove_link(&(dev->dev.kobj), linkname); in netdev_adjacent_sysfs_del()
8155 return (dev_list == &dev->adj_list.upper || in netdev_adjacent_is_neigh_list()
8156 dev_list == &dev->adj_list.lower) && in netdev_adjacent_is_neigh_list()
8171 adj->ref_nr += 1; in __netdev_adjacent_dev_insert()
8172 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d\n", in __netdev_adjacent_dev_insert()
8173 dev->name, adj_dev->name, adj->ref_nr); in __netdev_adjacent_dev_insert()
8180 return -ENOMEM; in __netdev_adjacent_dev_insert()
8182 adj->dev = adj_dev; in __netdev_adjacent_dev_insert()
8183 adj->master = master; in __netdev_adjacent_dev_insert()
8184 adj->ref_nr = 1; in __netdev_adjacent_dev_insert()
8185 adj->private = private; in __netdev_adjacent_dev_insert()
8186 adj->ignore = false; in __netdev_adjacent_dev_insert()
8187 netdev_hold(adj_dev, &adj->dev_tracker, GFP_KERNEL); in __netdev_adjacent_dev_insert()
8189 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d; dev_hold on %s\n", in __netdev_adjacent_dev_insert()
8190 dev->name, adj_dev->name, adj->ref_nr, adj_dev->name); in __netdev_adjacent_dev_insert()
8200 ret = sysfs_create_link(&(dev->dev.kobj), in __netdev_adjacent_dev_insert()
8201 &(adj_dev->dev.kobj), "master"); in __netdev_adjacent_dev_insert()
8205 list_add_rcu(&adj->list, dev_list); in __netdev_adjacent_dev_insert()
8207 list_add_tail_rcu(&adj->list, dev_list); in __netdev_adjacent_dev_insert()
8214 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list); in __netdev_adjacent_dev_insert()
8216 netdev_put(adj_dev, &adj->dev_tracker); in __netdev_adjacent_dev_insert()
8230 dev->name, adj_dev->name, ref_nr); in __netdev_adjacent_dev_remove()
8236 dev->name, adj_dev->name); in __netdev_adjacent_dev_remove()
8241 if (adj->ref_nr > ref_nr) { in __netdev_adjacent_dev_remove()
8242 pr_debug("adjacency: %s to %s ref_nr - %d = %d\n", in __netdev_adjacent_dev_remove()
8243 dev->name, adj_dev->name, ref_nr, in __netdev_adjacent_dev_remove()
8244 adj->ref_nr - ref_nr); in __netdev_adjacent_dev_remove()
8245 adj->ref_nr -= ref_nr; in __netdev_adjacent_dev_remove()
8249 if (adj->master) in __netdev_adjacent_dev_remove()
8250 sysfs_remove_link(&(dev->dev.kobj), "master"); in __netdev_adjacent_dev_remove()
8253 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list); in __netdev_adjacent_dev_remove()
8255 list_del_rcu(&adj->list); in __netdev_adjacent_dev_remove()
8257 adj_dev->name, dev->name, adj_dev->name); in __netdev_adjacent_dev_remove()
8258 netdev_put(adj_dev, &adj->dev_tracker); in __netdev_adjacent_dev_remove()
8300 &dev->adj_list.upper, in __netdev_adjacent_dev_link_neighbour()
8301 &upper_dev->adj_list.lower, in __netdev_adjacent_dev_link_neighbour()
8309 &dev->adj_list.upper, in __netdev_adjacent_dev_unlink_neighbour()
8310 &upper_dev->adj_list.lower); in __netdev_adjacent_dev_unlink_neighbour()
8335 return -EBUSY; in __netdev_upper_dev_link()
8339 return -EBUSY; in __netdev_upper_dev_link()
8341 if ((dev->lower_level + upper_dev->upper_level) > MAX_NEST_DEV) in __netdev_upper_dev_link()
8342 return -EMLINK; in __netdev_upper_dev_link()
8346 return -EEXIST; in __netdev_upper_dev_link()
8350 return master_dev == upper_dev ? -EEXIST : -EBUSY; in __netdev_upper_dev_link()
8386 * netdev_upper_dev_link - Add a link to the upper device
8411 * netdev_master_upper_dev_link - Add a master link to the upper device
8418 * Adds a link to device which is upper to this one. In this case, only
8419 * one master upper device can be linked, although other non-master devices
8472 * netdev_upper_dev_unlink - Removes a link to upper device
8497 adj = __netdev_find_adj(lower_dev, &upper_dev->adj_list.lower); in __netdev_adjacent_dev_set()
8499 adj->ignore = val; in __netdev_adjacent_dev_set()
8501 adj = __netdev_find_adj(upper_dev, &lower_dev->adj_list.upper); in __netdev_adjacent_dev_set()
8503 adj->ignore = val; in __netdev_adjacent_dev_set()
8586 * netdev_bonding_info_change - Dispatch event about slave change
8618 dev->offload_xstats_l3 = kzalloc(sizeof(*dev->offload_xstats_l3), in netdev_offload_xstats_enable_l3()
8620 if (!dev->offload_xstats_l3) in netdev_offload_xstats_enable_l3()
8621 return -ENOMEM; in netdev_offload_xstats_enable_l3()
8633 kfree(dev->offload_xstats_l3); in netdev_offload_xstats_enable_l3()
8634 dev->offload_xstats_l3 = NULL; in netdev_offload_xstats_enable_l3()
8645 return -EALREADY; in netdev_offload_xstats_enable()
8648 case NETDEV_OFFLOAD_XSTATS_TYPE_L3: in netdev_offload_xstats_enable()
8653 return -EINVAL; in netdev_offload_xstats_enable()
8666 kfree(dev->offload_xstats_l3); in netdev_offload_xstats_disable_l3()
8667 dev->offload_xstats_l3 = NULL; in netdev_offload_xstats_disable_l3()
8676 return -EALREADY; in netdev_offload_xstats_disable()
8679 case NETDEV_OFFLOAD_XSTATS_TYPE_L3: in netdev_offload_xstats_disable()
8685 return -EINVAL; in netdev_offload_xstats_disable()
8699 case NETDEV_OFFLOAD_XSTATS_TYPE_L3: in netdev_offload_xstats_get_ptr()
8700 return dev->offload_xstats_l3; in netdev_offload_xstats_get_ptr()
8728 dest->rx_packets += src->rx_packets; in netdev_hw_stats64_add()
8729 dest->tx_packets += src->tx_packets; in netdev_hw_stats64_add()
8730 dest->rx_bytes += src->rx_bytes; in netdev_hw_stats64_add()
8731 dest->tx_bytes += src->tx_bytes; in netdev_hw_stats64_add()
8732 dest->rx_errors += src->rx_errors; in netdev_hw_stats64_add()
8733 dest->tx_errors += src->tx_errors; in netdev_hw_stats64_add()
8734 dest->rx_dropped += src->rx_dropped; in netdev_hw_stats64_add()
8735 dest->tx_dropped += src->tx_dropped; in netdev_hw_stats64_add()
8736 dest->multicast += src->multicast; in netdev_hw_stats64_add()
8778 return -EINVAL; in netdev_offload_xstats_get_stats()
8815 report_delta->used = true; in netdev_offload_xstats_report_delta()
8816 netdev_hw_stats64_add(&report_delta->stats, stats); in netdev_offload_xstats_report_delta()
8823 report_used->used = true; in netdev_offload_xstats_report_used()
8844 * netdev_get_xmit_slave - Get the xmit slave of master device
8858 const struct net_device_ops *ops = dev->netdev_ops; in netdev_get_xmit_slave()
8860 if (!ops->ndo_get_xmit_slave) in netdev_get_xmit_slave()
8862 return ops->ndo_get_xmit_slave(dev, skb, all_slaves); in netdev_get_xmit_slave()
8869 const struct net_device_ops *ops = dev->netdev_ops; in netdev_sk_get_lower_dev()
8871 if (!ops->ndo_sk_get_lower_dev) in netdev_sk_get_lower_dev()
8873 return ops->ndo_sk_get_lower_dev(dev, sk); in netdev_sk_get_lower_dev()
8877 * netdev_sk_get_lowest_dev - Get the lowest device in chain given device and socket
8881 * %NULL is returned if no lower device is found.
8887 struct net_device *lower; in netdev_sk_get_lowest_dev() local
8889 lower = netdev_sk_get_lower_dev(dev, sk); in netdev_sk_get_lowest_dev()
8890 while (lower) { in netdev_sk_get_lowest_dev()
8891 dev = lower; in netdev_sk_get_lowest_dev()
8892 lower = netdev_sk_get_lower_dev(dev, sk); in netdev_sk_get_lowest_dev()
8905 list_for_each_entry(iter, &dev->adj_list.upper, list) { in netdev_adjacent_add_links()
8906 if (!net_eq(net, dev_net(iter->dev))) in netdev_adjacent_add_links()
8908 netdev_adjacent_sysfs_add(iter->dev, dev, in netdev_adjacent_add_links()
8909 &iter->dev->adj_list.lower); in netdev_adjacent_add_links()
8910 netdev_adjacent_sysfs_add(dev, iter->dev, in netdev_adjacent_add_links()
8911 &dev->adj_list.upper); in netdev_adjacent_add_links()
8914 list_for_each_entry(iter, &dev->adj_list.lower, list) { in netdev_adjacent_add_links()
8915 if (!net_eq(net, dev_net(iter->dev))) in netdev_adjacent_add_links()
8917 netdev_adjacent_sysfs_add(iter->dev, dev, in netdev_adjacent_add_links()
8918 &iter->dev->adj_list.upper); in netdev_adjacent_add_links()
8919 netdev_adjacent_sysfs_add(dev, iter->dev, in netdev_adjacent_add_links()
8920 &dev->adj_list.lower); in netdev_adjacent_add_links()
8930 list_for_each_entry(iter, &dev->adj_list.upper, list) { in netdev_adjacent_del_links()
8931 if (!net_eq(net, dev_net(iter->dev))) in netdev_adjacent_del_links()
8933 netdev_adjacent_sysfs_del(iter->dev, dev->name, in netdev_adjacent_del_links()
8934 &iter->dev->adj_list.lower); in netdev_adjacent_del_links()
8935 netdev_adjacent_sysfs_del(dev, iter->dev->name, in netdev_adjacent_del_links()
8936 &dev->adj_list.upper); in netdev_adjacent_del_links()
8939 list_for_each_entry(iter, &dev->adj_list.lower, list) { in netdev_adjacent_del_links()
8940 if (!net_eq(net, dev_net(iter->dev))) in netdev_adjacent_del_links()
8942 netdev_adjacent_sysfs_del(iter->dev, dev->name, in netdev_adjacent_del_links()
8943 &iter->dev->adj_list.upper); in netdev_adjacent_del_links()
8944 netdev_adjacent_sysfs_del(dev, iter->dev->name, in netdev_adjacent_del_links()
8945 &dev->adj_list.lower); in netdev_adjacent_del_links()
8955 list_for_each_entry(iter, &dev->adj_list.upper, list) { in netdev_adjacent_rename_links()
8956 if (!net_eq(net, dev_net(iter->dev))) in netdev_adjacent_rename_links()
8958 netdev_adjacent_sysfs_del(iter->dev, oldname, in netdev_adjacent_rename_links()
8959 &iter->dev->adj_list.lower); in netdev_adjacent_rename_links()
8960 netdev_adjacent_sysfs_add(iter->dev, dev, in netdev_adjacent_rename_links()
8961 &iter->dev->adj_list.lower); in netdev_adjacent_rename_links()
8964 list_for_each_entry(iter, &dev->adj_list.lower, list) { in netdev_adjacent_rename_links()
8965 if (!net_eq(net, dev_net(iter->dev))) in netdev_adjacent_rename_links()
8967 netdev_adjacent_sysfs_del(iter->dev, oldname, in netdev_adjacent_rename_links()
8968 &iter->dev->adj_list.upper); in netdev_adjacent_rename_links()
8969 netdev_adjacent_sysfs_add(iter->dev, dev, in netdev_adjacent_rename_links()
8970 &iter->dev->adj_list.upper); in netdev_adjacent_rename_links()
8977 struct netdev_adjacent *lower; in netdev_lower_dev_get_private() local
8981 lower = __netdev_find_adj(lower_dev, &dev->adj_list.lower); in netdev_lower_dev_get_private()
8982 if (!lower) in netdev_lower_dev_get_private()
8985 return lower->private; in netdev_lower_dev_get_private()
8991 * netdev_lower_state_changed - Dispatch event about lower device state change
9014 const struct net_device_ops *ops = dev->netdev_ops; in dev_change_rx_flags()
9016 if (ops->ndo_change_rx_flags) in dev_change_rx_flags()
9017 ops->ndo_change_rx_flags(dev, flags); in dev_change_rx_flags()
9022 unsigned int old_flags = dev->flags; in __dev_set_promiscuity()
9029 promiscuity = dev->promiscuity + inc; in __dev_set_promiscuity()
9037 return -EOVERFLOW; in __dev_set_promiscuity()
9043 WRITE_ONCE(dev->promiscuity, promiscuity); in __dev_set_promiscuity()
9045 WRITE_ONCE(dev->flags, flags); in __dev_set_promiscuity()
9047 dev->flags & IFF_PROMISC ? "entered" : "left"); in __dev_set_promiscuity()
9053 dev->name, (dev->flags & IFF_PROMISC), in __dev_set_promiscuity()
9069 * dev_set_promiscuity - update promiscuity count on a device
9081 unsigned int old_flags = dev->flags; in dev_set_promiscuity()
9087 if (dev->flags != old_flags) in dev_set_promiscuity()
9095 unsigned int old_flags = dev->flags, old_gflags = dev->gflags; in __dev_set_allmulti()
9100 allmulti = dev->allmulti + inc; in __dev_set_allmulti()
9108 return -EOVERFLOW; in __dev_set_allmulti()
9114 WRITE_ONCE(dev->allmulti, allmulti); in __dev_set_allmulti()
9116 WRITE_ONCE(dev->flags, flags); in __dev_set_allmulti()
9118 dev->flags & IFF_ALLMULTI ? "entered" : "left"); in __dev_set_allmulti()
9123 dev->gflags ^ old_gflags, 0, NULL); in __dev_set_allmulti()
9129 * dev_set_allmulti - update allmulti count on a device
9155 const struct net_device_ops *ops = dev->netdev_ops; in __dev_set_rx_mode()
9158 if (!(dev->flags&IFF_UP)) in __dev_set_rx_mode()
9164 if (!(dev->priv_flags & IFF_UNICAST_FLT)) { in __dev_set_rx_mode()
9168 if (!netdev_uc_empty(dev) && !dev->uc_promisc) { in __dev_set_rx_mode()
9170 dev->uc_promisc = true; in __dev_set_rx_mode()
9171 } else if (netdev_uc_empty(dev) && dev->uc_promisc) { in __dev_set_rx_mode()
9172 __dev_set_promiscuity(dev, -1, false); in __dev_set_rx_mode()
9173 dev->uc_promisc = false; in __dev_set_rx_mode()
9177 if (ops->ndo_set_rx_mode) in __dev_set_rx_mode()
9178 ops->ndo_set_rx_mode(dev); in __dev_set_rx_mode()
9189 * dev_get_flags - get flags reported to userspace
9198 flags = (READ_ONCE(dev->flags) & ~(IFF_PROMISC | in dev_get_flags()
9203 (READ_ONCE(dev->gflags) & (IFF_PROMISC | in dev_get_flags()
9222 unsigned int old_flags = dev->flags; in __dev_change_flags()
9231 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP | in __dev_change_flags()
9234 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC | in __dev_change_flags()
9260 if ((flags ^ dev->gflags) & IFF_PROMISC) { in __dev_change_flags()
9261 int inc = (flags & IFF_PROMISC) ? 1 : -1; in __dev_change_flags()
9262 unsigned int old_flags = dev->flags; in __dev_change_flags()
9264 dev->gflags ^= IFF_PROMISC; in __dev_change_flags()
9267 if (dev->flags != old_flags) in __dev_change_flags()
9275 if ((flags ^ dev->gflags) & IFF_ALLMULTI) { in __dev_change_flags()
9276 int inc = (flags & IFF_ALLMULTI) ? 1 : -1; in __dev_change_flags()
9278 dev->gflags ^= IFF_ALLMULTI; in __dev_change_flags()
9289 unsigned int changes = dev->flags ^ old_flags; in __dev_notify_flags()
9295 if (dev->flags & IFF_UP) in __dev_notify_flags()
9301 if (dev->flags & IFF_UP && in __dev_notify_flags()
9315 * dev_change_flags - change device settings
9327 unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags; in dev_change_flags()
9333 changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags); in dev_change_flags()
9341 const struct net_device_ops *ops = dev->netdev_ops; in __dev_set_mtu()
9343 if (ops->ndo_change_mtu) in __dev_set_mtu()
9344 return ops->ndo_change_mtu(dev, new_mtu); in __dev_set_mtu()
9346 /* Pairs with all the lockless reads of dev->mtu in the stack */ in __dev_set_mtu()
9347 WRITE_ONCE(dev->mtu, new_mtu); in __dev_set_mtu()
9356 if (new_mtu < 0 || new_mtu < dev->min_mtu) { in dev_validate_mtu()
9358 return -EINVAL; in dev_validate_mtu()
9361 if (dev->max_mtu > 0 && new_mtu > dev->max_mtu) { in dev_validate_mtu()
9363 return -EINVAL; in dev_validate_mtu()
9369 * dev_set_mtu_ext - Change maximum transfer unit
9381 if (new_mtu == dev->mtu) in dev_set_mtu_ext()
9389 return -ENODEV; in dev_set_mtu_ext()
9396 orig_mtu = dev->mtu; in dev_set_mtu_ext()
9423 net_err_ratelimited("%s: %s\n", dev->name, extack._msg); in dev_set_mtu()
9429 * dev_change_tx_queue_len - Change TX queue length of a netdevice
9435 unsigned int orig_len = dev->tx_queue_len; in dev_change_tx_queue_len()
9439 return -ERANGE; in dev_change_tx_queue_len()
9442 WRITE_ONCE(dev->tx_queue_len, new_len); in dev_change_tx_queue_len()
9456 WRITE_ONCE(dev->tx_queue_len, orig_len); in dev_change_tx_queue_len()
9461 * dev_set_group - Change group this device belongs to
9467 dev->group = new_group; in dev_set_group()
9471 * dev_pre_changeaddr_notify - Call NETDEV_PRE_CHANGEADDR.
9492 * dev_set_mac_address - Change Media Access Control Address
9502 const struct net_device_ops *ops = dev->netdev_ops; in dev_set_mac_address()
9505 if (!ops->ndo_set_mac_address) in dev_set_mac_address()
9506 return -EOPNOTSUPP; in dev_set_mac_address()
9507 if (sa->sa_family != dev->type) in dev_set_mac_address()
9508 return -EINVAL; in dev_set_mac_address()
9510 return -ENODEV; in dev_set_mac_address()
9511 err = dev_pre_changeaddr_notify(dev, sa->sa_data, extack); in dev_set_mac_address()
9514 if (memcmp(dev->dev_addr, sa->sa_data, dev->addr_len)) { in dev_set_mac_address()
9515 err = ops->ndo_set_mac_address(dev, sa); in dev_set_mac_address()
9519 dev->addr_assign_type = NET_ADDR_SET; in dev_set_mac_address()
9521 add_device_randomness(dev->dev_addr, dev->addr_len); in dev_set_mac_address()
9542 size_t size = sizeof(sa->sa_data_min); in dev_get_mac_address()
9551 ret = -ENODEV; in dev_get_mac_address()
9554 if (!dev->addr_len) in dev_get_mac_address()
9555 memset(sa->sa_data, 0, size); in dev_get_mac_address()
9557 memcpy(sa->sa_data, dev->dev_addr, in dev_get_mac_address()
9558 min_t(size_t, size, dev->addr_len)); in dev_get_mac_address()
9559 sa->sa_family = dev->type; in dev_get_mac_address()
9569 * dev_change_carrier - Change device carrier
9577 const struct net_device_ops *ops = dev->netdev_ops; in dev_change_carrier()
9579 if (!ops->ndo_change_carrier) in dev_change_carrier()
9580 return -EOPNOTSUPP; in dev_change_carrier()
9582 return -ENODEV; in dev_change_carrier()
9583 return ops->ndo_change_carrier(dev, new_carrier); in dev_change_carrier()
9587 * dev_get_phys_port_id - Get device physical port ID
9596 const struct net_device_ops *ops = dev->netdev_ops; in dev_get_phys_port_id()
9598 if (!ops->ndo_get_phys_port_id) in dev_get_phys_port_id()
9599 return -EOPNOTSUPP; in dev_get_phys_port_id()
9600 return ops->ndo_get_phys_port_id(dev, ppid); in dev_get_phys_port_id()
9604 * dev_get_phys_port_name - Get device physical port name
9614 const struct net_device_ops *ops = dev->netdev_ops; in dev_get_phys_port_name()
9617 if (ops->ndo_get_phys_port_name) { in dev_get_phys_port_name()
9618 err = ops->ndo_get_phys_port_name(dev, name, len); in dev_get_phys_port_name()
9619 if (err != -EOPNOTSUPP) in dev_get_phys_port_name()
9626 * dev_get_port_parent_id - Get the device's port parent identifier
9629 * @recurse: allow/disallow recursion to lower devices
9637 const struct net_device_ops *ops = dev->netdev_ops; in dev_get_port_parent_id()
9643 if (ops->ndo_get_port_parent_id) { in dev_get_port_parent_id()
9644 err = ops->ndo_get_port_parent_id(dev, ppid); in dev_get_port_parent_id()
9645 if (err != -EOPNOTSUPP) in dev_get_port_parent_id()
9650 if (!recurse || err != -EOPNOTSUPP) in dev_get_port_parent_id()
9660 return -EOPNOTSUPP; in dev_get_port_parent_id()
9668 * netdev_port_same_parent_id - Indicate if two network devices have
9687 * dev_change_proto_down - set carrier according to proto_down.
9694 if (!dev->change_proto_down) in dev_change_proto_down()
9695 return -EOPNOTSUPP; in dev_change_proto_down()
9697 return -ENODEV; in dev_change_proto_down()
9702 WRITE_ONCE(dev->proto_down, proto_down); in dev_change_proto_down()
9707 * dev_change_proto_down_reason - proto down reason
9722 proto_down_reason = dev->proto_down_reason; in dev_change_proto_down_reason()
9730 WRITE_ONCE(dev->proto_down_reason, proto_down_reason); in dev_change_proto_down_reason()
9747 return dev->netdev_ops->ndo_bpf ? XDP_MODE_DRV : XDP_MODE_SKB; in dev_xdp_mode()
9753 case XDP_MODE_SKB: in dev_xdp_bpf_op()
9755 case XDP_MODE_DRV: in dev_xdp_bpf_op()
9756 case XDP_MODE_HW: in dev_xdp_bpf_op()
9757 return dev->netdev_ops->ndo_bpf; in dev_xdp_bpf_op()
9766 return dev->xdp_state[mode].link; in dev_xdp_link()
9775 return link->link.prog; in dev_xdp_prog()
9776 return dev->xdp_state[mode].prog; in dev_xdp_prog()
9785 if (dev->xdp_state[i].prog || dev->xdp_state[i].link) in dev_xdp_prog_count()
9797 if (dev->xdp_state[i].prog && in dev_xdp_sb_prog_count()
9798 !dev->xdp_state[i].prog->aux->xdp_has_frags) in dev_xdp_sb_prog_count()
9805 if (!dev->netdev_ops->ndo_bpf) in dev_xdp_propagate()
9806 return -EOPNOTSUPP; in dev_xdp_propagate()
9808 if (dev->cfg->hds_config == ETHTOOL_TCP_DATA_SPLIT_ENABLED && in dev_xdp_propagate()
9809 bpf->command == XDP_SETUP_PROG && in dev_xdp_propagate()
9810 bpf->prog && !bpf->prog->aux->xdp_has_frags) { in dev_xdp_propagate()
9811 NL_SET_ERR_MSG(bpf->extack, in dev_xdp_propagate()
9812 "unable to propagate XDP to device using tcp-data-split"); in dev_xdp_propagate()
9813 return -EBUSY; in dev_xdp_propagate()
9817 NL_SET_ERR_MSG(bpf->extack, "unable to propagate XDP to device using memory provider"); in dev_xdp_propagate()
9818 return -EBUSY; in dev_xdp_propagate()
9821 return dev->netdev_ops->ndo_bpf(dev, bpf); in dev_xdp_propagate()
9829 return prog ? prog->aux->id : 0; in dev_xdp_prog_id()
9835 dev->xdp_state[mode].link = link; in dev_xdp_set_link()
9836 dev->xdp_state[mode].prog = NULL; in dev_xdp_set_link()
9842 dev->xdp_state[mode].link = NULL; in dev_xdp_set_prog()
9843 dev->xdp_state[mode].prog = prog; in dev_xdp_set_prog()
9853 if (dev->cfg->hds_config == ETHTOOL_TCP_DATA_SPLIT_ENABLED && in dev_xdp_install()
9854 prog && !prog->aux->xdp_has_frags) { in dev_xdp_install()
9855 NL_SET_ERR_MSG(extack, "unable to install XDP to device using tcp-data-split"); in dev_xdp_install()
9856 return -EBUSY; in dev_xdp_install()
9861 return -EBUSY; in dev_xdp_install()
9911 /* auto-detach link from net device */ in dev_xdp_uninstall()
9914 link->dev = NULL; in dev_xdp_uninstall()
9938 return -EINVAL; in dev_xdp_attach()
9942 return -EINVAL; in dev_xdp_attach()
9947 return -EINVAL; in dev_xdp_attach()
9953 return -EINVAL; in dev_xdp_attach()
9958 return -EINVAL; in dev_xdp_attach()
9965 return -EBUSY; in dev_xdp_attach()
9972 return -EEXIST; in dev_xdp_attach()
9980 return -EBUSY; in dev_xdp_attach()
9984 return -EEXIST; in dev_xdp_attach()
9989 new_prog = link->link.prog; in dev_xdp_attach()
9998 return -EBUSY; in dev_xdp_attach()
10002 return -EEXIST; in dev_xdp_attach()
10004 if (!offload && bpf_prog_is_offloaded(new_prog->aux)) { in dev_xdp_attach()
10006 return -EINVAL; in dev_xdp_attach()
10008 if (bpf_prog_is_dev_bound(new_prog->aux) && !bpf_offload_dev_match(new_prog, dev)) { in dev_xdp_attach()
10010 return -EINVAL; in dev_xdp_attach()
10012 if (bpf_prog_is_dev_bound(new_prog->aux) && mode == XDP_MODE_SKB) { in dev_xdp_attach()
10013 NL_SET_ERR_MSG(extack, "Can't attach device-bound programs in generic mode"); in dev_xdp_attach()
10014 return -EINVAL; in dev_xdp_attach()
10016 if (new_prog->expected_attach_type == BPF_XDP_DEVMAP) { in dev_xdp_attach()
10018 return -EINVAL; in dev_xdp_attach()
10020 if (new_prog->expected_attach_type == BPF_XDP_CPUMAP) { in dev_xdp_attach()
10022 return -EINVAL; in dev_xdp_attach()
10031 return -EOPNOTSUPP; in dev_xdp_attach()
10053 return dev_xdp_attach(dev, extack, link, NULL, NULL, link->flags); in dev_xdp_attach_link()
10065 mode = dev_xdp_mode(dev, link->flags); in dev_xdp_detach_link()
10067 return -EINVAL; in dev_xdp_detach_link()
10081 /* if racing with net_device's tear down, xdp_link->dev might be in bpf_xdp_link_release()
10082 * already NULL, in which case link was already auto-detached in bpf_xdp_link_release()
10084 if (xdp_link->dev) { in bpf_xdp_link_release()
10085 WARN_ON(dev_xdp_detach_link(xdp_link->dev, NULL, xdp_link)); in bpf_xdp_link_release()
10086 xdp_link->dev = NULL; in bpf_xdp_link_release()
10112 if (xdp_link->dev) in bpf_xdp_link_show_fdinfo()
10113 ifindex = xdp_link->dev->ifindex; in bpf_xdp_link_show_fdinfo()
10126 if (xdp_link->dev) in bpf_xdp_link_fill_link_info()
10127 ifindex = xdp_link->dev->ifindex; in bpf_xdp_link_fill_link_info()
10130 info->xdp.ifindex = ifindex; in bpf_xdp_link_fill_link_info()
10144 /* link might have been auto-released already, so fail */ in bpf_xdp_link_update()
10145 if (!xdp_link->dev) { in bpf_xdp_link_update()
10146 err = -ENOLINK; in bpf_xdp_link_update()
10150 if (old_prog && link->prog != old_prog) { in bpf_xdp_link_update()
10151 err = -EPERM; in bpf_xdp_link_update()
10154 old_prog = link->prog; in bpf_xdp_link_update()
10155 if (old_prog->type != new_prog->type || in bpf_xdp_link_update()
10156 old_prog->expected_attach_type != new_prog->expected_attach_type) { in bpf_xdp_link_update()
10157 err = -EINVAL; in bpf_xdp_link_update()
10162 /* no-op, don't disturb drivers */ in bpf_xdp_link_update()
10167 mode = dev_xdp_mode(xdp_link->dev, xdp_link->flags); in bpf_xdp_link_update()
10168 bpf_op = dev_xdp_bpf_op(xdp_link->dev, mode); in bpf_xdp_link_update()
10169 err = dev_xdp_install(xdp_link->dev, mode, bpf_op, NULL, in bpf_xdp_link_update()
10170 xdp_link->flags, new_prog); in bpf_xdp_link_update()
10174 old_prog = xchg(&link->prog, new_prog); in bpf_xdp_link_update()
10193 struct net *net = current->nsproxy->net_ns; in bpf_xdp_link_attach()
10201 dev = dev_get_by_index(net, attr->link_create.target_ifindex); in bpf_xdp_link_attach()
10204 return -EINVAL; in bpf_xdp_link_attach()
10209 err = -ENOMEM; in bpf_xdp_link_attach()
10213 bpf_link_init(&link->link, BPF_LINK_TYPE_XDP, &bpf_xdp_link_lops, prog); in bpf_xdp_link_attach()
10214 link->dev = dev; in bpf_xdp_link_attach()
10215 link->flags = attr->link_create.flags; in bpf_xdp_link_attach()
10217 err = bpf_link_prime(&link->link, &link_primer); in bpf_xdp_link_attach()
10227 link->dev = NULL; in bpf_xdp_link_attach()
10247 * dev_change_xdp_fd - set or clear a bpf program for a device rx path
10252 * @flags: xdp-related flags
10298 for (i = dev->real_num_rx_queues - 1; i >= 0; i--) in dev_get_min_mp_channel_count()
10299 if (dev->_rx[i].mp_params.mp_priv) in dev_get_min_mp_channel_count()
10307 * dev_index_reserve() - allocate an ifindex in a namespace
10315 * Return: a suitable unique value for a new device interface number or -errno.
10323 return -EINVAL; in dev_index_reserve()
10327 err = xa_alloc_cyclic(&net->dev_by_index, &ifindex, NULL, in dev_index_reserve()
10328 xa_limit_31b, &net->ifindex, GFP_KERNEL); in dev_index_reserve()
10330 err = xa_insert(&net->dev_by_index, ifindex, NULL, GFP_KERNEL); in dev_index_reserve()
10340 WARN_ON(xa_erase(&net->dev_by_index, ifindex)); in dev_index_release()
10359 list_add_tail(&dev->todo_list, &net_todo_list); in net_set_todo()
10362 static netdev_features_t netdev_sync_upper_features(struct net_device *lower, in netdev_sync_upper_features() argument
10371 if (!(upper->wanted_features & feature) in netdev_sync_upper_features()
10373 netdev_dbg(lower, "Dropping feature %pNF, upper dev %s has it off.\n", in netdev_sync_upper_features()
10374 &feature, upper->name); in netdev_sync_upper_features()
10383 struct net_device *lower, netdev_features_t features) in netdev_sync_lower_features() argument
10391 if (!(features & feature) && (lower->features & feature)) { in netdev_sync_lower_features()
10392 netdev_dbg(upper, "Disabling feature %pNF on lower dev %s.\n", in netdev_sync_lower_features()
10393 &feature, lower->name); in netdev_sync_lower_features()
10394 lower->wanted_features &= ~feature; in netdev_sync_lower_features()
10395 __netdev_update_features(lower); in netdev_sync_lower_features()
10397 if (unlikely(lower->features & feature)) in netdev_sync_lower_features()
10399 &feature, lower->name); in netdev_sync_lower_features()
10401 netdev_features_change(lower); in netdev_sync_lower_features()
10459 if ((features & dev->gso_partial_features) && in netdev_fix_features()
10463 features &= ~dev->gso_partial_features; in netdev_fix_features()
10478 /* LRO/HW-GRO features cannot be combined with RX-FCS */ in netdev_fix_features()
10481 netdev_dbg(dev, "Dropping LRO feature since RX-FCS is requested.\n"); in netdev_fix_features()
10486 netdev_dbg(dev, "Dropping HW-GRO feature since RX-FCS is requested.\n"); in netdev_fix_features()
10492 netdev_dbg(dev, "Dropping LRO feature since HW-GRO is requested.\n"); in netdev_fix_features()
10516 struct net_device *upper, *lower; in __netdev_update_features() local
10519 int err = -1; in __netdev_update_features()
10525 if (dev->netdev_ops->ndo_fix_features) in __netdev_update_features()
10526 features = dev->netdev_ops->ndo_fix_features(dev, features); in __netdev_update_features()
10535 if (dev->features == features) in __netdev_update_features()
10538 netdev_dbg(dev, "Features changed: %pNF -> %pNF\n", in __netdev_update_features()
10539 &dev->features, &features); in __netdev_update_features()
10541 if (dev->netdev_ops->ndo_set_features) in __netdev_update_features()
10542 err = dev->netdev_ops->ndo_set_features(dev, features); in __netdev_update_features()
10549 err, &features, &dev->features); in __netdev_update_features()
10550 /* return non-0 since some features might have changed and in __netdev_update_features()
10553 return -1; in __netdev_update_features()
10557 /* some features must be disabled on lower devices when disabled in __netdev_update_features()
10560 netdev_for_each_lower_dev(dev, lower, iter) in __netdev_update_features()
10561 netdev_sync_lower_features(dev, lower, features); in __netdev_update_features()
10564 netdev_features_t diff = features ^ dev->features; in __netdev_update_features()
10570 * Thus we need to update dev->features in __netdev_update_features()
10575 dev->features = features; in __netdev_update_features()
10584 dev->features = features; in __netdev_update_features()
10593 dev->features = features; in __netdev_update_features()
10600 dev->features = features; in __netdev_update_features()
10607 * netdev_update_features - recalculate device features
10610 * Recalculate dev->features set and send notifications if it
10622 * netdev_change_features - recalculate device features
10625 * Recalculate dev->features set and send notifications even
10627 * netdev_update_features() if also dev->vlan_features might
10639 * netif_stacked_transfer_operstate - transfer operstate
10640 * @rootdev: the root or lower level device to transfer state from
10650 if (rootdev->operstate == IF_OPER_DORMANT) in netif_stacked_transfer_operstate()
10655 if (rootdev->operstate == IF_OPER_TESTING) in netif_stacked_transfer_operstate()
10669 unsigned int i, count = dev->num_rx_queues; in netif_alloc_rx_queues()
10678 return -ENOMEM; in netif_alloc_rx_queues()
10680 dev->_rx = rx; in netif_alloc_rx_queues()
10685 /* XDP RX-queue setup */ in netif_alloc_rx_queues()
10694 while (i--) in netif_alloc_rx_queues()
10696 kvfree(dev->_rx); in netif_alloc_rx_queues()
10697 dev->_rx = NULL; in netif_alloc_rx_queues()
10703 unsigned int i, count = dev->num_rx_queues; in netif_free_rx_queues()
10706 if (!dev->_rx) in netif_free_rx_queues()
10710 xdp_rxq_info_unreg(&dev->_rx[i].xdp_rxq); in netif_free_rx_queues()
10712 kvfree(dev->_rx); in netif_free_rx_queues()
10719 spin_lock_init(&queue->_xmit_lock); in netdev_init_one_queue()
10720 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type); in netdev_init_one_queue()
10721 queue->xmit_lock_owner = -1; in netdev_init_one_queue()
10723 queue->dev = dev; in netdev_init_one_queue()
10725 dql_init(&queue->dql, HZ); in netdev_init_one_queue()
10731 kvfree(dev->_tx); in netif_free_tx_queues()
10736 unsigned int count = dev->num_tx_queues; in netif_alloc_netdev_queues()
10741 return -EINVAL; in netif_alloc_netdev_queues()
10745 return -ENOMEM; in netif_alloc_netdev_queues()
10747 dev->_tx = tx; in netif_alloc_netdev_queues()
10750 spin_lock_init(&dev->tx_global_lock); in netif_alloc_netdev_queues()
10759 for (i = 0; i < dev->num_tx_queues; i++) { in netif_tx_stop_all_queues()
10775 if (dev->netdev_ops->ndo_get_peer_dev && in netdev_do_alloc_pcpu_stats()
10776 dev->pcpu_stat_type != NETDEV_PCPU_STAT_TSTATS) in netdev_do_alloc_pcpu_stats()
10777 return -EOPNOTSUPP; in netdev_do_alloc_pcpu_stats()
10779 switch (dev->pcpu_stat_type) { in netdev_do_alloc_pcpu_stats()
10780 case NETDEV_PCPU_STAT_NONE: in netdev_do_alloc_pcpu_stats()
10782 case NETDEV_PCPU_STAT_LSTATS: in netdev_do_alloc_pcpu_stats()
10783 v = dev->lstats = netdev_alloc_pcpu_stats(struct pcpu_lstats); in netdev_do_alloc_pcpu_stats()
10785 case NETDEV_PCPU_STAT_TSTATS: in netdev_do_alloc_pcpu_stats()
10786 v = dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats); in netdev_do_alloc_pcpu_stats()
10788 case NETDEV_PCPU_STAT_DSTATS: in netdev_do_alloc_pcpu_stats()
10789 v = dev->dstats = netdev_alloc_pcpu_stats(struct pcpu_dstats); in netdev_do_alloc_pcpu_stats()
10792 return -EINVAL; in netdev_do_alloc_pcpu_stats()
10795 return v ? 0 : -ENOMEM; in netdev_do_alloc_pcpu_stats()
10800 switch (dev->pcpu_stat_type) { in netdev_do_free_pcpu_stats()
10801 case NETDEV_PCPU_STAT_NONE: in netdev_do_free_pcpu_stats()
10803 case NETDEV_PCPU_STAT_LSTATS: in netdev_do_free_pcpu_stats()
10804 free_percpu(dev->lstats); in netdev_do_free_pcpu_stats()
10806 case NETDEV_PCPU_STAT_TSTATS: in netdev_do_free_pcpu_stats()
10807 free_percpu(dev->tstats); in netdev_do_free_pcpu_stats()
10809 case NETDEV_PCPU_STAT_DSTATS: in netdev_do_free_pcpu_stats()
10810 free_percpu(dev->dstats); in netdev_do_free_pcpu_stats()
10817 struct phy_link_topology *topo = dev->link_topo; in netdev_free_phy_link_topology()
10820 xa_destroy(&topo->phys); in netdev_free_phy_link_topology()
10822 dev->link_topo = NULL; in netdev_free_phy_link_topology()
10827 * register_netdevice() - register a network device
10832 * Callers must hold the rtnl lock - you may want register_netdev()
10848 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED); in register_netdevice()
10851 ret = ethtool_check_ops(dev->ethtool_ops); in register_netdevice()
10856 xa_init_flags(&dev->ethtool->rss_ctx, XA_FLAGS_ALLOC1); in register_netdevice()
10857 mutex_init(&dev->ethtool->rss_lock); in register_netdevice()
10859 spin_lock_init(&dev->addr_list_lock); in register_netdevice()
10862 ret = dev_get_valid_name(net, dev, dev->name); in register_netdevice()
10866 ret = -ENOMEM; in register_netdevice()
10867 dev->name_node = netdev_name_node_head_alloc(dev); in register_netdevice()
10868 if (!dev->name_node) in register_netdevice()
10872 if (dev->netdev_ops->ndo_init) { in register_netdevice()
10873 ret = dev->netdev_ops->ndo_init(dev); in register_netdevice()
10876 ret = -EIO; in register_netdevice()
10881 if (((dev->hw_features | dev->features) & in register_netdevice()
10883 (!dev->netdev_ops->ndo_vlan_rx_add_vid || in register_netdevice()
10884 !dev->netdev_ops->ndo_vlan_rx_kill_vid)) { in register_netdevice()
10886 ret = -EINVAL; in register_netdevice()
10894 ret = dev_index_reserve(net, dev->ifindex); in register_netdevice()
10897 dev->ifindex = ret; in register_netdevice()
10902 dev->hw_features |= (NETIF_F_SOFT_FEATURES | NETIF_F_SOFT_FEATURES_OFF); in register_netdevice()
10903 dev->features |= NETIF_F_SOFT_FEATURES; in register_netdevice()
10905 if (dev->udp_tunnel_nic_info) { in register_netdevice()
10906 dev->features |= NETIF_F_RX_UDP_TUNNEL_PORT; in register_netdevice()
10907 dev->hw_features |= NETIF_F_RX_UDP_TUNNEL_PORT; in register_netdevice()
10910 dev->wanted_features = dev->features & dev->hw_features; in register_netdevice()
10912 if (!(dev->flags & IFF_LOOPBACK)) in register_netdevice()
10913 dev->hw_features |= NETIF_F_NOCACHE_COPY; in register_netdevice()
10920 if (dev->hw_features & NETIF_F_TSO) in register_netdevice()
10921 dev->hw_features |= NETIF_F_TSO_MANGLEID; in register_netdevice()
10922 if (dev->vlan_features & NETIF_F_TSO) in register_netdevice()
10923 dev->vlan_features |= NETIF_F_TSO_MANGLEID; in register_netdevice()
10924 if (dev->mpls_features & NETIF_F_TSO) in register_netdevice()
10925 dev->mpls_features |= NETIF_F_TSO_MANGLEID; in register_netdevice()
10926 if (dev->hw_enc_features & NETIF_F_TSO) in register_netdevice()
10927 dev->hw_enc_features |= NETIF_F_TSO_MANGLEID; in register_netdevice()
10931 dev->vlan_features |= NETIF_F_HIGHDMA; in register_netdevice()
10935 dev->hw_enc_features |= NETIF_F_SG | NETIF_F_GSO_PARTIAL; in register_netdevice()
10939 dev->mpls_features |= NETIF_F_SG; in register_netdevice()
10949 WRITE_ONCE(dev->reg_state, ret ? NETREG_UNREGISTERED : NETREG_REGISTERED); in register_netdevice()
10962 set_bit(__LINK_STATE_PRESENT, &dev->state); in register_netdevice()
10968 netdev_hold(dev, &dev->dev_registered_tracker, GFP_KERNEL); in register_netdevice()
10971 add_device_randomness(dev->dev_addr, dev->addr_len); in register_netdevice()
10977 if (dev->addr_assign_type == NET_ADDR_PERM) in register_netdevice()
10978 memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len); in register_netdevice()
10985 dev->needs_free_netdev = false; in register_netdevice()
10993 if (!dev->rtnl_link_ops || in register_netdevice()
10994 dev->rtnl_link_state == RTNL_LINK_INITIALIZED) in register_netdevice()
11003 dev_index_release(net, dev->ifindex); in register_netdevice()
11007 if (dev->netdev_ops->ndo_uninit) in register_netdevice()
11008 dev->netdev_ops->ndo_uninit(dev); in register_netdevice()
11009 if (dev->priv_destructor) in register_netdevice()
11010 dev->priv_destructor(dev); in register_netdevice()
11012 netdev_name_node_free(dev->name_node); in register_netdevice()
11026 dev->reg_state = NETREG_DUMMY; in init_dummy_netdev()
11029 set_bit(__LINK_STATE_PRESENT, &dev->state); in init_dummy_netdev()
11030 set_bit(__LINK_STATE_START, &dev->state); in init_dummy_netdev()
11039 * register_netdev - register a network device
11057 return -EINTR; in register_netdev()
11073 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i); in netdev_refcnt_read()
11076 return refcount_read(&dev->dev_refcnt); in netdev_refcnt_read()
11086 * netdev_wait_allrefs_any - wait until all references are gone.
11123 &dev->state)) { in netdev_wait_allrefs_any()
11156 dev->name, netdev_refcnt_read(dev)); in netdev_wait_allrefs_any()
11157 ref_tracker_dir_print(&dev->refcnt_tracker, 10); in netdev_wait_allrefs_any()
11203 list_del_init(&dev->unlink_list); in netdev_run_todo()
11204 dev->nested_level = dev->lower_level - 1; in netdev_run_todo()
11218 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) { in netdev_run_todo()
11220 list_del(&dev->todo_list); in netdev_run_todo()
11225 WRITE_ONCE(dev->reg_state, NETREG_UNREGISTERED); in netdev_run_todo()
11233 list_del(&dev->todo_list); in netdev_run_todo()
11237 BUG_ON(!list_empty(&dev->ptype_all)); in netdev_run_todo()
11238 BUG_ON(!list_empty(&dev->ptype_specific)); in netdev_run_todo()
11239 WARN_ON(rcu_access_pointer(dev->ip_ptr)); in netdev_run_todo()
11240 WARN_ON(rcu_access_pointer(dev->ip6_ptr)); in netdev_run_todo()
11243 if (dev->priv_destructor) in netdev_run_todo()
11244 dev->priv_destructor(dev); in netdev_run_todo()
11245 if (dev->needs_free_netdev) in netdev_run_todo()
11251 kobject_put(&dev->dev.kobj); in netdev_run_todo()
11257 /* Collate per-cpu network dstats statistics
11259 * Read per-cpu network statistics from dev->dstats and populate the related
11275 start = u64_stats_fetch_begin(&stats->syncp); in dev_fetch_dstats()
11276 rx_packets = u64_stats_read(&stats->rx_packets); in dev_fetch_dstats()
11277 rx_bytes = u64_stats_read(&stats->rx_bytes); in dev_fetch_dstats()
11278 rx_drops = u64_stats_read(&stats->rx_drops); in dev_fetch_dstats()
11279 tx_packets = u64_stats_read(&stats->tx_packets); in dev_fetch_dstats()
11280 tx_bytes = u64_stats_read(&stats->tx_bytes); in dev_fetch_dstats()
11281 tx_drops = u64_stats_read(&stats->tx_drops); in dev_fetch_dstats()
11282 } while (u64_stats_fetch_retry(&stats->syncp, start)); in dev_fetch_dstats()
11284 s->rx_packets += rx_packets; in dev_fetch_dstats()
11285 s->rx_bytes += rx_bytes; in dev_fetch_dstats()
11286 s->rx_dropped += rx_drops; in dev_fetch_dstats()
11287 s->tx_packets += tx_packets; in dev_fetch_dstats()
11288 s->tx_bytes += tx_bytes; in dev_fetch_dstats()
11289 s->tx_dropped += tx_drops; in dev_fetch_dstats()
11293 /* ndo_get_stats64 implementation for dtstats-based accounting.
11295 * Populate @s from dev->stats and dev->dstats. This is used internally by the
11296 * core for NETDEV_PCPU_STAT_DSTAT-type stats collection.
11301 netdev_stats_to_stats64(s, &dev->stats); in dev_get_dstats64()
11302 dev_fetch_dstats(s, dev->dstats); in dev_get_dstats64()
11322 sizeof(*stats64) - n * sizeof(u64)); in netdev_stats_to_stats64()
11334 if (p && cmpxchg(&dev->core_stats, NULL, p)) in netdev_core_stats_alloc()
11338 return READ_ONCE(dev->core_stats); in netdev_core_stats_alloc()
11344 struct net_device_core_stats __percpu *p = READ_ONCE(dev->core_stats); in netdev_core_stats_inc()
11359 * dev_get_stats - get network device statistics
11365 * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
11371 const struct net_device_ops *ops = dev->netdev_ops; in dev_get_stats()
11388 if (ops->ndo_get_stats64) { in dev_get_stats()
11390 ops->ndo_get_stats64(dev, storage); in dev_get_stats()
11391 } else if (ops->ndo_get_stats) { in dev_get_stats()
11392 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev)); in dev_get_stats()
11393 } else if (dev->pcpu_stat_type == NETDEV_PCPU_STAT_TSTATS) { in dev_get_stats()
11395 } else if (dev->pcpu_stat_type == NETDEV_PCPU_STAT_DSTATS) { in dev_get_stats()
11398 netdev_stats_to_stats64(storage, &dev->stats); in dev_get_stats()
11402 p = READ_ONCE(dev->core_stats); in dev_get_stats()
11409 storage->rx_dropped += READ_ONCE(core_stats->rx_dropped); in dev_get_stats()
11410 storage->tx_dropped += READ_ONCE(core_stats->tx_dropped); in dev_get_stats()
11411 storage->rx_nohandler += READ_ONCE(core_stats->rx_nohandler); in dev_get_stats()
11412 storage->rx_otherhost_dropped += READ_ONCE(core_stats->rx_otherhost_dropped); in dev_get_stats()
11420 * dev_fetch_sw_netstats - get per-cpu network device statistics
11422 * @netstats: per-cpu network stats to read from
11424 * Read per-cpu network statistics and populate the related fields in @s.
11438 start = u64_stats_fetch_begin(&stats->syncp); in dev_fetch_sw_netstats()
11439 rx_packets = u64_stats_read(&stats->rx_packets); in dev_fetch_sw_netstats()
11440 rx_bytes = u64_stats_read(&stats->rx_bytes); in dev_fetch_sw_netstats()
11441 tx_packets = u64_stats_read(&stats->tx_packets); in dev_fetch_sw_netstats()
11442 tx_bytes = u64_stats_read(&stats->tx_bytes); in dev_fetch_sw_netstats()
11443 } while (u64_stats_fetch_retry(&stats->syncp, start)); in dev_fetch_sw_netstats()
11445 s->rx_packets += rx_packets; in dev_fetch_sw_netstats()
11446 s->rx_bytes += rx_bytes; in dev_fetch_sw_netstats()
11447 s->tx_packets += tx_packets; in dev_fetch_sw_netstats()
11448 s->tx_bytes += tx_bytes; in dev_fetch_sw_netstats()
11454 * dev_get_tstats64 - ndo_get_stats64 implementation
11458 * Populate @s from dev->stats and dev->tstats. Can be used as
11463 netdev_stats_to_stats64(s, &dev->stats); in dev_get_tstats64()
11464 dev_fetch_sw_netstats(s, dev->tstats); in dev_get_tstats64()
11479 RCU_INIT_POINTER(queue->qdisc, &noop_qdisc); in dev_ingress_queue_create()
11480 RCU_INIT_POINTER(queue->qdisc_sleeping, &noop_qdisc); in dev_ingress_queue_create()
11481 rcu_assign_pointer(dev->ingress_queue, queue); in dev_ingress_queue_create()
11491 if (dev->ethtool_ops == &default_ethtool_ops) in netdev_set_default_ethtool_ops()
11492 dev->ethtool_ops = ops; in netdev_set_default_ethtool_ops()
11497 * netdev_sw_irq_coalesce_default_on() - enable SW IRQ coalescing by default
11505 WARN_ON(dev->reg_state == NETREG_REGISTERED); in netdev_sw_irq_coalesce_default_on()
11515 * alloc_netdev_mqs - allocate network device
11536 BUG_ON(strlen(name) >= sizeof(dev->name)); in alloc_netdev_mqs()
11555 dev->priv_len = sizeof_priv; in alloc_netdev_mqs()
11557 ref_tracker_dir_init(&dev->refcnt_tracker, 128, name); in alloc_netdev_mqs()
11559 dev->pcpu_refcnt = alloc_percpu(int); in alloc_netdev_mqs()
11560 if (!dev->pcpu_refcnt) in alloc_netdev_mqs()
11564 refcount_set(&dev->dev_refcnt, 1); in alloc_netdev_mqs()
11575 dev->gso_max_size = GSO_LEGACY_MAX_SIZE; in alloc_netdev_mqs()
11576 dev->xdp_zc_max_segs = 1; in alloc_netdev_mqs()
11577 dev->gso_max_segs = GSO_MAX_SEGS; in alloc_netdev_mqs()
11578 dev->gro_max_size = GRO_LEGACY_MAX_SIZE; in alloc_netdev_mqs()
11579 dev->gso_ipv4_max_size = GSO_LEGACY_MAX_SIZE; in alloc_netdev_mqs()
11580 dev->gro_ipv4_max_size = GRO_LEGACY_MAX_SIZE; in alloc_netdev_mqs()
11581 dev->tso_max_size = TSO_LEGACY_MAX_SIZE; in alloc_netdev_mqs()
11582 dev->tso_max_segs = TSO_MAX_SEGS; in alloc_netdev_mqs()
11583 dev->upper_level = 1; in alloc_netdev_mqs()
11584 dev->lower_level = 1; in alloc_netdev_mqs()
11586 dev->nested_level = 0; in alloc_netdev_mqs()
11587 INIT_LIST_HEAD(&dev->unlink_list); in alloc_netdev_mqs()
11590 INIT_LIST_HEAD(&dev->napi_list); in alloc_netdev_mqs()
11591 INIT_LIST_HEAD(&dev->unreg_list); in alloc_netdev_mqs()
11592 INIT_LIST_HEAD(&dev->close_list); in alloc_netdev_mqs()
11593 INIT_LIST_HEAD(&dev->link_watch_list); in alloc_netdev_mqs()
11594 INIT_LIST_HEAD(&dev->adj_list.upper); in alloc_netdev_mqs()
11595 INIT_LIST_HEAD(&dev->adj_list.lower); in alloc_netdev_mqs()
11596 INIT_LIST_HEAD(&dev->ptype_all); in alloc_netdev_mqs()
11597 INIT_LIST_HEAD(&dev->ptype_specific); in alloc_netdev_mqs()
11598 INIT_LIST_HEAD(&dev->net_notifier_list); in alloc_netdev_mqs()
11600 hash_init(dev->qdisc_hash); in alloc_netdev_mqs()
11603 mutex_init(&dev->lock); in alloc_netdev_mqs()
11605 dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM; in alloc_netdev_mqs()
11608 if (!dev->tx_queue_len) { in alloc_netdev_mqs()
11609 dev->priv_flags |= IFF_NO_QUEUE; in alloc_netdev_mqs()
11610 dev->tx_queue_len = DEFAULT_TX_QUEUE_LEN; in alloc_netdev_mqs()
11613 dev->num_tx_queues = txqs; in alloc_netdev_mqs()
11614 dev->real_num_tx_queues = txqs; in alloc_netdev_mqs()
11618 dev->num_rx_queues = rxqs; in alloc_netdev_mqs()
11619 dev->real_num_rx_queues = rxqs; in alloc_netdev_mqs()
11622 dev->ethtool = kzalloc(sizeof(*dev->ethtool), GFP_KERNEL_ACCOUNT); in alloc_netdev_mqs()
11623 if (!dev->ethtool) in alloc_netdev_mqs()
11626 dev->cfg = kzalloc(sizeof(*dev->cfg), GFP_KERNEL_ACCOUNT); in alloc_netdev_mqs()
11627 if (!dev->cfg) in alloc_netdev_mqs()
11629 dev->cfg_pending = dev->cfg; in alloc_netdev_mqs()
11631 napi_config_sz = array_size(maxqs, sizeof(*dev->napi_config)); in alloc_netdev_mqs()
11632 dev->napi_config = kvzalloc(napi_config_sz, GFP_KERNEL_ACCOUNT); in alloc_netdev_mqs()
11633 if (!dev->napi_config) in alloc_netdev_mqs()
11636 strscpy(dev->name, name); in alloc_netdev_mqs()
11637 dev->name_assign_type = name_assign_type; in alloc_netdev_mqs()
11638 dev->group = INIT_NETDEV_GROUP; in alloc_netdev_mqs()
11639 if (!dev->ethtool_ops) in alloc_netdev_mqs()
11640 dev->ethtool_ops = &default_ethtool_ops; in alloc_netdev_mqs()
11652 free_percpu(dev->pcpu_refcnt); in alloc_netdev_mqs()
11662 if (!list_empty(&dev->napi_list)) { in netdev_napi_exit()
11666 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list) in netdev_napi_exit()
11673 kvfree(dev->napi_config); in netdev_napi_exit()
11677 * free_netdev - free network device
11690 * handling may still be dismantling the device. Handle that case by in free_netdev()
11693 if (dev->reg_state == NETREG_UNREGISTERING) { in free_netdev()
11695 dev->needs_free_netdev = true; in free_netdev()
11699 WARN_ON(dev->cfg != dev->cfg_pending); in free_netdev()
11700 kfree(dev->cfg); in free_netdev()
11701 kfree(dev->ethtool); in free_netdev()
11705 kfree(rcu_dereference_protected(dev->ingress_queue, 1)); in free_netdev()
11712 ref_tracker_dir_exit(&dev->refcnt_tracker); in free_netdev()
11714 free_percpu(dev->pcpu_refcnt); in free_netdev()
11715 dev->pcpu_refcnt = NULL; in free_netdev()
11717 free_percpu(dev->core_stats); in free_netdev()
11718 dev->core_stats = NULL; in free_netdev()
11719 free_percpu(dev->xdp_bulkq); in free_netdev()
11720 dev->xdp_bulkq = NULL; in free_netdev()
11724 mutex_destroy(&dev->lock); in free_netdev()
11727 if (dev->reg_state == NETREG_UNINITIALIZED || in free_netdev()
11728 dev->reg_state == NETREG_DUMMY) { in free_netdev()
11733 BUG_ON(dev->reg_state != NETREG_UNREGISTERED); in free_netdev()
11734 WRITE_ONCE(dev->reg_state, NETREG_RELEASED); in free_netdev()
11737 put_device(&dev->dev); in free_netdev()
11742 * alloc_netdev_dummy - Allocate and initialize a dummy net device.
11755 * synchronize_net - Synchronize with packet receive processing
11775 mutex_lock(&dev->ethtool->rss_lock); in netdev_rss_contexts_free()
11776 xa_for_each(&dev->ethtool->rss_ctx, context, ctx) { in netdev_rss_contexts_free()
11781 rxfh.hfunc = ctx->hfunc; in netdev_rss_contexts_free()
11782 rxfh.input_xfrm = ctx->input_xfrm; in netdev_rss_contexts_free()
11786 xa_erase(&dev->ethtool->rss_ctx, context); in netdev_rss_contexts_free()
11787 if (dev->ethtool_ops->create_rxfh_context) in netdev_rss_contexts_free()
11788 dev->ethtool_ops->remove_rxfh_context(dev, ctx, in netdev_rss_contexts_free()
11791 dev->ethtool_ops->set_rxfh(dev, &rxfh, NULL); in netdev_rss_contexts_free()
11794 xa_destroy(&dev->ethtool->rss_ctx); in netdev_rss_contexts_free()
11795 mutex_unlock(&dev->ethtool->rss_lock); in netdev_rss_contexts_free()
11799 * unregister_netdevice_queue - remove device from the kernel
11816 list_move_tail(&dev->unreg_list, head); in unregister_netdevice_queue()
11820 list_add(&dev->unreg_list, &single); in unregister_netdevice_queue()
11844 if (dev->reg_state == NETREG_UNINITIALIZED) { in unregister_netdevice_many_notify()
11846 dev->name, dev); in unregister_netdevice_many_notify()
11849 list_del(&dev->unreg_list); in unregister_netdevice_many_notify()
11852 dev->dismantle = true; in unregister_netdevice_many_notify()
11853 BUG_ON(dev->reg_state != NETREG_REGISTERED); in unregister_netdevice_many_notify()
11858 list_add_tail(&dev->close_list, &close_head); in unregister_netdevice_many_notify()
11865 WRITE_ONCE(dev->reg_state, NETREG_UNREGISTERING); in unregister_netdevice_many_notify()
11889 if (!dev->rtnl_link_ops || in unregister_netdevice_many_notify()
11890 dev->rtnl_link_state == RTNL_LINK_INITIALIZED) in unregister_netdevice_many_notify()
11902 netdev_name_node_free(dev->name_node); in unregister_netdevice_many_notify()
11908 if (dev->netdev_ops->ndo_uninit) in unregister_netdevice_many_notify()
11909 dev->netdev_ops->ndo_uninit(dev); in unregister_netdevice_many_notify()
11911 mutex_destroy(&dev->ethtool->rss_lock); in unregister_netdevice_many_notify()
11933 netdev_put(dev, &dev->dev_registered_tracker); in unregister_netdevice_many_notify()
11943 * unregister_netdevice_many - unregister many devices
11956 * unregister_netdev - remove device from the kernel
11975 * __dev_change_net_namespace - move device to different nethost namespace
12001 err = -EINVAL; in __dev_change_net_namespace()
12002 if (dev->netns_local) in __dev_change_net_namespace()
12006 if (dev->reg_state != NETREG_REGISTERED) in __dev_change_net_namespace()
12017 err = -EEXIST; in __dev_change_net_namespace()
12018 if (netdev_name_in_use(net, dev->name)) { in __dev_change_net_namespace()
12027 err = -EEXIST; in __dev_change_net_namespace()
12029 if (netdev_name_in_use(net, name_node->name)) in __dev_change_net_namespace()
12039 err = dev_index_reserve(net, dev->ifindex); in __dev_change_net_namespace()
12040 if (err == -EBUSY) in __dev_change_net_namespace()
12065 * Note that dev->reg_state stays at NETREG_REGISTERED. in __dev_change_net_namespace()
12083 /* Send a netdev-removed uevent to the old namespace */ in __dev_change_net_namespace()
12084 kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE); in __dev_change_net_namespace()
12087 /* Move per-net netdevice notifiers that are following the netdevice */ in __dev_change_net_namespace()
12092 dev->ifindex = new_ifindex; in __dev_change_net_namespace()
12097 strscpy(dev->name, new_name, IFNAMSIZ); in __dev_change_net_namespace()
12102 dev_set_uevent_suppress(&dev->dev, 1); in __dev_change_net_namespace()
12103 err = device_rename(&dev->dev, dev->name); in __dev_change_net_namespace()
12104 dev_set_uevent_suppress(&dev->dev, 0); in __dev_change_net_namespace()
12107 /* Send a netdev-add uevent to the new namespace */ in __dev_change_net_namespace()
12108 kobject_uevent(&dev->dev.kobj, KOBJ_ADD); in __dev_change_net_namespace()
12111 /* Adapt owner in case owning user namespace of target network in __dev_change_net_namespace()
12149 list_skb = &sd->completion_queue; in dev_cpu_dead()
12151 list_skb = &(*list_skb)->next; in dev_cpu_dead()
12153 *list_skb = oldsd->completion_queue; in dev_cpu_dead()
12154 oldsd->completion_queue = NULL; in dev_cpu_dead()
12157 if (oldsd->output_queue) { in dev_cpu_dead()
12158 *sd->output_queue_tailp = oldsd->output_queue; in dev_cpu_dead()
12159 sd->output_queue_tailp = oldsd->output_queue_tailp; in dev_cpu_dead()
12160 oldsd->output_queue = NULL; in dev_cpu_dead()
12161 oldsd->output_queue_tailp = &oldsd->output_queue; in dev_cpu_dead()
12167 while (!list_empty(&oldsd->poll_list)) { in dev_cpu_dead()
12168 struct napi_struct *napi = list_first_entry(&oldsd->poll_list, in dev_cpu_dead()
12172 list_del_init(&napi->poll_list); in dev_cpu_dead()
12173 if (napi->poll == process_backlog) in dev_cpu_dead()
12174 napi->state &= NAPIF_STATE_THREADED; in dev_cpu_dead()
12184 remsd = oldsd->rps_ipi_list; in dev_cpu_dead()
12185 oldsd->rps_ipi_list = NULL; in dev_cpu_dead()
12192 while ((skb = __skb_dequeue(&oldsd->process_queue))) { in dev_cpu_dead()
12196 while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) { in dev_cpu_dead()
12205 * netdev_increment_features - increment feature set by one
12251 INIT_LIST_HEAD(&net->dev_base_head); in netdev_init()
12253 net->dev_name_head = netdev_create_hash(); in netdev_init()
12254 if (net->dev_name_head == NULL) in netdev_init()
12257 net->dev_index_head = netdev_create_hash(); in netdev_init()
12258 if (net->dev_index_head == NULL) in netdev_init()
12261 xa_init_flags(&net->dev_by_index, XA_FLAGS_ALLOC1); in netdev_init()
12263 RAW_INIT_NOTIFIER_HEAD(&net->netdev_chain); in netdev_init()
12268 kfree(net->dev_name_head); in netdev_init()
12270 return -ENOMEM; in netdev_init()
12274 * netdev_drivername - network driver for the device
12285 parent = dev->dev.parent; in netdev_drivername()
12289 driver = parent->driver; in netdev_drivername()
12290 if (driver && driver->name) in netdev_drivername()
12291 return driver->name; in netdev_drivername()
12298 if (dev && dev->dev.parent) { in __netdev_printk()
12299 dev_printk_emit(level[1] - '0', in __netdev_printk()
12300 dev->dev.parent, in __netdev_printk()
12302 dev_driver_string(dev->dev.parent), in __netdev_printk()
12303 dev_name(dev->dev.parent), in __netdev_printk()
12358 kfree(net->dev_name_head); in netdev_exit()
12359 kfree(net->dev_index_head); in netdev_exit()
12360 xa_destroy(&net->dev_by_index); in netdev_exit()
12362 WARN_ON_ONCE(!list_empty(&net->dev_base_head)); in netdev_exit()
12384 if (dev->netns_local) in default_device_exit_net()
12388 if (dev->rtnl_link_ops && !dev->rtnl_link_ops->netns_refund) in default_device_exit_net()
12392 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex); in default_device_exit_net()
12397 if (netdev_name_in_use(&init_net, name_node->name)) in default_device_exit_net()
12403 __func__, dev->name, err); in default_device_exit_net()
12428 if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink) in default_device_exit_batch()
12429 dev->rtnl_link_ops->dellink(dev, &dev_kill_list); in default_device_exit_batch()
12444 /* TX read-mostly hotpath */ in net_dev_struct_check()
12469 /* TXRX read-mostly hotpath */ in net_dev_struct_check()
12478 /* RX read-mostly hotpath */ in net_dev_struct_check()
12520 return -ENOMEM; in net_page_pool_create()
12536 struct napi_struct *napi = &sd->backlog; in backlog_napi_should_run()
12538 return test_bit(NAPI_STATE_SCHED_THREADED, &napi->state); in backlog_napi_should_run()
12545 napi_threaded_poll_loop(&sd->backlog); in run_backlog_napi()
12551 struct napi_struct *napi = &sd->backlog; in backlog_napi_setup()
12553 napi->thread = this_cpu_read(backlog_napi); in backlog_napi_setup()
12554 set_bit(NAPI_STATE_THREADED, &napi->state); in backlog_napi_setup()
12571 int i, rc = -ENOMEM; in net_dev_init()
12600 skb_queue_head_init(&sd->input_pkt_queue); in net_dev_init()
12601 skb_queue_head_init(&sd->process_queue); in net_dev_init()
12603 skb_queue_head_init(&sd->xfrm_backlog); in net_dev_init()
12605 INIT_LIST_HEAD(&sd->poll_list); in net_dev_init()
12606 sd->output_queue_tailp = &sd->output_queue; in net_dev_init()
12608 INIT_CSD(&sd->csd, rps_trigger_softirq, sd); in net_dev_init()
12609 sd->cpu = i; in net_dev_init()
12611 INIT_CSD(&sd->defer_csd, trigger_rx_softirq, sd); in net_dev_init()
12612 spin_lock_init(&sd->defer_lock); in net_dev_init()
12614 init_gro_hash(&sd->backlog); in net_dev_init()
12615 sd->backlog.poll = process_backlog; in net_dev_init()
12616 sd->backlog.weight = weight_p; in net_dev_init()
12617 INIT_LIST_HEAD(&sd->backlog.poll_list); in net_dev_init()