1 // SPDX-License-Identifier: GPL-2.0-or-later 2 3 #include <linux/export.h> 4 #include <linux/list.h> 5 #include <linux/netdevice.h> 6 #include <linux/rtnetlink.h> 7 #include <linux/spinlock.h> 8 #include <linux/workqueue.h> 9 #include <net/netdev_lock.h> 10 11 #include "dev.h" 12 13 static void netdev_work_proc(struct work_struct *work); 14 15 /* @netdev_work_lock protects: 16 * - @netdev_work_list 17 * - within the list entries (struct net_device fields): 18 * - work_node 19 * - work_tracker 20 * - work_pending 21 * - work_core_pending 22 */ 23 static LIST_HEAD(netdev_work_list); 24 static DEFINE_SPINLOCK(netdev_work_lock); 25 static DECLARE_WORK(netdev_work, netdev_work_proc); 26 27 static void netdev_work_enqueue(struct net_device *dev, unsigned long events, 28 unsigned long core) 29 { 30 if (!events && !core) 31 return; 32 33 spin_lock_bh(&netdev_work_lock); 34 if (list_empty(&dev->work_node)) { 35 list_add_tail(&dev->work_node, &netdev_work_list); 36 netdev_hold(dev, &dev->work_tracker, GFP_ATOMIC); 37 } 38 dev->work_pending |= events; 39 dev->work_core_pending |= core; 40 spin_unlock_bh(&netdev_work_lock); 41 42 schedule_work(&netdev_work); 43 } 44 45 static unsigned long 46 netdev_work_dequeue(struct net_device *dev, unsigned long *pending, 47 unsigned long mask) 48 { 49 unsigned long events; 50 51 spin_lock_bh(&netdev_work_lock); 52 events = *pending & mask; 53 *pending &= ~events; 54 if (!list_empty(&dev->work_node) && 55 !dev->work_pending && !dev->work_core_pending) { 56 list_del_init(&dev->work_node); 57 netdev_put(dev, &dev->work_tracker); 58 } 59 spin_unlock_bh(&netdev_work_lock); 60 61 return events; 62 } 63 64 void netdev_work_sched(struct net_device *dev, unsigned long events) 65 { 66 netdev_work_enqueue(dev, events, 0); 67 } 68 EXPORT_SYMBOL(netdev_work_sched); 69 70 /** 71 * netdev_work_cancel() - cancel selected work for a netdev 72 * @dev: net_device 73 * @mask: events to cancel 74 * 75 * Clear @mask from the device's work pending mask. If no work is left pending 76 * the device is dequeued and its ndo_work won't be called. 77 * 78 * No expectations on locking, but also no guarantees provided. If the caller 79 * wants to touch @dev afterwards (e.g. call the work that got canceled) 80 * they have to ensure @dev does not get freed. 81 * 82 * Returns: the subset of @mask that was actually pending, so the caller can run 83 * those events inline. 84 */ 85 unsigned long netdev_work_cancel(struct net_device *dev, unsigned long mask) 86 { 87 return netdev_work_dequeue(dev, &dev->work_pending, mask); 88 } 89 EXPORT_SYMBOL(netdev_work_cancel); 90 91 void __netdev_work_core_sched(struct net_device *dev, unsigned long events) 92 { 93 netdev_work_enqueue(dev, 0, events); 94 } 95 96 unsigned long 97 __netdev_work_core_cancel(struct net_device *dev, unsigned long mask) 98 { 99 return netdev_work_dequeue(dev, &dev->work_core_pending, mask); 100 } 101 102 static void netdev_work_run(struct net_device *dev, unsigned long events, 103 unsigned long core) 104 { 105 if (!netif_device_present(dev)) 106 return; 107 108 if (core & NETDEV_WORK_RX_MODE) 109 netif_rx_mode_run(dev); 110 if (events && dev->netdev_ops->ndo_work) 111 dev->netdev_ops->ndo_work(dev, events); 112 } 113 114 static void netdev_work_proc(struct work_struct *work) 115 { 116 rtnl_lock(); 117 118 while (true) { 119 unsigned long events = 0, core = 0; 120 netdevice_tracker tracker; 121 struct net_device *dev; 122 123 spin_lock_bh(&netdev_work_lock); 124 if (list_empty(&netdev_work_list)) { 125 spin_unlock_bh(&netdev_work_lock); 126 break; 127 } 128 dev = list_first_entry(&netdev_work_list, struct net_device, 129 work_node); 130 /* Take a temporary reference so @dev can't be freed while we 131 * drop the lock to grab its ops lock; the work reference is 132 * only released once we claim the work below. 133 * The re-locking dance is to ensure that ops lock is enough 134 * to ensure canceling work is not racy with dequeue. 135 */ 136 netdev_hold(dev, &tracker, GFP_ATOMIC); 137 spin_unlock_bh(&netdev_work_lock); 138 139 netdev_lock_ops(dev); 140 spin_lock_bh(&netdev_work_lock); 141 if (!list_empty(&dev->work_node)) { 142 list_del_init(&dev->work_node); 143 core = dev->work_core_pending; 144 dev->work_core_pending = 0; 145 events = dev->work_pending; 146 dev->work_pending = 0; 147 /* We took another ref above */ 148 netdev_put(dev, &dev->work_tracker); 149 150 if (!dev_isalive(dev)) 151 core = events = 0; 152 } 153 spin_unlock_bh(&netdev_work_lock); 154 155 netdev_work_run(dev, events, core); 156 netdev_unlock_ops(dev); 157 158 netdev_put(dev, &tracker); 159 } 160 161 rtnl_unlock(); 162 } 163