1 // SPDX-License-Identifier: GPL-2.0-or-later 2 3 #include <linux/export.h> 4 #include <linux/list.h> 5 #include <linux/netdevice.h> 6 #include <linux/rtnetlink.h> 7 #include <linux/spinlock.h> 8 #include <linux/workqueue.h> 9 #include <net/netdev_lock.h> 10 11 #include "dev.h" 12 13 static void netdev_work_proc(struct work_struct *work); 14 15 /* @netdev_work_lock protects: 16 * - @netdev_work_list 17 * - within the list entries (struct net_device fields): 18 * - work_node 19 * - work_tracker 20 * - work_pending 21 * - work_core_pending 22 */ 23 static LIST_HEAD(netdev_work_list); 24 static DEFINE_SPINLOCK(netdev_work_lock); 25 static DECLARE_WORK(netdev_work, netdev_work_proc); 26 27 static void netdev_work_enqueue(struct net_device *dev, unsigned long events, 28 unsigned long core) 29 { 30 if (!events && !core) 31 return; 32 33 spin_lock_bh(&netdev_work_lock); 34 if (!dev_isalive(dev)) { 35 spin_unlock_bh(&netdev_work_lock); 36 return; 37 } 38 if (list_empty(&dev->work_node)) { 39 list_add_tail(&dev->work_node, &netdev_work_list); 40 netdev_hold(dev, &dev->work_tracker, GFP_ATOMIC); 41 } 42 dev->work_pending |= events; 43 dev->work_core_pending |= core; 44 spin_unlock_bh(&netdev_work_lock); 45 46 schedule_work(&netdev_work); 47 } 48 49 static unsigned long 50 netdev_work_dequeue(struct net_device *dev, unsigned long *pending, 51 unsigned long mask) 52 { 53 unsigned long events; 54 55 spin_lock_bh(&netdev_work_lock); 56 events = *pending & mask; 57 *pending &= ~events; 58 if (!list_empty(&dev->work_node) && 59 !dev->work_pending && !dev->work_core_pending) { 60 list_del_init(&dev->work_node); 61 netdev_put(dev, &dev->work_tracker); 62 } 63 spin_unlock_bh(&netdev_work_lock); 64 65 return events; 66 } 67 68 void netdev_work_cancel_all(struct net_device *dev) 69 { 70 spin_lock_bh(&netdev_work_lock); 71 dev->work_pending = 0; 72 dev->work_core_pending = 0; 73 if (!list_empty(&dev->work_node)) { 74 list_del_init(&dev->work_node); 75 netdev_put(dev, &dev->work_tracker); 76 } 77 spin_unlock_bh(&netdev_work_lock); 78 } 79 80 void netdev_work_sched(struct net_device *dev, unsigned long events) 81 { 82 netdev_work_enqueue(dev, events, 0); 83 } 84 EXPORT_SYMBOL(netdev_work_sched); 85 86 /** 87 * netdev_work_cancel() - cancel selected work for a netdev 88 * @dev: net_device 89 * @mask: events to cancel 90 * 91 * Clear @mask from the device's work pending mask. If no work is left pending 92 * the device is dequeued and its ndo_work won't be called. 93 * 94 * No expectations on locking, but also no guarantees provided. If the caller 95 * wants to touch @dev afterwards (e.g. call the work that got canceled) 96 * they have to ensure @dev does not get freed. 97 * 98 * Returns: the subset of @mask that was actually pending, so the caller can run 99 * those events inline. 100 */ 101 unsigned long netdev_work_cancel(struct net_device *dev, unsigned long mask) 102 { 103 return netdev_work_dequeue(dev, &dev->work_pending, mask); 104 } 105 EXPORT_SYMBOL(netdev_work_cancel); 106 107 void __netdev_work_core_sched(struct net_device *dev, unsigned long events) 108 { 109 netdev_work_enqueue(dev, 0, events); 110 } 111 112 unsigned long 113 __netdev_work_core_cancel(struct net_device *dev, unsigned long mask) 114 { 115 return netdev_work_dequeue(dev, &dev->work_core_pending, mask); 116 } 117 118 static void netdev_work_run(struct net_device *dev, unsigned long events, 119 unsigned long core) 120 { 121 if (!netif_device_present(dev)) 122 return; 123 124 if (core & NETDEV_WORK_RX_MODE) 125 netif_rx_mode_run(dev); 126 if (events && dev->netdev_ops->ndo_work) 127 dev->netdev_ops->ndo_work(dev, events); 128 } 129 130 static void netdev_work_proc(struct work_struct *work) 131 { 132 rtnl_lock(); 133 134 while (true) { 135 unsigned long events = 0, core = 0; 136 netdevice_tracker tracker; 137 struct net_device *dev; 138 139 spin_lock_bh(&netdev_work_lock); 140 if (list_empty(&netdev_work_list)) { 141 spin_unlock_bh(&netdev_work_lock); 142 break; 143 } 144 dev = list_first_entry(&netdev_work_list, struct net_device, 145 work_node); 146 /* Take a temporary reference so @dev can't be freed while we 147 * drop the lock to grab its ops lock; the work reference is 148 * only released once we claim the work below. 149 * The re-locking dance is to ensure that ops lock is enough 150 * to ensure canceling work is not racy with dequeue. 151 */ 152 netdev_hold(dev, &tracker, GFP_ATOMIC); 153 spin_unlock_bh(&netdev_work_lock); 154 155 netdev_lock_ops(dev); 156 spin_lock_bh(&netdev_work_lock); 157 if (!list_empty(&dev->work_node)) { 158 list_del_init(&dev->work_node); 159 core = dev->work_core_pending; 160 dev->work_core_pending = 0; 161 events = dev->work_pending; 162 dev->work_pending = 0; 163 /* We took another ref above */ 164 netdev_put(dev, &dev->work_tracker); 165 166 if (!dev_isalive(dev)) 167 core = events = 0; 168 } 169 spin_unlock_bh(&netdev_work_lock); 170 171 netdev_work_run(dev, events, core); 172 netdev_unlock_ops(dev); 173 174 netdev_put(dev, &tracker); 175 } 176 177 rtnl_unlock(); 178 } 179