1 // SPDX-License-Identifier: GPL-2.0-or-later
2
3 #include <linux/export.h>
4 #include <linux/list.h>
5 #include <linux/netdevice.h>
6 #include <linux/rtnetlink.h>
7 #include <linux/spinlock.h>
8 #include <linux/workqueue.h>
9 #include <net/netdev_lock.h>
10
11 #include "dev.h"
12
13 static void netdev_work_proc(struct work_struct *work);
14
15 /* @netdev_work_lock protects:
16 * - @netdev_work_list
17 * - within the list entries (struct net_device fields):
18 * - work_node
19 * - work_tracker
20 * - work_pending
21 * - work_core_pending
22 */
23 static LIST_HEAD(netdev_work_list);
24 static DEFINE_SPINLOCK(netdev_work_lock);
25 static DECLARE_WORK(netdev_work, netdev_work_proc);
26
netdev_work_enqueue(struct net_device * dev,unsigned long events,unsigned long core)27 static void netdev_work_enqueue(struct net_device *dev, unsigned long events,
28 unsigned long core)
29 {
30 if (!events && !core)
31 return;
32
33 spin_lock_bh(&netdev_work_lock);
34 if (!dev_isalive(dev)) {
35 spin_unlock_bh(&netdev_work_lock);
36 return;
37 }
38 if (list_empty(&dev->work_node)) {
39 list_add_tail(&dev->work_node, &netdev_work_list);
40 netdev_hold(dev, &dev->work_tracker, GFP_ATOMIC);
41 }
42 dev->work_pending |= events;
43 dev->work_core_pending |= core;
44 spin_unlock_bh(&netdev_work_lock);
45
46 schedule_work(&netdev_work);
47 }
48
49 static unsigned long
netdev_work_dequeue(struct net_device * dev,unsigned long * pending,unsigned long mask)50 netdev_work_dequeue(struct net_device *dev, unsigned long *pending,
51 unsigned long mask)
52 {
53 unsigned long events;
54
55 spin_lock_bh(&netdev_work_lock);
56 events = *pending & mask;
57 *pending &= ~events;
58 if (!list_empty(&dev->work_node) &&
59 !dev->work_pending && !dev->work_core_pending) {
60 list_del_init(&dev->work_node);
61 netdev_put(dev, &dev->work_tracker);
62 }
63 spin_unlock_bh(&netdev_work_lock);
64
65 return events;
66 }
67
netdev_work_cancel_all(struct net_device * dev)68 void netdev_work_cancel_all(struct net_device *dev)
69 {
70 spin_lock_bh(&netdev_work_lock);
71 dev->work_pending = 0;
72 dev->work_core_pending = 0;
73 if (!list_empty(&dev->work_node)) {
74 list_del_init(&dev->work_node);
75 netdev_put(dev, &dev->work_tracker);
76 }
77 spin_unlock_bh(&netdev_work_lock);
78 }
79
netdev_work_sched(struct net_device * dev,unsigned long events)80 void netdev_work_sched(struct net_device *dev, unsigned long events)
81 {
82 netdev_work_enqueue(dev, events, 0);
83 }
84 EXPORT_SYMBOL(netdev_work_sched);
85
86 /**
87 * netdev_work_cancel() - cancel selected work for a netdev
88 * @dev: net_device
89 * @mask: events to cancel
90 *
91 * Clear @mask from the device's work pending mask. If no work is left pending
92 * the device is dequeued and its ndo_work won't be called.
93 *
94 * No expectations on locking, but also no guarantees provided. If the caller
95 * wants to touch @dev afterwards (e.g. call the work that got canceled)
96 * they have to ensure @dev does not get freed.
97 *
98 * Returns: the subset of @mask that was actually pending, so the caller can run
99 * those events inline.
100 */
netdev_work_cancel(struct net_device * dev,unsigned long mask)101 unsigned long netdev_work_cancel(struct net_device *dev, unsigned long mask)
102 {
103 return netdev_work_dequeue(dev, &dev->work_pending, mask);
104 }
105 EXPORT_SYMBOL(netdev_work_cancel);
106
__netdev_work_core_sched(struct net_device * dev,unsigned long events)107 void __netdev_work_core_sched(struct net_device *dev, unsigned long events)
108 {
109 netdev_work_enqueue(dev, 0, events);
110 }
111
112 unsigned long
__netdev_work_core_cancel(struct net_device * dev,unsigned long mask)113 __netdev_work_core_cancel(struct net_device *dev, unsigned long mask)
114 {
115 return netdev_work_dequeue(dev, &dev->work_core_pending, mask);
116 }
117
netdev_work_run(struct net_device * dev,unsigned long events,unsigned long core)118 static void netdev_work_run(struct net_device *dev, unsigned long events,
119 unsigned long core)
120 {
121 if (!netif_device_present(dev))
122 return;
123
124 if (core & NETDEV_WORK_RX_MODE)
125 netif_rx_mode_run(dev);
126 if (events && dev->netdev_ops->ndo_work)
127 dev->netdev_ops->ndo_work(dev, events);
128 }
129
netdev_work_proc(struct work_struct * work)130 static void netdev_work_proc(struct work_struct *work)
131 {
132 rtnl_lock();
133
134 while (true) {
135 unsigned long events = 0, core = 0;
136 netdevice_tracker tracker;
137 struct net_device *dev;
138
139 spin_lock_bh(&netdev_work_lock);
140 if (list_empty(&netdev_work_list)) {
141 spin_unlock_bh(&netdev_work_lock);
142 break;
143 }
144 dev = list_first_entry(&netdev_work_list, struct net_device,
145 work_node);
146 /* Take a temporary reference so @dev can't be freed while we
147 * drop the lock to grab its ops lock; the work reference is
148 * only released once we claim the work below.
149 * The re-locking dance is to ensure that ops lock is enough
150 * to ensure canceling work is not racy with dequeue.
151 */
152 netdev_hold(dev, &tracker, GFP_ATOMIC);
153 spin_unlock_bh(&netdev_work_lock);
154
155 netdev_lock_ops(dev);
156 spin_lock_bh(&netdev_work_lock);
157 if (!list_empty(&dev->work_node)) {
158 list_del_init(&dev->work_node);
159 core = dev->work_core_pending;
160 dev->work_core_pending = 0;
161 events = dev->work_pending;
162 dev->work_pending = 0;
163 /* We took another ref above */
164 netdev_put(dev, &dev->work_tracker);
165
166 if (!dev_isalive(dev))
167 core = events = 0;
168 }
169 spin_unlock_bh(&netdev_work_lock);
170
171 netdev_work_run(dev, events, core);
172 netdev_unlock_ops(dev);
173
174 netdev_put(dev, &tracker);
175 }
176
177 rtnl_unlock();
178 }
179