xref: /linux/net/core/dev.c (revision fdca4b030c78fa2773a94fda3a6dc3ce5558a727)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  *      NET3    Protocol independent device support routines.
4  *
5  *	Derived from the non IP parts of dev.c 1.0.19
6  *              Authors:	Ross Biro
7  *				Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
8  *				Mark Evans, <evansmp@uhura.aston.ac.uk>
9  *
10  *	Additional Authors:
11  *		Florian la Roche <rzsfl@rz.uni-sb.de>
12  *		Alan Cox <gw4pts@gw4pts.ampr.org>
13  *		David Hinds <dahinds@users.sourceforge.net>
14  *		Alexey Kuznetsov <kuznet@ms2.inr.ac.ru>
15  *		Adam Sulmicki <adam@cfar.umd.edu>
16  *              Pekka Riikonen <priikone@poesidon.pspt.fi>
17  *
18  *	Changes:
19  *              D.J. Barrow     :       Fixed bug where dev->refcnt gets set
20  *                                      to 2 if register_netdev gets called
21  *                                      before net_dev_init & also removed a
22  *                                      few lines of code in the process.
23  *		Alan Cox	:	device private ioctl copies fields back.
24  *		Alan Cox	:	Transmit queue code does relevant
25  *					stunts to keep the queue safe.
26  *		Alan Cox	:	Fixed double lock.
27  *		Alan Cox	:	Fixed promisc NULL pointer trap
28  *		????????	:	Support the full private ioctl range
29  *		Alan Cox	:	Moved ioctl permission check into
30  *					drivers
31  *		Tim Kordas	:	SIOCADDMULTI/SIOCDELMULTI
32  *		Alan Cox	:	100 backlog just doesn't cut it when
33  *					you start doing multicast video 8)
34  *		Alan Cox	:	Rewrote net_bh and list manager.
35  *              Alan Cox        :       Fix ETH_P_ALL echoback lengths.
36  *		Alan Cox	:	Took out transmit every packet pass
37  *					Saved a few bytes in the ioctl handler
38  *		Alan Cox	:	Network driver sets packet type before
39  *					calling netif_rx. Saves a function
40  *					call a packet.
41  *		Alan Cox	:	Hashed net_bh()
42  *		Richard Kooijman:	Timestamp fixes.
43  *		Alan Cox	:	Wrong field in SIOCGIFDSTADDR
44  *		Alan Cox	:	Device lock protection.
45  *              Alan Cox        :       Fixed nasty side effect of device close
46  *					changes.
47  *		Rudi Cilibrasi	:	Pass the right thing to
48  *					set_mac_address()
49  *		Dave Miller	:	32bit quantity for the device lock to
50  *					make it work out on a Sparc.
51  *		Bjorn Ekwall	:	Added KERNELD hack.
52  *		Alan Cox	:	Cleaned up the backlog initialise.
53  *		Craig Metz	:	SIOCGIFCONF fix if space for under
54  *					1 device.
55  *	    Thomas Bogendoerfer :	Return ENODEV for dev_open, if there
56  *					is no device open function.
57  *		Andi Kleen	:	Fix error reporting for SIOCGIFCONF
58  *	    Michael Chastain	:	Fix signed/unsigned for SIOCGIFCONF
59  *		Cyrus Durgin	:	Cleaned for KMOD
60  *		Adam Sulmicki   :	Bug Fix : Network Device Unload
61  *					A network device unload needs to purge
62  *					the backlog queue.
63  *	Paul Rusty Russell	:	SIOCSIFNAME
64  *              Pekka Riikonen  :	Netdev boot-time settings code
65  *              Andrew Morton   :       Make unregister_netdevice wait
66  *                                      indefinitely on dev->refcnt
67  *              J Hadi Salim    :       - Backlog queue sampling
68  *				        - netif_rx() feedback
69  */
70 
71 #include <linux/uaccess.h>
72 #include <linux/bitmap.h>
73 #include <linux/capability.h>
74 #include <linux/cpu.h>
75 #include <linux/types.h>
76 #include <linux/kernel.h>
77 #include <linux/hash.h>
78 #include <linux/slab.h>
79 #include <linux/sched.h>
80 #include <linux/sched/isolation.h>
81 #include <linux/sched/mm.h>
82 #include <linux/smpboot.h>
83 #include <linux/mutex.h>
84 #include <linux/rwsem.h>
85 #include <linux/string.h>
86 #include <linux/mm.h>
87 #include <linux/socket.h>
88 #include <linux/sockios.h>
89 #include <linux/errno.h>
90 #include <linux/interrupt.h>
91 #include <linux/if_ether.h>
92 #include <linux/netdevice.h>
93 #include <linux/etherdevice.h>
94 #include <linux/ethtool.h>
95 #include <linux/ethtool_netlink.h>
96 #include <linux/skbuff.h>
97 #include <linux/kthread.h>
98 #include <linux/bpf.h>
99 #include <linux/bpf_trace.h>
100 #include <net/net_namespace.h>
101 #include <net/sock.h>
102 #include <net/busy_poll.h>
103 #include <linux/rtnetlink.h>
104 #include <linux/stat.h>
105 #include <net/dsa.h>
106 #include <net/dst.h>
107 #include <net/dst_metadata.h>
108 #include <net/gro.h>
109 #include <net/netdev_queues.h>
110 #include <net/pkt_sched.h>
111 #include <net/pkt_cls.h>
112 #include <net/checksum.h>
113 #include <net/xfrm.h>
114 #include <net/tcx.h>
115 #include <linux/highmem.h>
116 #include <linux/init.h>
117 #include <linux/module.h>
118 #include <linux/netpoll.h>
119 #include <linux/rcupdate.h>
120 #include <linux/delay.h>
121 #include <net/iw_handler.h>
122 #include <asm/current.h>
123 #include <linux/audit.h>
124 #include <linux/dmaengine.h>
125 #include <linux/err.h>
126 #include <linux/ctype.h>
127 #include <linux/if_arp.h>
128 #include <linux/if_vlan.h>
129 #include <linux/ip.h>
130 #include <net/ip.h>
131 #include <net/mpls.h>
132 #include <net/route.h>
133 #include <linux/ipv6.h>
134 #include <net/ip6_route.h>
135 #include <linux/in.h>
136 #include <linux/jhash.h>
137 #include <linux/random.h>
138 #include <trace/events/napi.h>
139 #include <trace/events/net.h>
140 #include <trace/events/skb.h>
141 #include <trace/events/qdisc.h>
142 #include <trace/events/xdp.h>
143 #include <linux/inetdevice.h>
144 #include <linux/cpu_rmap.h>
145 #include <linux/static_key.h>
146 #include <linux/hashtable.h>
147 #include <linux/vmalloc.h>
148 #include <linux/if_macvlan.h>
149 #include <linux/errqueue.h>
150 #include <linux/hrtimer.h>
151 #include <linux/netfilter_netdev.h>
152 #include <linux/crash_dump.h>
153 #include <linux/sctp.h>
154 #include <net/udp_tunnel.h>
155 #include <linux/net_namespace.h>
156 #include <linux/indirect_call_wrapper.h>
157 #include <net/devlink.h>
158 #include <linux/pm_runtime.h>
159 #include <linux/prandom.h>
160 #include <linux/once_lite.h>
161 #include <net/netdev_lock.h>
162 #include <net/netdev_rx_queue.h>
163 #include <net/page_pool/types.h>
164 #include <net/page_pool/helpers.h>
165 #include <net/page_pool/memory_provider.h>
166 #include <net/rps.h>
167 #include <linux/phy_link_topology.h>
168 
169 #include "dev.h"
170 #include "devmem.h"
171 #include "net-sysfs.h"
172 
173 static DEFINE_SPINLOCK(ptype_lock);
174 struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
175 
176 static int netif_rx_internal(struct sk_buff *skb);
177 static int call_netdevice_notifiers_extack(unsigned long val,
178 					   struct net_device *dev,
179 					   struct netlink_ext_ack *extack);
180 
181 static DEFINE_MUTEX(ifalias_mutex);
182 
183 /* protects napi_hash addition/deletion and napi_gen_id */
184 static DEFINE_SPINLOCK(napi_hash_lock);
185 
186 static unsigned int napi_gen_id = NR_CPUS;
187 static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8);
188 
189 static inline void dev_base_seq_inc(struct net *net)
190 {
191 	unsigned int val = net->dev_base_seq + 1;
192 
193 	WRITE_ONCE(net->dev_base_seq, val ?: 1);
194 }
195 
196 static inline struct hlist_head *dev_name_hash(struct net *net, const char *name)
197 {
198 	unsigned int hash = full_name_hash(net, name, strnlen(name, IFNAMSIZ));
199 
200 	return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)];
201 }
202 
203 static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex)
204 {
205 	return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)];
206 }
207 
208 #ifndef CONFIG_PREEMPT_RT
209 
210 static DEFINE_STATIC_KEY_FALSE(use_backlog_threads_key);
211 
212 static int __init setup_backlog_napi_threads(char *arg)
213 {
214 	static_branch_enable(&use_backlog_threads_key);
215 	return 0;
216 }
217 early_param("thread_backlog_napi", setup_backlog_napi_threads);
218 
219 static bool use_backlog_threads(void)
220 {
221 	return static_branch_unlikely(&use_backlog_threads_key);
222 }
223 
224 #else
225 
226 static bool use_backlog_threads(void)
227 {
228 	return true;
229 }
230 
231 #endif
232 
233 static inline void backlog_lock_irq_save(struct softnet_data *sd,
234 					 unsigned long *flags)
235 {
236 	if (IS_ENABLED(CONFIG_PREEMPT_RT)) {
237 		spin_lock_irqsave(&sd->input_pkt_queue.lock, *flags);
238 	} else {
239 		local_irq_save(*flags);
240 		if (IS_ENABLED(CONFIG_RPS) || use_backlog_threads())
241 			spin_lock(&sd->input_pkt_queue.lock);
242 	}
243 }
244 
245 static inline void backlog_lock_irq_disable(struct softnet_data *sd)
246 {
247 	if (IS_ENABLED(CONFIG_RPS) || use_backlog_threads())
248 		spin_lock_irq(&sd->input_pkt_queue.lock);
249 	else
250 		local_irq_disable();
251 }
252 
253 static inline void backlog_unlock_irq_restore(struct softnet_data *sd,
254 					      unsigned long flags)
255 {
256 	if (IS_ENABLED(CONFIG_PREEMPT_RT)) {
257 		spin_unlock_irqrestore(&sd->input_pkt_queue.lock, flags);
258 	} else {
259 		if (IS_ENABLED(CONFIG_RPS) || use_backlog_threads())
260 			spin_unlock(&sd->input_pkt_queue.lock);
261 		local_irq_restore(flags);
262 	}
263 }
264 
265 static inline void backlog_unlock_irq_enable(struct softnet_data *sd)
266 {
267 	if (IS_ENABLED(CONFIG_RPS) || use_backlog_threads())
268 		spin_unlock_irq(&sd->input_pkt_queue.lock);
269 	else
270 		local_irq_enable();
271 }
272 
273 static struct netdev_name_node *netdev_name_node_alloc(struct net_device *dev,
274 						       const char *name)
275 {
276 	struct netdev_name_node *name_node;
277 
278 	name_node = kmalloc_obj(*name_node);
279 	if (!name_node)
280 		return NULL;
281 	INIT_HLIST_NODE(&name_node->hlist);
282 	name_node->dev = dev;
283 	name_node->name = name;
284 	return name_node;
285 }
286 
287 static struct netdev_name_node *
288 netdev_name_node_head_alloc(struct net_device *dev)
289 {
290 	struct netdev_name_node *name_node;
291 
292 	name_node = netdev_name_node_alloc(dev, dev->name);
293 	if (!name_node)
294 		return NULL;
295 	INIT_LIST_HEAD(&name_node->list);
296 	return name_node;
297 }
298 
299 static void netdev_name_node_free(struct netdev_name_node *name_node)
300 {
301 	kfree(name_node);
302 }
303 
304 static void netdev_name_node_add(struct net *net,
305 				 struct netdev_name_node *name_node)
306 {
307 	hlist_add_head_rcu(&name_node->hlist,
308 			   dev_name_hash(net, name_node->name));
309 }
310 
311 static void netdev_name_node_del(struct netdev_name_node *name_node)
312 {
313 	hlist_del_rcu(&name_node->hlist);
314 }
315 
316 static struct netdev_name_node *netdev_name_node_lookup(struct net *net,
317 							const char *name)
318 {
319 	struct hlist_head *head = dev_name_hash(net, name);
320 	struct netdev_name_node *name_node;
321 
322 	hlist_for_each_entry(name_node, head, hlist)
323 		if (!strcmp(name_node->name, name))
324 			return name_node;
325 	return NULL;
326 }
327 
328 static struct netdev_name_node *netdev_name_node_lookup_rcu(struct net *net,
329 							    const char *name)
330 {
331 	struct hlist_head *head = dev_name_hash(net, name);
332 	struct netdev_name_node *name_node;
333 
334 	hlist_for_each_entry_rcu(name_node, head, hlist)
335 		if (!strcmp(name_node->name, name))
336 			return name_node;
337 	return NULL;
338 }
339 
340 bool netdev_name_in_use(struct net *net, const char *name)
341 {
342 	return netdev_name_node_lookup(net, name);
343 }
344 EXPORT_SYMBOL(netdev_name_in_use);
345 
346 int netdev_name_node_alt_create(struct net_device *dev, const char *name)
347 {
348 	struct netdev_name_node *name_node;
349 	struct net *net = dev_net(dev);
350 
351 	name_node = netdev_name_node_lookup(net, name);
352 	if (name_node)
353 		return -EEXIST;
354 	name_node = netdev_name_node_alloc(dev, name);
355 	if (!name_node)
356 		return -ENOMEM;
357 	netdev_name_node_add(net, name_node);
358 	/* The node that holds dev->name acts as a head of per-device list. */
359 	list_add_tail_rcu(&name_node->list, &dev->name_node->list);
360 
361 	return 0;
362 }
363 
364 static void netdev_name_node_alt_free(struct rcu_head *head)
365 {
366 	struct netdev_name_node *name_node =
367 		container_of(head, struct netdev_name_node, rcu);
368 
369 	kfree(name_node->name);
370 	netdev_name_node_free(name_node);
371 }
372 
373 static void __netdev_name_node_alt_destroy(struct netdev_name_node *name_node)
374 {
375 	netdev_name_node_del(name_node);
376 	list_del_rcu(&name_node->list);
377 	call_rcu(&name_node->rcu, netdev_name_node_alt_free);
378 }
379 
380 int netdev_name_node_alt_destroy(struct net_device *dev, const char *name)
381 {
382 	struct netdev_name_node *name_node;
383 	struct net *net = dev_net(dev);
384 
385 	name_node = netdev_name_node_lookup(net, name);
386 	if (!name_node)
387 		return -ENOENT;
388 	/* lookup might have found our primary name or a name belonging
389 	 * to another device.
390 	 */
391 	if (name_node == dev->name_node || name_node->dev != dev)
392 		return -EINVAL;
393 
394 	__netdev_name_node_alt_destroy(name_node);
395 	return 0;
396 }
397 
398 static void netdev_name_node_alt_flush(struct net_device *dev)
399 {
400 	struct netdev_name_node *name_node, *tmp;
401 
402 	list_for_each_entry_safe(name_node, tmp, &dev->name_node->list, list) {
403 		list_del(&name_node->list);
404 		netdev_name_node_alt_free(&name_node->rcu);
405 	}
406 }
407 
408 /* Device list insertion */
409 static void list_netdevice(struct net_device *dev)
410 {
411 	struct netdev_name_node *name_node;
412 	struct net *net = dev_net(dev);
413 
414 	ASSERT_RTNL();
415 
416 	list_add_tail_rcu(&dev->dev_list, &net->dev_base_head);
417 	netdev_name_node_add(net, dev->name_node);
418 	hlist_add_head_rcu(&dev->index_hlist,
419 			   dev_index_hash(net, dev->ifindex));
420 
421 	netdev_for_each_altname(dev, name_node)
422 		netdev_name_node_add(net, name_node);
423 
424 	/* We reserved the ifindex, this can't fail */
425 	WARN_ON(xa_store(&net->dev_by_index, dev->ifindex, dev, GFP_KERNEL));
426 
427 	dev_base_seq_inc(net);
428 }
429 
430 /* Device list removal
431  * caller must respect a RCU grace period before freeing/reusing dev
432  */
433 static void unlist_netdevice(struct net_device *dev)
434 {
435 	struct netdev_name_node *name_node;
436 	struct net *net = dev_net(dev);
437 
438 	ASSERT_RTNL();
439 
440 	xa_erase(&net->dev_by_index, dev->ifindex);
441 
442 	netdev_for_each_altname(dev, name_node)
443 		netdev_name_node_del(name_node);
444 
445 	/* Unlink dev from the device chain */
446 	list_del_rcu(&dev->dev_list);
447 	netdev_name_node_del(dev->name_node);
448 	hlist_del_rcu(&dev->index_hlist);
449 
450 	dev_base_seq_inc(dev_net(dev));
451 }
452 
453 /*
454  *	Our notifier list
455  */
456 
457 static RAW_NOTIFIER_HEAD(netdev_chain);
458 
459 /*
460  *	Device drivers call our routines to queue packets here. We empty the
461  *	queue in the local softnet handler.
462  */
463 
464 DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data) = {
465 	.process_queue_bh_lock = INIT_LOCAL_LOCK(process_queue_bh_lock),
466 };
467 EXPORT_PER_CPU_SYMBOL(softnet_data);
468 
469 /* Page_pool has a lockless array/stack to alloc/recycle pages.
470  * PP consumers must pay attention to run APIs in the appropriate context
471  * (e.g. NAPI context).
472  */
473 DEFINE_PER_CPU(struct page_pool_bh, system_page_pool) = {
474 	.bh_lock = INIT_LOCAL_LOCK(bh_lock),
475 };
476 
477 #ifdef CONFIG_LOCKDEP
478 /*
479  * register_netdevice() inits txq->_xmit_lock and sets lockdep class
480  * according to dev->type
481  */
482 static const unsigned short netdev_lock_type[] = {
483 	 ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25,
484 	 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET,
485 	 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM,
486 	 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP,
487 	 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD,
488 	 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25,
489 	 ARPHRD_CAN, ARPHRD_MCTP,
490 	 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP,
491 	 ARPHRD_RAWHDLC, ARPHRD_RAWIP,
492 	 ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD,
493 	 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI,
494 	 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE,
495 	 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET,
496 	 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL,
497 	 ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM,
498 	 ARPHRD_IEEE80211_RADIOTAP,
499 	 ARPHRD_IEEE802154, ARPHRD_IEEE802154_MONITOR,
500 	 ARPHRD_PHONET, ARPHRD_PHONET_PIPE,
501 	 ARPHRD_CAIF, ARPHRD_IP6GRE, ARPHRD_NETLINK, ARPHRD_6LOWPAN,
502 	 ARPHRD_VSOCKMON,
503 	 ARPHRD_VOID, ARPHRD_NONE};
504 
505 static const char *const netdev_lock_name[] = {
506 	"_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25",
507 	"_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET",
508 	"_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM",
509 	"_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP",
510 	"_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD",
511 	"_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25",
512 	"_xmit_CAN", "_xmit_MCTP",
513 	"_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP",
514 	"_xmit_RAWHDLC", "_xmit_RAWIP",
515 	"_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD",
516 	"_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI",
517 	"_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE",
518 	"_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET",
519 	"_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL",
520 	"_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM",
521 	"_xmit_IEEE80211_RADIOTAP",
522 	"_xmit_IEEE802154", "_xmit_IEEE802154_MONITOR",
523 	"_xmit_PHONET", "_xmit_PHONET_PIPE",
524 	"_xmit_CAIF", "_xmit_IP6GRE", "_xmit_NETLINK", "_xmit_6LOWPAN",
525 	"_xmit_VSOCKMON",
526 	"_xmit_VOID", "_xmit_NONE"};
527 
528 static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)];
529 static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)];
530 
531 static inline unsigned short netdev_lock_pos(unsigned short dev_type)
532 {
533 	int i;
534 
535 	for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++)
536 		if (netdev_lock_type[i] == dev_type)
537 			return i;
538 	/* the last key is used by default */
539 	WARN_ONCE(1, "netdev_lock_pos() could not find dev_type=%u\n", dev_type);
540 	return ARRAY_SIZE(netdev_lock_type) - 1;
541 }
542 
543 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
544 						 unsigned short dev_type)
545 {
546 	int i;
547 
548 	i = netdev_lock_pos(dev_type);
549 	lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i],
550 				   netdev_lock_name[i]);
551 }
552 
553 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
554 {
555 	int i;
556 
557 	i = netdev_lock_pos(dev->type);
558 	lockdep_set_class_and_name(&dev->addr_list_lock,
559 				   &netdev_addr_lock_key[i],
560 				   netdev_lock_name[i]);
561 }
562 #else
563 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock,
564 						 unsigned short dev_type)
565 {
566 }
567 
568 static inline void netdev_set_addr_lockdep_class(struct net_device *dev)
569 {
570 }
571 #endif
572 
573 #ifdef CONFIG_PROVE_LOCKING
574 static int netdev_lock_cmp_fn(const struct lockdep_map *a,
575 			      const struct lockdep_map *b)
576 {
577 	if (a == b)
578 		return 0;
579 
580 	/* @a and @b are of same lock class.
581 	 * cmp_fn won't be called for devices of different classes.
582 	 *
583 	 * For the same class only allow nesting under the protection
584 	 * of rtnl_lock. Note that we can't use lockdep_rtnl_is_held()
585 	 * here, it always answers UNKNOWN from within lockdep.
586 	 */
587 	return rtnl_is_locked() ? -1 : 1;
588 }
589 
590 /* A virtual device can be locked before the physical device it leases
591  * queues from, see netdev_nl_queue_create_doit(). Keep the two kinds
592  * in separate classes so the dependency graph enforces the order;
593  * netdev_lock_cmp_fn() then only has to rule on same-class nesting.
594  * Other virtual devices stay in the default class.
595  */
596 void netdev_set_instance_lock_class(struct net_device *dev)
597 {
598 	static struct lock_class_key netdev_virt_instance_lock_key;
599 
600 	if (!netdev_can_create_queue(dev, NULL))
601 		return;
602 
603 	lockdep_set_class(&dev->lock, &netdev_virt_instance_lock_key);
604 	lock_set_cmp_fn(&dev->lock, netdev_lock_cmp_fn, NULL);
605 }
606 EXPORT_SYMBOL_GPL(netdev_set_instance_lock_class);
607 #endif
608 
609 /*******************************************************************************
610  *
611  *		Protocol management and registration routines
612  *
613  *******************************************************************************/
614 
615 
616 /*
617  *	Add a protocol ID to the list. Now that the input handler is
618  *	smarter we can dispense with all the messy stuff that used to be
619  *	here.
620  *
621  *	BEWARE!!! Protocol handlers, mangling input packets,
622  *	MUST BE last in hash buckets and checking protocol handlers
623  *	MUST start from promiscuous ptype_all chain in net_bh.
624  *	It is true now, do not change it.
625  *	Explanation follows: if protocol handler, mangling packet, will
626  *	be the first on list, it is not able to sense, that packet
627  *	is cloned and should be copied-on-write, so that it will
628  *	change it and subsequent readers will get broken packet.
629  *							--ANK (980803)
630  */
631 
632 static inline struct list_head *ptype_head(const struct packet_type *pt)
633 {
634 	if (pt->type == htons(ETH_P_ALL)) {
635 		if (!pt->af_packet_net && !pt->dev)
636 			return NULL;
637 
638 		return pt->dev ? &pt->dev->ptype_all :
639 				 &pt->af_packet_net->ptype_all;
640 	}
641 
642 	if (pt->dev)
643 		return &pt->dev->ptype_specific;
644 
645 	return pt->af_packet_net ? &pt->af_packet_net->ptype_specific :
646 				 &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK];
647 }
648 
649 /**
650  *	dev_add_pack - add packet handler
651  *	@pt: packet type declaration
652  *
653  *	Add a protocol handler to the networking stack. The passed &packet_type
654  *	is linked into kernel lists and may not be freed until it has been
655  *	removed from the kernel lists.
656  *
657  *	This call does not sleep therefore it can not
658  *	guarantee all CPU's that are in middle of receiving packets
659  *	will see the new packet type (until the next received packet).
660  */
661 
662 void dev_add_pack(struct packet_type *pt)
663 {
664 	struct list_head *head = ptype_head(pt);
665 
666 	if (WARN_ON_ONCE(!head))
667 		return;
668 
669 	spin_lock(&ptype_lock);
670 	list_add_rcu(&pt->list, head);
671 	spin_unlock(&ptype_lock);
672 }
673 EXPORT_SYMBOL(dev_add_pack);
674 
675 /**
676  *	__dev_remove_pack	 - remove packet handler
677  *	@pt: packet type declaration
678  *
679  *	Remove a protocol handler that was previously added to the kernel
680  *	protocol handlers by dev_add_pack(). The passed &packet_type is removed
681  *	from the kernel lists and can be freed or reused once this function
682  *	returns.
683  *
684  *      The packet type might still be in use by receivers
685  *	and must not be freed until after all the CPU's have gone
686  *	through a quiescent state.
687  */
688 void __dev_remove_pack(struct packet_type *pt)
689 {
690 	struct list_head *head = ptype_head(pt);
691 	struct packet_type *pt1;
692 
693 	if (!head)
694 		return;
695 
696 	spin_lock(&ptype_lock);
697 
698 	list_for_each_entry(pt1, head, list) {
699 		if (pt == pt1) {
700 			list_del_rcu(&pt->list);
701 			goto out;
702 		}
703 	}
704 
705 	pr_warn("dev_remove_pack: %p not found\n", pt);
706 out:
707 	spin_unlock(&ptype_lock);
708 }
709 EXPORT_SYMBOL(__dev_remove_pack);
710 
711 /**
712  *	dev_remove_pack	 - remove packet handler
713  *	@pt: packet type declaration
714  *
715  *	Remove a protocol handler that was previously added to the kernel
716  *	protocol handlers by dev_add_pack(). The passed &packet_type is removed
717  *	from the kernel lists and can be freed or reused once this function
718  *	returns.
719  *
720  *	This call sleeps to guarantee that no CPU is looking at the packet
721  *	type after return.
722  */
723 void dev_remove_pack(struct packet_type *pt)
724 {
725 	__dev_remove_pack(pt);
726 
727 	synchronize_net();
728 }
729 EXPORT_SYMBOL(dev_remove_pack);
730 
731 
732 /*******************************************************************************
733  *
734  *			    Device Interface Subroutines
735  *
736  *******************************************************************************/
737 
738 /**
739  *	dev_get_iflink	- get 'iflink' value of a interface
740  *	@dev: targeted interface
741  *
742  *	Indicates the ifindex the interface is linked to.
743  *	Physical interfaces have the same 'ifindex' and 'iflink' values.
744  */
745 
746 int dev_get_iflink(const struct net_device *dev)
747 {
748 	if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink)
749 		return dev->netdev_ops->ndo_get_iflink(dev);
750 
751 	return READ_ONCE(dev->ifindex);
752 }
753 EXPORT_SYMBOL(dev_get_iflink);
754 
755 /**
756  *	dev_fill_metadata_dst - Retrieve tunnel egress information.
757  *	@dev: targeted interface
758  *	@skb: The packet.
759  *
760  *	For better visibility of tunnel traffic OVS needs to retrieve
761  *	egress tunnel information for a packet. Following API allows
762  *	user to get this info.
763  */
764 int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb)
765 {
766 	struct ip_tunnel_info *info;
767 
768 	if (!dev->netdev_ops  || !dev->netdev_ops->ndo_fill_metadata_dst)
769 		return -EINVAL;
770 
771 	info = skb_tunnel_info_unclone(skb);
772 	if (!info)
773 		return -ENOMEM;
774 	if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX)))
775 		return -EINVAL;
776 
777 	return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb);
778 }
779 EXPORT_SYMBOL_GPL(dev_fill_metadata_dst);
780 
781 static struct net_device_path *dev_fwd_path(struct net_device_path_stack *stack)
782 {
783 	if (stack->num_paths + 1 > NET_DEVICE_PATH_STACK_MAX)
784 		return NULL;
785 
786 	return &stack->path[stack->num_paths];
787 }
788 
789 void dev_fill_forward_path_release(struct net_device_path_stack *stack)
790 {
791 	struct net_device_path *path;
792 	int k;
793 
794 	if (stack->num_paths == 0)
795 		return;
796 
797 	for (k = stack->num_paths - 1; k >= 0; k--) {
798 		path = &stack->path[k];
799 		switch (path->type) {
800 		case DEV_PATH_TUN:
801 			dst_release(path->tun.dst);
802 			break;
803 		default:
804 			break;
805 		}
806 	}
807 }
808 EXPORT_SYMBOL_GPL(dev_fill_forward_path_release);
809 
810 int dev_fill_forward_path(struct net_device_path_ctx *ctx,
811 			  struct net_device_path_stack *stack)
812 {
813 	const struct net_device *last_dev;
814 	struct net_device_path *path;
815 	int ret = 0;
816 
817 	stack->num_paths = 0;
818 	while (ctx->dev && ctx->dev->netdev_ops->ndo_fill_forward_path) {
819 		last_dev = ctx->dev;
820 		path = dev_fwd_path(stack);
821 		if (!path)
822 			goto err_out;
823 
824 		memset(path, 0, sizeof(struct net_device_path));
825 		ret = ctx->dev->netdev_ops->ndo_fill_forward_path(ctx, path);
826 		if (ret < 0)
827 			goto err_out;
828 
829 		stack->num_paths++;
830 		if (last_dev == ctx->dev)
831 			goto err_out;
832 	}
833 
834 	if (!ctx->dev)
835 		return ret;
836 
837 	path = dev_fwd_path(stack);
838 	if (!path)
839 		goto err_out;
840 
841 	path->type = DEV_PATH_ETHERNET;
842 	path->dev = ctx->dev;
843 	stack->num_paths++;
844 
845 	return 0;
846 err_out:
847 	dev_fill_forward_path_release(stack);
848 
849 	return -1;
850 }
851 EXPORT_SYMBOL_GPL(dev_fill_forward_path);
852 
853 /* must be called under rcu_read_lock(), as we dont take a reference */
854 static struct napi_struct *napi_by_id(unsigned int napi_id)
855 {
856 	unsigned int hash = napi_id % HASH_SIZE(napi_hash);
857 	struct napi_struct *napi;
858 
859 	hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node)
860 		if (napi->napi_id == napi_id)
861 			return napi;
862 
863 	return NULL;
864 }
865 
866 /* must be called under rcu_read_lock(), as we dont take a reference */
867 static struct napi_struct *
868 netdev_napi_by_id(struct net *net, unsigned int napi_id)
869 {
870 	struct napi_struct *napi;
871 
872 	napi = napi_by_id(napi_id);
873 	if (!napi)
874 		return NULL;
875 
876 	if (WARN_ON_ONCE(!napi->dev))
877 		return NULL;
878 	if (!net_eq(net, dev_net(napi->dev)))
879 		return NULL;
880 
881 	return napi;
882 }
883 
884 /**
885  *	netdev_napi_by_id_lock() - find a device by NAPI ID and lock it
886  *	@net: the applicable net namespace
887  *	@napi_id: ID of a NAPI of a target device
888  *
889  *	Find a NAPI instance with @napi_id. Lock its device.
890  *	The device must be in %NETREG_REGISTERED state for lookup to succeed.
891  *	netdev_unlock() must be called to release it.
892  *
893  *	Return: pointer to NAPI, its device with lock held, NULL if not found.
894  */
895 struct napi_struct *
896 netdev_napi_by_id_lock(struct net *net, unsigned int napi_id)
897 {
898 	struct napi_struct *napi;
899 	struct net_device *dev;
900 
901 	rcu_read_lock();
902 	napi = netdev_napi_by_id(net, napi_id);
903 	if (!napi || READ_ONCE(napi->dev->reg_state) != NETREG_REGISTERED) {
904 		rcu_read_unlock();
905 		return NULL;
906 	}
907 
908 	dev = napi->dev;
909 	dev_hold(dev);
910 	rcu_read_unlock();
911 
912 	dev = __netdev_put_lock(dev, net);
913 	if (!dev)
914 		return NULL;
915 
916 	rcu_read_lock();
917 	napi = netdev_napi_by_id(net, napi_id);
918 	if (napi && napi->dev != dev)
919 		napi = NULL;
920 	rcu_read_unlock();
921 
922 	if (!napi)
923 		netdev_unlock(dev);
924 	return napi;
925 }
926 
927 /**
928  *	__dev_get_by_name	- find a device by its name
929  *	@net: the applicable net namespace
930  *	@name: name to find
931  *
932  *	Find an interface by name. Must be called under RTNL semaphore.
933  *	If the name is found a pointer to the device is returned.
934  *	If the name is not found then %NULL is returned. The
935  *	reference counters are not incremented so the caller must be
936  *	careful with locks.
937  */
938 
939 struct net_device *__dev_get_by_name(struct net *net, const char *name)
940 {
941 	struct netdev_name_node *node_name;
942 
943 	node_name = netdev_name_node_lookup(net, name);
944 	return node_name ? node_name->dev : NULL;
945 }
946 EXPORT_SYMBOL(__dev_get_by_name);
947 
948 /**
949  * dev_get_by_name_rcu	- find a device by its name
950  * @net: the applicable net namespace
951  * @name: name to find
952  *
953  * Find an interface by name.
954  * If the name is found a pointer to the device is returned.
955  * If the name is not found then %NULL is returned.
956  * The reference counters are not incremented so the caller must be
957  * careful with locks. The caller must hold RCU lock.
958  */
959 
960 struct net_device *dev_get_by_name_rcu(struct net *net, const char *name)
961 {
962 	struct netdev_name_node *node_name;
963 
964 	node_name = netdev_name_node_lookup_rcu(net, name);
965 	return node_name ? node_name->dev : NULL;
966 }
967 EXPORT_SYMBOL(dev_get_by_name_rcu);
968 
969 /* Deprecated for new users, call netdev_get_by_name() instead */
970 struct net_device *dev_get_by_name(struct net *net, const char *name)
971 {
972 	struct net_device *dev;
973 
974 	rcu_read_lock();
975 	dev = dev_get_by_name_rcu(net, name);
976 	dev_hold(dev);
977 	rcu_read_unlock();
978 	return dev;
979 }
980 EXPORT_SYMBOL(dev_get_by_name);
981 
982 /**
983  *	netdev_get_by_name() - find a device by its name
984  *	@net: the applicable net namespace
985  *	@name: name to find
986  *	@tracker: tracking object for the acquired reference
987  *	@gfp: allocation flags for the tracker
988  *
989  *	Find an interface by name. This can be called from any
990  *	context and does its own locking. The returned handle has
991  *	the usage count incremented and the caller must use netdev_put() to
992  *	release it when it is no longer needed. %NULL is returned if no
993  *	matching device is found.
994  */
995 struct net_device *netdev_get_by_name(struct net *net, const char *name,
996 				      netdevice_tracker *tracker, gfp_t gfp)
997 {
998 	struct net_device *dev;
999 
1000 	dev = dev_get_by_name(net, name);
1001 	if (dev)
1002 		netdev_tracker_alloc(dev, tracker, gfp);
1003 	return dev;
1004 }
1005 EXPORT_SYMBOL(netdev_get_by_name);
1006 
1007 /**
1008  *	__dev_get_by_index - find a device by its ifindex
1009  *	@net: the applicable net namespace
1010  *	@ifindex: index of device
1011  *
1012  *	Search for an interface by index. Returns %NULL if the device
1013  *	is not found or a pointer to the device. The device has not
1014  *	had its reference counter increased so the caller must be careful
1015  *	about locking. The caller must hold the RTNL semaphore.
1016  */
1017 
1018 struct net_device *__dev_get_by_index(struct net *net, int ifindex)
1019 {
1020 	struct net_device *dev;
1021 	struct hlist_head *head = dev_index_hash(net, ifindex);
1022 
1023 	hlist_for_each_entry(dev, head, index_hlist)
1024 		if (dev->ifindex == ifindex)
1025 			return dev;
1026 
1027 	return NULL;
1028 }
1029 EXPORT_SYMBOL(__dev_get_by_index);
1030 
1031 /**
1032  *	dev_get_by_index_rcu - find a device by its ifindex
1033  *	@net: the applicable net namespace
1034  *	@ifindex: index of device
1035  *
1036  *	Search for an interface by index. Returns %NULL if the device
1037  *	is not found or a pointer to the device. The device has not
1038  *	had its reference counter increased so the caller must be careful
1039  *	about locking. The caller must hold RCU lock.
1040  */
1041 
1042 struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex)
1043 {
1044 	struct net_device *dev;
1045 	struct hlist_head *head = dev_index_hash(net, ifindex);
1046 
1047 	hlist_for_each_entry_rcu(dev, head, index_hlist)
1048 		if (dev->ifindex == ifindex)
1049 			return dev;
1050 
1051 	return NULL;
1052 }
1053 EXPORT_SYMBOL(dev_get_by_index_rcu);
1054 
1055 /* Deprecated for new users, call netdev_get_by_index() instead */
1056 struct net_device *dev_get_by_index(struct net *net, int ifindex)
1057 {
1058 	struct net_device *dev;
1059 
1060 	rcu_read_lock();
1061 	dev = dev_get_by_index_rcu(net, ifindex);
1062 	dev_hold(dev);
1063 	rcu_read_unlock();
1064 	return dev;
1065 }
1066 EXPORT_SYMBOL(dev_get_by_index);
1067 
1068 /**
1069  *	netdev_get_by_index() - find a device by its ifindex
1070  *	@net: the applicable net namespace
1071  *	@ifindex: index of device
1072  *	@tracker: tracking object for the acquired reference
1073  *	@gfp: allocation flags for the tracker
1074  *
1075  *	Search for an interface by index. Returns NULL if the device
1076  *	is not found or a pointer to the device. The device returned has
1077  *	had a reference added and the pointer is safe until the user calls
1078  *	netdev_put() to indicate they have finished with it.
1079  */
1080 struct net_device *netdev_get_by_index(struct net *net, int ifindex,
1081 				       netdevice_tracker *tracker, gfp_t gfp)
1082 {
1083 	struct net_device *dev;
1084 
1085 	dev = dev_get_by_index(net, ifindex);
1086 	if (dev)
1087 		netdev_tracker_alloc(dev, tracker, gfp);
1088 	return dev;
1089 }
1090 EXPORT_SYMBOL(netdev_get_by_index);
1091 
1092 /**
1093  *	dev_get_by_napi_id - find a device by napi_id
1094  *	@napi_id: ID of the NAPI struct
1095  *
1096  *	Search for an interface by NAPI ID. Returns %NULL if the device
1097  *	is not found or a pointer to the device. The device has not had
1098  *	its reference counter increased so the caller must be careful
1099  *	about locking. The caller must hold RCU lock.
1100  */
1101 struct net_device *dev_get_by_napi_id(unsigned int napi_id)
1102 {
1103 	struct napi_struct *napi;
1104 
1105 	WARN_ON_ONCE(!rcu_read_lock_held());
1106 
1107 	if (!napi_id_valid(napi_id))
1108 		return NULL;
1109 
1110 	napi = napi_by_id(napi_id);
1111 
1112 	return napi ? napi->dev : NULL;
1113 }
1114 
1115 /* Release the held reference on the net_device, and if the net_device
1116  * is still registered try to lock the instance lock. If device is being
1117  * unregistered NULL will be returned (but the reference has been released,
1118  * either way!)
1119  *
1120  * This helper is intended for locking net_device after it has been looked up
1121  * using a lockless lookup helper. Lock prevents the instance from going away.
1122  */
1123 struct net_device *
1124 netdev_put_lock(struct net_device *dev, struct net *net,
1125 		netdevice_tracker *tracker)
1126 {
1127 	netdev_lock(dev);
1128 	if (dev->reg_state > NETREG_REGISTERED ||
1129 	    dev->moving_ns || !net_eq(dev_net(dev), net)) {
1130 		netdev_unlock(dev);
1131 		netdev_put(dev, tracker);
1132 		return NULL;
1133 	}
1134 	netdev_put(dev, tracker);
1135 	return dev;
1136 }
1137 
1138 static struct net_device *
1139 __netdev_put_lock_ops_compat(struct net_device *dev, struct net *net)
1140 {
1141 	netdev_lock_ops_compat(dev);
1142 	if (dev->reg_state > NETREG_REGISTERED ||
1143 	    dev->moving_ns || !net_eq(dev_net(dev), net)) {
1144 		netdev_unlock_ops_compat(dev);
1145 		dev_put(dev);
1146 		return NULL;
1147 	}
1148 	dev_put(dev);
1149 	return dev;
1150 }
1151 
1152 /**
1153  *	netdev_get_by_index_lock() - find a device by its ifindex
1154  *	@net: the applicable net namespace
1155  *	@ifindex: index of device
1156  *
1157  *	Search for an interface by index. If a valid device
1158  *	with @ifindex is found it will be returned with netdev->lock held.
1159  *	netdev_unlock() must be called to release it.
1160  *
1161  *	Return: pointer to a device with lock held, NULL if not found.
1162  */
1163 struct net_device *netdev_get_by_index_lock(struct net *net, int ifindex)
1164 {
1165 	struct net_device *dev;
1166 
1167 	dev = dev_get_by_index(net, ifindex);
1168 	if (!dev)
1169 		return NULL;
1170 
1171 	return __netdev_put_lock(dev, net);
1172 }
1173 
1174 struct net_device *
1175 netdev_get_by_index_lock_ops_compat(struct net *net, int ifindex)
1176 {
1177 	struct net_device *dev;
1178 
1179 	dev = dev_get_by_index(net, ifindex);
1180 	if (!dev)
1181 		return NULL;
1182 
1183 	return __netdev_put_lock_ops_compat(dev, net);
1184 }
1185 
1186 struct net_device *
1187 netdev_xa_find_lock(struct net *net, struct net_device *dev,
1188 		    unsigned long *index)
1189 {
1190 	if (dev)
1191 		netdev_unlock(dev);
1192 
1193 	do {
1194 		rcu_read_lock();
1195 		dev = xa_find(&net->dev_by_index, index, ULONG_MAX, XA_PRESENT);
1196 		if (!dev) {
1197 			rcu_read_unlock();
1198 			return NULL;
1199 		}
1200 		dev_hold(dev);
1201 		rcu_read_unlock();
1202 
1203 		dev = __netdev_put_lock(dev, net);
1204 		if (dev)
1205 			return dev;
1206 
1207 		(*index)++;
1208 	} while (true);
1209 }
1210 
1211 struct net_device *
1212 netdev_xa_find_lock_ops_compat(struct net *net, struct net_device *dev,
1213 			       unsigned long *index)
1214 {
1215 	if (dev)
1216 		netdev_unlock_ops_compat(dev);
1217 
1218 	do {
1219 		rcu_read_lock();
1220 		dev = xa_find(&net->dev_by_index, index, ULONG_MAX, XA_PRESENT);
1221 		if (!dev) {
1222 			rcu_read_unlock();
1223 			return NULL;
1224 		}
1225 		dev_hold(dev);
1226 		rcu_read_unlock();
1227 
1228 		dev = __netdev_put_lock_ops_compat(dev, net);
1229 		if (dev)
1230 			return dev;
1231 
1232 		(*index)++;
1233 	} while (true);
1234 }
1235 
1236 static DEFINE_SEQLOCK(netdev_rename_lock);
1237 
1238 void netdev_copy_name(struct net_device *dev, char *name)
1239 {
1240 	unsigned int seq;
1241 
1242 	do {
1243 		seq = read_seqbegin(&netdev_rename_lock);
1244 		strscpy(name, dev->name, IFNAMSIZ);
1245 	} while (read_seqretry(&netdev_rename_lock, seq));
1246 }
1247 
1248 /**
1249  *	netdev_get_name - get a netdevice name, knowing its ifindex.
1250  *	@net: network namespace
1251  *	@name: a pointer to the buffer where the name will be stored.
1252  *	@ifindex: the ifindex of the interface to get the name from.
1253  */
1254 int netdev_get_name(struct net *net, char *name, int ifindex)
1255 {
1256 	struct net_device *dev;
1257 	int ret;
1258 
1259 	rcu_read_lock();
1260 
1261 	dev = dev_get_by_index_rcu(net, ifindex);
1262 	if (!dev) {
1263 		ret = -ENODEV;
1264 		goto out;
1265 	}
1266 
1267 	netdev_copy_name(dev, name);
1268 
1269 	ret = 0;
1270 out:
1271 	rcu_read_unlock();
1272 	return ret;
1273 }
1274 
1275 static bool dev_addr_cmp(struct net_device *dev, unsigned short type,
1276 			 const char *ha)
1277 {
1278 	return dev->type == type && !memcmp(dev->dev_addr, ha, dev->addr_len);
1279 }
1280 
1281 /**
1282  *	dev_getbyhwaddr_rcu - find a device by its hardware address
1283  *	@net: the applicable net namespace
1284  *	@type: media type of device
1285  *	@ha: hardware address
1286  *
1287  *	Search for an interface by MAC address. Returns NULL if the device
1288  *	is not found or a pointer to the device.
1289  *	The caller must hold RCU.
1290  *	The returned device has not had its ref count increased
1291  *	and the caller must therefore be careful about locking
1292  *
1293  */
1294 
1295 struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
1296 				       const char *ha)
1297 {
1298 	struct net_device *dev;
1299 
1300 	for_each_netdev_rcu(net, dev)
1301 		if (dev_addr_cmp(dev, type, ha))
1302 			return dev;
1303 
1304 	return NULL;
1305 }
1306 EXPORT_SYMBOL(dev_getbyhwaddr_rcu);
1307 
1308 /**
1309  * dev_getbyhwaddr() - find a device by its hardware address
1310  * @net: the applicable net namespace
1311  * @type: media type of device
1312  * @ha: hardware address
1313  *
1314  * Similar to dev_getbyhwaddr_rcu(), but the owner needs to hold
1315  * rtnl_lock.
1316  *
1317  * Context: rtnl_lock() must be held.
1318  * Return: pointer to the net_device, or NULL if not found
1319  */
1320 struct net_device *dev_getbyhwaddr(struct net *net, unsigned short type,
1321 				   const char *ha)
1322 {
1323 	struct net_device *dev;
1324 
1325 	ASSERT_RTNL();
1326 	for_each_netdev(net, dev)
1327 		if (dev_addr_cmp(dev, type, ha))
1328 			return dev;
1329 
1330 	return NULL;
1331 }
1332 EXPORT_SYMBOL(dev_getbyhwaddr);
1333 
1334 struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type)
1335 {
1336 	struct net_device *dev, *ret = NULL;
1337 
1338 	rcu_read_lock();
1339 	for_each_netdev_rcu(net, dev)
1340 		if (dev->type == type) {
1341 			dev_hold(dev);
1342 			ret = dev;
1343 			break;
1344 		}
1345 	rcu_read_unlock();
1346 	return ret;
1347 }
1348 EXPORT_SYMBOL(dev_getfirstbyhwtype);
1349 
1350 /**
1351  * netdev_get_by_flags_rcu - find any device with given flags
1352  * @net: the applicable net namespace
1353  * @tracker: tracking object for the acquired reference
1354  * @if_flags: IFF_* values
1355  * @mask: bitmask of bits in if_flags to check
1356  *
1357  * Search for any interface with the given flags.
1358  *
1359  * Context: rcu_read_lock() must be held.
1360  * Returns: NULL if a device is not found or a pointer to the device.
1361  */
1362 struct net_device *netdev_get_by_flags_rcu(struct net *net, netdevice_tracker *tracker,
1363 					   unsigned short if_flags, unsigned short mask)
1364 {
1365 	struct net_device *dev;
1366 
1367 	for_each_netdev_rcu(net, dev) {
1368 		if (((READ_ONCE(dev->flags) ^ if_flags) & mask) == 0) {
1369 			netdev_hold(dev, tracker, GFP_ATOMIC);
1370 			return dev;
1371 		}
1372 	}
1373 
1374 	return NULL;
1375 }
1376 
1377 /**
1378  *	dev_valid_name - check if name is okay for network device
1379  *	@name: name string
1380  *
1381  *	Network device names need to be valid file names to
1382  *	allow sysfs to work.  We also disallow any kind of
1383  *	whitespace.
1384  */
1385 bool dev_valid_name(const char *name)
1386 {
1387 	if (*name == '\0')
1388 		return false;
1389 	if (strnlen(name, IFNAMSIZ) == IFNAMSIZ)
1390 		return false;
1391 	if (!strcmp(name, ".") || !strcmp(name, ".."))
1392 		return false;
1393 
1394 	while (*name) {
1395 		if (*name == '/' || *name == ':' || isspace(*name))
1396 			return false;
1397 		name++;
1398 	}
1399 	return true;
1400 }
1401 EXPORT_SYMBOL(dev_valid_name);
1402 
1403 /**
1404  *	__dev_alloc_name - allocate a name for a device
1405  *	@net: network namespace to allocate the device name in
1406  *	@name: name format string
1407  *	@res: result name string
1408  *
1409  *	Passed a format string - eg "lt%d" it will try and find a suitable
1410  *	id. It scans list of devices to build up a free map, then chooses
1411  *	the first empty slot. The caller must hold the dev_base or rtnl lock
1412  *	while allocating the name and adding the device in order to avoid
1413  *	duplicates.
1414  *	Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1415  *	Returns the number of the unit assigned or a negative errno code.
1416  */
1417 
1418 static int __dev_alloc_name(struct net *net, const char *name, char *res)
1419 {
1420 	int i = 0;
1421 	const char *p;
1422 	const int max_netdevices = 8*PAGE_SIZE;
1423 	unsigned long *inuse;
1424 	struct net_device *d;
1425 	char buf[IFNAMSIZ];
1426 
1427 	/* Verify the string as this thing may have come from the user.
1428 	 * There must be one "%d" and no other "%" characters.
1429 	 */
1430 	p = strchr(name, '%');
1431 	if (!p || p[1] != 'd' || strchr(p + 2, '%'))
1432 		return -EINVAL;
1433 
1434 	/* Use one page as a bit array of possible slots */
1435 	inuse = bitmap_zalloc(max_netdevices, GFP_ATOMIC);
1436 	if (!inuse)
1437 		return -ENOMEM;
1438 
1439 	for_each_netdev(net, d) {
1440 		struct netdev_name_node *name_node;
1441 
1442 		netdev_for_each_altname(d, name_node) {
1443 			if (!sscanf(name_node->name, name, &i))
1444 				continue;
1445 			if (i < 0 || i >= max_netdevices)
1446 				continue;
1447 
1448 			/* avoid cases where sscanf is not exact inverse of printf */
1449 			snprintf(buf, IFNAMSIZ, name, i);
1450 			if (!strncmp(buf, name_node->name, IFNAMSIZ))
1451 				__set_bit(i, inuse);
1452 		}
1453 		if (!sscanf(d->name, name, &i))
1454 			continue;
1455 		if (i < 0 || i >= max_netdevices)
1456 			continue;
1457 
1458 		/* avoid cases where sscanf is not exact inverse of printf */
1459 		snprintf(buf, IFNAMSIZ, name, i);
1460 		if (!strncmp(buf, d->name, IFNAMSIZ))
1461 			__set_bit(i, inuse);
1462 	}
1463 
1464 	i = find_first_zero_bit(inuse, max_netdevices);
1465 	bitmap_free(inuse);
1466 	if (i == max_netdevices)
1467 		return -ENFILE;
1468 
1469 	/* 'res' and 'name' could overlap, use 'buf' as an intermediate buffer */
1470 	strscpy(buf, name, IFNAMSIZ);
1471 	snprintf(res, IFNAMSIZ, buf, i);
1472 	return i;
1473 }
1474 
1475 /* Returns negative errno or allocated unit id (see __dev_alloc_name()) */
1476 static int dev_prep_valid_name(struct net *net, struct net_device *dev,
1477 			       const char *want_name, char *out_name,
1478 			       int dup_errno)
1479 {
1480 	if (!dev_valid_name(want_name))
1481 		return -EINVAL;
1482 
1483 	if (strchr(want_name, '%'))
1484 		return __dev_alloc_name(net, want_name, out_name);
1485 
1486 	if (netdev_name_in_use(net, want_name))
1487 		return -dup_errno;
1488 	if (out_name != want_name)
1489 		strscpy(out_name, want_name, IFNAMSIZ);
1490 	return 0;
1491 }
1492 
1493 /**
1494  *	dev_alloc_name - allocate a name for a device
1495  *	@dev: device
1496  *	@name: name format string
1497  *
1498  *	Passed a format string - eg "lt%d" it will try and find a suitable
1499  *	id. It scans list of devices to build up a free map, then chooses
1500  *	the first empty slot. The caller must hold the dev_base or rtnl lock
1501  *	while allocating the name and adding the device in order to avoid
1502  *	duplicates.
1503  *	Limited to bits_per_byte * page size devices (ie 32K on most platforms).
1504  *	Returns the number of the unit assigned or a negative errno code.
1505  */
1506 
1507 int dev_alloc_name(struct net_device *dev, const char *name)
1508 {
1509 	return dev_prep_valid_name(dev_net(dev), dev, name, dev->name, ENFILE);
1510 }
1511 EXPORT_SYMBOL(dev_alloc_name);
1512 
1513 static int dev_get_valid_name(struct net *net, struct net_device *dev,
1514 			      const char *name)
1515 {
1516 	int ret;
1517 
1518 	ret = dev_prep_valid_name(net, dev, name, dev->name, EEXIST);
1519 	return ret < 0 ? ret : 0;
1520 }
1521 
1522 int netif_change_name(struct net_device *dev, const char *newname)
1523 {
1524 	struct net *net = dev_net(dev);
1525 	unsigned char old_assign_type;
1526 	char oldname[IFNAMSIZ];
1527 	int err = 0;
1528 	int ret;
1529 
1530 	ASSERT_RTNL_NET(net);
1531 
1532 	if (!strncmp(newname, dev->name, IFNAMSIZ))
1533 		return 0;
1534 
1535 	memcpy(oldname, dev->name, IFNAMSIZ);
1536 
1537 	write_seqlock_bh(&netdev_rename_lock);
1538 	err = dev_get_valid_name(net, dev, newname);
1539 	write_sequnlock_bh(&netdev_rename_lock);
1540 
1541 	if (err < 0)
1542 		return err;
1543 
1544 	if (oldname[0] && !strchr(oldname, '%'))
1545 		netdev_info(dev, "renamed from %s%s\n", oldname,
1546 			    dev->flags & IFF_UP ? " (while UP)" : "");
1547 
1548 	old_assign_type = dev->name_assign_type;
1549 	WRITE_ONCE(dev->name_assign_type, NET_NAME_RENAMED);
1550 
1551 rollback:
1552 	ret = device_rename(&dev->dev, dev->name);
1553 	if (ret) {
1554 		write_seqlock_bh(&netdev_rename_lock);
1555 		memcpy(dev->name, oldname, IFNAMSIZ);
1556 		write_sequnlock_bh(&netdev_rename_lock);
1557 		WRITE_ONCE(dev->name_assign_type, old_assign_type);
1558 		return ret;
1559 	}
1560 
1561 	netdev_adjacent_rename_links(dev, oldname);
1562 
1563 	netdev_name_node_del(dev->name_node);
1564 
1565 	synchronize_net();
1566 
1567 	netdev_name_node_add(net, dev->name_node);
1568 
1569 	ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev);
1570 	ret = notifier_to_errno(ret);
1571 
1572 	if (ret) {
1573 		/* err >= 0 after dev_alloc_name() or stores the first errno */
1574 		if (err >= 0) {
1575 			err = ret;
1576 			write_seqlock_bh(&netdev_rename_lock);
1577 			memcpy(dev->name, oldname, IFNAMSIZ);
1578 			write_sequnlock_bh(&netdev_rename_lock);
1579 			memcpy(oldname, newname, IFNAMSIZ);
1580 			WRITE_ONCE(dev->name_assign_type, old_assign_type);
1581 			old_assign_type = NET_NAME_RENAMED;
1582 			goto rollback;
1583 		} else {
1584 			netdev_err(dev, "name change rollback failed: %d\n",
1585 				   ret);
1586 		}
1587 	}
1588 
1589 	return err;
1590 }
1591 
1592 int netif_set_alias(struct net_device *dev, const char *alias, size_t len)
1593 {
1594 	struct dev_ifalias *new_alias = NULL;
1595 
1596 	if (len >= IFALIASZ)
1597 		return -EINVAL;
1598 
1599 	if (len) {
1600 		new_alias = kmalloc(sizeof(*new_alias) + len + 1, GFP_KERNEL);
1601 		if (!new_alias)
1602 			return -ENOMEM;
1603 
1604 		memcpy(new_alias->ifalias, alias, len);
1605 		new_alias->ifalias[len] = 0;
1606 	}
1607 
1608 	mutex_lock(&ifalias_mutex);
1609 	new_alias = rcu_replace_pointer(dev->ifalias, new_alias,
1610 					mutex_is_locked(&ifalias_mutex));
1611 	mutex_unlock(&ifalias_mutex);
1612 
1613 	if (new_alias)
1614 		kfree_rcu(new_alias, rcuhead);
1615 
1616 	return len;
1617 }
1618 
1619 /**
1620  *	dev_get_alias - get ifalias of a device
1621  *	@dev: device
1622  *	@name: buffer to store name of ifalias
1623  *	@len: size of buffer
1624  *
1625  *	get ifalias for a device.  Caller must make sure dev cannot go
1626  *	away,  e.g. rcu read lock or own a reference count to device.
1627  */
1628 int dev_get_alias(const struct net_device *dev, char *name, size_t len)
1629 {
1630 	const struct dev_ifalias *alias;
1631 	int ret = 0;
1632 
1633 	rcu_read_lock();
1634 	alias = rcu_dereference(dev->ifalias);
1635 	if (alias)
1636 		ret = snprintf(name, len, "%s", alias->ifalias);
1637 	rcu_read_unlock();
1638 
1639 	return ret;
1640 }
1641 
1642 /**
1643  *	netdev_features_change - device changes features
1644  *	@dev: device to cause notification
1645  *
1646  *	Called to indicate a device has changed features.
1647  */
1648 void netdev_features_change(struct net_device *dev)
1649 {
1650 	call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev);
1651 }
1652 EXPORT_SYMBOL(netdev_features_change);
1653 
1654 void netif_state_change(struct net_device *dev)
1655 {
1656 	netdev_assert_locked_ops_compat_or_invisible(dev);
1657 
1658 	if (dev->flags & IFF_UP) {
1659 		struct netdev_notifier_change_info change_info = {
1660 			.info.dev = dev,
1661 		};
1662 
1663 		call_netdevice_notifiers_info(NETDEV_CHANGE,
1664 					      &change_info.info);
1665 		rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL, 0, NULL);
1666 	}
1667 }
1668 
1669 /**
1670  * __netdev_notify_peers - notify network peers about existence of @dev,
1671  * to be called when rtnl lock is already held.
1672  * @dev: network device
1673  *
1674  * Generate traffic such that interested network peers are aware of
1675  * @dev, such as by generating a gratuitous ARP. This may be used when
1676  * a device wants to inform the rest of the network about some sort of
1677  * reconfiguration such as a failover event or virtual machine
1678  * migration.
1679  */
1680 void __netdev_notify_peers(struct net_device *dev)
1681 {
1682 	ASSERT_RTNL();
1683 	call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev);
1684 	call_netdevice_notifiers(NETDEV_RESEND_IGMP, dev);
1685 }
1686 EXPORT_SYMBOL(__netdev_notify_peers);
1687 
1688 /**
1689  * netdev_notify_peers - notify network peers about existence of @dev
1690  * @dev: network device
1691  *
1692  * Generate traffic such that interested network peers are aware of
1693  * @dev, such as by generating a gratuitous ARP. This may be used when
1694  * a device wants to inform the rest of the network about some sort of
1695  * reconfiguration such as a failover event or virtual machine
1696  * migration.
1697  */
1698 void netdev_notify_peers(struct net_device *dev)
1699 {
1700 	rtnl_lock();
1701 	__netdev_notify_peers(dev);
1702 	rtnl_unlock();
1703 }
1704 EXPORT_SYMBOL(netdev_notify_peers);
1705 
1706 static int napi_threaded_poll(void *data);
1707 
1708 static int napi_kthread_create(struct napi_struct *n)
1709 {
1710 	int err = 0;
1711 
1712 	/* Create and wake up the kthread once to put it in
1713 	 * TASK_INTERRUPTIBLE mode to avoid the blocked task
1714 	 * warning and work with loadavg.
1715 	 */
1716 	n->thread = kthread_run(napi_threaded_poll, n, "napi/%s-%d",
1717 				n->dev->name, n->napi_id);
1718 	if (IS_ERR(n->thread)) {
1719 		err = PTR_ERR(n->thread);
1720 		pr_err("kthread_run failed with err %d\n", err);
1721 		n->thread = NULL;
1722 	}
1723 
1724 	return err;
1725 }
1726 
1727 static int __dev_open(struct net_device *dev, struct netlink_ext_ack *extack)
1728 {
1729 	const struct net_device_ops *ops = dev->netdev_ops;
1730 	int ret;
1731 
1732 	ASSERT_RTNL();
1733 	dev_addr_check(dev);
1734 
1735 	if (!netif_device_present(dev)) {
1736 		/* may be detached because parent is runtime-suspended */
1737 		if (dev->dev.parent)
1738 			pm_runtime_resume(dev->dev.parent);
1739 		if (!netif_device_present(dev))
1740 			return -ENODEV;
1741 	}
1742 
1743 	/* Block netpoll from trying to do any rx path servicing.
1744 	 * If we don't do this there is a chance ndo_poll_controller
1745 	 * or ndo_poll may be running while we open the device
1746 	 */
1747 	netpoll_poll_disable(dev);
1748 
1749 	ret = call_netdevice_notifiers_extack(NETDEV_PRE_UP, dev, extack);
1750 	ret = notifier_to_errno(ret);
1751 	if (ret)
1752 		return ret;
1753 
1754 	set_bit(__LINK_STATE_START, &dev->state);
1755 
1756 	netdev_assert_locked_ops_compat(dev);
1757 
1758 	if (ops->ndo_validate_addr)
1759 		ret = ops->ndo_validate_addr(dev);
1760 
1761 	if (!ret && ops->ndo_open)
1762 		ret = ops->ndo_open(dev);
1763 
1764 	netpoll_poll_enable(dev);
1765 
1766 	if (ret)
1767 		clear_bit(__LINK_STATE_START, &dev->state);
1768 	else {
1769 		netif_set_up(dev, true);
1770 		dev_set_rx_mode(dev);
1771 		dev_activate(dev);
1772 		add_device_randomness(dev->dev_addr, dev->addr_len);
1773 	}
1774 
1775 	return ret;
1776 }
1777 
1778 int netif_open(struct net_device *dev, struct netlink_ext_ack *extack)
1779 {
1780 	int ret;
1781 
1782 	if (dev->flags & IFF_UP)
1783 		return 0;
1784 
1785 	ret = __dev_open(dev, extack);
1786 	if (ret < 0)
1787 		return ret;
1788 
1789 	rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP | IFF_RUNNING, GFP_KERNEL, 0, NULL);
1790 	call_netdevice_notifiers(NETDEV_UP, dev);
1791 
1792 	return ret;
1793 }
1794 EXPORT_SYMBOL(netif_open);
1795 
1796 static void __dev_close_many(struct list_head *head)
1797 {
1798 	struct net_device *dev;
1799 
1800 	ASSERT_RTNL();
1801 	might_sleep();
1802 
1803 	list_for_each_entry(dev, head, close_list) {
1804 		/* Temporarily disable netpoll until the interface is down */
1805 		netpoll_poll_disable(dev);
1806 
1807 		call_netdevice_notifiers(NETDEV_GOING_DOWN, dev);
1808 
1809 		clear_bit(__LINK_STATE_START, &dev->state);
1810 
1811 		/* Synchronize to scheduled poll. We cannot touch poll list, it
1812 		 * can be even on different cpu. So just clear netif_running().
1813 		 *
1814 		 * dev->stop() will invoke napi_disable() on all of it's
1815 		 * napi_struct instances on this device.
1816 		 */
1817 		smp_mb__after_atomic(); /* Commit netif_running(). */
1818 	}
1819 
1820 	dev_deactivate_many(head, true);
1821 
1822 	list_for_each_entry(dev, head, close_list) {
1823 		const struct net_device_ops *ops = dev->netdev_ops;
1824 
1825 		/*
1826 		 *	Call the device specific close. This cannot fail.
1827 		 *	Only if device is UP
1828 		 *
1829 		 *	We allow it to be called even after a DETACH hot-plug
1830 		 *	event.
1831 		 */
1832 
1833 		netdev_assert_locked_ops_compat(dev);
1834 
1835 		if (ops->ndo_stop)
1836 			ops->ndo_stop(dev);
1837 
1838 		netif_rx_mode_cancel_retry(dev);
1839 		netif_set_up(dev, false);
1840 		netpoll_poll_enable(dev);
1841 	}
1842 }
1843 
1844 static void __dev_close(struct net_device *dev)
1845 {
1846 	LIST_HEAD(single);
1847 
1848 	list_add(&dev->close_list, &single);
1849 	__dev_close_many(&single);
1850 	list_del(&single);
1851 }
1852 
1853 void netif_close_many(struct list_head *head, bool unlink)
1854 {
1855 	struct net_device *dev, *tmp;
1856 
1857 	/* Remove the devices that don't need to be closed */
1858 	list_for_each_entry_safe(dev, tmp, head, close_list)
1859 		if (!(dev->flags & IFF_UP))
1860 			list_del_init(&dev->close_list);
1861 
1862 	__dev_close_many(head);
1863 
1864 	list_for_each_entry_safe(dev, tmp, head, close_list) {
1865 		netdev_assert_locked_ops_compat(dev);
1866 		rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP | IFF_RUNNING, GFP_KERNEL, 0, NULL);
1867 		call_netdevice_notifiers(NETDEV_DOWN, dev);
1868 		if (unlink)
1869 			list_del_init(&dev->close_list);
1870 	}
1871 }
1872 EXPORT_SYMBOL_NS_GPL(netif_close_many, "NETDEV_INTERNAL");
1873 
1874 void netif_close(struct net_device *dev)
1875 {
1876 	if (dev->flags & IFF_UP) {
1877 		LIST_HEAD(single);
1878 
1879 		list_add(&dev->close_list, &single);
1880 		netif_close_many(&single, true);
1881 		list_del(&single);
1882 	}
1883 }
1884 EXPORT_SYMBOL(netif_close);
1885 
1886 void netif_disable_lro(struct net_device *dev)
1887 {
1888 	struct net_device *lower_dev;
1889 	struct list_head *iter;
1890 
1891 	dev->wanted_features &= ~NETIF_F_LRO;
1892 	netdev_update_features(dev);
1893 
1894 	if (unlikely(dev->features & NETIF_F_LRO))
1895 		netdev_WARN(dev, "failed to disable LRO!\n");
1896 
1897 	netdev_for_each_lower_dev(dev, lower_dev, iter) {
1898 		netdev_lock_ops(lower_dev);
1899 		netif_disable_lro(lower_dev);
1900 		netdev_unlock_ops(lower_dev);
1901 	}
1902 }
1903 
1904 /**
1905  *	dev_disable_gro_hw - disable HW Generic Receive Offload on a device
1906  *	@dev: device
1907  *
1908  *	Disable HW Generic Receive Offload (GRO_HW) on a net device.  Must be
1909  *	called under RTNL.  This is needed if Generic XDP is installed on
1910  *	the device.
1911  */
1912 static void dev_disable_gro_hw(struct net_device *dev)
1913 {
1914 	dev->wanted_features &= ~NETIF_F_GRO_HW;
1915 	netdev_update_features(dev);
1916 
1917 	if (unlikely(dev->features & NETIF_F_GRO_HW))
1918 		netdev_WARN(dev, "failed to disable GRO_HW!\n");
1919 }
1920 
1921 const char *netdev_cmd_to_name(enum netdev_cmd cmd)
1922 {
1923 #define N(val) 						\
1924 	case NETDEV_##val:				\
1925 		return "NETDEV_" __stringify(val);
1926 	switch (cmd) {
1927 	N(UP) N(DOWN) N(REBOOT) N(CHANGE) N(REGISTER) N(UNREGISTER)
1928 	N(CHANGEMTU) N(CHANGEADDR) N(GOING_DOWN) N(CHANGENAME) N(FEAT_CHANGE)
1929 	N(BONDING_FAILOVER) N(PRE_UP) N(PRE_TYPE_CHANGE) N(POST_TYPE_CHANGE)
1930 	N(POST_INIT) N(PRE_UNINIT) N(RELEASE) N(NOTIFY_PEERS) N(JOIN)
1931 	N(CHANGEUPPER) N(RESEND_IGMP) N(PRECHANGEMTU) N(CHANGEINFODATA)
1932 	N(BONDING_INFO) N(PRECHANGEUPPER) N(CHANGELOWERSTATE)
1933 	N(UDP_TUNNEL_PUSH_INFO) N(UDP_TUNNEL_DROP_INFO) N(CHANGE_TX_QUEUE_LEN)
1934 	N(CVLAN_FILTER_PUSH_INFO) N(CVLAN_FILTER_DROP_INFO)
1935 	N(SVLAN_FILTER_PUSH_INFO) N(SVLAN_FILTER_DROP_INFO)
1936 	N(PRE_CHANGEADDR) N(OFFLOAD_XSTATS_ENABLE) N(OFFLOAD_XSTATS_DISABLE)
1937 	N(OFFLOAD_XSTATS_REPORT_USED) N(OFFLOAD_XSTATS_REPORT_DELTA)
1938 	N(XDP_FEAT_CHANGE)
1939 	}
1940 #undef N
1941 	return "UNKNOWN_NETDEV_EVENT";
1942 }
1943 EXPORT_SYMBOL_GPL(netdev_cmd_to_name);
1944 
1945 static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val,
1946 				   struct net_device *dev)
1947 {
1948 	struct netdev_notifier_info info = {
1949 		.dev = dev,
1950 	};
1951 
1952 	return nb->notifier_call(nb, val, &info);
1953 }
1954 
1955 static int call_netdevice_register_notifiers(struct notifier_block *nb,
1956 					     struct net_device *dev)
1957 {
1958 	int err;
1959 
1960 	err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev);
1961 	err = notifier_to_errno(err);
1962 	if (err)
1963 		return err;
1964 
1965 	if (!(dev->flags & IFF_UP))
1966 		return 0;
1967 
1968 	call_netdevice_notifier(nb, NETDEV_UP, dev);
1969 	return 0;
1970 }
1971 
1972 static void call_netdevice_unregister_notifiers(struct notifier_block *nb,
1973 						struct net_device *dev)
1974 {
1975 	if (dev->flags & IFF_UP) {
1976 		netdev_lock_ops(dev);
1977 		call_netdevice_notifier(nb, NETDEV_GOING_DOWN,
1978 					dev);
1979 		call_netdevice_notifier(nb, NETDEV_DOWN, dev);
1980 		netdev_unlock_ops(dev);
1981 	}
1982 	call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev);
1983 }
1984 
1985 static int call_netdevice_register_net_notifiers(struct notifier_block *nb,
1986 						 struct net *net)
1987 {
1988 	struct net_device *dev;
1989 	int err;
1990 
1991 	for_each_netdev(net, dev) {
1992 		netdev_lock_ops(dev);
1993 		err = call_netdevice_register_notifiers(nb, dev);
1994 		netdev_unlock_ops(dev);
1995 		if (err)
1996 			goto rollback;
1997 	}
1998 	return 0;
1999 
2000 rollback:
2001 	for_each_netdev_continue_reverse(net, dev)
2002 		call_netdevice_unregister_notifiers(nb, dev);
2003 	return err;
2004 }
2005 
2006 static void call_netdevice_unregister_net_notifiers(struct notifier_block *nb,
2007 						    struct net *net)
2008 {
2009 	struct net_device *dev;
2010 
2011 	for_each_netdev(net, dev)
2012 		call_netdevice_unregister_notifiers(nb, dev);
2013 }
2014 
2015 static int dev_boot_phase = 1;
2016 
2017 /**
2018  * register_netdevice_notifier - register a network notifier block
2019  * @nb: notifier
2020  *
2021  * Register a notifier to be called when network device events occur.
2022  * The notifier passed is linked into the kernel structures and must
2023  * not be reused until it has been unregistered. A negative errno code
2024  * is returned on a failure.
2025  *
2026  * When registered all registration and up events are replayed
2027  * to the new notifier to allow device to have a race free
2028  * view of the network device list.
2029  */
2030 
2031 int register_netdevice_notifier(struct notifier_block *nb)
2032 {
2033 	struct net *net;
2034 	int err;
2035 
2036 	/* Close race with setup_net() and cleanup_net() */
2037 	down_write(&pernet_ops_rwsem);
2038 
2039 	/* When RTNL is removed, we need protection for netdev_chain. */
2040 	rtnl_lock();
2041 
2042 	err = raw_notifier_chain_register(&netdev_chain, nb);
2043 	if (err)
2044 		goto unlock;
2045 	if (dev_boot_phase)
2046 		goto unlock;
2047 	for_each_net(net) {
2048 		__rtnl_net_lock(net);
2049 		err = call_netdevice_register_net_notifiers(nb, net);
2050 		__rtnl_net_unlock(net);
2051 		if (err)
2052 			goto rollback;
2053 	}
2054 
2055 unlock:
2056 	rtnl_unlock();
2057 	up_write(&pernet_ops_rwsem);
2058 	return err;
2059 
2060 rollback:
2061 	for_each_net_continue_reverse(net) {
2062 		__rtnl_net_lock(net);
2063 		call_netdevice_unregister_net_notifiers(nb, net);
2064 		__rtnl_net_unlock(net);
2065 	}
2066 
2067 	raw_notifier_chain_unregister(&netdev_chain, nb);
2068 	goto unlock;
2069 }
2070 EXPORT_SYMBOL(register_netdevice_notifier);
2071 
2072 /**
2073  * unregister_netdevice_notifier - unregister a network notifier block
2074  * @nb: notifier
2075  *
2076  * Unregister a notifier previously registered by
2077  * register_netdevice_notifier(). The notifier is unlinked into the
2078  * kernel structures and may then be reused. A negative errno code
2079  * is returned on a failure.
2080  *
2081  * After unregistering unregister and down device events are synthesized
2082  * for all devices on the device list to the removed notifier to remove
2083  * the need for special case cleanup code.
2084  */
2085 
2086 int unregister_netdevice_notifier(struct notifier_block *nb)
2087 {
2088 	struct net *net;
2089 	int err;
2090 
2091 	/* Close race with setup_net() and cleanup_net() */
2092 	down_write(&pernet_ops_rwsem);
2093 	rtnl_lock();
2094 	err = raw_notifier_chain_unregister(&netdev_chain, nb);
2095 	if (err)
2096 		goto unlock;
2097 
2098 	for_each_net(net) {
2099 		__rtnl_net_lock(net);
2100 		call_netdevice_unregister_net_notifiers(nb, net);
2101 		__rtnl_net_unlock(net);
2102 	}
2103 
2104 unlock:
2105 	rtnl_unlock();
2106 	up_write(&pernet_ops_rwsem);
2107 	return err;
2108 }
2109 EXPORT_SYMBOL(unregister_netdevice_notifier);
2110 
2111 static int __register_netdevice_notifier_net(struct net *net,
2112 					     struct notifier_block *nb,
2113 					     bool ignore_call_fail)
2114 {
2115 	int err;
2116 
2117 	err = raw_notifier_chain_register(&net->netdev_chain, nb);
2118 	if (err)
2119 		return err;
2120 	if (dev_boot_phase)
2121 		return 0;
2122 
2123 	err = call_netdevice_register_net_notifiers(nb, net);
2124 	if (err && !ignore_call_fail)
2125 		goto chain_unregister;
2126 
2127 	return 0;
2128 
2129 chain_unregister:
2130 	raw_notifier_chain_unregister(&net->netdev_chain, nb);
2131 	return err;
2132 }
2133 
2134 static int __unregister_netdevice_notifier_net(struct net *net,
2135 					       struct notifier_block *nb)
2136 {
2137 	int err;
2138 
2139 	err = raw_notifier_chain_unregister(&net->netdev_chain, nb);
2140 	if (err)
2141 		return err;
2142 
2143 	call_netdevice_unregister_net_notifiers(nb, net);
2144 	return 0;
2145 }
2146 
2147 /**
2148  * register_netdevice_notifier_net - register a per-netns network notifier block
2149  * @net: network namespace
2150  * @nb: notifier
2151  *
2152  * Register a notifier to be called when network device events occur.
2153  * The notifier passed is linked into the kernel structures and must
2154  * not be reused until it has been unregistered. A negative errno code
2155  * is returned on a failure.
2156  *
2157  * When registered all registration and up events are replayed
2158  * to the new notifier to allow device to have a race free
2159  * view of the network device list.
2160  */
2161 
2162 int register_netdevice_notifier_net(struct net *net, struct notifier_block *nb)
2163 {
2164 	int err;
2165 
2166 	rtnl_net_lock(net);
2167 	err = __register_netdevice_notifier_net(net, nb, false);
2168 	rtnl_net_unlock(net);
2169 
2170 	return err;
2171 }
2172 EXPORT_SYMBOL(register_netdevice_notifier_net);
2173 
2174 /**
2175  * unregister_netdevice_notifier_net - unregister a per-netns
2176  *                                     network notifier block
2177  * @net: network namespace
2178  * @nb: notifier
2179  *
2180  * Unregister a notifier previously registered by
2181  * register_netdevice_notifier_net(). The notifier is unlinked from the
2182  * kernel structures and may then be reused. A negative errno code
2183  * is returned on a failure.
2184  *
2185  * After unregistering unregister and down device events are synthesized
2186  * for all devices on the device list to the removed notifier to remove
2187  * the need for special case cleanup code.
2188  */
2189 
2190 int unregister_netdevice_notifier_net(struct net *net,
2191 				      struct notifier_block *nb)
2192 {
2193 	int err;
2194 
2195 	rtnl_net_lock(net);
2196 	err = __unregister_netdevice_notifier_net(net, nb);
2197 	rtnl_net_unlock(net);
2198 
2199 	return err;
2200 }
2201 EXPORT_SYMBOL(unregister_netdevice_notifier_net);
2202 
2203 static void __move_netdevice_notifier_net(struct net *src_net,
2204 					  struct net *dst_net,
2205 					  struct notifier_block *nb)
2206 {
2207 	__unregister_netdevice_notifier_net(src_net, nb);
2208 	__register_netdevice_notifier_net(dst_net, nb, true);
2209 }
2210 
2211 static void rtnl_net_dev_lock(struct net_device *dev)
2212 {
2213 	bool again;
2214 
2215 	do {
2216 		struct net *net;
2217 
2218 		again = false;
2219 
2220 		/* netns might be being dismantled. */
2221 		rcu_read_lock();
2222 		net = dev_net_rcu(dev);
2223 		net_passive_inc(net);
2224 		rcu_read_unlock();
2225 
2226 		rtnl_net_lock(net);
2227 
2228 #ifdef CONFIG_NET_NS
2229 		/* dev might have been moved to another netns. */
2230 		if (!net_eq(net, rcu_access_pointer(dev->nd_net.net))) {
2231 			rtnl_net_unlock(net);
2232 			net_passive_dec(net);
2233 			again = true;
2234 		}
2235 #endif
2236 	} while (again);
2237 }
2238 
2239 static void rtnl_net_dev_unlock(struct net_device *dev)
2240 {
2241 	struct net *net = dev_net(dev);
2242 
2243 	rtnl_net_unlock(net);
2244 	net_passive_dec(net);
2245 }
2246 
2247 int register_netdevice_notifier_dev_net(struct net_device *dev,
2248 					struct notifier_block *nb,
2249 					struct netdev_net_notifier *nn)
2250 {
2251 	int err;
2252 
2253 	rtnl_net_dev_lock(dev);
2254 	err = __register_netdevice_notifier_net(dev_net(dev), nb, false);
2255 	if (!err) {
2256 		nn->nb = nb;
2257 		list_add(&nn->list, &dev->net_notifier_list);
2258 	}
2259 	rtnl_net_dev_unlock(dev);
2260 
2261 	return err;
2262 }
2263 EXPORT_SYMBOL(register_netdevice_notifier_dev_net);
2264 
2265 int unregister_netdevice_notifier_dev_net(struct net_device *dev,
2266 					  struct notifier_block *nb,
2267 					  struct netdev_net_notifier *nn)
2268 {
2269 	int err;
2270 
2271 	rtnl_net_dev_lock(dev);
2272 	list_del(&nn->list);
2273 	err = __unregister_netdevice_notifier_net(dev_net(dev), nb);
2274 	rtnl_net_dev_unlock(dev);
2275 
2276 	return err;
2277 }
2278 EXPORT_SYMBOL(unregister_netdevice_notifier_dev_net);
2279 
2280 static void move_netdevice_notifiers_dev_net(struct net_device *dev,
2281 					     struct net *net)
2282 {
2283 	struct netdev_net_notifier *nn;
2284 
2285 	list_for_each_entry(nn, &dev->net_notifier_list, list)
2286 		__move_netdevice_notifier_net(dev_net(dev), net, nn->nb);
2287 }
2288 
2289 /**
2290  *	call_netdevice_notifiers_info - call all network notifier blocks
2291  *	@val: value passed unmodified to notifier function
2292  *	@info: notifier information data
2293  *
2294  *	Call all network notifier blocks.  Parameters and return value
2295  *	are as for raw_notifier_call_chain().
2296  */
2297 
2298 int call_netdevice_notifiers_info(unsigned long val,
2299 				  struct netdev_notifier_info *info)
2300 {
2301 	struct net *net = dev_net(info->dev);
2302 	int ret;
2303 
2304 	ASSERT_RTNL();
2305 
2306 	/* Run per-netns notifier block chain first, then run the global one.
2307 	 * Hopefully, one day, the global one is going to be removed after
2308 	 * all notifier block registrators get converted to be per-netns.
2309 	 */
2310 	ret = raw_notifier_call_chain(&net->netdev_chain, val, info);
2311 	if (ret & NOTIFY_STOP_MASK)
2312 		return ret;
2313 	return raw_notifier_call_chain(&netdev_chain, val, info);
2314 }
2315 
2316 /**
2317  *	call_netdevice_notifiers_info_robust - call per-netns notifier blocks
2318  *	                                       for and rollback on error
2319  *	@val_up: value passed unmodified to notifier function
2320  *	@val_down: value passed unmodified to the notifier function when
2321  *	           recovering from an error on @val_up
2322  *	@info: notifier information data
2323  *
2324  *	Call all per-netns network notifier blocks, but not notifier blocks on
2325  *	the global notifier chain. Parameters and return value are as for
2326  *	raw_notifier_call_chain_robust().
2327  */
2328 
2329 static int
2330 call_netdevice_notifiers_info_robust(unsigned long val_up,
2331 				     unsigned long val_down,
2332 				     struct netdev_notifier_info *info)
2333 {
2334 	struct net *net = dev_net(info->dev);
2335 
2336 	ASSERT_RTNL();
2337 
2338 	return raw_notifier_call_chain_robust(&net->netdev_chain,
2339 					      val_up, val_down, info);
2340 }
2341 
2342 static int call_netdevice_notifiers_extack(unsigned long val,
2343 					   struct net_device *dev,
2344 					   struct netlink_ext_ack *extack)
2345 {
2346 	struct netdev_notifier_info info = {
2347 		.dev = dev,
2348 		.extack = extack,
2349 	};
2350 
2351 	return call_netdevice_notifiers_info(val, &info);
2352 }
2353 
2354 /**
2355  *	call_netdevice_notifiers - call all network notifier blocks
2356  *      @val: value passed unmodified to notifier function
2357  *      @dev: net_device pointer passed unmodified to notifier function
2358  *
2359  *	Call all network notifier blocks.  Parameters and return value
2360  *	are as for raw_notifier_call_chain().
2361  */
2362 
2363 int call_netdevice_notifiers(unsigned long val, struct net_device *dev)
2364 {
2365 	return call_netdevice_notifiers_extack(val, dev, NULL);
2366 }
2367 EXPORT_SYMBOL(call_netdevice_notifiers);
2368 
2369 /**
2370  *	call_netdevice_notifiers_mtu - call all network notifier blocks
2371  *	@val: value passed unmodified to notifier function
2372  *	@dev: net_device pointer passed unmodified to notifier function
2373  *	@arg: additional u32 argument passed to the notifier function
2374  *
2375  *	Call all network notifier blocks.  Parameters and return value
2376  *	are as for raw_notifier_call_chain().
2377  */
2378 static int call_netdevice_notifiers_mtu(unsigned long val,
2379 					struct net_device *dev, u32 arg)
2380 {
2381 	struct netdev_notifier_info_ext info = {
2382 		.info.dev = dev,
2383 		.ext.mtu = arg,
2384 	};
2385 
2386 	BUILD_BUG_ON(offsetof(struct netdev_notifier_info_ext, info) != 0);
2387 
2388 	return call_netdevice_notifiers_info(val, &info.info);
2389 }
2390 
2391 #ifdef CONFIG_NET_INGRESS
2392 static DEFINE_STATIC_KEY_FALSE(ingress_needed_key);
2393 
2394 void net_inc_ingress_queue(void)
2395 {
2396 	static_branch_inc(&ingress_needed_key);
2397 }
2398 EXPORT_SYMBOL_GPL(net_inc_ingress_queue);
2399 
2400 void net_dec_ingress_queue(void)
2401 {
2402 	static_branch_dec(&ingress_needed_key);
2403 }
2404 EXPORT_SYMBOL_GPL(net_dec_ingress_queue);
2405 #endif
2406 
2407 #ifdef CONFIG_NET_EGRESS
2408 static DEFINE_STATIC_KEY_FALSE(egress_needed_key);
2409 
2410 void net_inc_egress_queue(void)
2411 {
2412 	static_branch_inc(&egress_needed_key);
2413 }
2414 EXPORT_SYMBOL_GPL(net_inc_egress_queue);
2415 
2416 void net_dec_egress_queue(void)
2417 {
2418 	static_branch_dec(&egress_needed_key);
2419 }
2420 EXPORT_SYMBOL_GPL(net_dec_egress_queue);
2421 #endif
2422 
2423 #ifdef CONFIG_NET_CLS_ACT
2424 DEFINE_STATIC_KEY_FALSE(tcf_sw_enabled_key);
2425 EXPORT_SYMBOL(tcf_sw_enabled_key);
2426 #endif
2427 
2428 DEFINE_STATIC_KEY_FALSE(netstamp_needed_key);
2429 EXPORT_SYMBOL(netstamp_needed_key);
2430 #ifdef CONFIG_JUMP_LABEL
2431 static atomic_t netstamp_needed_deferred;
2432 static atomic_t netstamp_wanted;
2433 static void netstamp_clear(struct work_struct *work)
2434 {
2435 	int deferred = atomic_xchg(&netstamp_needed_deferred, 0);
2436 	int wanted;
2437 
2438 	wanted = atomic_add_return(deferred, &netstamp_wanted);
2439 	if (wanted > 0)
2440 		static_branch_enable(&netstamp_needed_key);
2441 	else
2442 		static_branch_disable(&netstamp_needed_key);
2443 }
2444 static DECLARE_WORK(netstamp_work, netstamp_clear);
2445 #endif
2446 
2447 void net_enable_timestamp(void)
2448 {
2449 #ifdef CONFIG_JUMP_LABEL
2450 	int wanted = atomic_read(&netstamp_wanted);
2451 
2452 	while (wanted > 0) {
2453 		if (atomic_try_cmpxchg(&netstamp_wanted, &wanted, wanted + 1))
2454 			return;
2455 	}
2456 	atomic_inc(&netstamp_needed_deferred);
2457 	schedule_work(&netstamp_work);
2458 #else
2459 	static_branch_inc(&netstamp_needed_key);
2460 #endif
2461 }
2462 EXPORT_SYMBOL(net_enable_timestamp);
2463 
2464 void net_disable_timestamp(void)
2465 {
2466 #ifdef CONFIG_JUMP_LABEL
2467 	int wanted = atomic_read(&netstamp_wanted);
2468 
2469 	while (wanted > 1) {
2470 		if (atomic_try_cmpxchg(&netstamp_wanted, &wanted, wanted - 1))
2471 			return;
2472 	}
2473 	atomic_dec(&netstamp_needed_deferred);
2474 	schedule_work(&netstamp_work);
2475 #else
2476 	static_branch_dec(&netstamp_needed_key);
2477 #endif
2478 }
2479 EXPORT_SYMBOL(net_disable_timestamp);
2480 
2481 static inline void net_timestamp_set(struct sk_buff *skb)
2482 {
2483 	skb->tstamp = 0;
2484 	skb->tstamp_type = SKB_CLOCK_REALTIME;
2485 	if (static_branch_unlikely(&netstamp_needed_key))
2486 		skb->tstamp = ktime_get_real();
2487 }
2488 
2489 #define net_timestamp_check(COND, SKB)				\
2490 	if (static_branch_unlikely(&netstamp_needed_key)) {	\
2491 		if ((COND) && !(SKB)->tstamp)			\
2492 			(SKB)->tstamp = ktime_get_real();	\
2493 	}							\
2494 
2495 bool is_skb_forwardable(const struct net_device *dev, const struct sk_buff *skb)
2496 {
2497 	return __is_skb_forwardable(dev, skb, true);
2498 }
2499 EXPORT_SYMBOL_GPL(is_skb_forwardable);
2500 
2501 static int __dev_forward_skb2(struct net_device *dev, struct sk_buff *skb,
2502 			      bool check_mtu)
2503 {
2504 	enum skb_drop_reason reason;
2505 	int ret;
2506 
2507 	ret = ____dev_forward_skb(dev, skb, check_mtu);
2508 	if (unlikely(ret))
2509 		return ret;
2510 
2511 	reason = eth_type_trans_careful(skb, dev);
2512 	if (unlikely(reason)) {
2513 		dev_core_stats_rx_dropped_inc(dev);
2514 		kfree_skb_reason(skb, reason);
2515 		return NET_RX_DROP;
2516 	}
2517 	skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN);
2518 
2519 	return 0;
2520 }
2521 
2522 int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
2523 {
2524 	return __dev_forward_skb2(dev, skb, true);
2525 }
2526 EXPORT_SYMBOL_GPL(__dev_forward_skb);
2527 
2528 /**
2529  * dev_forward_skb - loopback an skb to another netif
2530  *
2531  * @dev: destination network device
2532  * @skb: buffer to forward
2533  *
2534  * return values:
2535  *	NET_RX_SUCCESS	(no congestion)
2536  *	NET_RX_DROP     (packet was dropped, but freed)
2537  *
2538  * dev_forward_skb can be used for injecting an skb from the
2539  * start_xmit function of one device into the receive queue
2540  * of another device.
2541  *
2542  * The receiving device may be in another namespace, so
2543  * we have to clear all information in the skb that could
2544  * impact namespace isolation.
2545  */
2546 int dev_forward_skb(struct net_device *dev, struct sk_buff *skb)
2547 {
2548 	return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb);
2549 }
2550 EXPORT_SYMBOL_GPL(dev_forward_skb);
2551 
2552 int dev_forward_skb_nomtu(struct net_device *dev, struct sk_buff *skb)
2553 {
2554 	return __dev_forward_skb2(dev, skb, false) ?: netif_rx_internal(skb);
2555 }
2556 
2557 static int deliver_skb(struct sk_buff *skb,
2558 		       struct packet_type *pt_prev,
2559 		       struct net_device *orig_dev)
2560 {
2561 	if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC)))
2562 		return -ENOMEM;
2563 	refcount_inc(&skb->users);
2564 	return pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
2565 }
2566 
2567 static inline void deliver_ptype_list_skb(struct sk_buff *skb,
2568 					  struct packet_type **pt,
2569 					  struct net_device *orig_dev,
2570 					  __be16 type,
2571 					  struct list_head *ptype_list)
2572 {
2573 	struct packet_type *ptype, *pt_prev = *pt;
2574 
2575 	list_for_each_entry_rcu(ptype, ptype_list, list) {
2576 		if (ptype->type != type)
2577 			continue;
2578 		if (unlikely(pt_prev))
2579 			deliver_skb(skb, pt_prev, orig_dev);
2580 		pt_prev = ptype;
2581 	}
2582 	*pt = pt_prev;
2583 }
2584 
2585 static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb)
2586 {
2587 	if (!ptype->af_packet_priv || !skb->sk)
2588 		return false;
2589 
2590 	if (ptype->id_match)
2591 		return ptype->id_match(ptype, skb->sk);
2592 	else if ((struct sock *)ptype->af_packet_priv == skb->sk)
2593 		return true;
2594 
2595 	return false;
2596 }
2597 
2598 /**
2599  * dev_nit_active_rcu - return true if any network interface taps are in use
2600  *
2601  * The caller must hold the RCU lock
2602  *
2603  * @dev: network device to check for the presence of taps
2604  */
2605 bool dev_nit_active_rcu(const struct net_device *dev)
2606 {
2607 	/* Callers may hold either RCU or RCU BH lock */
2608 	WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held());
2609 
2610 	return !list_empty(&dev_net(dev)->ptype_all) ||
2611 	       !list_empty(&dev->ptype_all);
2612 }
2613 EXPORT_SYMBOL_GPL(dev_nit_active_rcu);
2614 
2615 /*
2616  *	Support routine. Sends outgoing frames to any network
2617  *	taps currently in use.
2618  */
2619 
2620 void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev)
2621 {
2622 	struct packet_type *ptype, *pt_prev = NULL;
2623 	struct list_head *ptype_list;
2624 	struct sk_buff *skb2 = NULL;
2625 
2626 	rcu_read_lock();
2627 	ptype_list = &dev_net_rcu(dev)->ptype_all;
2628 again:
2629 	list_for_each_entry_rcu(ptype, ptype_list, list) {
2630 		if (READ_ONCE(ptype->ignore_outgoing))
2631 			continue;
2632 
2633 		/* Never send packets back to the socket
2634 		 * they originated from - MvS (miquels@drinkel.ow.org)
2635 		 */
2636 		if (skb_loop_sk(ptype, skb))
2637 			continue;
2638 
2639 		if (unlikely(pt_prev)) {
2640 			deliver_skb(skb2, pt_prev, skb->dev);
2641 			pt_prev = ptype;
2642 			continue;
2643 		}
2644 
2645 		/* need to clone skb, done only once */
2646 		skb2 = skb_clone(skb, GFP_ATOMIC);
2647 		if (!skb2)
2648 			goto out_unlock;
2649 
2650 		net_timestamp_set(skb2);
2651 
2652 		/* skb->nh should be correctly
2653 		 * set by sender, so that the second statement is
2654 		 * just protection against buggy protocols.
2655 		 */
2656 		skb_reset_mac_header(skb2);
2657 
2658 		if (skb_network_header(skb2) < skb2->data ||
2659 		    skb_network_header(skb2) > skb_tail_pointer(skb2)) {
2660 			net_crit_ratelimited("protocol %04x is buggy, dev %s\n",
2661 					     ntohs(skb2->protocol),
2662 					     dev->name);
2663 			skb_reset_network_header(skb2);
2664 		}
2665 
2666 		skb2->transport_header = skb2->network_header;
2667 		skb2->pkt_type = PACKET_OUTGOING;
2668 		pt_prev = ptype;
2669 	}
2670 
2671 	if (ptype_list != &dev->ptype_all) {
2672 		ptype_list = &dev->ptype_all;
2673 		goto again;
2674 	}
2675 out_unlock:
2676 	if (pt_prev) {
2677 		if (!skb_orphan_frags_rx(skb2, GFP_ATOMIC))
2678 			pt_prev->func(skb2, skb->dev, pt_prev, skb->dev);
2679 		else
2680 			kfree_skb(skb2);
2681 	}
2682 	rcu_read_unlock();
2683 }
2684 EXPORT_SYMBOL_GPL(dev_queue_xmit_nit);
2685 
2686 /**
2687  * netif_setup_tc - Handle tc mappings on real_num_tx_queues change
2688  * @dev: Network device
2689  * @txq: number of queues available
2690  *
2691  * If real_num_tx_queues is changed the tc mappings may no longer be
2692  * valid. To resolve this verify the tc mapping remains valid and if
2693  * not NULL the mapping. With no priorities mapping to this
2694  * offset/count pair it will no longer be used. In the worst case TC0
2695  * is invalid nothing can be done so disable priority mappings. If is
2696  * expected that drivers will fix this mapping if they can before
2697  * calling netif_set_real_num_tx_queues.
2698  */
2699 static void netif_setup_tc(struct net_device *dev, unsigned int txq)
2700 {
2701 	struct netdev_tc_txq res;
2702 	int i;
2703 
2704 	res.combined = READ_ONCE(dev->tc_to_txq[0].combined);
2705 
2706 	/* If TC0 is invalidated disable TC mapping */
2707 	if (res.offset + res.count > txq) {
2708 		netdev_warn(dev, "Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n");
2709 		WRITE_ONCE(dev->num_tc, 0);
2710 		return;
2711 	}
2712 
2713 	/* Invalidated prio to tc mappings set to TC0 */
2714 	for (i = 1; i < TC_BITMASK + 1; i++) {
2715 		int q = netdev_get_prio_tc_map(dev, i);
2716 
2717 		res.combined = READ_ONCE(dev->tc_to_txq[q].combined);
2718 		if (res.offset + res.count > txq) {
2719 			netdev_warn(dev, "Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n",
2720 				    i, q);
2721 			netdev_set_prio_tc_map(dev, i, 0);
2722 		}
2723 	}
2724 }
2725 
2726 int netdev_txq_to_tc(struct net_device *dev, unsigned int txq)
2727 {
2728 	if (READ_ONCE(dev->num_tc)) {
2729 		struct netdev_tc_txq *tc = &dev->tc_to_txq[0];
2730 		int i;
2731 
2732 		/* walk through the TCs and see if it falls into any of them */
2733 		for (i = 0; i < TC_MAX_QUEUE; i++, tc++) {
2734 			struct netdev_tc_txq res;
2735 
2736 			res.combined = READ_ONCE(tc->combined);
2737 			if ((txq - res.offset) < res.count)
2738 				return i;
2739 		}
2740 
2741 		/* didn't find it, just return -1 to indicate no match */
2742 		return -1;
2743 	}
2744 
2745 	return 0;
2746 }
2747 EXPORT_SYMBOL(netdev_txq_to_tc);
2748 
2749 #ifdef CONFIG_XPS
2750 static struct static_key xps_needed __read_mostly;
2751 static struct static_key xps_rxqs_needed __read_mostly;
2752 static DEFINE_MUTEX(xps_map_mutex);
2753 #define xmap_dereference(P)		\
2754 	rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex))
2755 
2756 static bool remove_xps_queue(struct xps_dev_maps *dev_maps,
2757 			     struct xps_dev_maps *old_maps, int tci, u16 index)
2758 {
2759 	struct xps_map *map = NULL;
2760 	int pos;
2761 
2762 	map = xmap_dereference(dev_maps->attr_map[tci]);
2763 	if (!map)
2764 		return false;
2765 
2766 	for (pos = map->len; pos--;) {
2767 		if (map->queues[pos] != index)
2768 			continue;
2769 
2770 		if (map->len > 1) {
2771 			map->queues[pos] = map->queues[--map->len];
2772 			break;
2773 		}
2774 
2775 		if (old_maps)
2776 			RCU_INIT_POINTER(old_maps->attr_map[tci], NULL);
2777 		RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL);
2778 		kfree_rcu(map, rcu);
2779 		return false;
2780 	}
2781 
2782 	return true;
2783 }
2784 
2785 static bool remove_xps_queue_cpu(struct net_device *dev,
2786 				 struct xps_dev_maps *dev_maps,
2787 				 int cpu, u16 offset, u16 count)
2788 {
2789 	int num_tc = dev_maps->num_tc;
2790 	bool active = false;
2791 	int tci;
2792 
2793 	for (tci = cpu * num_tc; num_tc--; tci++) {
2794 		int i, j;
2795 
2796 		for (i = count, j = offset; i--; j++) {
2797 			if (!remove_xps_queue(dev_maps, NULL, tci, j))
2798 				break;
2799 		}
2800 
2801 		active |= i < 0;
2802 	}
2803 
2804 	return active;
2805 }
2806 
2807 static void reset_xps_maps(struct net_device *dev,
2808 			   struct xps_dev_maps *dev_maps,
2809 			   enum xps_map_type type)
2810 {
2811 	static_key_slow_dec_cpuslocked(&xps_needed);
2812 	if (type == XPS_RXQS)
2813 		static_key_slow_dec_cpuslocked(&xps_rxqs_needed);
2814 
2815 	RCU_INIT_POINTER(dev->xps_maps[type], NULL);
2816 
2817 	kfree_rcu(dev_maps, rcu);
2818 }
2819 
2820 static void clean_xps_maps(struct net_device *dev, enum xps_map_type type,
2821 			   u16 offset, u16 count)
2822 {
2823 	struct xps_dev_maps *dev_maps;
2824 	bool active = false;
2825 	int i, j;
2826 
2827 	dev_maps = xmap_dereference(dev->xps_maps[type]);
2828 	if (!dev_maps)
2829 		return;
2830 
2831 	for (j = 0; j < dev_maps->nr_ids; j++)
2832 		active |= remove_xps_queue_cpu(dev, dev_maps, j, offset, count);
2833 	if (!active)
2834 		reset_xps_maps(dev, dev_maps, type);
2835 
2836 	if (type == XPS_CPUS) {
2837 		for (i = offset + (count - 1); count--; i--)
2838 			netdev_queue_numa_node_write(
2839 				netdev_get_tx_queue(dev, i), NUMA_NO_NODE);
2840 	}
2841 }
2842 
2843 static void netif_reset_xps_queues(struct net_device *dev, u16 offset,
2844 				   u16 count)
2845 {
2846 	if (!static_key_false(&xps_needed))
2847 		return;
2848 
2849 	cpus_read_lock();
2850 	mutex_lock(&xps_map_mutex);
2851 
2852 	if (static_key_false(&xps_rxqs_needed))
2853 		clean_xps_maps(dev, XPS_RXQS, offset, count);
2854 
2855 	clean_xps_maps(dev, XPS_CPUS, offset, count);
2856 
2857 	mutex_unlock(&xps_map_mutex);
2858 	cpus_read_unlock();
2859 }
2860 
2861 static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index)
2862 {
2863 	netif_reset_xps_queues(dev, index, dev->num_tx_queues - index);
2864 }
2865 
2866 static struct xps_map *expand_xps_map(struct xps_map *map, int attr_index,
2867 				      u16 index, bool is_rxqs_map)
2868 {
2869 	struct xps_map *new_map;
2870 	int alloc_len = XPS_MIN_MAP_ALLOC;
2871 	int i, pos;
2872 
2873 	for (pos = 0; map && pos < map->len; pos++) {
2874 		if (map->queues[pos] != index)
2875 			continue;
2876 		return map;
2877 	}
2878 
2879 	/* Need to add tx-queue to this CPU's/rx-queue's existing map */
2880 	if (map) {
2881 		if (pos < map->alloc_len)
2882 			return map;
2883 
2884 		alloc_len = map->alloc_len * 2;
2885 	}
2886 
2887 	/* Need to allocate new map to store tx-queue on this CPU's/rx-queue's
2888 	 *  map
2889 	 */
2890 	if (is_rxqs_map)
2891 		new_map = kzalloc(XPS_MAP_SIZE(alloc_len), GFP_KERNEL);
2892 	else
2893 		new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL,
2894 				       cpu_to_node(attr_index));
2895 	if (!new_map)
2896 		return NULL;
2897 
2898 	for (i = 0; i < pos; i++)
2899 		new_map->queues[i] = map->queues[i];
2900 	new_map->alloc_len = alloc_len;
2901 	new_map->len = pos;
2902 
2903 	return new_map;
2904 }
2905 
2906 /* Copy xps maps at a given index */
2907 static void xps_copy_dev_maps(struct xps_dev_maps *dev_maps,
2908 			      struct xps_dev_maps *new_dev_maps, int index,
2909 			      int tc, bool skip_tc)
2910 {
2911 	int i, tci = index * dev_maps->num_tc;
2912 	struct xps_map *map;
2913 
2914 	/* copy maps belonging to foreign traffic classes */
2915 	for (i = 0; i < dev_maps->num_tc; i++, tci++) {
2916 		if (i == tc && skip_tc)
2917 			continue;
2918 
2919 		/* fill in the new device map from the old device map */
2920 		map = xmap_dereference(dev_maps->attr_map[tci]);
2921 		RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
2922 	}
2923 }
2924 
2925 /* Must be called under cpus_read_lock */
2926 int __netif_set_xps_queue(struct net_device *dev, const unsigned long *mask,
2927 			  u16 index, enum xps_map_type type)
2928 {
2929 	struct xps_dev_maps *dev_maps, *new_dev_maps = NULL, *old_dev_maps = NULL;
2930 	int maps_sz, num_tc = 1, tc = 0, dev_num_tc;
2931 	const unsigned long *online_mask = NULL;
2932 	bool active = false, copy = false;
2933 	int i, j, tci, numa_node_id = -2;
2934 	struct xps_map *map, *new_map;
2935 	unsigned int nr_ids;
2936 
2937 	WARN_ON_ONCE(index >= dev->num_tx_queues);
2938 
2939 	dev_num_tc = READ_ONCE(dev->num_tc);
2940 	if (dev_num_tc) {
2941 		/* Do not allow XPS on subordinate device directly */
2942 		num_tc = dev_num_tc;
2943 		if (num_tc < 0)
2944 			return -EINVAL;
2945 
2946 		/* If queue belongs to subordinate dev use its map */
2947 		dev = netdev_get_tx_queue(dev, index)->sb_dev ? : dev;
2948 
2949 		tc = netdev_txq_to_tc(dev, index);
2950 		if (tc < 0 || tc >= num_tc)
2951 			return -EINVAL;
2952 	}
2953 
2954 	mutex_lock(&xps_map_mutex);
2955 
2956 	dev_maps = xmap_dereference(dev->xps_maps[type]);
2957 	if (type == XPS_RXQS) {
2958 		maps_sz = XPS_RXQ_DEV_MAPS_SIZE(num_tc, dev->num_rx_queues);
2959 		nr_ids = dev->num_rx_queues;
2960 	} else {
2961 		maps_sz = XPS_CPU_DEV_MAPS_SIZE(num_tc);
2962 		if (num_possible_cpus() > 1)
2963 			online_mask = cpumask_bits(cpu_online_mask);
2964 		nr_ids = nr_cpu_ids;
2965 	}
2966 
2967 	if (maps_sz < L1_CACHE_BYTES)
2968 		maps_sz = L1_CACHE_BYTES;
2969 
2970 	/* The old dev_maps could be larger or smaller than the one we're
2971 	 * setting up now, as dev->num_tc or nr_ids could have been updated in
2972 	 * between. We could try to be smart, but let's be safe instead and only
2973 	 * copy foreign traffic classes if the two map sizes match.
2974 	 */
2975 	if (dev_maps &&
2976 	    dev_maps->num_tc == num_tc && dev_maps->nr_ids == nr_ids)
2977 		copy = true;
2978 
2979 	/* allocate memory for queue storage */
2980 	for (j = -1; j = netif_attrmask_next_and(j, online_mask, mask, nr_ids),
2981 	     j < nr_ids;) {
2982 		if (!new_dev_maps) {
2983 			new_dev_maps = kzalloc(maps_sz, GFP_KERNEL);
2984 			if (!new_dev_maps) {
2985 				mutex_unlock(&xps_map_mutex);
2986 				return -ENOMEM;
2987 			}
2988 
2989 			new_dev_maps->nr_ids = nr_ids;
2990 			new_dev_maps->num_tc = num_tc;
2991 		}
2992 
2993 		tci = j * num_tc + tc;
2994 		map = copy ? xmap_dereference(dev_maps->attr_map[tci]) : NULL;
2995 
2996 		map = expand_xps_map(map, j, index, type == XPS_RXQS);
2997 		if (!map)
2998 			goto error;
2999 
3000 		RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map);
3001 	}
3002 
3003 	if (!new_dev_maps)
3004 		goto out_no_new_maps;
3005 
3006 	if (!dev_maps) {
3007 		/* Increment static keys at most once per type */
3008 		static_key_slow_inc_cpuslocked(&xps_needed);
3009 		if (type == XPS_RXQS)
3010 			static_key_slow_inc_cpuslocked(&xps_rxqs_needed);
3011 	}
3012 
3013 	for (j = 0; j < nr_ids; j++) {
3014 		bool skip_tc = false;
3015 
3016 		tci = j * num_tc + tc;
3017 		if (netif_attr_test_mask(j, mask, nr_ids) &&
3018 		    netif_attr_test_online(j, online_mask, nr_ids)) {
3019 			/* add tx-queue to CPU/rx-queue maps */
3020 			int pos = 0;
3021 
3022 			skip_tc = true;
3023 
3024 			map = xmap_dereference(new_dev_maps->attr_map[tci]);
3025 			while ((pos < map->len) && (map->queues[pos] != index))
3026 				pos++;
3027 
3028 			if (pos == map->len)
3029 				map->queues[map->len++] = index;
3030 #ifdef CONFIG_NUMA
3031 			if (type == XPS_CPUS) {
3032 				if (numa_node_id == -2)
3033 					numa_node_id = cpu_to_node(j);
3034 				else if (numa_node_id != cpu_to_node(j))
3035 					numa_node_id = -1;
3036 			}
3037 #endif
3038 		}
3039 
3040 		if (copy)
3041 			xps_copy_dev_maps(dev_maps, new_dev_maps, j, tc,
3042 					  skip_tc);
3043 	}
3044 
3045 	rcu_assign_pointer(dev->xps_maps[type], new_dev_maps);
3046 
3047 	/* Cleanup old maps */
3048 	if (!dev_maps)
3049 		goto out_no_old_maps;
3050 
3051 	for (j = 0; j < dev_maps->nr_ids; j++) {
3052 		for (i = dev_maps->num_tc, tci = j * dev_maps->num_tc; i--; tci++) {
3053 			map = xmap_dereference(dev_maps->attr_map[tci]);
3054 			if (!map)
3055 				continue;
3056 
3057 			if (copy) {
3058 				new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
3059 				if (map == new_map)
3060 					continue;
3061 			}
3062 
3063 			RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL);
3064 			kfree_rcu(map, rcu);
3065 		}
3066 	}
3067 
3068 	old_dev_maps = dev_maps;
3069 
3070 out_no_old_maps:
3071 	dev_maps = new_dev_maps;
3072 	active = true;
3073 
3074 out_no_new_maps:
3075 	if (type == XPS_CPUS)
3076 		/* update Tx queue numa node */
3077 		netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index),
3078 					     (numa_node_id >= 0) ?
3079 					     numa_node_id : NUMA_NO_NODE);
3080 
3081 	if (!dev_maps)
3082 		goto out_no_maps;
3083 
3084 	/* removes tx-queue from unused CPUs/rx-queues */
3085 	for (j = 0; j < dev_maps->nr_ids; j++) {
3086 		tci = j * dev_maps->num_tc;
3087 
3088 		for (i = 0; i < dev_maps->num_tc; i++, tci++) {
3089 			if (i == tc &&
3090 			    netif_attr_test_mask(j, mask, dev_maps->nr_ids) &&
3091 			    netif_attr_test_online(j, online_mask, dev_maps->nr_ids))
3092 				continue;
3093 
3094 			active |= remove_xps_queue(dev_maps,
3095 						   copy ? old_dev_maps : NULL,
3096 						   tci, index);
3097 		}
3098 	}
3099 
3100 	if (old_dev_maps)
3101 		kfree_rcu(old_dev_maps, rcu);
3102 
3103 	/* free map if not active */
3104 	if (!active)
3105 		reset_xps_maps(dev, dev_maps, type);
3106 
3107 out_no_maps:
3108 	mutex_unlock(&xps_map_mutex);
3109 
3110 	return 0;
3111 error:
3112 	/* remove any maps that we added */
3113 	for (j = 0; j < nr_ids; j++) {
3114 		for (i = num_tc, tci = j * num_tc; i--; tci++) {
3115 			new_map = xmap_dereference(new_dev_maps->attr_map[tci]);
3116 			map = copy ?
3117 			      xmap_dereference(dev_maps->attr_map[tci]) :
3118 			      NULL;
3119 			if (new_map && new_map != map)
3120 				kfree(new_map);
3121 		}
3122 	}
3123 
3124 	mutex_unlock(&xps_map_mutex);
3125 
3126 	kfree(new_dev_maps);
3127 	return -ENOMEM;
3128 }
3129 EXPORT_SYMBOL_GPL(__netif_set_xps_queue);
3130 
3131 int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
3132 			u16 index)
3133 {
3134 	int ret;
3135 
3136 	cpus_read_lock();
3137 	ret =  __netif_set_xps_queue(dev, cpumask_bits(mask), index, XPS_CPUS);
3138 	cpus_read_unlock();
3139 
3140 	return ret;
3141 }
3142 EXPORT_SYMBOL(netif_set_xps_queue);
3143 
3144 #endif
3145 static void netdev_unbind_all_sb_channels(struct net_device *dev)
3146 {
3147 	struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues];
3148 
3149 	/* Unbind any subordinate channels */
3150 	while (txq-- != &dev->_tx[0]) {
3151 		if (txq->sb_dev)
3152 			netdev_unbind_sb_channel(dev, txq->sb_dev);
3153 	}
3154 }
3155 
3156 void netdev_reset_tc(struct net_device *dev)
3157 {
3158 	int i;
3159 
3160 #ifdef CONFIG_XPS
3161 	netif_reset_xps_queues_gt(dev, 0);
3162 #endif
3163 	netdev_unbind_all_sb_channels(dev);
3164 
3165 	/* Reset TC configuration of device */
3166 	WRITE_ONCE(dev->num_tc, 0);
3167 	for (i = 0; i < TC_MAX_QUEUE; i++)
3168 		WRITE_ONCE(dev->tc_to_txq[i].combined, 0);
3169 	for (i = 0; i <= TC_BITMASK; i++)
3170 		WRITE_ONCE(dev->prio_tc_map[i], 0);
3171 }
3172 EXPORT_SYMBOL(netdev_reset_tc);
3173 
3174 int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset)
3175 {
3176 	struct netdev_tc_txq res = {
3177 		.count = count,
3178 		.offset = offset,
3179 	};
3180 
3181 	if (tc >= READ_ONCE(dev->num_tc))
3182 		return -EINVAL;
3183 
3184 #ifdef CONFIG_XPS
3185 	netif_reset_xps_queues(dev, offset, count);
3186 #endif
3187 	WRITE_ONCE(dev->tc_to_txq[tc].combined, res.combined);
3188 	return 0;
3189 }
3190 EXPORT_SYMBOL(netdev_set_tc_queue);
3191 
3192 int netdev_set_num_tc(struct net_device *dev, u8 num_tc)
3193 {
3194 	if (num_tc > TC_MAX_QUEUE)
3195 		return -EINVAL;
3196 
3197 #ifdef CONFIG_XPS
3198 	netif_reset_xps_queues_gt(dev, 0);
3199 #endif
3200 	netdev_unbind_all_sb_channels(dev);
3201 
3202 	WRITE_ONCE(dev->num_tc, num_tc);
3203 	return 0;
3204 }
3205 EXPORT_SYMBOL(netdev_set_num_tc);
3206 
3207 void netdev_unbind_sb_channel(struct net_device *dev,
3208 			      struct net_device *sb_dev)
3209 {
3210 	struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues];
3211 	int i;
3212 
3213 #ifdef CONFIG_XPS
3214 	netif_reset_xps_queues_gt(sb_dev, 0);
3215 #endif
3216 	for (i = 0; i < TC_MAX_QUEUE; i++)
3217 		WRITE_ONCE(sb_dev->tc_to_txq[i].combined, 0);
3218 	for (i = 0; i <= TC_BITMASK; i++)
3219 		WRITE_ONCE(sb_dev->prio_tc_map[i], 0);
3220 
3221 	while (txq-- != &dev->_tx[0]) {
3222 		if (txq->sb_dev == sb_dev)
3223 			txq->sb_dev = NULL;
3224 	}
3225 }
3226 EXPORT_SYMBOL(netdev_unbind_sb_channel);
3227 
3228 int netdev_bind_sb_channel_queue(struct net_device *dev,
3229 				 struct net_device *sb_dev,
3230 				 u8 tc, u16 count, u16 offset)
3231 {
3232 	/* Make certain the sb_dev and dev are already configured */
3233 	if (READ_ONCE(sb_dev->num_tc) >= 0 || tc >= READ_ONCE(dev->num_tc))
3234 		return -EINVAL;
3235 
3236 	/* We cannot hand out queues we don't have */
3237 	if ((offset + count) > dev->real_num_tx_queues)
3238 		return -EINVAL;
3239 
3240 	/* Record the mapping */
3241 	struct netdev_tc_txq res = {
3242 		.count = count,
3243 		.offset = offset,
3244 	};
3245 
3246 	WRITE_ONCE(sb_dev->tc_to_txq[tc].combined, res.combined);
3247 
3248 	/* Provide a way for Tx queue to find the tc_to_txq map or
3249 	 * XPS map for itself.
3250 	 */
3251 	while (count--)
3252 		netdev_get_tx_queue(dev, count + offset)->sb_dev = sb_dev;
3253 
3254 	return 0;
3255 }
3256 EXPORT_SYMBOL(netdev_bind_sb_channel_queue);
3257 
3258 int netdev_set_sb_channel(struct net_device *dev, u16 channel)
3259 {
3260 	/* Do not use a multiqueue device to represent a subordinate channel */
3261 	if (netif_is_multiqueue(dev))
3262 		return -ENODEV;
3263 
3264 	/* We allow channels 1 - 32767 to be used for subordinate channels.
3265 	 * Channel 0 is meant to be "native" mode and used only to represent
3266 	 * the main root device. We allow writing 0 to reset the device back
3267 	 * to normal mode after being used as a subordinate channel.
3268 	 */
3269 	if (channel > S16_MAX)
3270 		return -EINVAL;
3271 
3272 	WRITE_ONCE(dev->num_tc, -channel);
3273 
3274 	return 0;
3275 }
3276 EXPORT_SYMBOL(netdev_set_sb_channel);
3277 
3278 /*
3279  * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues
3280  * greater than real_num_tx_queues stale skbs on the qdisc must be flushed.
3281  */
3282 int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq)
3283 {
3284 	bool disabling;
3285 	int rc;
3286 
3287 	disabling = txq < dev->real_num_tx_queues;
3288 
3289 	if (txq < 1 || txq > dev->num_tx_queues)
3290 		return -EINVAL;
3291 
3292 	if (dev->reg_state == NETREG_REGISTERED ||
3293 	    dev->reg_state == NETREG_UNREGISTERING) {
3294 		netdev_assert_locked_ops_compat(dev);
3295 
3296 		rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues,
3297 						  txq);
3298 		if (rc)
3299 			return rc;
3300 
3301 		if (READ_ONCE(dev->num_tc))
3302 			netif_setup_tc(dev, txq);
3303 
3304 		net_shaper_set_real_num_tx_queues(dev, txq);
3305 
3306 		dev_qdisc_change_real_num_tx(dev, txq);
3307 
3308 		dev->real_num_tx_queues = txq;
3309 
3310 		if (disabling) {
3311 			synchronize_net();
3312 			qdisc_reset_all_tx_gt(dev, txq);
3313 #ifdef CONFIG_XPS
3314 			netif_reset_xps_queues_gt(dev, txq);
3315 #endif
3316 		}
3317 	} else {
3318 		dev->real_num_tx_queues = txq;
3319 	}
3320 
3321 	return 0;
3322 }
3323 EXPORT_SYMBOL(netif_set_real_num_tx_queues);
3324 
3325 /**
3326  *	netif_set_real_num_rx_queues - set actual number of RX queues used
3327  *	@dev: Network device
3328  *	@rxq: Actual number of RX queues
3329  *
3330  *	This must be called either with the rtnl_lock held or before
3331  *	registration of the net device.  Returns 0 on success, or a
3332  *	negative error code.  If called before registration, it always
3333  *	succeeds.
3334  */
3335 int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq)
3336 {
3337 	int rc;
3338 
3339 	if (rxq < 1 || rxq > dev->num_rx_queues)
3340 		return -EINVAL;
3341 
3342 	if (dev->reg_state == NETREG_REGISTERED) {
3343 		netdev_assert_locked_ops_compat(dev);
3344 
3345 		rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues,
3346 						  rxq);
3347 		if (rc)
3348 			return rc;
3349 	}
3350 
3351 	dev->real_num_rx_queues = rxq;
3352 	return 0;
3353 }
3354 EXPORT_SYMBOL(netif_set_real_num_rx_queues);
3355 
3356 /**
3357  *	netif_set_real_num_queues - set actual number of RX and TX queues used
3358  *	@dev: Network device
3359  *	@txq: Actual number of TX queues
3360  *	@rxq: Actual number of RX queues
3361  *
3362  *	Set the real number of both TX and RX queues.
3363  *	Does nothing if the number of queues is already correct.
3364  */
3365 int netif_set_real_num_queues(struct net_device *dev,
3366 			      unsigned int txq, unsigned int rxq)
3367 {
3368 	unsigned int old_rxq = dev->real_num_rx_queues;
3369 	int err;
3370 
3371 	if (txq < 1 || txq > dev->num_tx_queues ||
3372 	    rxq < 1 || rxq > dev->num_rx_queues)
3373 		return -EINVAL;
3374 
3375 	/* Start from increases, so the error path only does decreases -
3376 	 * decreases can't fail.
3377 	 */
3378 	if (rxq > dev->real_num_rx_queues) {
3379 		err = netif_set_real_num_rx_queues(dev, rxq);
3380 		if (err)
3381 			return err;
3382 	}
3383 	if (txq > dev->real_num_tx_queues) {
3384 		err = netif_set_real_num_tx_queues(dev, txq);
3385 		if (err)
3386 			goto undo_rx;
3387 	}
3388 	if (rxq < dev->real_num_rx_queues)
3389 		WARN_ON(netif_set_real_num_rx_queues(dev, rxq));
3390 	if (txq < dev->real_num_tx_queues)
3391 		WARN_ON(netif_set_real_num_tx_queues(dev, txq));
3392 
3393 	return 0;
3394 undo_rx:
3395 	WARN_ON(netif_set_real_num_rx_queues(dev, old_rxq));
3396 	return err;
3397 }
3398 EXPORT_SYMBOL(netif_set_real_num_queues);
3399 
3400 /**
3401  * netif_set_tso_max_size() - set the max size of TSO frames supported
3402  * @dev:	netdev to update
3403  * @size:	max skb->len of a TSO frame
3404  *
3405  * Set the limit on the size of TSO super-frames the device can handle.
3406  * Unless explicitly set the stack will assume the value of
3407  * %GSO_LEGACY_MAX_SIZE.
3408  */
3409 void netif_set_tso_max_size(struct net_device *dev, unsigned int size)
3410 {
3411 	dev->tso_max_size = min(GSO_MAX_SIZE, size);
3412 	if (size < READ_ONCE(dev->gso_max_size))
3413 		netif_set_gso_max_size(dev, size);
3414 	if (size < READ_ONCE(dev->gso_ipv4_max_size))
3415 		netif_set_gso_ipv4_max_size(dev, size);
3416 }
3417 EXPORT_SYMBOL(netif_set_tso_max_size);
3418 
3419 /**
3420  * netif_set_tso_max_segs() - set the max number of segs supported for TSO
3421  * @dev:	netdev to update
3422  * @segs:	max number of TCP segments
3423  *
3424  * Set the limit on the number of TCP segments the device can generate from
3425  * a single TSO super-frame.
3426  * Unless explicitly set the stack will assume the value of %GSO_MAX_SEGS.
3427  */
3428 void netif_set_tso_max_segs(struct net_device *dev, unsigned int segs)
3429 {
3430 	dev->tso_max_segs = segs;
3431 	if (segs < READ_ONCE(dev->gso_max_segs))
3432 		netif_set_gso_max_segs(dev, segs);
3433 }
3434 EXPORT_SYMBOL(netif_set_tso_max_segs);
3435 
3436 /**
3437  * netif_inherit_tso_max() - copy all TSO limits from a lower device to an upper
3438  * @to:		netdev to update
3439  * @from:	netdev from which to copy the limits
3440  */
3441 void netif_inherit_tso_max(struct net_device *to, const struct net_device *from)
3442 {
3443 	netif_set_tso_max_size(to, from->tso_max_size);
3444 	netif_set_tso_max_segs(to, from->tso_max_segs);
3445 }
3446 EXPORT_SYMBOL(netif_inherit_tso_max);
3447 
3448 /**
3449  * netif_get_num_default_rss_queues - default number of RSS queues
3450  *
3451  * Default value is the number of physical cores if there are only 1 or 2, or
3452  * divided by 2 if there are more.
3453  */
3454 int netif_get_num_default_rss_queues(void)
3455 {
3456 	cpumask_var_t cpus;
3457 	int cpu, count = 0;
3458 
3459 	if (unlikely(is_kdump_kernel() || !zalloc_cpumask_var(&cpus, GFP_KERNEL)))
3460 		return 1;
3461 
3462 	cpumask_copy(cpus, cpu_online_mask);
3463 	for_each_cpu(cpu, cpus) {
3464 		++count;
3465 		cpumask_andnot(cpus, cpus, topology_sibling_cpumask(cpu));
3466 	}
3467 	free_cpumask_var(cpus);
3468 
3469 	return count > 2 ? DIV_ROUND_UP(count, 2) : count;
3470 }
3471 EXPORT_SYMBOL(netif_get_num_default_rss_queues);
3472 
3473 static void __netif_reschedule(struct Qdisc *q)
3474 {
3475 	struct softnet_data *sd;
3476 	unsigned long flags;
3477 
3478 	local_irq_save(flags);
3479 	sd = this_cpu_ptr(&softnet_data);
3480 	q->next_sched = NULL;
3481 	*sd->output_queue_tailp = q;
3482 	sd->output_queue_tailp = &q->next_sched;
3483 	raise_softirq_irqoff(NET_TX_SOFTIRQ);
3484 	local_irq_restore(flags);
3485 }
3486 
3487 void __netif_schedule(struct Qdisc *q)
3488 {
3489 	if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state)))
3490 		return;
3491 
3492 	/* If q->defer_list is not empty, at least one thread is
3493 	 * in __dev_xmit_skb() before llist_del_all(&q->defer_list).
3494 	 * This thread will attempt to run the queue.
3495 	 */
3496 	if (!llist_empty(&q->defer_list))
3497 		return;
3498 
3499 	if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state))
3500 		__netif_reschedule(q);
3501 }
3502 EXPORT_SYMBOL(__netif_schedule);
3503 
3504 struct dev_kfree_skb_cb {
3505 	enum skb_drop_reason reason;
3506 };
3507 
3508 static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb)
3509 {
3510 	return (struct dev_kfree_skb_cb *)skb->cb;
3511 }
3512 
3513 void netif_schedule_queue(struct netdev_queue *txq)
3514 {
3515 	rcu_read_lock();
3516 	if (!netif_xmit_stopped(txq)) {
3517 		struct Qdisc *q = rcu_dereference(txq->qdisc);
3518 
3519 		__netif_schedule(q);
3520 	}
3521 	rcu_read_unlock();
3522 }
3523 EXPORT_SYMBOL(netif_schedule_queue);
3524 
3525 void netif_tx_wake_queue(struct netdev_queue *dev_queue)
3526 {
3527 	if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) {
3528 		struct Qdisc *q;
3529 
3530 		rcu_read_lock();
3531 		q = rcu_dereference(dev_queue->qdisc);
3532 		__netif_schedule(q);
3533 		rcu_read_unlock();
3534 	}
3535 }
3536 EXPORT_SYMBOL(netif_tx_wake_queue);
3537 
3538 void dev_kfree_skb_irq_reason(struct sk_buff *skb, enum skb_drop_reason reason)
3539 {
3540 	unsigned long flags;
3541 
3542 	if (unlikely(!skb))
3543 		return;
3544 
3545 	if (likely(refcount_read(&skb->users) == 1)) {
3546 		smp_rmb();
3547 		refcount_set(&skb->users, 0);
3548 	} else if (likely(!refcount_dec_and_test(&skb->users))) {
3549 		return;
3550 	}
3551 	get_kfree_skb_cb(skb)->reason = reason;
3552 	local_irq_save(flags);
3553 	skb->next = __this_cpu_read(softnet_data.completion_queue);
3554 	__this_cpu_write(softnet_data.completion_queue, skb);
3555 	raise_softirq_irqoff(NET_TX_SOFTIRQ);
3556 	local_irq_restore(flags);
3557 }
3558 EXPORT_SYMBOL(dev_kfree_skb_irq_reason);
3559 
3560 void dev_kfree_skb_any_reason(struct sk_buff *skb, enum skb_drop_reason reason)
3561 {
3562 	if (in_hardirq() || irqs_disabled())
3563 		dev_kfree_skb_irq_reason(skb, reason);
3564 	else
3565 		kfree_skb_reason(skb, reason);
3566 }
3567 EXPORT_SYMBOL(dev_kfree_skb_any_reason);
3568 
3569 
3570 /**
3571  * netif_device_detach - mark device as removed
3572  * @dev: network device
3573  *
3574  * Mark device as removed from system and therefore no longer available.
3575  */
3576 void netif_device_detach(struct net_device *dev)
3577 {
3578 	if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) &&
3579 	    netif_running(dev)) {
3580 		netif_tx_stop_all_queues(dev);
3581 	}
3582 }
3583 EXPORT_SYMBOL(netif_device_detach);
3584 
3585 /**
3586  * netif_device_attach - mark device as attached
3587  * @dev: network device
3588  *
3589  * Mark device as attached from system and restart if needed.
3590  */
3591 void netif_device_attach(struct net_device *dev)
3592 {
3593 	if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) &&
3594 	    netif_running(dev)) {
3595 		netif_tx_wake_all_queues(dev);
3596 		netdev_watchdog_up(dev);
3597 	}
3598 }
3599 EXPORT_SYMBOL(netif_device_attach);
3600 
3601 /*
3602  * Returns a Tx hash based on the given packet descriptor a Tx queues' number
3603  * to be used as a distribution range.
3604  */
3605 static u16 skb_tx_hash(const struct net_device *dev,
3606 		       const struct net_device *sb_dev,
3607 		       struct sk_buff *skb)
3608 {
3609 	u32 hash;
3610 	u16 qoffset = 0;
3611 	u16 qcount = dev->real_num_tx_queues;
3612 
3613 	if (READ_ONCE(dev->num_tc)) {
3614 		u8 tc = netdev_get_prio_tc_map(dev, skb->priority);
3615 		struct netdev_tc_txq res;
3616 
3617 		res.combined = READ_ONCE(sb_dev->tc_to_txq[tc].combined);
3618 		qoffset = res.offset;
3619 		qcount = res.count;
3620 		if (unlikely(!qcount)) {
3621 			net_warn_ratelimited("%s: invalid qcount, qoffset %u for tc %u\n",
3622 					     sb_dev->name, qoffset, tc);
3623 			qoffset = 0;
3624 			qcount = dev->real_num_tx_queues;
3625 		}
3626 	}
3627 
3628 	if (skb_rx_queue_recorded(skb)) {
3629 		DEBUG_NET_WARN_ON_ONCE(qcount == 0);
3630 		hash = skb_get_rx_queue(skb);
3631 		if (hash >= qoffset)
3632 			hash -= qoffset;
3633 		while (unlikely(hash >= qcount))
3634 			hash -= qcount;
3635 		return hash + qoffset;
3636 	}
3637 
3638 	return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset;
3639 }
3640 
3641 void skb_warn_bad_offload(const struct sk_buff *skb)
3642 {
3643 	static const netdev_features_t null_features;
3644 	struct net_device *dev = skb->dev;
3645 	const char *name = "";
3646 
3647 	if (!net_ratelimit())
3648 		return;
3649 
3650 	if (dev) {
3651 		if (dev->dev.parent)
3652 			name = dev_driver_string(dev->dev.parent);
3653 		else
3654 			name = netdev_name(dev);
3655 	}
3656 	skb_dump(KERN_WARNING, skb, false);
3657 	WARN(1, "%s: caps=(%pNF, %pNF)\n",
3658 	     name, dev ? &dev->features : &null_features,
3659 	     skb->sk ? &skb->sk->sk_route_caps : &null_features);
3660 }
3661 
3662 /*
3663  * Invalidate hardware checksum when packet is to be mangled, and
3664  * complete checksum manually on outgoing path.
3665  */
3666 int skb_checksum_help(struct sk_buff *skb)
3667 {
3668 	__wsum csum;
3669 	int ret = 0, offset;
3670 
3671 	if (skb->ip_summed == CHECKSUM_COMPLETE)
3672 		goto out_set_summed;
3673 
3674 	if (unlikely(skb_is_gso(skb))) {
3675 		skb_warn_bad_offload(skb);
3676 		return -EINVAL;
3677 	}
3678 
3679 	if (!skb_frags_readable(skb)) {
3680 		return -EFAULT;
3681 	}
3682 
3683 	/* Before computing a checksum, we should make sure no frag could
3684 	 * be modified by an external entity : checksum could be wrong.
3685 	 */
3686 	if (skb_has_shared_frag(skb)) {
3687 		ret = __skb_linearize(skb);
3688 		if (ret)
3689 			goto out;
3690 	}
3691 
3692 	offset = skb_checksum_start_offset(skb);
3693 	ret = -EINVAL;
3694 	if (unlikely(offset >= skb_headlen(skb))) {
3695 		DO_ONCE_LITE(skb_dump, KERN_ERR, skb, false);
3696 		WARN_ONCE(true, "offset (%d) >= skb_headlen() (%u)\n",
3697 			  offset, skb_headlen(skb));
3698 		goto out;
3699 	}
3700 	csum = skb_checksum(skb, offset, skb->len - offset, 0);
3701 
3702 	offset += skb->csum_offset;
3703 	if (unlikely(offset + sizeof(__sum16) > skb_headlen(skb))) {
3704 		DO_ONCE_LITE(skb_dump, KERN_ERR, skb, false);
3705 		WARN_ONCE(true, "offset+2 (%zu) > skb_headlen() (%u)\n",
3706 			  offset + sizeof(__sum16), skb_headlen(skb));
3707 		goto out;
3708 	}
3709 	ret = skb_ensure_writable(skb, offset + sizeof(__sum16));
3710 	if (ret)
3711 		goto out;
3712 
3713 	*(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0;
3714 out_set_summed:
3715 	skb->ip_summed = CHECKSUM_NONE;
3716 out:
3717 	return ret;
3718 }
3719 EXPORT_SYMBOL(skb_checksum_help);
3720 
3721 #ifdef CONFIG_NET_CRC32C
3722 int skb_crc32c_csum_help(struct sk_buff *skb)
3723 {
3724 	u32 crc;
3725 	int ret = 0, offset, start;
3726 
3727 	if (skb->ip_summed != CHECKSUM_PARTIAL)
3728 		goto out;
3729 
3730 	if (unlikely(skb_is_gso(skb)))
3731 		goto out;
3732 
3733 	/* Before computing a checksum, we should make sure no frag could
3734 	 * be modified by an external entity : checksum could be wrong.
3735 	 */
3736 	if (unlikely(skb_has_shared_frag(skb))) {
3737 		ret = __skb_linearize(skb);
3738 		if (ret)
3739 			goto out;
3740 	}
3741 	start = skb_checksum_start_offset(skb);
3742 	offset = start + offsetof(struct sctphdr, checksum);
3743 	if (WARN_ON_ONCE(offset >= skb_headlen(skb))) {
3744 		ret = -EINVAL;
3745 		goto out;
3746 	}
3747 
3748 	ret = skb_ensure_writable(skb, offset + sizeof(__le32));
3749 	if (ret)
3750 		goto out;
3751 
3752 	crc = ~skb_crc32c(skb, start, skb->len - start, ~0);
3753 	*(__le32 *)(skb->data + offset) = cpu_to_le32(crc);
3754 	skb_reset_csum_not_inet(skb);
3755 out:
3756 	return ret;
3757 }
3758 EXPORT_SYMBOL(skb_crc32c_csum_help);
3759 #endif /* CONFIG_NET_CRC32C */
3760 
3761 __be16 skb_network_protocol(struct sk_buff *skb, int *depth)
3762 {
3763 	__be16 type = skb->protocol;
3764 
3765 	/* Tunnel gso handlers can set protocol to ethernet. */
3766 	if (type == htons(ETH_P_TEB)) {
3767 		struct ethhdr *eth;
3768 
3769 		if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr))))
3770 			return 0;
3771 
3772 		eth = (struct ethhdr *)skb->data;
3773 		type = eth->h_proto;
3774 	}
3775 
3776 	return vlan_get_protocol_and_depth(skb, type, depth);
3777 }
3778 
3779 
3780 /* Take action when hardware reception checksum errors are detected. */
3781 #ifdef CONFIG_BUG
3782 static void do_netdev_rx_csum_fault(struct net_device *dev, struct sk_buff *skb)
3783 {
3784 	netdev_err(dev, "hw csum failure\n");
3785 	skb_dump(KERN_ERR, skb, true);
3786 	dump_stack();
3787 }
3788 
3789 void netdev_rx_csum_fault(struct net_device *dev, struct sk_buff *skb)
3790 {
3791 	DO_ONCE_LITE(do_netdev_rx_csum_fault, dev, skb);
3792 }
3793 EXPORT_SYMBOL(netdev_rx_csum_fault);
3794 #endif
3795 
3796 /* XXX: check that highmem exists at all on the given machine. */
3797 static int illegal_highdma(struct net_device *dev, struct sk_buff *skb)
3798 {
3799 #ifdef CONFIG_HIGHMEM
3800 	int i;
3801 
3802 	if (!(dev->features & NETIF_F_HIGHDMA)) {
3803 		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
3804 			skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3805 			struct page *page = skb_frag_page(frag);
3806 
3807 			if (page && PageHighMem(page))
3808 				return 1;
3809 		}
3810 	}
3811 #endif
3812 	return 0;
3813 }
3814 
3815 /* If MPLS offload request, verify we are testing hardware MPLS features
3816  * instead of standard features for the netdev.
3817  */
3818 #if IS_ENABLED(CONFIG_NET_MPLS_GSO)
3819 static netdev_features_t net_mpls_features(struct sk_buff *skb,
3820 					   netdev_features_t features,
3821 					   __be16 type)
3822 {
3823 	if (eth_p_mpls(type))
3824 		features &= skb->dev->mpls_features;
3825 
3826 	return features;
3827 }
3828 #else
3829 static netdev_features_t net_mpls_features(struct sk_buff *skb,
3830 					   netdev_features_t features,
3831 					   __be16 type)
3832 {
3833 	return features;
3834 }
3835 #endif
3836 
3837 static netdev_features_t harmonize_features(struct sk_buff *skb,
3838 	netdev_features_t features)
3839 {
3840 	__be16 type;
3841 
3842 	type = skb_network_protocol(skb, NULL);
3843 	features = net_mpls_features(skb, features, type);
3844 
3845 	if (skb->ip_summed != CHECKSUM_NONE &&
3846 	    !can_checksum_protocol(features, type)) {
3847 		features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
3848 	}
3849 	if (illegal_highdma(skb->dev, skb))
3850 		features &= ~NETIF_F_SG;
3851 
3852 	return features;
3853 }
3854 
3855 netdev_features_t passthru_features_check(struct sk_buff *skb,
3856 					  struct net_device *dev,
3857 					  netdev_features_t features)
3858 {
3859 	return features;
3860 }
3861 EXPORT_SYMBOL(passthru_features_check);
3862 
3863 static netdev_features_t dflt_features_check(struct sk_buff *skb,
3864 					     struct net_device *dev,
3865 					     netdev_features_t features)
3866 {
3867 	return vlan_features_check(skb, features);
3868 }
3869 
3870 static bool __skb_has_ipv6_ext_hdr(const struct sk_buff *skb, int nhoff)
3871 {
3872 	const struct ipv6hdr *ip6h;
3873 	struct ipv6hdr _ip6h;
3874 
3875 	ip6h = skb_header_pointer(skb, nhoff, sizeof(_ip6h), &_ip6h);
3876 	return ip6h && ip6h->version == 6 && ipv6_ext_hdr(ip6h->nexthdr);
3877 }
3878 
3879 static bool skb_has_ipv6_extension_hdr(const struct sk_buff *skb)
3880 {
3881 	if (vlan_get_protocol(skb) == htons(ETH_P_IPV6)) {
3882 		if (__skb_has_ipv6_ext_hdr(skb, skb_network_offset(skb)))
3883 			return true;
3884 
3885 		/* Catch tunnels without skb->encapsulation (e.g., virtio). */
3886 		if (!skb->encapsulation &&
3887 		    skb_transport_header_was_set(skb) &&
3888 		    skb_network_header_len(skb) != sizeof(struct ipv6hdr))
3889 			return true;
3890 	}
3891 
3892 	/* Tunnels without an inner network header, such as SCTP-in-UDP or
3893 	 * PSP, have no inner IP header and thus no inner extension header.
3894 	 */
3895 	if (skb->encapsulation && skb_inner_network_header_was_set(skb) &&
3896 	    __skb_has_ipv6_ext_hdr(skb, skb_inner_network_offset(skb)))
3897 		return true;
3898 
3899 	return false;
3900 }
3901 
3902 static unsigned int
3903 netif_get_gso_max_size(const struct net_device *dev, const struct sk_buff *skb)
3904 {
3905 	/* pairs with WRITE_ONCE() in netif_set_gso(_ipv4)_max_size() */
3906 	return vlan_get_protocol(skb) == htons(ETH_P_IPV6) ?
3907 	       READ_ONCE(dev->gso_max_size) :
3908 	       READ_ONCE(dev->gso_ipv4_max_size);
3909 }
3910 
3911 static bool
3912 gso_within_dev_limits(const struct sk_buff *skb, const struct net_device *dev)
3913 {
3914 	if (skb_shinfo(skb)->gso_segs > READ_ONCE(dev->gso_max_segs))
3915 		return false;
3916 
3917 	if (unlikely(skb->len >= netif_get_gso_max_size(dev, skb)))
3918 		return false;
3919 
3920 	return true;
3921 }
3922 
3923 static netdev_features_t gso_features_check(const struct sk_buff *skb,
3924 					    struct net_device *dev,
3925 					    netdev_features_t features,
3926 					    bool check_limits)
3927 {
3928 	if (check_limits && !gso_within_dev_limits(skb, dev))
3929 		return features & ~NETIF_F_GSO_MASK;
3930 
3931 	if (!skb_shinfo(skb)->gso_type) {
3932 		skb_warn_bad_offload(skb);
3933 		return features & ~NETIF_F_GSO_MASK;
3934 	}
3935 
3936 	/* Support for GSO partial features requires software
3937 	 * intervention before we can actually process the packets
3938 	 * so we need to strip support for any partial features now
3939 	 * and we can pull them back in after we have partially
3940 	 * segmented the frame.
3941 	 */
3942 	if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL))
3943 		features &= ~dev->gso_partial_features;
3944 
3945 	/* Make sure to clear the IPv4 ID mangling feature if the IPv4 header
3946 	 * has the potential to be fragmented so that TSO does not generate
3947 	 * segments with the same ID. For encapsulated packets, the ID mangling
3948 	 * feature is guaranteed not to use the same ID for the outer IPv4
3949 	 * headers of the generated segments if the headers have the potential
3950 	 * to be fragmented, so there is no need to clear the IPv4 ID mangling
3951 	 * feature (see the section about NETIF_F_TSO_MANGLEID in
3952 	 * segmentation-offloads.rst).
3953 	 */
3954 	if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
3955 		const struct iphdr *iph;
3956 		struct iphdr _iph;
3957 		int nhoff = skb->encapsulation ?
3958 			    skb_inner_network_offset(skb) :
3959 			    skb_network_offset(skb);
3960 
3961 		iph = skb_header_pointer(skb, nhoff, sizeof(_iph), &_iph);
3962 
3963 		if (!iph || !(iph->frag_off & htons(IP_DF)))
3964 			features &= ~dev->mangleid_features;
3965 	}
3966 
3967 	/* NETIF_F_IPV6_CSUM does not support IPv6 extension headers,
3968 	 * so neither does TSO that depends on it.
3969 	 */
3970 	if (features & NETIF_F_IPV6_CSUM &&
3971 	    skb_has_ipv6_extension_hdr(skb))
3972 		features &= ~(NETIF_F_IPV6_CSUM | NETIF_F_TSO6 | NETIF_F_GSO_UDP_L4);
3973 
3974 	return features;
3975 }
3976 
3977 netdev_features_t
3978 __netif_skb_features(struct sk_buff *skb, bool check_gso_limits)
3979 {
3980 	struct net_device *dev = skb->dev;
3981 	netdev_features_t features = dev->features;
3982 
3983 	if (skb_is_gso(skb))
3984 		features = gso_features_check(skb, dev, features,
3985 					      check_gso_limits);
3986 
3987 	/* If encapsulation offload request, verify we are testing
3988 	 * hardware encapsulation features instead of standard
3989 	 * features for the netdev
3990 	 */
3991 	if (skb->encapsulation)
3992 		features &= dev->hw_enc_features;
3993 
3994 	if (skb_vlan_tagged(skb))
3995 		features = netdev_intersect_features(features,
3996 						     dev->vlan_features |
3997 						     NETIF_F_HW_VLAN_CTAG_TX |
3998 						     NETIF_F_HW_VLAN_STAG_TX);
3999 
4000 	if (dev->netdev_ops->ndo_features_check)
4001 		features &= dev->netdev_ops->ndo_features_check(skb, dev,
4002 								features);
4003 	else
4004 		features &= dflt_features_check(skb, dev, features);
4005 
4006 	return harmonize_features(skb, features);
4007 }
4008 EXPORT_SYMBOL(__netif_skb_features);
4009 
4010 static unsigned int
4011 skb_gso_output_max_segs(struct sk_buff *skb, struct net_device *dev)
4012 {
4013 	unsigned int mss = skb_shinfo(skb)->gso_size;
4014 	unsigned int gso_max_size, hdr_len, max_segs;
4015 	netdev_features_t features;
4016 	struct tcphdr _tcph, *th;
4017 
4018 	if (!skb_is_gso(skb) || !skb_is_gso_tcp(skb) ||
4019 	    skb->encapsulation || skb_has_frag_list(skb))
4020 		return 0;
4021 
4022 	/*
4023 	 * The transport offset can be unset or stale, so hdr_len is only taken
4024 	 * from a header at the checksum start.
4025 	 */
4026 	if (!skb_transport_header_was_set(skb) ||
4027 	    skb_checksum_start(skb) != skb_transport_header(skb))
4028 		return 0;
4029 
4030 	th = skb_header_pointer(skb, skb_transport_offset(skb), sizeof(_tcph),
4031 				&_tcph);
4032 	if (!th || __tcp_hdrlen(th) < sizeof(*th))
4033 		return 0;
4034 
4035 	hdr_len = skb_transport_header(skb) - skb_mac_header(skb) +
4036 		  __tcp_hdrlen(th);
4037 
4038 	/* The output is still a GSO skb: recheck without the limit checks */
4039 	features = __netif_skb_features(skb, false) | NETIF_F_GSO_ROBUST;
4040 	if (!net_gso_ok(features, skb_shinfo(skb)->gso_type))
4041 		return 0;
4042 
4043 	gso_max_size = netif_get_gso_max_size(dev, skb);
4044 	gso_max_size = min(gso_max_size, GSO_LEGACY_MAX_SIZE);
4045 
4046 	/*
4047 	 * gso_within_dev_limits() accepts gso_segs == gso_max_segs but
4048 	 * rejects skb->len >= gso_max_size, so only the size limit needs the
4049 	 * - 1; the inner min() keeps that subtraction from wrapping.
4050 	 */
4051 	max_segs = (gso_max_size - min(gso_max_size, hdr_len + 1)) / mss;
4052 	max_segs = min(max_segs, READ_ONCE(dev->gso_max_segs));
4053 
4054 	return max_segs;
4055 }
4056 
4057 static int xmit_one(struct sk_buff *skb, struct net_device *dev,
4058 		    struct netdev_queue *txq, bool more)
4059 {
4060 	unsigned int len;
4061 	int rc;
4062 
4063 	if (dev_nit_active_rcu(dev))
4064 		dev_queue_xmit_nit(skb, dev);
4065 
4066 	len = skb->len;
4067 	trace_net_dev_start_xmit(skb, dev);
4068 	rc = netdev_start_xmit(skb, dev, txq, more);
4069 	trace_net_dev_xmit(skb, rc, dev, len);
4070 
4071 	return rc;
4072 }
4073 
4074 struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev,
4075 				    struct netdev_queue *txq, int *ret)
4076 {
4077 	struct sk_buff *skb = first;
4078 	int rc = NETDEV_TX_OK;
4079 
4080 	while (skb) {
4081 		struct sk_buff *next = skb->next;
4082 
4083 		skb_mark_not_on_list(skb);
4084 		rc = xmit_one(skb, dev, txq, next != NULL);
4085 		if (unlikely(!dev_xmit_complete(rc))) {
4086 			skb->next = next;
4087 			goto out;
4088 		}
4089 
4090 		skb = next;
4091 		if (netif_tx_queue_stopped(txq) && skb) {
4092 			rc = NETDEV_TX_BUSY;
4093 			break;
4094 		}
4095 	}
4096 
4097 out:
4098 	*ret = rc;
4099 	return skb;
4100 }
4101 
4102 static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb,
4103 					  netdev_features_t features)
4104 {
4105 	if (skb_vlan_tag_present(skb) &&
4106 	    !vlan_hw_offload_capable(features, skb->vlan_proto))
4107 		skb = __vlan_hwaccel_push_inside(skb);
4108 	return skb;
4109 }
4110 
4111 int skb_csum_hwoffload_help(struct sk_buff *skb,
4112 			    const netdev_features_t features)
4113 {
4114 	if (unlikely(skb_csum_is_sctp(skb)))
4115 		return !!(features & NETIF_F_SCTP_CRC) ? 0 :
4116 			skb_crc32c_csum_help(skb);
4117 
4118 	if (features & NETIF_F_HW_CSUM)
4119 		return 0;
4120 
4121 	if (features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) {
4122 		if (skb_has_ipv6_extension_hdr(skb))
4123 			goto sw_checksum;
4124 
4125 		switch (skb->csum_offset) {
4126 		case offsetof(struct tcphdr, check):
4127 		case offsetof(struct udphdr, check):
4128 			return 0;
4129 		}
4130 	}
4131 
4132 sw_checksum:
4133 	return skb_checksum_help(skb);
4134 }
4135 EXPORT_SYMBOL(skb_csum_hwoffload_help);
4136 
4137 /* Checks if this SKB belongs to an HW offloaded socket
4138  * and whether any SW fallbacks are required based on dev.
4139  * Check decrypted mark in case skb_orphan() cleared socket.
4140  */
4141 static struct sk_buff *sk_validate_xmit_skb(struct sk_buff *skb,
4142 					    struct net_device *dev)
4143 {
4144 #ifdef CONFIG_SOCK_VALIDATE_XMIT
4145 	struct sk_buff *(*sk_validate)(struct sock *sk, struct net_device *dev,
4146 				       struct sk_buff *skb);
4147 	struct sock *sk = skb->sk;
4148 
4149 	sk_validate = NULL;
4150 	if (sk) {
4151 		if (sk_fullsock(sk))
4152 			sk_validate = sk->sk_validate_xmit_skb;
4153 		else if (sk_is_inet(sk) && sk->sk_state == TCP_TIME_WAIT)
4154 			sk_validate = inet_twsk(sk)->tw_validate_xmit_skb;
4155 	}
4156 
4157 	if (sk_validate) {
4158 		skb = sk_validate(sk, dev, skb);
4159 	} else if (unlikely(skb_is_decrypted(skb))) {
4160 		pr_warn_ratelimited("unencrypted skb with no associated socket - dropping\n");
4161 		kfree_skb(skb);
4162 		skb = NULL;
4163 	}
4164 #endif
4165 
4166 	return skb;
4167 }
4168 
4169 static struct sk_buff *validate_xmit_unreadable_skb(struct sk_buff *skb,
4170 						    struct net_device *dev)
4171 {
4172 	struct skb_shared_info *shinfo;
4173 	struct net_iov *niov;
4174 
4175 	if (likely(skb_frags_readable(skb) ||
4176 		   dev->netmem_tx == NETMEM_TX_NO_DMA))
4177 		goto out;
4178 
4179 	if (dev->netmem_tx == NETMEM_TX_NONE)
4180 		goto out_free;
4181 
4182 	shinfo = skb_shinfo(skb);
4183 
4184 	if (shinfo->nr_frags > 0) {
4185 		niov = netmem_to_net_iov(skb_frag_netmem(&shinfo->frags[0]));
4186 		if (net_is_devmem_iov(niov) &&
4187 		    READ_ONCE(net_devmem_iov_binding(niov)->dev) != dev)
4188 			goto out_free;
4189 	}
4190 
4191 out:
4192 	return skb;
4193 
4194 out_free:
4195 	kfree_skb(skb);
4196 	return NULL;
4197 }
4198 
4199 /* Returns the skb on success, NULL if dropped, or ERR_PTR(-EINPROGRESS)
4200  * if stolen by async xfrm crypto (delivered via xfrm_dev_resume()).
4201  */
4202 static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev, bool *again)
4203 {
4204 	netdev_features_t features;
4205 
4206 	skb = validate_xmit_unreadable_skb(skb, dev);
4207 	if (unlikely(!skb))
4208 		goto out_null;
4209 
4210 	/* Only fraglist GSO, whose segments already carry valid checksums,
4211 	 * can be handed to a device as CHECKSUM_UNNECESSARY. Any other GSO
4212 	 * skb has no csum_start for the device to segment from. NONE means
4213 	 * the same on transmit, but makes netif_needs_gso() send it to the
4214 	 * GSO code, which turns TCP into CHECKSUM_PARTIAL for TSO and
4215 	 * segments anything the device cannot offload in software.
4216 	 */
4217 	if (unlikely(skb->ip_summed == CHECKSUM_UNNECESSARY &&
4218 		     skb_is_gso(skb) &&
4219 		     !(skb_shinfo(skb)->gso_type & SKB_GSO_FRAGLIST)))
4220 		skb->ip_summed = CHECKSUM_NONE;
4221 
4222 	features = netif_skb_features(skb);
4223 	skb = validate_xmit_vlan(skb, features);
4224 	if (unlikely(!skb))
4225 		goto out_null;
4226 
4227 	skb = sk_validate_xmit_skb(skb, dev);
4228 	if (unlikely(!skb))
4229 		goto out_null;
4230 
4231 	if (netif_needs_gso(skb, features)) {
4232 		unsigned int max_segs = 0;
4233 		struct sk_buff *segs;
4234 
4235 		if (unlikely(!gso_within_dev_limits(skb, dev)))
4236 			max_segs = skb_gso_output_max_segs(skb, dev);
4237 
4238 		segs = __skb_gso_segment(skb, features, true, max_segs);
4239 		if (IS_ERR(segs)) {
4240 			goto out_kfree_skb;
4241 		} else if (segs) {
4242 			consume_skb(skb);
4243 			skb = segs;
4244 		}
4245 	} else {
4246 		if (skb_needs_linearize(skb, features) &&
4247 		    __skb_linearize(skb))
4248 			goto out_kfree_skb;
4249 
4250 		/* If packet is not checksummed and device does not
4251 		 * support checksumming for this protocol, complete
4252 		 * checksumming here.
4253 		 */
4254 		if (skb->ip_summed == CHECKSUM_PARTIAL) {
4255 			if (skb->encapsulation)
4256 				skb_set_inner_transport_header(skb,
4257 							       skb_checksum_start_offset(skb));
4258 			else
4259 				skb_set_transport_header(skb,
4260 							 skb_checksum_start_offset(skb));
4261 			if (skb_csum_hwoffload_help(skb, features))
4262 				goto out_kfree_skb;
4263 		}
4264 	}
4265 
4266 	skb = validate_xmit_xfrm(skb, features, again);
4267 
4268 	return skb;
4269 
4270 out_kfree_skb:
4271 	kfree_skb(skb);
4272 out_null:
4273 	dev_core_stats_tx_dropped_inc(dev);
4274 	return NULL;
4275 }
4276 
4277 struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev, bool *again)
4278 {
4279 	struct sk_buff *next, *head = NULL, *tail;
4280 
4281 	for (; skb != NULL; skb = next) {
4282 		next = skb->next;
4283 		skb_mark_not_on_list(skb);
4284 
4285 		/* in case skb won't be segmented, point to itself */
4286 		skb->prev = skb;
4287 
4288 		skb = validate_xmit_skb(skb, dev, again);
4289 		if (IS_ERR_OR_NULL(skb))
4290 			continue;
4291 
4292 		if (!head)
4293 			head = skb;
4294 		else
4295 			tail->next = skb;
4296 		/* If skb was segmented, skb->prev points to
4297 		 * the last segment. If not, it still contains skb.
4298 		 */
4299 		tail = skb->prev;
4300 	}
4301 	return head;
4302 }
4303 EXPORT_SYMBOL_GPL(validate_xmit_skb_list);
4304 
4305 static enum skb_drop_reason qdisc_pkt_len_segs_init(struct sk_buff *skb)
4306 {
4307 	struct skb_shared_info *shinfo = skb_shinfo(skb);
4308 	unsigned int hdr_len, tlen;
4309 	u32 gso_segs;
4310 
4311 	qdisc_skb_cb(skb)->pkt_len = skb->len;
4312 	if (!shinfo->gso_size) {
4313 		qdisc_skb_cb(skb)->pkt_segs = 1;
4314 		return SKB_NOT_DROPPED_YET;
4315 	}
4316 
4317 	qdisc_skb_cb(skb)->pkt_segs = gso_segs = shinfo->gso_segs;
4318 
4319 	/* To get more precise estimation of bytes sent on wire,
4320 	 * we add to pkt_len the headers size of all segments
4321 	 */
4322 
4323 	/* mac layer + network layer */
4324 	if (!skb->encapsulation) {
4325 		if (unlikely(!skb_transport_header_was_set(skb)))
4326 			return SKB_NOT_DROPPED_YET;
4327 		hdr_len = skb_transport_offset(skb);
4328 	} else {
4329 		hdr_len = skb_inner_transport_offset(skb);
4330 	}
4331 	/* + transport layer */
4332 	if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) {
4333 		const struct tcphdr *th;
4334 
4335 		/* A header in front of skb->data gives a negative offset. */
4336 		if ((int)hdr_len < 0 ||
4337 		    !pskb_may_pull(skb, hdr_len + sizeof(struct tcphdr)))
4338 			return SKB_DROP_REASON_SKB_BAD_GSO;
4339 
4340 		th = (const struct tcphdr *)(skb->data + hdr_len);
4341 		tlen = __tcp_hdrlen(th);
4342 		if (tlen < sizeof(*th))
4343 			return SKB_DROP_REASON_SKB_BAD_GSO;
4344 		hdr_len += tlen;
4345 		if (!pskb_may_pull(skb, hdr_len))
4346 			return SKB_DROP_REASON_SKB_BAD_GSO;
4347 	} else if (shinfo->gso_type & SKB_GSO_UDP_L4) {
4348 		if (!pskb_may_pull(skb, hdr_len + sizeof(struct udphdr)))
4349 			return SKB_DROP_REASON_SKB_BAD_GSO;
4350 		hdr_len += sizeof(struct udphdr);
4351 	}
4352 
4353 	/* prior pskb_may_pull() might have changed skb->head. */
4354 	shinfo = skb_shinfo(skb);
4355 	if (unlikely(shinfo->gso_type & SKB_GSO_DODGY)) {
4356 		int payload = skb->len - hdr_len;
4357 
4358 		/* Malicious packet. */
4359 		if (payload <= 0)
4360 			return SKB_DROP_REASON_SKB_BAD_GSO;
4361 		gso_segs = DIV_ROUND_UP(payload, shinfo->gso_size);
4362 		/* shinfo->gso_segs is a 16bit field.
4363 		 * Also make sure untrusted gso_size/hdr_len can not inflate
4364 		 * qdisc_pkt_len() beyond QDISC_PKT_LEN_MAX, or deficit
4365 		 * schedulers could spin in their refill loops.
4366 		 */
4367 		if (unlikely(gso_segs > GSO_MAX_SEGS ||
4368 			     skb->len + (u64)(gso_segs - 1) * hdr_len >
4369 			     QDISC_PKT_LEN_MAX))
4370 			return SKB_DROP_REASON_SKB_BAD_GSO;
4371 		shinfo->gso_segs = gso_segs;
4372 		qdisc_skb_cb(skb)->pkt_segs = gso_segs;
4373 	}
4374 	qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len;
4375 	return SKB_NOT_DROPPED_YET;
4376 }
4377 
4378 static int dev_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *q,
4379 			     struct sk_buff **to_free,
4380 			     struct netdev_queue *txq)
4381 {
4382 	int rc;
4383 
4384 	rc = q->enqueue(skb, q, to_free) & NET_XMIT_MASK;
4385 	if (rc == NET_XMIT_SUCCESS)
4386 		trace_qdisc_enqueue(q, txq, skb);
4387 	return rc;
4388 }
4389 
4390 static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q,
4391 				 struct net_device *dev,
4392 				 struct netdev_queue *txq)
4393 {
4394 	struct sk_buff *next, *to_free = NULL, *to_free2 = NULL;
4395 	spinlock_t *root_lock = qdisc_lock(q);
4396 	struct llist_node *ll_list, *first_n;
4397 	unsigned long defer_count = 0;
4398 	int rc;
4399 
4400 	qdisc_calculate_pkt_len(skb, q);
4401 
4402 	tcf_set_qdisc_drop_reason(skb, QDISC_DROP_GENERIC);
4403 
4404 	if (q->flags & TCQ_F_NOLOCK) {
4405 		if (q->flags & TCQ_F_CAN_BYPASS && nolock_qdisc_is_empty(q) &&
4406 		    qdisc_run_begin(q)) {
4407 			/* Retest nolock_qdisc_is_empty() within the protection
4408 			 * of q->seqlock to protect from racing with requeuing.
4409 			 */
4410 			if (unlikely(!nolock_qdisc_is_empty(q))) {
4411 				rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
4412 				__qdisc_run(q);
4413 				to_free2 = qdisc_run_end(q);
4414 
4415 				goto free_skbs;
4416 			}
4417 
4418 			qdisc_bstats_cpu_update(q, skb);
4419 			if (sch_direct_xmit(skb, q, dev, txq, NULL, true) &&
4420 			    !nolock_qdisc_is_empty(q))
4421 				__qdisc_run(q);
4422 
4423 			to_free2 = qdisc_run_end(q);
4424 			rc = NET_XMIT_SUCCESS;
4425 			goto free_skbs;
4426 		}
4427 
4428 		rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
4429 		to_free2 = qdisc_run(q);
4430 		goto free_skbs;
4431 	}
4432 
4433 	/* Open code llist_add(&skb->ll_node, &q->defer_list) + queue limit.
4434 	 * In the try_cmpxchg() loop, we want to increment q->defer_count
4435 	 * at most once to limit the number of skbs in defer_list.
4436 	 * We perform the defer_count increment only if the list is not empty,
4437 	 * because some arches have slow atomic_long_inc_return().
4438 	 */
4439 	first_n = READ_ONCE(q->defer_list.first);
4440 	do {
4441 		if (first_n && !defer_count) {
4442 			defer_count = atomic_long_inc_return(&q->defer_count);
4443 			if (unlikely(defer_count > READ_ONCE(net_hotdata.qdisc_max_burst))) {
4444 				kfree_skb_reason(skb, SKB_DROP_REASON_QDISC_BURST_DROP);
4445 				return NET_XMIT_DROP;
4446 			}
4447 		}
4448 		skb->ll_node.next = first_n;
4449 	} while (!try_cmpxchg(&q->defer_list.first, &first_n, &skb->ll_node));
4450 
4451 	/* If defer_list was not empty, we know the cpu which queued
4452 	 * the first skb will process the whole list for us.
4453 	 */
4454 	if (first_n)
4455 		return NET_XMIT_SUCCESS;
4456 
4457 	spin_lock(root_lock);
4458 
4459 	ll_list = llist_del_all(&q->defer_list);
4460 	/* There is a small race because we clear defer_count not atomically
4461 	 * with the prior llist_del_all(). This means defer_list could grow
4462 	 * over qdisc_max_burst.
4463 	 */
4464 	atomic_long_set(&q->defer_count, 0);
4465 
4466 	ll_list = llist_reverse_order(ll_list);
4467 
4468 	if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) {
4469 		llist_for_each_entry_safe(skb, next, ll_list, ll_node)
4470 			__qdisc_drop(skb, &to_free);
4471 		rc = NET_XMIT_DROP;
4472 		goto unlock;
4473 	}
4474 	if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) &&
4475 	    !llist_next(ll_list) && qdisc_run_begin(q)) {
4476 		/*
4477 		 * This is a work-conserving queue; there are no old skbs
4478 		 * waiting to be sent out; and the qdisc is not running -
4479 		 * xmit the skb directly.
4480 		 */
4481 
4482 		DEBUG_NET_WARN_ON_ONCE(skb != llist_entry(ll_list,
4483 							  struct sk_buff,
4484 							  ll_node));
4485 		qdisc_bstats_update(q, skb);
4486 		if (sch_direct_xmit(skb, q, dev, txq, root_lock, true))
4487 			__qdisc_run(q);
4488 		to_free2 = qdisc_run_end(q);
4489 		rc = NET_XMIT_SUCCESS;
4490 	} else {
4491 		int count = 0;
4492 
4493 		llist_for_each_entry_safe(skb, next, ll_list, ll_node) {
4494 			if (next) {
4495 				prefetch(next);
4496 				prefetch(&next->priority);
4497 				skb_mark_not_on_list(skb);
4498 			}
4499 			rc = dev_qdisc_enqueue(skb, q, &to_free, txq);
4500 			count++;
4501 		}
4502 		to_free2 = qdisc_run(q);
4503 		if (count != 1)
4504 			rc = NET_XMIT_SUCCESS;
4505 	}
4506 unlock:
4507 	spin_unlock(root_lock);
4508 
4509 free_skbs:
4510 	tcf_kfree_skb_list(to_free, q, txq, dev);
4511 	tcf_kfree_skb_list(to_free2, q, txq, dev);
4512 	return rc;
4513 }
4514 
4515 #if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
4516 static void skb_update_prio(struct sk_buff *skb)
4517 {
4518 	const struct netprio_map *map;
4519 	const struct sock *sk;
4520 	unsigned int prioidx;
4521 
4522 	if (skb->priority)
4523 		return;
4524 	map = rcu_dereference_bh(skb->dev->priomap);
4525 	if (!map)
4526 		return;
4527 	sk = skb_to_full_sk(skb);
4528 	if (!sk)
4529 		return;
4530 
4531 	prioidx = sock_cgroup_prioidx(&sk->sk_cgrp_data);
4532 
4533 	if (prioidx < map->priomap_len)
4534 		skb->priority = map->priomap[prioidx];
4535 }
4536 #else
4537 #define skb_update_prio(skb)
4538 #endif
4539 
4540 /**
4541  *	dev_loopback_xmit - loop back @skb
4542  *	@net: network namespace this loopback is happening in
4543  *	@sk:  sk needed to be a netfilter okfn
4544  *	@skb: buffer to transmit
4545  */
4546 int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *skb)
4547 {
4548 	skb_reset_mac_header(skb);
4549 	__skb_pull(skb, skb_network_offset(skb));
4550 	skb->pkt_type = PACKET_LOOPBACK;
4551 	if (skb->ip_summed == CHECKSUM_NONE)
4552 		skb->ip_summed = CHECKSUM_UNNECESSARY;
4553 	DEBUG_NET_WARN_ON_ONCE(!skb_dst(skb));
4554 	skb_dst_force(skb);
4555 	netif_rx(skb);
4556 	return 0;
4557 }
4558 EXPORT_SYMBOL(dev_loopback_xmit);
4559 
4560 #ifdef CONFIG_NET_EGRESS
4561 static struct netdev_queue *
4562 netdev_tx_queue_mapping(struct net_device *dev, struct sk_buff *skb)
4563 {
4564 	int queue = skb_get_queue_mapping(skb);
4565 	int capped;
4566 
4567 	capped = netdev_cap_txqueue(dev, queue);
4568 	if (unlikely(capped != queue))
4569 		skb_set_queue_mapping(skb, capped);
4570 
4571 	return netdev_get_tx_queue(dev, capped);
4572 }
4573 
4574 #ifndef CONFIG_PREEMPT_RT
4575 static bool netdev_xmit_txqueue_skipped(void)
4576 {
4577 	return __this_cpu_read(softnet_data.xmit.skip_txqueue);
4578 }
4579 
4580 bool netdev_xmit_skip_txqueue(bool skip)
4581 {
4582 	bool prev = netdev_xmit_txqueue_skipped();
4583 
4584 	__this_cpu_write(softnet_data.xmit.skip_txqueue, skip);
4585 
4586 	return prev;
4587 }
4588 EXPORT_SYMBOL_GPL(netdev_xmit_skip_txqueue);
4589 
4590 #else
4591 static bool netdev_xmit_txqueue_skipped(void)
4592 {
4593 	return current->net_xmit.skip_txqueue;
4594 }
4595 
4596 bool netdev_xmit_skip_txqueue(bool skip)
4597 {
4598 	bool prev = netdev_xmit_txqueue_skipped();
4599 
4600 	current->net_xmit.skip_txqueue = skip;
4601 
4602 	return prev;
4603 }
4604 EXPORT_SYMBOL_GPL(netdev_xmit_skip_txqueue);
4605 #endif
4606 #endif /* CONFIG_NET_EGRESS */
4607 
4608 #ifdef CONFIG_NET_XGRESS
4609 static int tc_run(struct tcx_entry *entry, struct sk_buff *skb,
4610 		  enum skb_drop_reason *drop_reason)
4611 {
4612 	int ret = TC_ACT_UNSPEC;
4613 #ifdef CONFIG_NET_CLS_ACT
4614 	struct mini_Qdisc *miniq = rcu_dereference_bh(entry->miniq);
4615 	struct tcf_result res;
4616 
4617 	if (!miniq)
4618 		return ret;
4619 
4620 	/* Global bypass */
4621 	if (!static_branch_likely(&tcf_sw_enabled_key))
4622 		return ret;
4623 
4624 	/* Block-wise bypass */
4625 	if (tcf_block_bypass_sw(miniq->block))
4626 		return ret;
4627 
4628 	tc_skb_cb(skb)->mru = 0;
4629 	qdisc_skb_cb(skb)->post_ct = false;
4630 	tcf_set_drop_reason(skb, *drop_reason);
4631 
4632 	mini_qdisc_bstats_cpu_update(miniq, skb);
4633 	ret = tcf_classify(skb, miniq->block, miniq->filter_list, &res, false);
4634 	/* Only tcf related quirks below. */
4635 	switch (ret) {
4636 	case TC_ACT_SHOT:
4637 		*drop_reason = tcf_get_drop_reason(skb);
4638 		mini_qdisc_qstats_cpu_drop(miniq);
4639 		break;
4640 	case TC_ACT_OK:
4641 	case TC_ACT_RECLASSIFY:
4642 		skb->tc_index = TC_H_MIN(res.classid);
4643 		break;
4644 	}
4645 #endif /* CONFIG_NET_CLS_ACT */
4646 	return ret;
4647 }
4648 
4649 static DEFINE_STATIC_KEY_FALSE(tcx_needed_key);
4650 
4651 void tcx_inc(void)
4652 {
4653 	static_branch_inc(&tcx_needed_key);
4654 }
4655 
4656 void tcx_dec(void)
4657 {
4658 	static_branch_dec(&tcx_needed_key);
4659 }
4660 
4661 static __always_inline enum tcx_action_base
4662 tcx_run(const struct bpf_mprog_entry *entry, struct sk_buff *skb,
4663 	const bool needs_mac)
4664 {
4665 	const struct bpf_mprog_fp *fp;
4666 	const struct bpf_prog *prog;
4667 	int ret = TCX_NEXT;
4668 
4669 	if (needs_mac)
4670 		__skb_push(skb, skb->mac_len);
4671 	bpf_mprog_foreach_prog(entry, fp, prog) {
4672 		bpf_compute_data_pointers(skb);
4673 		ret = bpf_prog_run(prog, skb);
4674 		if (ret != TCX_NEXT)
4675 			break;
4676 	}
4677 	if (needs_mac)
4678 		__skb_pull(skb, skb->mac_len);
4679 	return tcx_action_code(skb, ret);
4680 }
4681 
4682 static __always_inline struct sk_buff *
4683 sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret,
4684 		   struct net_device *orig_dev, bool *another)
4685 {
4686 	struct bpf_mprog_entry *entry = rcu_dereference_bh(skb->dev->tcx_ingress);
4687 	enum skb_drop_reason drop_reason = SKB_DROP_REASON_TC_INGRESS;
4688 	struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
4689 	int sch_ret;
4690 
4691 	if (!entry)
4692 		return skb;
4693 
4694 	bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
4695 	if (unlikely(*pt_prev)) {
4696 		*ret = deliver_skb(skb, *pt_prev, orig_dev);
4697 		*pt_prev = NULL;
4698 	}
4699 
4700 	qdisc_pkt_len_segs_init(skb);
4701 	tcx_set_ingress(skb, true);
4702 
4703 	if (static_branch_unlikely(&tcx_needed_key)) {
4704 		sch_ret = tcx_run(entry, skb, true);
4705 		if (sch_ret != TC_ACT_UNSPEC)
4706 			goto ingress_verdict;
4707 	}
4708 	sch_ret = tc_run(tcx_entry(entry), skb, &drop_reason);
4709 ingress_verdict:
4710 	switch (sch_ret) {
4711 	case TC_ACT_REDIRECT:
4712 		/* skb_mac_header check was done by BPF, so we can safely
4713 		 * push the L2 header back before redirecting to another
4714 		 * netdev.
4715 		 */
4716 		__skb_push(skb, skb->mac_len);
4717 		if (skb_do_redirect(skb) == -EAGAIN) {
4718 			__skb_pull(skb, skb->mac_len);
4719 			*another = true;
4720 			break;
4721 		}
4722 		*ret = NET_RX_SUCCESS;
4723 		bpf_net_ctx_clear(bpf_net_ctx);
4724 		return NULL;
4725 	case TC_ACT_SHOT:
4726 		kfree_skb_reason(skb, drop_reason);
4727 		*ret = NET_RX_DROP;
4728 		bpf_net_ctx_clear(bpf_net_ctx);
4729 		return NULL;
4730 	/* used by tc_run */
4731 	case TC_ACT_STOLEN:
4732 	case TC_ACT_QUEUED:
4733 	case TC_ACT_TRAP:
4734 		consume_skb(skb);
4735 		fallthrough;
4736 	case TC_ACT_CONSUMED:
4737 		*ret = NET_RX_SUCCESS;
4738 		bpf_net_ctx_clear(bpf_net_ctx);
4739 		return NULL;
4740 	}
4741 	bpf_net_ctx_clear(bpf_net_ctx);
4742 
4743 	return skb;
4744 }
4745 
4746 static __always_inline struct sk_buff *
4747 sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev)
4748 {
4749 	struct bpf_mprog_entry *entry = rcu_dereference_bh(dev->tcx_egress);
4750 	enum skb_drop_reason drop_reason = SKB_DROP_REASON_TC_EGRESS;
4751 	struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
4752 	int sch_ret;
4753 
4754 	if (!entry)
4755 		return skb;
4756 
4757 	bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
4758 
4759 	/* qdisc_skb_cb(skb)->pkt_len & tcx_set_ingress() was
4760 	 * already set by the caller.
4761 	 */
4762 	if (static_branch_unlikely(&tcx_needed_key)) {
4763 		sch_ret = tcx_run(entry, skb, false);
4764 		if (sch_ret != TC_ACT_UNSPEC)
4765 			goto egress_verdict;
4766 	}
4767 	sch_ret = tc_run(tcx_entry(entry), skb, &drop_reason);
4768 egress_verdict:
4769 	switch (sch_ret) {
4770 	case TC_ACT_REDIRECT:
4771 		/* No need to push/pop skb's mac_header here on egress! */
4772 		skb_do_redirect(skb);
4773 		*ret = NET_XMIT_SUCCESS;
4774 		bpf_net_ctx_clear(bpf_net_ctx);
4775 		return NULL;
4776 	case TC_ACT_SHOT:
4777 		kfree_skb_reason(skb, drop_reason);
4778 		*ret = NET_XMIT_DROP;
4779 		bpf_net_ctx_clear(bpf_net_ctx);
4780 		return NULL;
4781 	/* used by tc_run */
4782 	case TC_ACT_STOLEN:
4783 	case TC_ACT_QUEUED:
4784 	case TC_ACT_TRAP:
4785 		consume_skb(skb);
4786 		fallthrough;
4787 	case TC_ACT_CONSUMED:
4788 		*ret = NET_XMIT_SUCCESS;
4789 		bpf_net_ctx_clear(bpf_net_ctx);
4790 		return NULL;
4791 	}
4792 	bpf_net_ctx_clear(bpf_net_ctx);
4793 
4794 	return skb;
4795 }
4796 #else
4797 static __always_inline struct sk_buff *
4798 sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret,
4799 		   struct net_device *orig_dev, bool *another)
4800 {
4801 	return skb;
4802 }
4803 
4804 static __always_inline struct sk_buff *
4805 sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev)
4806 {
4807 	return skb;
4808 }
4809 #endif /* CONFIG_NET_XGRESS */
4810 
4811 #ifdef CONFIG_XPS
4812 static int __get_xps_queue_idx(struct net_device *dev, struct sk_buff *skb,
4813 			       struct xps_dev_maps *dev_maps, unsigned int tci)
4814 {
4815 	int tc = netdev_get_prio_tc_map(dev, skb->priority);
4816 	struct xps_map *map;
4817 	int queue_index = -1;
4818 
4819 	if (tc >= dev_maps->num_tc || tci >= dev_maps->nr_ids)
4820 		return queue_index;
4821 
4822 	tci *= dev_maps->num_tc;
4823 	tci += tc;
4824 
4825 	map = rcu_dereference(dev_maps->attr_map[tci]);
4826 	if (map) {
4827 		if (map->len == 1)
4828 			queue_index = map->queues[0];
4829 		else
4830 			queue_index = map->queues[reciprocal_scale(
4831 						skb_get_hash(skb), map->len)];
4832 		if (unlikely(queue_index >= dev->real_num_tx_queues))
4833 			queue_index = -1;
4834 	}
4835 	return queue_index;
4836 }
4837 #endif
4838 
4839 static int get_xps_queue(struct net_device *dev, struct net_device *sb_dev,
4840 			 struct sk_buff *skb)
4841 {
4842 #ifdef CONFIG_XPS
4843 	struct xps_dev_maps *dev_maps;
4844 	struct sock *sk = skb->sk;
4845 	int queue_index = -1;
4846 
4847 	if (!static_key_false(&xps_needed))
4848 		return -1;
4849 
4850 	rcu_read_lock();
4851 	if (!static_key_false(&xps_rxqs_needed))
4852 		goto get_cpus_map;
4853 
4854 	dev_maps = rcu_dereference(sb_dev->xps_maps[XPS_RXQS]);
4855 	if (dev_maps) {
4856 		int tci = sk_rx_queue_get(sk);
4857 
4858 		if (tci >= 0)
4859 			queue_index = __get_xps_queue_idx(dev, skb, dev_maps,
4860 							  tci);
4861 	}
4862 
4863 get_cpus_map:
4864 	if (queue_index < 0) {
4865 		dev_maps = rcu_dereference(sb_dev->xps_maps[XPS_CPUS]);
4866 		if (dev_maps) {
4867 			unsigned int tci = skb->sender_cpu - 1;
4868 
4869 			queue_index = __get_xps_queue_idx(dev, skb, dev_maps,
4870 							  tci);
4871 		}
4872 	}
4873 	rcu_read_unlock();
4874 
4875 	return queue_index;
4876 #else
4877 	return -1;
4878 #endif
4879 }
4880 
4881 u16 dev_pick_tx_zero(struct net_device *dev, struct sk_buff *skb,
4882 		     struct net_device *sb_dev)
4883 {
4884 	return 0;
4885 }
4886 EXPORT_SYMBOL(dev_pick_tx_zero);
4887 
4888 int sk_tx_queue_get(const struct sock *sk)
4889 {
4890 	int resel, val;
4891 
4892 	if (!sk)
4893 		return -1;
4894 	/* Paired with WRITE_ONCE() in sk_tx_queue_clear()
4895 	 * and sk_tx_queue_set().
4896 	 */
4897 	val = READ_ONCE(sk->sk_tx_queue_mapping);
4898 
4899 	if (val == NO_QUEUE_MAPPING)
4900 		return -1;
4901 
4902 	if (!sk_fullsock(sk))
4903 		return val;
4904 
4905 	resel = READ_ONCE(sock_net(sk)->core.sysctl_txq_reselection);
4906 	if (resel && time_is_before_jiffies(
4907 			READ_ONCE(sk->sk_tx_queue_mapping_jiffies) + resel))
4908 		return -1;
4909 
4910 	return val;
4911 }
4912 EXPORT_SYMBOL(sk_tx_queue_get);
4913 
4914 u16 netdev_pick_tx(struct net_device *dev, struct sk_buff *skb,
4915 		     struct net_device *sb_dev)
4916 {
4917 	struct sock *sk = skb->sk;
4918 	int queue_index = sk_tx_queue_get(sk);
4919 
4920 	sb_dev = sb_dev ? : dev;
4921 
4922 	if (queue_index < 0 || skb->ooo_okay ||
4923 	    queue_index >= dev->real_num_tx_queues) {
4924 		int new_index = get_xps_queue(dev, sb_dev, skb);
4925 
4926 		if (new_index < 0)
4927 			new_index = skb_tx_hash(dev, sb_dev, skb);
4928 
4929 		if (sk && sk_fullsock(sk) &&
4930 		    rcu_access_pointer(sk->sk_dst_cache))
4931 			sk_tx_queue_set(sk, new_index);
4932 
4933 		queue_index = new_index;
4934 	}
4935 
4936 	return queue_index;
4937 }
4938 EXPORT_SYMBOL(netdev_pick_tx);
4939 
4940 struct netdev_queue *netdev_core_pick_tx(struct net_device *dev,
4941 					 struct sk_buff *skb,
4942 					 struct net_device *sb_dev)
4943 {
4944 	int queue_index = 0;
4945 
4946 #ifdef CONFIG_XPS
4947 	u32 sender_cpu = skb->sender_cpu - 1;
4948 
4949 	if (sender_cpu >= (u32)NR_CPUS)
4950 		skb->sender_cpu = raw_smp_processor_id() + 1;
4951 #endif
4952 
4953 	if (dev->real_num_tx_queues != 1) {
4954 		const struct net_device_ops *ops = dev->netdev_ops;
4955 
4956 		if (ops->ndo_select_queue)
4957 			queue_index = ops->ndo_select_queue(dev, skb, sb_dev);
4958 		else
4959 			queue_index = netdev_pick_tx(dev, skb, sb_dev);
4960 
4961 		queue_index = netdev_cap_txqueue(dev, queue_index);
4962 	}
4963 
4964 	skb_set_queue_mapping(skb, queue_index);
4965 	return netdev_get_tx_queue(dev, queue_index);
4966 }
4967 
4968 /**
4969  * __dev_queue_xmit() - transmit a buffer
4970  * @skb:	buffer to transmit
4971  * @sb_dev:	suboordinate device used for L2 forwarding offload
4972  *
4973  * Queue a buffer for transmission to a network device. The caller must
4974  * have set the device and priority and built the buffer before calling
4975  * this function. The function can be called from an interrupt.
4976  *
4977  * When calling this method, interrupts MUST be enabled. This is because
4978  * the BH enable code must have IRQs enabled so that it will not deadlock.
4979  *
4980  * Regardless of the return value, the skb is consumed, so it is currently
4981  * difficult to retry a send to this method. (You can bump the ref count
4982  * before sending to hold a reference for retry if you are careful.)
4983  *
4984  * Return:
4985  * * 0				- buffer successfully transmitted
4986  * * positive qdisc return code	- NET_XMIT_DROP etc.
4987  * * negative errno		- other errors
4988  */
4989 int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev)
4990 {
4991 	struct net_device *dev = skb->dev;
4992 	struct netdev_queue *txq = NULL;
4993 	enum skb_drop_reason reason;
4994 	int cpu, rc = -ENOMEM;
4995 	bool again = false;
4996 	struct Qdisc *q;
4997 
4998 	skb_reset_mac_header(skb);
4999 	skb_assert_len(skb);
5000 
5001 	if (unlikely(skb_shinfo(skb)->tx_flags &
5002 		     (SKBTX_SCHED_TSTAMP | SKBTX_BPF)))
5003 		__skb_tstamp_tx(skb, NULL, NULL, skb->sk, SCM_TSTAMP_SCHED);
5004 
5005 	reason = qdisc_pkt_len_segs_init(skb);
5006 	if (unlikely(reason)) {
5007 		dev_core_stats_tx_dropped_inc(dev);
5008 		kfree_skb_reason(skb, reason);
5009 		return -EINVAL;
5010 	}
5011 	/* Disable soft irqs for various locks below. Also
5012 	 * stops preemption for RCU.
5013 	 */
5014 	rcu_read_lock_bh();
5015 
5016 	skb_update_prio(skb);
5017 
5018 	tcx_set_ingress(skb, false);
5019 #ifdef CONFIG_NET_EGRESS
5020 	if (static_branch_unlikely(&egress_needed_key)) {
5021 		bool skip_txq;
5022 
5023 		if (nf_hook_egress_active()) {
5024 			skb = nf_hook_egress(skb, &rc, dev);
5025 			if (!skb)
5026 				goto out;
5027 		}
5028 
5029 		skip_txq = netdev_xmit_skip_txqueue(false);
5030 
5031 		nf_skip_egress(skb, true);
5032 		skb = sch_handle_egress(skb, &rc, dev);
5033 		if (!skb) {
5034 			netdev_xmit_skip_txqueue(skip_txq);
5035 			goto out;
5036 		}
5037 		nf_skip_egress(skb, false);
5038 
5039 		if (netdev_xmit_skip_txqueue(skip_txq))
5040 			txq = netdev_tx_queue_mapping(dev, skb);
5041 	}
5042 #endif
5043 	/* If device/qdisc don't need skb->dst, release it right now while
5044 	 * its hot in this cpu cache.
5045 	 */
5046 	if (dev->priv_flags & IFF_XMIT_DST_RELEASE)
5047 		skb_dst_drop(skb);
5048 	else
5049 		skb_dst_force(skb);
5050 
5051 	if (!txq)
5052 		txq = netdev_core_pick_tx(dev, skb, sb_dev);
5053 
5054 	q = rcu_dereference_bh(txq->qdisc);
5055 
5056 	trace_net_dev_queue(skb);
5057 	if (q->enqueue) {
5058 		rc = __dev_xmit_skb(skb, q, dev, txq);
5059 		goto out;
5060 	}
5061 
5062 	/* The device has no queue. Common case for software devices:
5063 	 * loopback, all the sorts of tunnels...
5064 
5065 	 * Really, it is unlikely that netif_tx_lock protection is necessary
5066 	 * here.  (f.e. loopback and IP tunnels are clean ignoring statistics
5067 	 * counters.)
5068 	 * However, it is possible, that they rely on protection
5069 	 * made by us here.
5070 
5071 	 * Check this and shot the lock. It is not prone from deadlocks.
5072 	 *Either shot noqueue qdisc, it is even simpler 8)
5073 	 */
5074 	if (unlikely(!(dev->flags & IFF_UP))) {
5075 		reason = SKB_DROP_REASON_DEV_READY;
5076 		goto drop;
5077 	}
5078 
5079 	cpu = smp_processor_id(); /* ok because BHs are off */
5080 
5081 	if (likely(!netif_tx_owned(txq, cpu))) {
5082 		bool is_list = false;
5083 
5084 		if (dev_xmit_recursion())
5085 			goto recursion_alert;
5086 
5087 		skb = validate_xmit_skb(skb, dev, &again);
5088 		if (IS_ERR_OR_NULL(skb)) {
5089 			if (PTR_ERR(skb) == -EINPROGRESS)
5090 				rc = NET_XMIT_SUCCESS;
5091 			goto out;
5092 		}
5093 
5094 		HARD_TX_LOCK(dev, txq, cpu);
5095 
5096 		if (!netif_xmit_stopped(txq)) {
5097 			is_list = !!skb->next;
5098 
5099 			dev_xmit_recursion_inc();
5100 			skb = dev_hard_start_xmit(skb, dev, txq, &rc);
5101 			dev_xmit_recursion_dec();
5102 
5103 			/* GSO segments a single SKB into a list of frames.
5104 			 * TCP expects error to mean none of the data was sent.
5105 			 */
5106 			if (is_list)
5107 				rc = NETDEV_TX_OK;
5108 		}
5109 		HARD_TX_UNLOCK(dev, txq);
5110 		if (!skb) /* xmit completed */
5111 			goto out;
5112 
5113 		net_crit_ratelimited("Virtual device %s asks to queue packet!\n",
5114 				     dev->name);
5115 		/* NETDEV_TX_BUSY or queue was stopped */
5116 		if (!is_list)
5117 			rc = -ENETDOWN;
5118 	} else {
5119 		/* Recursion is detected! It is possible unfortunately. */
5120 recursion_alert:
5121 		net_crit_ratelimited("Dead loop on virtual device %s (net %llu), fix it urgently!\n",
5122 				     dev->name, dev_net(dev)->net_cookie);
5123 
5124 		rc = -ENETDOWN;
5125 	}
5126 
5127 	reason = SKB_DROP_REASON_RECURSION_LIMIT;
5128 drop:
5129 	rcu_read_unlock_bh();
5130 
5131 	dev_core_stats_tx_dropped_inc(dev);
5132 	kfree_skb_list_reason(skb, reason);
5133 	return rc;
5134 out:
5135 	rcu_read_unlock_bh();
5136 	return rc;
5137 }
5138 EXPORT_SYMBOL(__dev_queue_xmit);
5139 
5140 int __dev_direct_xmit(struct sk_buff *skb, u16 queue_id)
5141 {
5142 	struct net_device *dev = skb->dev;
5143 	struct sk_buff *orig_skb = skb;
5144 	struct netdev_queue *txq;
5145 	int ret = NETDEV_TX_BUSY;
5146 	bool again = false;
5147 
5148 	if (unlikely(!netif_running(dev) ||
5149 		     !netif_carrier_ok(dev)))
5150 		goto drop;
5151 
5152 	skb = validate_xmit_skb_list(skb, dev, &again);
5153 	if (skb != orig_skb)
5154 		goto drop;
5155 
5156 	skb_set_queue_mapping(skb, queue_id);
5157 	txq = skb_get_tx_queue(dev, skb);
5158 
5159 	local_bh_disable();
5160 
5161 	dev_xmit_recursion_inc();
5162 	HARD_TX_LOCK(dev, txq, smp_processor_id());
5163 	if (!netif_xmit_frozen_or_drv_stopped(txq))
5164 		ret = netdev_start_xmit(skb, dev, txq, false);
5165 	HARD_TX_UNLOCK(dev, txq);
5166 	dev_xmit_recursion_dec();
5167 
5168 	local_bh_enable();
5169 	return ret;
5170 drop:
5171 	dev_core_stats_tx_dropped_inc(dev);
5172 	kfree_skb_list(skb);
5173 	return NET_XMIT_DROP;
5174 }
5175 EXPORT_SYMBOL(__dev_direct_xmit);
5176 
5177 /*************************************************************************
5178  *			Receiver routines
5179  *************************************************************************/
5180 static DEFINE_PER_CPU(struct task_struct *, backlog_napi);
5181 
5182 int weight_p __read_mostly = 64;           /* old backlog weight */
5183 int dev_weight_rx_bias __read_mostly = 1;  /* bias for backlog weight */
5184 int dev_weight_tx_bias __read_mostly = 1;  /* bias for output_queue quota */
5185 
5186 /* Called with irq disabled */
5187 static inline void ____napi_schedule(struct softnet_data *sd,
5188 				     struct napi_struct *napi)
5189 {
5190 	struct task_struct *thread;
5191 
5192 	lockdep_assert_irqs_disabled();
5193 
5194 	if (test_bit(NAPI_STATE_THREADED, &napi->state)) {
5195 		/* Paired with smp_mb__before_atomic() in
5196 		 * napi_enable()/netif_set_threaded().
5197 		 * Use READ_ONCE() to guarantee a complete
5198 		 * read on napi->thread. Only call
5199 		 * wake_up_process() when it's not NULL.
5200 		 */
5201 		thread = READ_ONCE(napi->thread);
5202 		if (thread) {
5203 			if (use_backlog_threads() && thread == raw_cpu_read(backlog_napi))
5204 				goto use_local_napi;
5205 
5206 			set_bit(NAPI_STATE_SCHED_THREADED, &napi->state);
5207 			wake_up_process(thread);
5208 			return;
5209 		}
5210 	}
5211 
5212 use_local_napi:
5213 	DEBUG_NET_WARN_ON_ONCE(!list_empty(&napi->poll_list));
5214 	list_add_tail(&napi->poll_list, &sd->poll_list);
5215 	WRITE_ONCE(napi->list_owner, smp_processor_id());
5216 	/* If not called from net_rx_action()
5217 	 * we have to raise NET_RX_SOFTIRQ.
5218 	 */
5219 	if (!sd->in_net_rx_action)
5220 		raise_softirq_irqoff(NET_RX_SOFTIRQ);
5221 }
5222 
5223 #ifdef CONFIG_RPS
5224 
5225 struct static_key_false rps_needed __read_mostly;
5226 EXPORT_SYMBOL(rps_needed);
5227 struct static_key_false rfs_needed __read_mostly;
5228 EXPORT_SYMBOL(rfs_needed);
5229 
5230 static u32 rfs_slot(u32 hash, rps_tag_ptr tag_ptr)
5231 {
5232 	return hash_32(hash, rps_tag_to_log(tag_ptr));
5233 }
5234 
5235 #ifdef CONFIG_RFS_ACCEL
5236 /**
5237  * rps_flow_is_active - check whether the flow is recently active.
5238  * @rflow: Specific flow to check activity.
5239  * @log: ilog2(hashsize).
5240  * @cpu: CPU saved in @rflow.
5241  *
5242  * If the CPU has processed many packets since the flow's last activity
5243  * (beyond 10 times the table size), the flow is considered stale.
5244  *
5245  * Return: true if flow was recently active.
5246  */
5247 static bool rps_flow_is_active(struct rps_dev_flow *rflow,
5248 			       u8 log,
5249 			       unsigned int cpu)
5250 {
5251 	unsigned int flow_last_active;
5252 	unsigned int sd_input_head;
5253 
5254 	if (cpu >= nr_cpu_ids)
5255 		return false;
5256 
5257 	sd_input_head = READ_ONCE(per_cpu(softnet_data, cpu).input_queue_head);
5258 	flow_last_active = READ_ONCE(rflow->last_qtail);
5259 
5260 	return (int)(sd_input_head - flow_last_active) <
5261 		(int)(10 << log);
5262 }
5263 #endif
5264 
5265 static struct rps_dev_flow *
5266 set_rps_cpu(struct net_device *dev, struct sk_buff *skb,
5267 	    struct rps_dev_flow *rflow, u16 next_cpu, u32 hash)
5268 {
5269 	if (next_cpu < nr_cpu_ids) {
5270 		u32 head;
5271 #ifdef CONFIG_RFS_ACCEL
5272 		struct netdev_rx_queue *rxqueue;
5273 		struct rps_dev_flow *flow_table;
5274 		struct rps_dev_flow *old_rflow;
5275 		struct rps_dev_flow *tmp_rflow;
5276 		rps_tag_ptr q_tag_ptr;
5277 		unsigned int tmp_cpu;
5278 		u16 rxq_index;
5279 		u32 flow_id;
5280 		int rc;
5281 
5282 		/* Should we steer this flow to a different hardware queue? */
5283 		if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap ||
5284 		    !(dev->features & NETIF_F_NTUPLE))
5285 			goto out;
5286 		rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu);
5287 		if (rxq_index == skb_get_rx_queue(skb))
5288 			goto out;
5289 
5290 		rxqueue = dev->_rx + rxq_index;
5291 		q_tag_ptr = READ_ONCE(rxqueue->rps_flow_table);
5292 		if (!q_tag_ptr)
5293 			goto out;
5294 
5295 		flow_id = rfs_slot(hash, q_tag_ptr);
5296 		flow_table = rps_tag_to_table(q_tag_ptr);
5297 		tmp_rflow = flow_table + flow_id;
5298 		tmp_cpu = READ_ONCE(tmp_rflow->cpu);
5299 
5300 		if (READ_ONCE(tmp_rflow->filter) != RPS_NO_FILTER) {
5301 			if (rps_flow_is_active(tmp_rflow,
5302 					       rps_tag_to_log(q_tag_ptr),
5303 					       tmp_cpu)) {
5304 				if (hash != READ_ONCE(tmp_rflow->hash) ||
5305 				    next_cpu == tmp_cpu)
5306 					goto out;
5307 			}
5308 		}
5309 
5310 		rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb,
5311 							rxq_index, flow_id);
5312 		if (rc < 0)
5313 			goto out;
5314 
5315 		old_rflow = rflow;
5316 		rflow = tmp_rflow;
5317 		WRITE_ONCE(rflow->filter, rc);
5318 		WRITE_ONCE(rflow->hash, hash);
5319 
5320 		if (old_rflow->filter == rc)
5321 			WRITE_ONCE(old_rflow->filter, RPS_NO_FILTER);
5322 	out:
5323 #endif
5324 		head = READ_ONCE(per_cpu(softnet_data, next_cpu).input_queue_head);
5325 		rps_input_queue_tail_save(&rflow->last_qtail, head);
5326 	}
5327 
5328 	WRITE_ONCE(rflow->cpu, next_cpu);
5329 	return rflow;
5330 }
5331 
5332 /*
5333  * get_rps_cpu is called from netif_receive_skb and returns the target
5334  * CPU from the RPS map of the receiving queue for a given skb.
5335  * rcu_read_lock must be held on entry.
5336  */
5337 static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb,
5338 		       struct rps_dev_flow **rflowp)
5339 {
5340 	struct netdev_rx_queue *rxqueue = dev->_rx;
5341 	rps_tag_ptr global_tag_ptr, q_tag_ptr;
5342 	struct rps_map *map;
5343 	int cpu = -1;
5344 	u32 tcpu;
5345 	u32 hash;
5346 
5347 	if (skb_rx_queue_recorded(skb)) {
5348 		u16 index = skb_get_rx_queue(skb);
5349 
5350 		if (unlikely(index >= dev->real_num_rx_queues)) {
5351 			WARN_ONCE(dev->real_num_rx_queues > 1,
5352 				  "%s received packet on queue %u, but number "
5353 				  "of RX queues is %u\n",
5354 				  dev->name, index, dev->real_num_rx_queues);
5355 			goto done;
5356 		}
5357 		rxqueue += index;
5358 	}
5359 
5360 	/* Avoid computing hash if RFS/RPS is not active for this rxqueue */
5361 
5362 	q_tag_ptr = READ_ONCE(rxqueue->rps_flow_table);
5363 	map = rcu_dereference(rxqueue->rps_map);
5364 	if (!q_tag_ptr && !map)
5365 		goto done;
5366 
5367 	skb_reset_network_header(skb);
5368 	hash = skb_get_hash(skb);
5369 	if (!hash)
5370 		goto done;
5371 
5372 	global_tag_ptr = READ_ONCE(net_hotdata.rps_sock_flow_table);
5373 	if (q_tag_ptr && global_tag_ptr) {
5374 		struct rps_sock_flow_table *sock_flow_table;
5375 		struct rps_dev_flow *flow_table;
5376 		struct rps_dev_flow *rflow;
5377 		u32 next_cpu;
5378 		u32 flow_id;
5379 		u32 ident;
5380 
5381 		/* First check into global flow table if there is a match.
5382 		 * This READ_ONCE() pairs with WRITE_ONCE() from rps_record_sock_flow().
5383 		 */
5384 		flow_id = hash & rps_tag_to_mask(global_tag_ptr);
5385 		sock_flow_table = rps_tag_to_table(global_tag_ptr);
5386 		ident = READ_ONCE(sock_flow_table[flow_id].ent);
5387 		if ((ident ^ hash) & ~net_hotdata.rps_cpu_mask)
5388 			goto try_rps;
5389 
5390 		next_cpu = ident & net_hotdata.rps_cpu_mask;
5391 
5392 		/* OK, now we know there is a match,
5393 		 * we can look at the local (per receive queue) flow table
5394 		 */
5395 		flow_id = rfs_slot(hash, q_tag_ptr);
5396 		flow_table = rps_tag_to_table(q_tag_ptr);
5397 		rflow = flow_table + flow_id;
5398 		tcpu = rflow->cpu;
5399 
5400 		/*
5401 		 * If the desired CPU (where last recvmsg was done) is
5402 		 * different from current CPU (one in the rx-queue flow
5403 		 * table entry), switch if one of the following holds:
5404 		 *   - Current CPU is unset (>= nr_cpu_ids).
5405 		 *   - Current CPU is offline.
5406 		 *   - The current CPU's queue tail has advanced beyond the
5407 		 *     last packet that was enqueued using this table entry.
5408 		 *     This guarantees that all previous packets for the flow
5409 		 *     have been dequeued, thus preserving in order delivery.
5410 		 */
5411 		if (unlikely(tcpu != next_cpu) &&
5412 		    (tcpu >= nr_cpu_ids || !cpu_online(tcpu) ||
5413 		     ((int)(READ_ONCE(per_cpu(softnet_data, tcpu).input_queue_head) -
5414 		      rflow->last_qtail)) >= 0)) {
5415 			tcpu = next_cpu;
5416 			rflow = set_rps_cpu(dev, skb, rflow, next_cpu, hash);
5417 		}
5418 
5419 		if (tcpu < nr_cpu_ids && cpu_online(tcpu)) {
5420 			*rflowp = rflow;
5421 			cpu = tcpu;
5422 			goto done;
5423 		}
5424 	}
5425 
5426 try_rps:
5427 
5428 	if (map) {
5429 		tcpu = map->cpus[reciprocal_scale(hash, map->len)];
5430 		if (cpu_online(tcpu)) {
5431 			cpu = tcpu;
5432 			goto done;
5433 		}
5434 	}
5435 
5436 done:
5437 	return cpu;
5438 }
5439 
5440 #ifdef CONFIG_RFS_ACCEL
5441 
5442 /**
5443  * rps_may_expire_flow - check whether an RFS hardware filter may be removed
5444  * @dev: Device on which the filter was set
5445  * @rxq_index: RX queue index
5446  * @flow_id: Flow ID passed to ndo_rx_flow_steer()
5447  * @filter_id: Filter ID returned by ndo_rx_flow_steer()
5448  *
5449  * Drivers that implement ndo_rx_flow_steer() should periodically call
5450  * this function for each installed filter and remove the filters for
5451  * which it returns %true.
5452  */
5453 bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index,
5454 			 u32 flow_id, u16 filter_id)
5455 {
5456 	struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index;
5457 	struct rps_dev_flow *flow_table;
5458 	struct rps_dev_flow *rflow;
5459 	rps_tag_ptr q_tag_ptr;
5460 	bool expire = true;
5461 	u8 log;
5462 
5463 	rcu_read_lock();
5464 	q_tag_ptr = READ_ONCE(rxqueue->rps_flow_table);
5465 	log = rps_tag_to_log(q_tag_ptr);
5466 	if (q_tag_ptr && flow_id < (1UL << log)) {
5467 		unsigned int cpu;
5468 
5469 		flow_table = rps_tag_to_table(q_tag_ptr);
5470 		rflow = flow_table + flow_id;
5471 		cpu = READ_ONCE(rflow->cpu);
5472 		if (READ_ONCE(rflow->filter) == filter_id &&
5473 		    rps_flow_is_active(rflow, log, cpu))
5474 			expire = false;
5475 	}
5476 	rcu_read_unlock();
5477 	return expire;
5478 }
5479 EXPORT_SYMBOL(rps_may_expire_flow);
5480 
5481 #endif /* CONFIG_RFS_ACCEL */
5482 
5483 /* Called from hardirq (IPI) context */
5484 static void rps_trigger_softirq(void *data)
5485 {
5486 	struct softnet_data *sd = data;
5487 
5488 	____napi_schedule(sd, &sd->backlog);
5489 	/* Pairs with READ_ONCE() in softnet_seq_show() */
5490 	WRITE_ONCE(sd->received_rps, sd->received_rps + 1);
5491 }
5492 
5493 #endif /* CONFIG_RPS */
5494 
5495 /* Called from hardirq (IPI) context */
5496 static void trigger_rx_softirq(void *data)
5497 {
5498 	struct softnet_data *sd = data;
5499 
5500 	__raise_softirq_irqoff(NET_RX_SOFTIRQ);
5501 	smp_store_release(&sd->defer_ipi_scheduled, 0);
5502 }
5503 
5504 /*
5505  * After we queued a packet into sd->input_pkt_queue,
5506  * we need to make sure this queue is serviced soon.
5507  *
5508  * - If this is another cpu queue, link it to our rps_ipi_list,
5509  *   and make sure we will process rps_ipi_list from net_rx_action().
5510  *
5511  * - If this is our own queue, NAPI schedule our backlog.
5512  *   Note that this also raises NET_RX_SOFTIRQ.
5513  */
5514 static void napi_schedule_rps(struct softnet_data *sd)
5515 {
5516 	struct softnet_data *mysd = this_cpu_ptr(&softnet_data);
5517 
5518 #ifdef CONFIG_RPS
5519 	if (sd != mysd) {
5520 		if (use_backlog_threads()) {
5521 			__napi_schedule_irqoff(&sd->backlog);
5522 			return;
5523 		}
5524 
5525 		sd->rps_ipi_next = mysd->rps_ipi_list;
5526 		mysd->rps_ipi_list = sd;
5527 
5528 		/* If not called from net_rx_action() or napi_threaded_poll()
5529 		 * we have to raise NET_RX_SOFTIRQ.
5530 		 */
5531 		if (!mysd->in_net_rx_action && !mysd->in_napi_threaded_poll)
5532 			__raise_softirq_irqoff(NET_RX_SOFTIRQ);
5533 		return;
5534 	}
5535 #endif /* CONFIG_RPS */
5536 	__napi_schedule_irqoff(&mysd->backlog);
5537 }
5538 
5539 void kick_defer_list_purge(unsigned int cpu)
5540 {
5541 	struct softnet_data *sd = &per_cpu(softnet_data, cpu);
5542 	unsigned long flags;
5543 
5544 	if (use_backlog_threads()) {
5545 		backlog_lock_irq_save(sd, &flags);
5546 
5547 		if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state))
5548 			__napi_schedule_irqoff(&sd->backlog);
5549 
5550 		backlog_unlock_irq_restore(sd, flags);
5551 
5552 	} else if (!cmpxchg(&sd->defer_ipi_scheduled, 0, 1)) {
5553 		if (smp_call_function_single_async(cpu, &sd->defer_csd))
5554 			WRITE_ONCE(sd->defer_ipi_scheduled, 0);
5555 	}
5556 }
5557 
5558 #ifdef CONFIG_NET_FLOW_LIMIT
5559 int netdev_flow_limit_table_len __read_mostly = (1 << 12);
5560 #endif
5561 
5562 static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen,
5563 			   int max_backlog)
5564 {
5565 #ifdef CONFIG_NET_FLOW_LIMIT
5566 	unsigned int old_flow, new_flow;
5567 	const struct softnet_data *sd;
5568 	struct sd_flow_limit *fl;
5569 
5570 	if (likely(qlen < (max_backlog >> 1)))
5571 		return false;
5572 
5573 	sd = this_cpu_ptr(&softnet_data);
5574 
5575 	rcu_read_lock();
5576 	fl = rcu_dereference(sd->flow_limit);
5577 	if (fl) {
5578 		new_flow = hash_32(skb_get_hash(skb), fl->log_buckets);
5579 		old_flow = fl->history[fl->history_head];
5580 		fl->history[fl->history_head] = new_flow;
5581 
5582 		fl->history_head++;
5583 		fl->history_head &= FLOW_LIMIT_HISTORY - 1;
5584 
5585 		if (likely(fl->buckets[old_flow]))
5586 			fl->buckets[old_flow]--;
5587 
5588 		if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) {
5589 			/* Pairs with READ_ONCE() in softnet_seq_show() */
5590 			WRITE_ONCE(fl->count, fl->count + 1);
5591 			rcu_read_unlock();
5592 			return true;
5593 		}
5594 	}
5595 	rcu_read_unlock();
5596 #endif
5597 	return false;
5598 }
5599 
5600 /*
5601  * enqueue_to_backlog is called to queue an skb to a per CPU backlog
5602  * queue (may be a remote CPU queue).
5603  */
5604 static int enqueue_to_backlog(struct sk_buff *skb, int cpu,
5605 			      unsigned int *qtail)
5606 {
5607 	enum skb_drop_reason reason;
5608 	struct softnet_data *sd;
5609 	unsigned long flags;
5610 	unsigned int qlen;
5611 	int max_backlog;
5612 	u32 tail;
5613 
5614 	reason = SKB_DROP_REASON_DEV_READY;
5615 
5616 	sd = &per_cpu(softnet_data, cpu);
5617 
5618 	qlen = skb_queue_len_lockless(&sd->input_pkt_queue);
5619 	max_backlog = READ_ONCE(net_hotdata.max_backlog);
5620 	if (unlikely(qlen > max_backlog) ||
5621 	    skb_flow_limit(skb, qlen, max_backlog))
5622 		goto cpu_backlog_drop;
5623 	backlog_lock_irq_save(sd, &flags);
5624 	qlen = skb_queue_len(&sd->input_pkt_queue);
5625 	if (likely(qlen <= max_backlog)) {
5626 		if (unlikely(!netif_running(skb->dev))) {
5627 			backlog_unlock_irq_restore(sd, flags);
5628 			goto bad_dev;
5629 		}
5630 		if (!qlen) {
5631 			/* Schedule NAPI for backlog device. We can use
5632 			 * non atomic operation as we own the queue lock.
5633 			 */
5634 			if (!__test_and_set_bit(NAPI_STATE_SCHED,
5635 						&sd->backlog.state))
5636 				napi_schedule_rps(sd);
5637 		}
5638 		__skb_queue_tail(&sd->input_pkt_queue, skb);
5639 		tail = rps_input_queue_tail_incr(sd);
5640 		backlog_unlock_irq_restore(sd, flags);
5641 
5642 		/* save the tail outside of the critical section */
5643 		rps_input_queue_tail_save(qtail, tail);
5644 		return NET_RX_SUCCESS;
5645 	}
5646 
5647 	backlog_unlock_irq_restore(sd, flags);
5648 
5649 cpu_backlog_drop:
5650 	reason = SKB_DROP_REASON_CPU_BACKLOG;
5651 	numa_drop_add(&sd->drop_counters, 1);
5652 bad_dev:
5653 	dev_core_stats_rx_dropped_inc(skb->dev);
5654 	kfree_skb_reason(skb, reason);
5655 	return NET_RX_DROP;
5656 }
5657 
5658 static struct netdev_rx_queue *netif_get_rxqueue(struct sk_buff *skb)
5659 {
5660 	struct net_device *dev = skb->dev;
5661 	struct netdev_rx_queue *rxqueue;
5662 
5663 	rxqueue = dev->_rx;
5664 
5665 	if (skb_rx_queue_recorded(skb)) {
5666 		u16 index = skb_get_rx_queue(skb);
5667 
5668 		if (unlikely(index >= dev->real_num_rx_queues)) {
5669 			WARN_ONCE(dev->real_num_rx_queues > 1,
5670 				  "%s received packet on queue %u, but number "
5671 				  "of RX queues is %u\n",
5672 				  dev->name, index, dev->real_num_rx_queues);
5673 
5674 			return rxqueue; /* Return first rxqueue */
5675 		}
5676 		rxqueue += index;
5677 	}
5678 	return rxqueue;
5679 }
5680 
5681 u32 bpf_prog_run_generic_xdp(struct sk_buff *skb, struct xdp_buff *xdp,
5682 			     const struct bpf_prog *xdp_prog)
5683 {
5684 	void *orig_data, *orig_data_end, *hard_start;
5685 	struct netdev_rx_queue *rxqueue;
5686 	bool orig_bcast, orig_host;
5687 	u32 mac_len, frame_sz;
5688 	__be16 orig_eth_type;
5689 	struct ethhdr *eth;
5690 	u32 metalen, act;
5691 	int off;
5692 
5693 	/* The XDP program wants to see the packet starting at the MAC
5694 	 * header.
5695 	 */
5696 	mac_len = skb->data - skb_mac_header(skb);
5697 	hard_start = skb->data - skb_headroom(skb);
5698 
5699 	/* SKB "head" area always have tailroom for skb_shared_info */
5700 	frame_sz = (void *)skb_end_pointer(skb) - hard_start;
5701 	frame_sz += SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
5702 
5703 	rxqueue = netif_get_rxqueue(skb);
5704 	xdp_init_buff(xdp, frame_sz, &rxqueue->xdp_rxq);
5705 	xdp_prepare_buff(xdp, hard_start, skb_headroom(skb) - mac_len,
5706 			 skb_headlen(skb) + mac_len, true);
5707 	if (skb_is_nonlinear(skb)) {
5708 		skb_shinfo(skb)->xdp_frags_size = skb->data_len;
5709 		xdp_buff_set_frags_flag(xdp);
5710 	} else {
5711 		xdp_buff_clear_frags_flag(xdp);
5712 	}
5713 
5714 	orig_data_end = xdp->data_end;
5715 	orig_data = xdp->data;
5716 	eth = (struct ethhdr *)xdp->data;
5717 	orig_host = ether_addr_equal_64bits(eth->h_dest, skb->dev->dev_addr);
5718 	orig_bcast = is_multicast_ether_addr_64bits(eth->h_dest);
5719 	orig_eth_type = eth->h_proto;
5720 
5721 	act = bpf_prog_run_xdp(xdp_prog, xdp);
5722 
5723 	/* check if bpf_xdp_adjust_head was used */
5724 	off = xdp->data - orig_data;
5725 	if (off) {
5726 		if (off > 0)
5727 			__skb_pull(skb, off);
5728 		else if (off < 0)
5729 			__skb_push(skb, -off);
5730 
5731 		skb->mac_header += off;
5732 		skb_reset_network_header(skb);
5733 	}
5734 
5735 	/* check if bpf_xdp_adjust_tail was used */
5736 	off = xdp->data_end - orig_data_end;
5737 	if (off != 0) {
5738 		skb_set_tail_pointer(skb, xdp->data_end - xdp->data);
5739 		skb->len += off; /* positive on grow, negative on shrink */
5740 	}
5741 
5742 	/* XDP frag metadata (e.g. nr_frags) are updated in eBPF helpers
5743 	 * (e.g. bpf_xdp_adjust_tail). Remove the old fragment contribution
5744 	 * from skb->len before updating data_len, then add the new one back.
5745 	 */
5746 	skb->len -= skb->data_len;
5747 	if (xdp_buff_has_frags(xdp)) {
5748 		skb->data_len = skb_shinfo(skb)->xdp_frags_size;
5749 		skb->len += skb->data_len;
5750 	} else {
5751 		skb->data_len = 0;
5752 	}
5753 
5754 	/* check if XDP changed eth hdr such SKB needs update */
5755 	eth = (struct ethhdr *)xdp->data;
5756 	if ((orig_eth_type != eth->h_proto) ||
5757 	    (orig_host != ether_addr_equal_64bits(eth->h_dest,
5758 						  skb->dev->dev_addr)) ||
5759 	    (orig_bcast != is_multicast_ether_addr_64bits(eth->h_dest))) {
5760 		__skb_push(skb, ETH_HLEN);
5761 		skb->pkt_type = PACKET_HOST;
5762 		skb->protocol = eth_type_trans(skb, skb->dev);
5763 	}
5764 
5765 	/* Redirect/Tx gives L2 packet, code that will reuse skb must __skb_pull
5766 	 * before calling us again on redirect path. We do not call do_redirect
5767 	 * as we leave that up to the caller.
5768 	 *
5769 	 * Caller is responsible for managing lifetime of skb (i.e. calling
5770 	 * kfree_skb in response to actions it cannot handle/XDP_DROP).
5771 	 */
5772 	switch (act) {
5773 	case XDP_REDIRECT:
5774 	case XDP_TX:
5775 		__skb_push(skb, mac_len);
5776 		break;
5777 	case XDP_PASS:
5778 		metalen = xdp->data - xdp->data_meta;
5779 		if (metalen)
5780 			skb_metadata_set(skb, metalen);
5781 		break;
5782 	}
5783 
5784 	return act;
5785 }
5786 
5787 static int
5788 netif_skb_check_for_xdp(struct sk_buff **pskb, const struct bpf_prog *prog)
5789 {
5790 	struct sk_buff *skb = *pskb;
5791 	int err, hroom, troom;
5792 
5793 	local_lock_nested_bh(&system_page_pool.bh_lock);
5794 	err = skb_cow_data_for_xdp(this_cpu_read(system_page_pool.pool), pskb, prog);
5795 	local_unlock_nested_bh(&system_page_pool.bh_lock);
5796 	if (!err)
5797 		return 0;
5798 
5799 	/* In case we have to go down the path and also linearize,
5800 	 * then lets do the pskb_expand_head() work just once here.
5801 	 */
5802 	hroom = XDP_PACKET_HEADROOM - skb_headroom(skb);
5803 	troom = skb->tail + skb->data_len - skb->end;
5804 	err = pskb_expand_head(skb,
5805 			       hroom > 0 ? ALIGN(hroom, NET_SKB_PAD) : 0,
5806 			       troom > 0 ? troom + 128 : 0, GFP_ATOMIC);
5807 	if (err)
5808 		return err;
5809 
5810 	return skb_linearize(skb);
5811 }
5812 
5813 static u32 netif_receive_generic_xdp(struct sk_buff **pskb,
5814 				     struct xdp_buff *xdp,
5815 				     const struct bpf_prog *xdp_prog)
5816 {
5817 	struct sk_buff *skb = *pskb;
5818 	u32 mac_len, act = XDP_DROP;
5819 
5820 	/* Reinjected packets coming from act_mirred or similar should
5821 	 * not get XDP generic processing.
5822 	 */
5823 	if (skb_is_redirected(skb))
5824 		return XDP_PASS;
5825 
5826 	/* XDP packets must have sufficient headroom of XDP_PACKET_HEADROOM
5827 	 * bytes. This is the guarantee that also native XDP provides,
5828 	 * thus we need to do it here as well.
5829 	 */
5830 	mac_len = skb->data - skb_mac_header(skb);
5831 	__skb_push(skb, mac_len);
5832 
5833 	if (skb_cloned(skb) || skb_is_nonlinear(skb) ||
5834 	    skb_headroom(skb) < XDP_PACKET_HEADROOM) {
5835 		if (netif_skb_check_for_xdp(pskb, xdp_prog))
5836 			goto do_drop;
5837 	}
5838 
5839 	__skb_pull(*pskb, mac_len);
5840 
5841 	act = bpf_prog_run_generic_xdp(*pskb, xdp, xdp_prog);
5842 	switch (act) {
5843 	case XDP_REDIRECT:
5844 	case XDP_TX:
5845 	case XDP_PASS:
5846 		break;
5847 	default:
5848 		bpf_warn_invalid_xdp_action((*pskb)->dev, xdp_prog, act);
5849 		fallthrough;
5850 	case XDP_ABORTED:
5851 		trace_xdp_exception((*pskb)->dev, xdp_prog, act);
5852 		fallthrough;
5853 	case XDP_DROP:
5854 	do_drop:
5855 		kfree_skb(*pskb);
5856 		break;
5857 	}
5858 
5859 	return act;
5860 }
5861 
5862 /* When doing generic XDP we have to bypass the qdisc layer and the
5863  * network taps in order to match in-driver-XDP behavior. This also means
5864  * that XDP packets are able to starve other packets going through a qdisc,
5865  * and DDOS attacks will be more effective. In-driver-XDP use dedicated TX
5866  * queues, so they do not have this starvation issue.
5867  */
5868 void generic_xdp_tx(struct sk_buff *skb, const struct bpf_prog *xdp_prog)
5869 {
5870 	struct net_device *dev = skb->dev;
5871 	struct netdev_queue *txq;
5872 	bool free_skb = true;
5873 	int cpu, rc;
5874 
5875 	txq = netdev_core_pick_tx(dev, skb, NULL);
5876 	cpu = smp_processor_id();
5877 	HARD_TX_LOCK(dev, txq, cpu);
5878 	if (!netif_xmit_frozen_or_drv_stopped(txq)) {
5879 		rc = netdev_start_xmit(skb, dev, txq, 0);
5880 		if (dev_xmit_complete(rc))
5881 			free_skb = false;
5882 	}
5883 	HARD_TX_UNLOCK(dev, txq);
5884 	if (free_skb) {
5885 		trace_xdp_exception(dev, xdp_prog, XDP_TX);
5886 		dev_core_stats_tx_dropped_inc(dev);
5887 		kfree_skb(skb);
5888 	}
5889 }
5890 
5891 static DEFINE_STATIC_KEY_FALSE(generic_xdp_needed_key);
5892 
5893 int do_xdp_generic(const struct bpf_prog *xdp_prog, struct sk_buff **pskb)
5894 {
5895 	struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
5896 
5897 	if (xdp_prog) {
5898 		struct xdp_buff xdp;
5899 		u32 act;
5900 		int err;
5901 
5902 		bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
5903 		act = netif_receive_generic_xdp(pskb, &xdp, xdp_prog);
5904 		if (act != XDP_PASS) {
5905 			switch (act) {
5906 			case XDP_REDIRECT:
5907 				err = xdp_do_generic_redirect((*pskb)->dev, *pskb,
5908 							      &xdp, xdp_prog);
5909 				if (err)
5910 					goto out_redir;
5911 				break;
5912 			case XDP_TX:
5913 				generic_xdp_tx(*pskb, xdp_prog);
5914 				break;
5915 			}
5916 			bpf_net_ctx_clear(bpf_net_ctx);
5917 			return XDP_DROP;
5918 		}
5919 		bpf_net_ctx_clear(bpf_net_ctx);
5920 	}
5921 	return XDP_PASS;
5922 out_redir:
5923 	bpf_net_ctx_clear(bpf_net_ctx);
5924 	kfree_skb_reason(*pskb, SKB_DROP_REASON_XDP);
5925 	return XDP_DROP;
5926 }
5927 EXPORT_SYMBOL_GPL(do_xdp_generic);
5928 
5929 static int netif_rx_internal(struct sk_buff *skb)
5930 {
5931 	int ret;
5932 
5933 	net_timestamp_check(READ_ONCE(net_hotdata.tstamp_prequeue), skb);
5934 
5935 	trace_netif_rx(skb);
5936 
5937 #ifdef CONFIG_RPS
5938 	if (static_branch_unlikely(&rps_needed)) {
5939 		struct rps_dev_flow voidflow, *rflow = &voidflow;
5940 		int cpu;
5941 
5942 		rcu_read_lock();
5943 
5944 		cpu = get_rps_cpu(skb->dev, skb, &rflow);
5945 		if (cpu < 0)
5946 			cpu = smp_processor_id();
5947 
5948 		ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
5949 
5950 		rcu_read_unlock();
5951 	} else
5952 #endif
5953 	{
5954 		unsigned int qtail;
5955 
5956 		ret = enqueue_to_backlog(skb, smp_processor_id(), &qtail);
5957 	}
5958 	return ret;
5959 }
5960 
5961 /**
5962  *	__netif_rx	-	Slightly optimized version of netif_rx
5963  *	@skb: buffer to post
5964  *
5965  *	This behaves as netif_rx except that it does not disable bottom halves.
5966  *	As a result this function may only be invoked from the interrupt context
5967  *	(either hard or soft interrupt).
5968  */
5969 int __netif_rx(struct sk_buff *skb)
5970 {
5971 	int ret;
5972 
5973 	lockdep_assert_once(hardirq_count() | softirq_count());
5974 
5975 	trace_netif_rx_entry(skb);
5976 	ret = netif_rx_internal(skb);
5977 	trace_netif_rx_exit(ret);
5978 	return ret;
5979 }
5980 EXPORT_SYMBOL(__netif_rx);
5981 
5982 /**
5983  *	netif_rx	-	post buffer to the network code
5984  *	@skb: buffer to post
5985  *
5986  *	This function receives a packet from a device driver and queues it for
5987  *	the upper (protocol) levels to process via the backlog NAPI device. It
5988  *	always succeeds. The buffer may be dropped during processing for
5989  *	congestion control or by the protocol layers.
5990  *	The network buffer is passed via the backlog NAPI device. Modern NIC
5991  *	driver should use NAPI and GRO.
5992  *	This function can used from interrupt and from process context. The
5993  *	caller from process context must not disable interrupts before invoking
5994  *	this function.
5995  *
5996  *	return values:
5997  *	NET_RX_SUCCESS	(no congestion)
5998  *	NET_RX_DROP     (packet was dropped)
5999  *
6000  */
6001 int netif_rx(struct sk_buff *skb)
6002 {
6003 	bool need_bh_off = !(hardirq_count() | softirq_count());
6004 	int ret;
6005 
6006 	if (need_bh_off)
6007 		local_bh_disable();
6008 	trace_netif_rx_entry(skb);
6009 	ret = netif_rx_internal(skb);
6010 	trace_netif_rx_exit(ret);
6011 	if (need_bh_off)
6012 		local_bh_enable();
6013 	return ret;
6014 }
6015 EXPORT_SYMBOL(netif_rx);
6016 
6017 static __latent_entropy void net_tx_action(void)
6018 {
6019 	struct softnet_data *sd = this_cpu_ptr(&softnet_data);
6020 
6021 	if (sd->completion_queue) {
6022 		struct sk_buff *clist;
6023 
6024 		local_irq_disable();
6025 		clist = sd->completion_queue;
6026 		sd->completion_queue = NULL;
6027 		local_irq_enable();
6028 
6029 		while (clist) {
6030 			struct sk_buff *skb = clist;
6031 
6032 			clist = clist->next;
6033 
6034 			WARN_ON(refcount_read(&skb->users));
6035 			if (likely(get_kfree_skb_cb(skb)->reason == SKB_CONSUMED))
6036 				trace_consume_skb(skb, net_tx_action);
6037 			else
6038 				trace_kfree_skb(skb, net_tx_action,
6039 						get_kfree_skb_cb(skb)->reason, NULL);
6040 
6041 			if (skb->fclone != SKB_FCLONE_UNAVAILABLE)
6042 				__kfree_skb(skb);
6043 			else
6044 				__napi_kfree_skb(skb,
6045 						 get_kfree_skb_cb(skb)->reason);
6046 		}
6047 	}
6048 
6049 	if (sd->output_queue) {
6050 		struct Qdisc *head;
6051 
6052 		local_irq_disable();
6053 		head = sd->output_queue;
6054 		sd->output_queue = NULL;
6055 		sd->output_queue_tailp = &sd->output_queue;
6056 		local_irq_enable();
6057 
6058 		rcu_read_lock();
6059 
6060 		while (head) {
6061 			spinlock_t *root_lock = NULL;
6062 			struct sk_buff *to_free;
6063 			struct Qdisc *q = head;
6064 
6065 			head = head->next_sched;
6066 
6067 			/* We need to make sure head->next_sched is read
6068 			 * before clearing __QDISC_STATE_SCHED
6069 			 */
6070 			smp_mb__before_atomic();
6071 
6072 			if (!(q->flags & TCQ_F_NOLOCK)) {
6073 				root_lock = qdisc_lock(q);
6074 				spin_lock(root_lock);
6075 			} else if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED,
6076 						     &q->state))) {
6077 				/* There is a synchronize_net() between
6078 				 * STATE_DEACTIVATED flag being set and
6079 				 * qdisc_reset()/some_qdisc_is_busy() in
6080 				 * dev_deactivate(), so we can safely bail out
6081 				 * early here to avoid data race between
6082 				 * qdisc_deactivate() and some_qdisc_is_busy()
6083 				 * for lockless qdisc.
6084 				 */
6085 				clear_bit(__QDISC_STATE_SCHED, &q->state);
6086 				continue;
6087 			}
6088 
6089 			clear_bit(__QDISC_STATE_SCHED, &q->state);
6090 			to_free = qdisc_run(q);
6091 			if (root_lock)
6092 				spin_unlock(root_lock);
6093 			tcf_kfree_skb_list(to_free, q, NULL, qdisc_dev(q));
6094 		}
6095 
6096 		rcu_read_unlock();
6097 	}
6098 
6099 	xfrm_dev_backlog(sd);
6100 }
6101 
6102 /**
6103  *	netdev_is_rx_handler_busy - check if receive handler is registered
6104  *	@dev: device to check
6105  *
6106  *	Check if a receive handler is already registered for a given device.
6107  *	Return true if there one.
6108  *
6109  *	The caller must hold the rtnl_mutex.
6110  */
6111 bool netdev_is_rx_handler_busy(struct net_device *dev)
6112 {
6113 	ASSERT_RTNL();
6114 	return dev && rtnl_dereference(dev->rx_handler);
6115 }
6116 EXPORT_SYMBOL_GPL(netdev_is_rx_handler_busy);
6117 
6118 /**
6119  *	netdev_rx_handler_register - register receive handler
6120  *	@dev: device to register a handler for
6121  *	@rx_handler: receive handler to register
6122  *	@rx_handler_data: data pointer that is used by rx handler
6123  *
6124  *	Register a receive handler for a device. This handler will then be
6125  *	called from __netif_receive_skb. A negative errno code is returned
6126  *	on a failure.
6127  *
6128  *	The caller must hold the rtnl_mutex.
6129  *
6130  *	For a general description of rx_handler, see enum rx_handler_result.
6131  */
6132 int netdev_rx_handler_register(struct net_device *dev,
6133 			       rx_handler_func_t *rx_handler,
6134 			       void *rx_handler_data)
6135 {
6136 	if (netdev_is_rx_handler_busy(dev))
6137 		return -EBUSY;
6138 
6139 	if (dev->priv_flags & IFF_NO_RX_HANDLER)
6140 		return -EINVAL;
6141 
6142 	/* Note: rx_handler_data must be set before rx_handler */
6143 	rcu_assign_pointer(dev->rx_handler_data, rx_handler_data);
6144 	rcu_assign_pointer(dev->rx_handler, rx_handler);
6145 
6146 	return 0;
6147 }
6148 EXPORT_SYMBOL_GPL(netdev_rx_handler_register);
6149 
6150 /**
6151  *	netdev_rx_handler_unregister - unregister receive handler
6152  *	@dev: device to unregister a handler from
6153  *
6154  *	Unregister a receive handler from a device.
6155  *
6156  *	The caller must hold the rtnl_mutex.
6157  */
6158 void netdev_rx_handler_unregister(struct net_device *dev)
6159 {
6160 
6161 	ASSERT_RTNL();
6162 	RCU_INIT_POINTER(dev->rx_handler, NULL);
6163 	/* a reader seeing a non NULL rx_handler in a rcu_read_lock()
6164 	 * section has a guarantee to see a non NULL rx_handler_data
6165 	 * as well.
6166 	 */
6167 	synchronize_net();
6168 	RCU_INIT_POINTER(dev->rx_handler_data, NULL);
6169 }
6170 EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister);
6171 
6172 /*
6173  * Limit the use of PFMEMALLOC reserves to those protocols that implement
6174  * the special handling of PFMEMALLOC skbs.
6175  */
6176 static bool skb_pfmemalloc_protocol(struct sk_buff *skb)
6177 {
6178 	switch (skb->protocol) {
6179 	case htons(ETH_P_ARP):
6180 	case htons(ETH_P_IP):
6181 	case htons(ETH_P_IPV6):
6182 	case htons(ETH_P_8021Q):
6183 	case htons(ETH_P_8021AD):
6184 		return true;
6185 	default:
6186 		return false;
6187 	}
6188 }
6189 
6190 static inline int nf_ingress(struct sk_buff *skb, struct packet_type **pt_prev,
6191 			     int *ret, struct net_device *orig_dev)
6192 {
6193 	if (nf_hook_ingress_active(skb)) {
6194 		int ingress_retval;
6195 
6196 		if (unlikely(*pt_prev)) {
6197 			*ret = deliver_skb(skb, *pt_prev, orig_dev);
6198 			*pt_prev = NULL;
6199 		}
6200 
6201 		rcu_read_lock();
6202 		ingress_retval = nf_hook_ingress(skb);
6203 		rcu_read_unlock();
6204 		return ingress_retval;
6205 	}
6206 	return 0;
6207 }
6208 
6209 static int __netif_receive_skb_core(struct sk_buff **pskb, bool pfmemalloc,
6210 				    struct packet_type **ppt_prev)
6211 {
6212 	enum skb_drop_reason drop_reason = SKB_DROP_REASON_UNHANDLED_PROTO;
6213 	struct packet_type *ptype, *pt_prev;
6214 	rx_handler_func_t *rx_handler;
6215 	struct sk_buff *skb = *pskb;
6216 	struct net_device *orig_dev;
6217 	bool deliver_exact = false;
6218 	int ret = NET_RX_DROP;
6219 	__be16 type;
6220 
6221 	net_timestamp_check(!READ_ONCE(net_hotdata.tstamp_prequeue), skb);
6222 
6223 	trace_netif_receive_skb(skb);
6224 
6225 	orig_dev = skb->dev;
6226 
6227 	skb_reset_network_header(skb);
6228 #if !defined(CONFIG_DEBUG_NET)
6229 	/* We plan to no longer reset the transport header here.
6230 	 * Give some time to fuzzers and dev build to catch bugs
6231 	 * in network stacks.
6232 	 */
6233 	if (!skb_transport_header_was_set(skb))
6234 		skb_reset_transport_header(skb);
6235 #endif
6236 	skb_reset_mac_len(skb);
6237 
6238 	pt_prev = NULL;
6239 
6240 another_round:
6241 	skb->skb_iif = skb->dev->ifindex;
6242 
6243 	__this_cpu_inc(softnet_data.processed);
6244 
6245 	if (static_branch_unlikely(&generic_xdp_needed_key)) {
6246 		int ret2;
6247 
6248 		migrate_disable();
6249 		ret2 = do_xdp_generic(rcu_dereference(skb->dev->xdp_prog),
6250 				      &skb);
6251 		migrate_enable();
6252 
6253 		if (ret2 != XDP_PASS) {
6254 			ret = NET_RX_DROP;
6255 			goto out;
6256 		}
6257 	}
6258 
6259 	if (eth_type_vlan(skb->protocol)) {
6260 		skb = skb_vlan_untag(skb);
6261 		if (unlikely(!skb))
6262 			goto out;
6263 	}
6264 
6265 	if (skb_skip_tc_classify(skb))
6266 		goto skip_classify;
6267 
6268 	if (pfmemalloc)
6269 		goto skip_taps;
6270 
6271 	list_for_each_entry_rcu(ptype, &dev_net_rcu(skb->dev)->ptype_all,
6272 				list) {
6273 		if (unlikely(pt_prev))
6274 			ret = deliver_skb(skb, pt_prev, orig_dev);
6275 		pt_prev = ptype;
6276 	}
6277 
6278 	list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) {
6279 		if (unlikely(pt_prev))
6280 			ret = deliver_skb(skb, pt_prev, orig_dev);
6281 		pt_prev = ptype;
6282 	}
6283 
6284 skip_taps:
6285 #ifdef CONFIG_NET_INGRESS
6286 	if (static_branch_unlikely(&ingress_needed_key)) {
6287 		bool another = false;
6288 
6289 		nf_skip_egress(skb, true);
6290 		skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev,
6291 					 &another);
6292 		if (another)
6293 			goto another_round;
6294 		if (!skb)
6295 			goto out;
6296 
6297 		nf_skip_egress(skb, false);
6298 		if (nf_ingress(skb, &pt_prev, &ret, orig_dev) < 0)
6299 			goto out;
6300 	}
6301 #endif
6302 	skb_reset_redirect(skb);
6303 skip_classify:
6304 	if (pfmemalloc && !skb_pfmemalloc_protocol(skb)) {
6305 		drop_reason = SKB_DROP_REASON_PFMEMALLOC;
6306 		goto drop;
6307 	}
6308 
6309 	if (skb_vlan_tag_present(skb)) {
6310 		if (unlikely(pt_prev)) {
6311 			ret = deliver_skb(skb, pt_prev, orig_dev);
6312 			pt_prev = NULL;
6313 		}
6314 		if (vlan_do_receive(&skb))
6315 			goto another_round;
6316 		else if (unlikely(!skb))
6317 			goto out;
6318 	}
6319 
6320 	rx_handler = rcu_dereference(skb->dev->rx_handler);
6321 	if (rx_handler) {
6322 		if (unlikely(pt_prev)) {
6323 			ret = deliver_skb(skb, pt_prev, orig_dev);
6324 			pt_prev = NULL;
6325 		}
6326 		switch (rx_handler(&skb)) {
6327 		case RX_HANDLER_CONSUMED:
6328 			ret = NET_RX_SUCCESS;
6329 			goto out;
6330 		case RX_HANDLER_ANOTHER:
6331 			goto another_round;
6332 		case RX_HANDLER_EXACT:
6333 			deliver_exact = true;
6334 			break;
6335 		case RX_HANDLER_PASS:
6336 			break;
6337 		default:
6338 			BUG();
6339 		}
6340 	}
6341 
6342 	if (unlikely(skb_vlan_tag_present(skb)) && !netdev_uses_dsa(skb->dev)) {
6343 		unsigned int priority_tag_depth = 1;
6344 
6345 check_vlan_id:
6346 		if (skb_vlan_tag_get_id(skb)) {
6347 			/* Vlan id is non 0 and vlan_do_receive() above couldn't
6348 			 * find vlan device.
6349 			 */
6350 			skb->pkt_type = PACKET_OTHERHOST;
6351 		} else if (eth_type_vlan(skb->protocol)) {
6352 			/* Outer header is 802.1P with vlan 0, inner header is
6353 			 * 802.1Q or 802.1AD and vlan_do_receive() above could
6354 			 * not find vlan dev for vlan id 0.
6355 			 */
6356 			if (unlikely(priority_tag_depth >= VLAN_MAX_DEPTH)) {
6357 				drop_reason = SKB_DROP_REASON_RECURSION_LIMIT;
6358 				goto drop;
6359 			}
6360 
6361 			priority_tag_depth++;
6362 			__vlan_hwaccel_clear_tag(skb);
6363 			skb = skb_vlan_untag(skb);
6364 			if (unlikely(!skb))
6365 				goto out;
6366 			if (vlan_do_receive(&skb))
6367 				/* After stripping off 802.1P header with vlan 0
6368 				 * vlan dev is found for inner header.
6369 				 */
6370 				goto another_round;
6371 			else if (unlikely(!skb))
6372 				goto out;
6373 			else
6374 				/* We have stripped outer 802.1P vlan 0 header.
6375 				 * But could not find vlan dev.
6376 				 * check again for vlan id to set OTHERHOST.
6377 				 */
6378 				goto check_vlan_id;
6379 		}
6380 		/* Note: we might in the future use prio bits
6381 		 * and set skb->priority like in vlan_do_receive()
6382 		 * For the time being, just ignore Priority Code Point
6383 		 */
6384 		__vlan_hwaccel_clear_tag(skb);
6385 	}
6386 
6387 	type = skb->protocol;
6388 
6389 	/* deliver only exact match when indicated */
6390 	if (likely(!deliver_exact)) {
6391 		deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
6392 				       &ptype_base[ntohs(type) &
6393 						   PTYPE_HASH_MASK]);
6394 
6395 		/* orig_dev and skb->dev could belong to different netns;
6396 		 * Even in such case we need to traverse only the list
6397 		 * coming from skb->dev, as the ptype owner (packet socket)
6398 		 * will use dev_net(skb->dev) to do namespace filtering.
6399 		 */
6400 		deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
6401 				       &dev_net_rcu(skb->dev)->ptype_specific);
6402 	}
6403 
6404 	deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
6405 			       &orig_dev->ptype_specific);
6406 
6407 	if (unlikely(skb->dev != orig_dev)) {
6408 		deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type,
6409 				       &skb->dev->ptype_specific);
6410 	}
6411 
6412 	if (pt_prev) {
6413 		*ppt_prev = pt_prev;
6414 	} else {
6415 drop:
6416 		if (!deliver_exact)
6417 			dev_core_stats_rx_dropped_inc(skb->dev);
6418 		else
6419 			dev_core_stats_rx_nohandler_inc(skb->dev);
6420 
6421 		kfree_skb_reason(skb, drop_reason);
6422 		/* Jamal, now you will not able to escape explaining
6423 		 * me how you were going to use this. :-)
6424 		 */
6425 		ret = NET_RX_DROP;
6426 	}
6427 
6428 out:
6429 	/* The invariant here is that if *ppt_prev is not NULL
6430 	 * then skb should also be non-NULL.
6431 	 *
6432 	 * Apparently *ppt_prev assignment above holds this invariant due to
6433 	 * skb dereferencing near it.
6434 	 */
6435 	*pskb = skb;
6436 	return ret;
6437 }
6438 
6439 static int __netif_receive_skb_one_core(struct sk_buff *skb, bool pfmemalloc)
6440 {
6441 	struct net_device *orig_dev = skb->dev;
6442 	struct packet_type *pt_prev = NULL;
6443 	int ret;
6444 
6445 	ret = __netif_receive_skb_core(&skb, pfmemalloc, &pt_prev);
6446 	if (pt_prev)
6447 		ret = INDIRECT_CALL_INET(pt_prev->func, ipv6_rcv, ip_rcv, skb,
6448 					 skb->dev, pt_prev, orig_dev);
6449 	return ret;
6450 }
6451 
6452 /**
6453  *	netif_receive_skb_core - special purpose version of netif_receive_skb
6454  *	@skb: buffer to process
6455  *
6456  *	More direct receive version of netif_receive_skb().  It should
6457  *	only be used by callers that have a need to skip RPS and Generic XDP.
6458  *	Caller must also take care of handling if ``(page_is_)pfmemalloc``.
6459  *
6460  *	This function may only be called from softirq context and interrupts
6461  *	should be enabled.
6462  *
6463  *	Return values (usually ignored):
6464  *	NET_RX_SUCCESS: no congestion
6465  *	NET_RX_DROP: packet was dropped
6466  */
6467 int netif_receive_skb_core(struct sk_buff *skb)
6468 {
6469 	int ret;
6470 
6471 	rcu_read_lock();
6472 	ret = __netif_receive_skb_one_core(skb, false);
6473 	rcu_read_unlock();
6474 
6475 	return ret;
6476 }
6477 EXPORT_SYMBOL(netif_receive_skb_core);
6478 
6479 static inline void __netif_receive_skb_list_ptype(struct list_head *head,
6480 						  struct packet_type *pt_prev,
6481 						  struct net_device *orig_dev)
6482 {
6483 	struct sk_buff *skb, *next;
6484 
6485 	if (!pt_prev)
6486 		return;
6487 	if (list_empty(head))
6488 		return;
6489 	if (pt_prev->list_func != NULL)
6490 		INDIRECT_CALL_INET(pt_prev->list_func, ipv6_list_rcv,
6491 				   ip_list_rcv, head, pt_prev, orig_dev);
6492 	else
6493 		list_for_each_entry_safe(skb, next, head, list) {
6494 			skb_list_del_init(skb);
6495 			pt_prev->func(skb, skb->dev, pt_prev, orig_dev);
6496 		}
6497 }
6498 
6499 static void __netif_receive_skb_list_core(struct list_head *head, bool pfmemalloc)
6500 {
6501 	/* Fast-path assumptions:
6502 	 * - There is no RX handler.
6503 	 * - Only one packet_type matches.
6504 	 * If either of these fails, we will end up doing some per-packet
6505 	 * processing in-line, then handling the 'last ptype' for the whole
6506 	 * sublist.  This can't cause out-of-order delivery to any single ptype,
6507 	 * because the 'last ptype' must be constant across the sublist, and all
6508 	 * other ptypes are handled per-packet.
6509 	 */
6510 	/* Current (common) ptype of sublist */
6511 	struct packet_type *pt_curr = NULL;
6512 	/* Current (common) orig_dev of sublist */
6513 	struct net_device *od_curr = NULL;
6514 	struct sk_buff *skb, *next;
6515 	LIST_HEAD(sublist);
6516 
6517 	list_for_each_entry_safe(skb, next, head, list) {
6518 		struct net_device *orig_dev = skb->dev;
6519 		struct packet_type *pt_prev = NULL;
6520 
6521 		skb_list_del_init(skb);
6522 		__netif_receive_skb_core(&skb, pfmemalloc, &pt_prev);
6523 		if (!pt_prev)
6524 			continue;
6525 		if (pt_curr != pt_prev || od_curr != orig_dev) {
6526 			/* dispatch old sublist */
6527 			__netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr);
6528 			/* start new sublist */
6529 			INIT_LIST_HEAD(&sublist);
6530 			pt_curr = pt_prev;
6531 			od_curr = orig_dev;
6532 		}
6533 		list_add_tail(&skb->list, &sublist);
6534 	}
6535 
6536 	/* dispatch final sublist */
6537 	__netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr);
6538 }
6539 
6540 static int __netif_receive_skb(struct sk_buff *skb)
6541 {
6542 	int ret;
6543 
6544 	if (sk_memalloc_socks() && skb_pfmemalloc(skb)) {
6545 		unsigned int noreclaim_flag;
6546 
6547 		/*
6548 		 * PFMEMALLOC skbs are special, they should
6549 		 * - be delivered to SOCK_MEMALLOC sockets only
6550 		 * - stay away from userspace
6551 		 * - have bounded memory usage
6552 		 *
6553 		 * Use PF_MEMALLOC as this saves us from propagating the allocation
6554 		 * context down to all allocation sites.
6555 		 */
6556 		noreclaim_flag = memalloc_noreclaim_save();
6557 		ret = __netif_receive_skb_one_core(skb, true);
6558 		memalloc_noreclaim_restore(noreclaim_flag);
6559 	} else
6560 		ret = __netif_receive_skb_one_core(skb, false);
6561 
6562 	return ret;
6563 }
6564 
6565 static void __netif_receive_skb_list(struct list_head *head)
6566 {
6567 	unsigned long noreclaim_flag = 0;
6568 	struct sk_buff *skb, *next;
6569 	bool pfmemalloc = false; /* Is current sublist PF_MEMALLOC? */
6570 
6571 	list_for_each_entry_safe(skb, next, head, list) {
6572 		if ((sk_memalloc_socks() && skb_pfmemalloc(skb)) != pfmemalloc) {
6573 			struct list_head sublist;
6574 
6575 			/* Handle the previous sublist */
6576 			list_cut_before(&sublist, head, &skb->list);
6577 			if (!list_empty(&sublist))
6578 				__netif_receive_skb_list_core(&sublist, pfmemalloc);
6579 			pfmemalloc = !pfmemalloc;
6580 			/* See comments in __netif_receive_skb */
6581 			if (pfmemalloc)
6582 				noreclaim_flag = memalloc_noreclaim_save();
6583 			else
6584 				memalloc_noreclaim_restore(noreclaim_flag);
6585 		}
6586 	}
6587 	/* Handle the remaining sublist */
6588 	if (!list_empty(head))
6589 		__netif_receive_skb_list_core(head, pfmemalloc);
6590 	/* Restore pflags */
6591 	if (pfmemalloc)
6592 		memalloc_noreclaim_restore(noreclaim_flag);
6593 }
6594 
6595 static int generic_xdp_install(struct net_device *dev, struct netdev_bpf *xdp)
6596 {
6597 	struct bpf_prog *old = rtnl_dereference(dev->xdp_prog);
6598 	struct bpf_prog *new = xdp->prog;
6599 	int ret = 0;
6600 
6601 	switch (xdp->command) {
6602 	case XDP_SETUP_PROG:
6603 		rcu_assign_pointer(dev->xdp_prog, new);
6604 		if (old)
6605 			bpf_prog_put(old);
6606 
6607 		if (old && !new) {
6608 			static_branch_dec(&generic_xdp_needed_key);
6609 		} else if (new && !old) {
6610 			static_branch_inc(&generic_xdp_needed_key);
6611 			netif_disable_lro(dev);
6612 			dev_disable_gro_hw(dev);
6613 		}
6614 		break;
6615 
6616 	default:
6617 		ret = -EINVAL;
6618 		break;
6619 	}
6620 
6621 	return ret;
6622 }
6623 
6624 static int netif_receive_skb_internal(struct sk_buff *skb)
6625 {
6626 	int ret;
6627 
6628 	net_timestamp_check(READ_ONCE(net_hotdata.tstamp_prequeue), skb);
6629 
6630 	if (skb_defer_rx_timestamp(skb))
6631 		return NET_RX_SUCCESS;
6632 
6633 	rcu_read_lock();
6634 #ifdef CONFIG_RPS
6635 	if (static_branch_unlikely(&rps_needed)) {
6636 		struct rps_dev_flow voidflow, *rflow = &voidflow;
6637 		int cpu = get_rps_cpu(skb->dev, skb, &rflow);
6638 
6639 		if (cpu >= 0) {
6640 			ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
6641 			rcu_read_unlock();
6642 			return ret;
6643 		}
6644 	}
6645 #endif
6646 	ret = __netif_receive_skb(skb);
6647 	rcu_read_unlock();
6648 	return ret;
6649 }
6650 
6651 void netif_receive_skb_list_internal(struct list_head *head)
6652 {
6653 	struct sk_buff *skb, *next;
6654 	LIST_HEAD(sublist);
6655 
6656 	list_for_each_entry_safe(skb, next, head, list) {
6657 		net_timestamp_check(READ_ONCE(net_hotdata.tstamp_prequeue),
6658 				    skb);
6659 		skb_list_del_init(skb);
6660 		if (!skb_defer_rx_timestamp(skb))
6661 			list_add_tail(&skb->list, &sublist);
6662 	}
6663 	list_splice_init(&sublist, head);
6664 
6665 	rcu_read_lock();
6666 #ifdef CONFIG_RPS
6667 	if (static_branch_unlikely(&rps_needed)) {
6668 		list_for_each_entry_safe(skb, next, head, list) {
6669 			struct rps_dev_flow voidflow, *rflow = &voidflow;
6670 			int cpu = get_rps_cpu(skb->dev, skb, &rflow);
6671 
6672 			if (cpu >= 0) {
6673 				/* Will be handled, remove from list */
6674 				skb_list_del_init(skb);
6675 				enqueue_to_backlog(skb, cpu, &rflow->last_qtail);
6676 			}
6677 		}
6678 	}
6679 #endif
6680 	__netif_receive_skb_list(head);
6681 	rcu_read_unlock();
6682 }
6683 
6684 /**
6685  *	netif_receive_skb - process receive buffer from network
6686  *	@skb: buffer to process
6687  *
6688  *	netif_receive_skb() is the main receive data processing function.
6689  *	It always succeeds. The buffer may be dropped during processing
6690  *	for congestion control or by the protocol layers.
6691  *
6692  *	This function may only be called from softirq context and interrupts
6693  *	should be enabled.
6694  *
6695  *	Return values (usually ignored):
6696  *	NET_RX_SUCCESS: no congestion
6697  *	NET_RX_DROP: packet was dropped
6698  */
6699 int netif_receive_skb(struct sk_buff *skb)
6700 {
6701 	int ret;
6702 
6703 	trace_netif_receive_skb_entry(skb);
6704 
6705 	ret = netif_receive_skb_internal(skb);
6706 	trace_netif_receive_skb_exit(ret);
6707 
6708 	return ret;
6709 }
6710 EXPORT_SYMBOL(netif_receive_skb);
6711 
6712 /**
6713  *	netif_receive_skb_list - process many receive buffers from network
6714  *	@head: list of skbs to process.
6715  *
6716  *	Since return value of netif_receive_skb() is normally ignored, and
6717  *	wouldn't be meaningful for a list, this function returns void.
6718  *
6719  *	This function may only be called from softirq context and interrupts
6720  *	should be enabled.
6721  */
6722 void netif_receive_skb_list(struct list_head *head)
6723 {
6724 	struct sk_buff *skb;
6725 
6726 	if (list_empty(head))
6727 		return;
6728 	if (trace_netif_receive_skb_list_entry_enabled()) {
6729 		list_for_each_entry(skb, head, list)
6730 			trace_netif_receive_skb_list_entry(skb);
6731 	}
6732 	netif_receive_skb_list_internal(head);
6733 	trace_netif_receive_skb_list_exit(0);
6734 }
6735 EXPORT_SYMBOL(netif_receive_skb_list);
6736 
6737 /* Network device is going away, flush any packets still pending */
6738 static void flush_backlog(struct work_struct *work)
6739 {
6740 	struct sk_buff *skb, *tmp;
6741 	struct sk_buff_head list;
6742 	struct softnet_data *sd;
6743 
6744 	__skb_queue_head_init(&list);
6745 	local_bh_disable();
6746 	sd = this_cpu_ptr(&softnet_data);
6747 
6748 	backlog_lock_irq_disable(sd);
6749 	skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) {
6750 		if (READ_ONCE(skb->dev->reg_state) == NETREG_UNREGISTERING) {
6751 			__skb_unlink(skb, &sd->input_pkt_queue);
6752 			__skb_queue_tail(&list, skb);
6753 			rps_input_queue_head_incr(sd);
6754 		}
6755 	}
6756 	backlog_unlock_irq_enable(sd);
6757 
6758 	local_lock_nested_bh(&softnet_data.process_queue_bh_lock);
6759 	skb_queue_walk_safe(&sd->process_queue, skb, tmp) {
6760 		if (READ_ONCE(skb->dev->reg_state) == NETREG_UNREGISTERING) {
6761 			__skb_unlink(skb, &sd->process_queue);
6762 			__skb_queue_tail(&list, skb);
6763 			rps_input_queue_head_incr(sd);
6764 		}
6765 	}
6766 	local_unlock_nested_bh(&softnet_data.process_queue_bh_lock);
6767 	local_bh_enable();
6768 
6769 	__skb_queue_purge_reason(&list, SKB_DROP_REASON_DEV_READY);
6770 }
6771 
6772 static bool flush_required(int cpu)
6773 {
6774 #if IS_ENABLED(CONFIG_RPS)
6775 	struct softnet_data *sd = &per_cpu(softnet_data, cpu);
6776 	bool do_flush;
6777 
6778 	backlog_lock_irq_disable(sd);
6779 
6780 	/* as insertion into process_queue happens with the rps lock held,
6781 	 * process_queue access may race only with dequeue
6782 	 */
6783 	do_flush = !skb_queue_empty(&sd->input_pkt_queue) ||
6784 		   !skb_queue_empty_lockless(&sd->process_queue);
6785 	backlog_unlock_irq_enable(sd);
6786 
6787 	return do_flush;
6788 #endif
6789 	/* without RPS we can't safely check input_pkt_queue: during a
6790 	 * concurrent remote skb_queue_splice() we can detect as empty both
6791 	 * input_pkt_queue and process_queue even if the latter could end-up
6792 	 * containing a lot of packets.
6793 	 */
6794 	return true;
6795 }
6796 
6797 struct flush_backlogs {
6798 	cpumask_t		flush_cpus;
6799 	struct work_struct	w[];
6800 };
6801 
6802 static struct flush_backlogs *flush_backlogs_alloc(void)
6803 {
6804 	return kmalloc_flex(struct flush_backlogs, w, nr_cpu_ids);
6805 }
6806 
6807 static struct flush_backlogs *flush_backlogs_fallback;
6808 static DEFINE_MUTEX(flush_backlogs_mutex);
6809 
6810 static void flush_all_backlogs(void)
6811 {
6812 	struct flush_backlogs *ptr = flush_backlogs_alloc();
6813 	unsigned int cpu;
6814 
6815 	if (!ptr) {
6816 		mutex_lock(&flush_backlogs_mutex);
6817 		ptr = flush_backlogs_fallback;
6818 	}
6819 	cpumask_clear(&ptr->flush_cpus);
6820 
6821 	cpus_read_lock();
6822 
6823 	for_each_online_cpu(cpu) {
6824 		if (flush_required(cpu)) {
6825 			INIT_WORK(&ptr->w[cpu], flush_backlog);
6826 			queue_work_on(cpu, system_highpri_wq, &ptr->w[cpu]);
6827 			__cpumask_set_cpu(cpu, &ptr->flush_cpus);
6828 		}
6829 	}
6830 
6831 	/* we can have in flight packet[s] on the cpus we are not flushing,
6832 	 * synchronize_net() in unregister_netdevice_many() will take care of
6833 	 * them.
6834 	 */
6835 	for_each_cpu(cpu, &ptr->flush_cpus)
6836 		flush_work(&ptr->w[cpu]);
6837 
6838 	cpus_read_unlock();
6839 
6840 	if (ptr != flush_backlogs_fallback)
6841 		kfree(ptr);
6842 	else
6843 		mutex_unlock(&flush_backlogs_mutex);
6844 }
6845 
6846 static void net_rps_send_ipi(struct softnet_data *remsd)
6847 {
6848 #ifdef CONFIG_RPS
6849 	while (remsd) {
6850 		struct softnet_data *next = remsd->rps_ipi_next;
6851 
6852 		if (cpu_online(remsd->cpu))
6853 			smp_call_function_single_async(remsd->cpu, &remsd->csd);
6854 		remsd = next;
6855 	}
6856 #endif
6857 }
6858 
6859 /*
6860  * net_rps_action_and_irq_enable sends any pending IPI's for rps.
6861  * Note: called with local irq disabled, but exits with local irq enabled.
6862  */
6863 static void net_rps_action_and_irq_enable(struct softnet_data *sd)
6864 {
6865 #ifdef CONFIG_RPS
6866 	struct softnet_data *remsd = sd->rps_ipi_list;
6867 
6868 	if (!use_backlog_threads() && remsd) {
6869 		sd->rps_ipi_list = NULL;
6870 
6871 		local_irq_enable();
6872 
6873 		/* Send pending IPI's to kick RPS processing on remote cpus. */
6874 		net_rps_send_ipi(remsd);
6875 	} else
6876 #endif
6877 		local_irq_enable();
6878 }
6879 
6880 static bool sd_has_rps_ipi_waiting(struct softnet_data *sd)
6881 {
6882 #ifdef CONFIG_RPS
6883 	return !use_backlog_threads() && sd->rps_ipi_list;
6884 #else
6885 	return false;
6886 #endif
6887 }
6888 
6889 static int process_backlog(struct napi_struct *napi, int quota)
6890 {
6891 	struct softnet_data *sd = container_of(napi, struct softnet_data, backlog);
6892 	bool again = true;
6893 	int work = 0;
6894 
6895 	/* Check if we have pending ipi, its better to send them now,
6896 	 * not waiting net_rx_action() end.
6897 	 */
6898 	if (sd_has_rps_ipi_waiting(sd)) {
6899 		local_irq_disable();
6900 		net_rps_action_and_irq_enable(sd);
6901 	}
6902 
6903 	napi->weight = READ_ONCE(net_hotdata.dev_rx_weight);
6904 	while (again) {
6905 		struct sk_buff *skb;
6906 
6907 		local_lock_nested_bh(&softnet_data.process_queue_bh_lock);
6908 		while ((skb = __skb_dequeue(&sd->process_queue))) {
6909 			local_unlock_nested_bh(&softnet_data.process_queue_bh_lock);
6910 			rcu_read_lock();
6911 			__netif_receive_skb(skb);
6912 			rcu_read_unlock();
6913 			if (++work >= quota) {
6914 				rps_input_queue_head_add(sd, work);
6915 				return work;
6916 			}
6917 
6918 			local_lock_nested_bh(&softnet_data.process_queue_bh_lock);
6919 		}
6920 		local_unlock_nested_bh(&softnet_data.process_queue_bh_lock);
6921 
6922 		backlog_lock_irq_disable(sd);
6923 		if (skb_queue_empty(&sd->input_pkt_queue)) {
6924 			/*
6925 			 * Inline a custom version of __napi_complete().
6926 			 * only current cpu owns and manipulates this napi,
6927 			 * and NAPI_STATE_SCHED is the only possible flag set
6928 			 * on backlog.
6929 			 * We can use a plain write instead of clear_bit(),
6930 			 * and we dont need an smp_mb() memory barrier.
6931 			 */
6932 			napi->state &= NAPIF_STATE_THREADED;
6933 			again = false;
6934 		} else {
6935 			local_lock_nested_bh(&softnet_data.process_queue_bh_lock);
6936 			skb_queue_splice_tail_init(&sd->input_pkt_queue,
6937 						   &sd->process_queue);
6938 			local_unlock_nested_bh(&softnet_data.process_queue_bh_lock);
6939 		}
6940 		backlog_unlock_irq_enable(sd);
6941 	}
6942 
6943 	if (work)
6944 		rps_input_queue_head_add(sd, work);
6945 	return work;
6946 }
6947 
6948 /**
6949  * __napi_schedule - schedule for receive
6950  * @n: entry to schedule
6951  *
6952  * The entry's receive function will be scheduled to run.
6953  * Consider using __napi_schedule_irqoff() if hard irqs are masked.
6954  */
6955 void __napi_schedule(struct napi_struct *n)
6956 {
6957 	unsigned long flags;
6958 
6959 	local_irq_save(flags);
6960 	____napi_schedule(this_cpu_ptr(&softnet_data), n);
6961 	local_irq_restore(flags);
6962 }
6963 EXPORT_SYMBOL(__napi_schedule);
6964 
6965 /**
6966  *	napi_schedule_prep - check if napi can be scheduled
6967  *	@n: napi context
6968  *
6969  * Test if NAPI routine is already running, and if not mark
6970  * it as running.  This is used as a condition variable to
6971  * insure only one NAPI poll instance runs.  We also make
6972  * sure there is no pending NAPI disable.
6973  */
6974 bool napi_schedule_prep(struct napi_struct *n)
6975 {
6976 	unsigned long new, val = READ_ONCE(n->state);
6977 
6978 	do {
6979 		if (unlikely(val & NAPIF_STATE_DISABLE))
6980 			return false;
6981 		new = val | NAPIF_STATE_SCHED;
6982 
6983 		/* Sets STATE_MISSED bit if STATE_SCHED was already set
6984 		 * This was suggested by Alexander Duyck, as compiler
6985 		 * emits better code than :
6986 		 * if (val & NAPIF_STATE_SCHED)
6987 		 *     new |= NAPIF_STATE_MISSED;
6988 		 */
6989 		new |= (val & NAPIF_STATE_SCHED) / NAPIF_STATE_SCHED *
6990 						   NAPIF_STATE_MISSED;
6991 	} while (!try_cmpxchg(&n->state, &val, new));
6992 
6993 	return !(val & NAPIF_STATE_SCHED);
6994 }
6995 EXPORT_SYMBOL(napi_schedule_prep);
6996 
6997 /**
6998  * __napi_schedule_irqoff - schedule for receive
6999  * @n: entry to schedule
7000  *
7001  * Variant of __napi_schedule() assuming hard irqs are masked.
7002  *
7003  * On PREEMPT_RT enabled kernels this maps to __napi_schedule()
7004  * because the interrupt disabled assumption might not be true
7005  * due to force-threaded interrupts and spinlock substitution.
7006  */
7007 void __napi_schedule_irqoff(struct napi_struct *n)
7008 {
7009 	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
7010 		____napi_schedule(this_cpu_ptr(&softnet_data), n);
7011 	else
7012 		__napi_schedule(n);
7013 }
7014 EXPORT_SYMBOL(__napi_schedule_irqoff);
7015 
7016 bool napi_complete_done(struct napi_struct *n, int work_done)
7017 {
7018 	unsigned long flags, val, new, timeout = 0;
7019 	bool ret = true;
7020 
7021 	/*
7022 	 * 1) Don't let napi dequeue from the cpu poll list
7023 	 *    just in case its running on a different cpu.
7024 	 * 2) If we are busy polling, do nothing here, we have
7025 	 *    the guarantee we will be called later.
7026 	 */
7027 	if (unlikely(n->state & (NAPIF_STATE_NPSVC |
7028 				 NAPIF_STATE_IN_BUSY_POLL)))
7029 		return false;
7030 
7031 	if (work_done) {
7032 		if (n->gro.bitmask)
7033 			timeout = napi_get_gro_flush_timeout(n);
7034 		n->defer_hard_irqs_count = napi_get_defer_hard_irqs(n);
7035 	}
7036 	if (n->defer_hard_irqs_count > 0) {
7037 		n->defer_hard_irqs_count--;
7038 		timeout = napi_get_gro_flush_timeout(n);
7039 		if (timeout)
7040 			ret = false;
7041 	}
7042 
7043 	/*
7044 	 * When the NAPI instance uses a timeout and keeps postponing
7045 	 * it, we need to bound somehow the time packets are kept in
7046 	 * the GRO layer.
7047 	 */
7048 	gro_flush_normal(&n->gro, !!timeout);
7049 
7050 	if (unlikely(!list_empty(&n->poll_list))) {
7051 		/* If n->poll_list is not empty, we need to mask irqs */
7052 		local_irq_save(flags);
7053 		list_del_init(&n->poll_list);
7054 		local_irq_restore(flags);
7055 	}
7056 	WRITE_ONCE(n->list_owner, -1);
7057 
7058 	val = READ_ONCE(n->state);
7059 	do {
7060 		WARN_ON_ONCE(!(val & NAPIF_STATE_SCHED));
7061 
7062 		new = val & ~(NAPIF_STATE_MISSED | NAPIF_STATE_SCHED |
7063 			      NAPIF_STATE_SCHED_THREADED |
7064 			      NAPIF_STATE_PREFER_BUSY_POLL);
7065 
7066 		/* If STATE_MISSED was set, leave STATE_SCHED set,
7067 		 * because we will call napi->poll() one more time.
7068 		 * This C code was suggested by Alexander Duyck to help gcc.
7069 		 */
7070 		new |= (val & NAPIF_STATE_MISSED) / NAPIF_STATE_MISSED *
7071 						    NAPIF_STATE_SCHED;
7072 	} while (!try_cmpxchg(&n->state, &val, new));
7073 
7074 	if (unlikely(val & NAPIF_STATE_MISSED)) {
7075 		__napi_schedule(n);
7076 		return false;
7077 	}
7078 
7079 	if (timeout)
7080 		hrtimer_start(&n->timer, ns_to_ktime(timeout),
7081 			      HRTIMER_MODE_REL_PINNED);
7082 	return ret;
7083 }
7084 EXPORT_SYMBOL(napi_complete_done);
7085 
7086 static void __skb_defer_free_flush(struct skb_defer_node *sdn, int budget)
7087 {
7088 	struct llist_node *free_list;
7089 	struct sk_buff *skb, *next;
7090 	long n = 0;
7091 
7092 	if (llist_empty(&sdn->defer_list))
7093 		return;
7094 	free_list = llist_del_all(&sdn->defer_list);
7095 
7096 	llist_for_each_entry_safe(skb, next, free_list, ll_node) {
7097 		prefetch(next);
7098 		napi_consume_skb(skb, budget);
7099 		n++;
7100 	}
7101 	/* Subtract exactly what was drained. Never store an absolute value:
7102 	 * mixing one with the relative updates in skb_attempt_defer_free()
7103 	 * lets the counter go negative, and it is read as unsigned there.
7104 	 */
7105 	atomic_long_sub(n, &sdn->defer_count);
7106 }
7107 
7108 void skb_defer_node_flush(struct skb_defer_node *sdn)
7109 {
7110 	__skb_defer_free_flush(sdn, 0);
7111 }
7112 
7113 static void skb_defer_free_flush(void)
7114 {
7115 	struct skb_defer_node *sdn;
7116 	int node;
7117 
7118 	for_each_online_node(node) {
7119 		sdn = this_cpu_ptr(net_hotdata.skb_defer_nodes) + node;
7120 		__skb_defer_free_flush(sdn, 1);
7121 	}
7122 }
7123 
7124 #if defined(CONFIG_NET_RX_BUSY_POLL)
7125 
7126 enum {
7127 	NAPI_F_PREFER_BUSY_POLL	= 1,
7128 	NAPI_F_END_ON_RESCHED	= 2,
7129 };
7130 
7131 static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock,
7132 			   unsigned flags, u16 budget)
7133 {
7134 	struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
7135 	unsigned long timeout = 0;
7136 	int rc;
7137 
7138 	/* Busy polling means there is a high chance device driver hard irq
7139 	 * could not grab NAPI_STATE_SCHED, and that NAPI_STATE_MISSED was
7140 	 * set in napi_schedule_prep().
7141 	 * Since we either call napi->poll() once more or start the timer,
7142 	 * we can safely clear NAPI_STATE_MISSED.
7143 	 *
7144 	 * Note: x86 could use a single "lock and ..." instruction
7145 	 * to perform these two clear_bit()
7146 	 */
7147 	clear_bit(NAPI_STATE_MISSED, &napi->state);
7148 	clear_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state);
7149 
7150 	local_bh_disable();
7151 	bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
7152 
7153 	if (flags & NAPI_F_PREFER_BUSY_POLL) {
7154 		napi->defer_hard_irqs_count = napi_get_defer_hard_irqs(napi);
7155 		if (napi->defer_hard_irqs_count)
7156 			timeout = napi_get_gro_flush_timeout(napi);
7157 	}
7158 	if (timeout) {
7159 		netpoll_poll_unlock(have_poll_lock);
7160 
7161 		/* Drain aged GRO packets before clearing SCHED since the NAPI
7162 		 * won't run again until after the timer fires. When HZ < 1000,
7163 		 * GRO age comparison is too coarse, so flush everything.
7164 		 */
7165 		gro_flush_normal(&napi->gro, HZ >= 1000);
7166 
7167 		clear_bit(NAPI_STATE_SCHED, &napi->state);
7168 		hrtimer_start(&napi->timer, ns_to_ktime(timeout),
7169 			      HRTIMER_MODE_REL_PINNED);
7170 	} else {
7171 		/* Use driver poll to re-enable device interrupts. */
7172 		rc = napi->poll(napi, budget);
7173 		/* Unless rc == budget we no longer own the NAPI instance,
7174 		 * IRQ may fire on another CPU, poll this NAPI, and enter GRO.
7175 		 */
7176 		trace_napi_poll(napi, rc, budget);
7177 		netpoll_poll_unlock(have_poll_lock);
7178 		if (rc == budget) {
7179 			gro_normal_list(&napi->gro);
7180 			__napi_schedule(napi);
7181 		}
7182 	}
7183 
7184 	bpf_net_ctx_clear(bpf_net_ctx);
7185 	local_bh_enable();
7186 }
7187 
7188 static void __napi_busy_loop(unsigned int napi_id,
7189 		      bool (*loop_end)(void *, unsigned long),
7190 		      void *loop_end_arg, unsigned flags, u16 budget)
7191 {
7192 	unsigned long start_time = loop_end ? busy_loop_current_time() : 0;
7193 	int (*napi_poll)(struct napi_struct *napi, int budget);
7194 	struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
7195 	void *have_poll_lock = NULL;
7196 	struct napi_struct *napi;
7197 
7198 	WARN_ON_ONCE(!rcu_read_lock_held());
7199 
7200 restart:
7201 	napi_poll = NULL;
7202 
7203 	napi = napi_by_id(napi_id);
7204 	if (!napi)
7205 		return;
7206 
7207 	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
7208 		preempt_disable();
7209 	for (;;) {
7210 		int work = 0;
7211 
7212 		local_bh_disable();
7213 		bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
7214 		if (!napi_poll) {
7215 			unsigned long val = READ_ONCE(napi->state);
7216 
7217 			/* If multiple threads are competing for this napi,
7218 			 * we avoid dirtying napi->state as much as we can.
7219 			 */
7220 			if (val & (NAPIF_STATE_DISABLE | NAPIF_STATE_SCHED |
7221 				   NAPIF_STATE_IN_BUSY_POLL)) {
7222 				if (flags & NAPI_F_PREFER_BUSY_POLL)
7223 					set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
7224 				goto count;
7225 			}
7226 			if (cmpxchg(&napi->state, val,
7227 				    val | NAPIF_STATE_IN_BUSY_POLL |
7228 					  NAPIF_STATE_SCHED) != val) {
7229 				if (flags & NAPI_F_PREFER_BUSY_POLL)
7230 					set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
7231 				goto count;
7232 			}
7233 			have_poll_lock = netpoll_poll_lock(napi);
7234 			napi_poll = napi->poll;
7235 		}
7236 		work = napi_poll(napi, budget);
7237 		trace_napi_poll(napi, work, budget);
7238 		gro_normal_list(&napi->gro);
7239 count:
7240 		if (work > 0)
7241 			__NET_ADD_STATS(dev_net(napi->dev),
7242 					LINUX_MIB_BUSYPOLLRXPACKETS, work);
7243 		skb_defer_free_flush();
7244 		bpf_net_ctx_clear(bpf_net_ctx);
7245 		local_bh_enable();
7246 
7247 		if (!loop_end || loop_end(loop_end_arg, start_time))
7248 			break;
7249 
7250 		if (unlikely(need_resched())) {
7251 			if (flags & NAPI_F_END_ON_RESCHED)
7252 				break;
7253 			if (napi_poll)
7254 				busy_poll_stop(napi, have_poll_lock, flags, budget);
7255 			if (!IS_ENABLED(CONFIG_PREEMPT_RT))
7256 				preempt_enable();
7257 			rcu_read_unlock();
7258 			cond_resched();
7259 			rcu_read_lock();
7260 			if (loop_end(loop_end_arg, start_time))
7261 				return;
7262 			goto restart;
7263 		}
7264 		cpu_relax();
7265 	}
7266 	if (napi_poll)
7267 		busy_poll_stop(napi, have_poll_lock, flags, budget);
7268 	if (!IS_ENABLED(CONFIG_PREEMPT_RT))
7269 		preempt_enable();
7270 }
7271 
7272 void napi_busy_loop_rcu(unsigned int napi_id,
7273 			bool (*loop_end)(void *, unsigned long),
7274 			void *loop_end_arg, bool prefer_busy_poll, u16 budget)
7275 {
7276 	unsigned flags = NAPI_F_END_ON_RESCHED;
7277 
7278 	if (prefer_busy_poll)
7279 		flags |= NAPI_F_PREFER_BUSY_POLL;
7280 
7281 	__napi_busy_loop(napi_id, loop_end, loop_end_arg, flags, budget);
7282 }
7283 
7284 void napi_busy_loop(unsigned int napi_id,
7285 		    bool (*loop_end)(void *, unsigned long),
7286 		    void *loop_end_arg, bool prefer_busy_poll, u16 budget)
7287 {
7288 	unsigned flags = prefer_busy_poll ? NAPI_F_PREFER_BUSY_POLL : 0;
7289 
7290 	rcu_read_lock();
7291 	__napi_busy_loop(napi_id, loop_end, loop_end_arg, flags, budget);
7292 	rcu_read_unlock();
7293 }
7294 EXPORT_SYMBOL(napi_busy_loop);
7295 
7296 void napi_suspend_irqs(unsigned int napi_id)
7297 {
7298 	struct napi_struct *napi;
7299 
7300 	rcu_read_lock();
7301 	napi = napi_by_id(napi_id);
7302 	if (napi) {
7303 		unsigned long timeout = napi_get_irq_suspend_timeout(napi);
7304 
7305 		if (timeout)
7306 			hrtimer_start(&napi->timer, ns_to_ktime(timeout),
7307 				      HRTIMER_MODE_REL_PINNED);
7308 	}
7309 	rcu_read_unlock();
7310 }
7311 
7312 void napi_resume_irqs(unsigned int napi_id)
7313 {
7314 	struct napi_struct *napi;
7315 
7316 	rcu_read_lock();
7317 	napi = napi_by_id(napi_id);
7318 	if (napi) {
7319 		/* If irq_suspend_timeout is set to 0 between the call to
7320 		 * napi_suspend_irqs and now, the original value still
7321 		 * determines the safety timeout as intended and napi_watchdog
7322 		 * will resume irq processing.
7323 		 */
7324 		if (napi_get_irq_suspend_timeout(napi)) {
7325 			local_bh_disable();
7326 			napi_schedule(napi);
7327 			local_bh_enable();
7328 		}
7329 	}
7330 	rcu_read_unlock();
7331 }
7332 
7333 #endif /* CONFIG_NET_RX_BUSY_POLL */
7334 
7335 static void __napi_hash_add_with_id(struct napi_struct *napi,
7336 				    unsigned int napi_id)
7337 {
7338 	napi->gro.cached_napi_id = napi_id;
7339 
7340 	WRITE_ONCE(napi->napi_id, napi_id);
7341 	hlist_add_head_rcu(&napi->napi_hash_node,
7342 			   &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]);
7343 }
7344 
7345 static void napi_hash_add_with_id(struct napi_struct *napi,
7346 				  unsigned int napi_id)
7347 {
7348 	unsigned long flags;
7349 
7350 	spin_lock_irqsave(&napi_hash_lock, flags);
7351 	WARN_ON_ONCE(napi_by_id(napi_id));
7352 	__napi_hash_add_with_id(napi, napi_id);
7353 	spin_unlock_irqrestore(&napi_hash_lock, flags);
7354 }
7355 
7356 static void napi_hash_add(struct napi_struct *napi)
7357 {
7358 	unsigned long flags;
7359 
7360 	if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state))
7361 		return;
7362 
7363 	spin_lock_irqsave(&napi_hash_lock, flags);
7364 
7365 	/* 0..NR_CPUS range is reserved for sender_cpu use */
7366 	do {
7367 		if (unlikely(!napi_id_valid(++napi_gen_id)))
7368 			napi_gen_id = MIN_NAPI_ID;
7369 	} while (napi_by_id(napi_gen_id));
7370 
7371 	__napi_hash_add_with_id(napi, napi_gen_id);
7372 
7373 	spin_unlock_irqrestore(&napi_hash_lock, flags);
7374 }
7375 
7376 /* Warning : caller is responsible to make sure rcu grace period
7377  * is respected before freeing memory containing @napi
7378  */
7379 static void napi_hash_del(struct napi_struct *napi)
7380 {
7381 	unsigned long flags;
7382 
7383 	spin_lock_irqsave(&napi_hash_lock, flags);
7384 
7385 	hlist_del_init_rcu(&napi->napi_hash_node);
7386 
7387 	spin_unlock_irqrestore(&napi_hash_lock, flags);
7388 }
7389 
7390 static enum hrtimer_restart napi_watchdog(struct hrtimer *timer)
7391 {
7392 	struct napi_struct *napi;
7393 
7394 	napi = container_of(timer, struct napi_struct, timer);
7395 
7396 	/* Note : we use a relaxed variant of napi_schedule_prep() not setting
7397 	 * NAPI_STATE_MISSED, since we do not react to a device IRQ.
7398 	 */
7399 	if (!napi_disable_pending(napi) &&
7400 	    !test_and_set_bit(NAPI_STATE_SCHED, &napi->state)) {
7401 		clear_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state);
7402 		__napi_schedule_irqoff(napi);
7403 	}
7404 
7405 	return HRTIMER_NORESTART;
7406 }
7407 
7408 static void napi_stop_kthread(struct napi_struct *napi)
7409 {
7410 	unsigned long val, new;
7411 
7412 	/* Instance is disabled, we can immediately stop the kthread */
7413 	if (test_bit(NAPI_STATE_DISABLED, &napi->state)) {
7414 		clear_bit(NAPI_STATE_THREADED, &napi->state);
7415 		clear_bit(NAPI_STATE_THREADED_BUSY_POLL, &napi->state);
7416 		goto stop;
7417 	}
7418 
7419 	/* Wait until the napi STATE_THREADED is unset. */
7420 	while (true) {
7421 		val = READ_ONCE(napi->state);
7422 
7423 		/* If napi kthread own this napi or the napi is idle,
7424 		 * STATE_THREADED can be unset here.
7425 		 */
7426 		if ((val & NAPIF_STATE_SCHED_THREADED) ||
7427 		    !(val & NAPIF_STATE_SCHED)) {
7428 			new = val & (~(NAPIF_STATE_THREADED |
7429 				       NAPIF_STATE_THREADED_BUSY_POLL));
7430 		} else {
7431 			msleep(20);
7432 			continue;
7433 		}
7434 
7435 		if (try_cmpxchg(&napi->state, &val, new))
7436 			break;
7437 	}
7438 
7439 	/* Once STATE_THREADED is unset, wait for SCHED_THREADED to be unset by
7440 	 * the kthread.
7441 	 */
7442 	while (true) {
7443 		if (!test_bit(NAPI_STATE_SCHED_THREADED, &napi->state))
7444 			break;
7445 
7446 		msleep(20);
7447 	}
7448 
7449 stop:
7450 	kthread_stop(napi->thread);
7451 	napi->thread = NULL;
7452 }
7453 
7454 static void napi_set_threaded_state(struct napi_struct *napi,
7455 				    enum netdev_napi_threaded threaded_mode)
7456 {
7457 	bool threaded = threaded_mode != NETDEV_NAPI_THREADED_DISABLED;
7458 	bool busy_poll = threaded_mode == NETDEV_NAPI_THREADED_BUSY_POLL;
7459 
7460 	assign_bit(NAPI_STATE_THREADED, &napi->state, threaded);
7461 	assign_bit(NAPI_STATE_THREADED_BUSY_POLL, &napi->state, busy_poll);
7462 }
7463 
7464 int napi_set_threaded(struct napi_struct *napi,
7465 		      enum netdev_napi_threaded threaded)
7466 {
7467 	if (threaded) {
7468 		if (!napi->thread) {
7469 			int err = napi_kthread_create(napi);
7470 
7471 			if (err)
7472 				return err;
7473 		}
7474 	}
7475 
7476 	if (napi->config)
7477 		napi->config->threaded = threaded;
7478 
7479 	/* Setting/unsetting threaded mode on a napi might not immediately
7480 	 * take effect, if the current napi instance is actively being
7481 	 * polled. In this case, the switch between threaded mode and
7482 	 * softirq mode will happen in the next round of napi_schedule().
7483 	 * This should not cause hiccups/stalls to the live traffic.
7484 	 */
7485 	if (!threaded && napi->thread) {
7486 		napi_stop_kthread(napi);
7487 	} else {
7488 		/* Make sure kthread is created before THREADED bit is set. */
7489 		smp_mb__before_atomic();
7490 		napi_set_threaded_state(napi, threaded);
7491 	}
7492 
7493 	return 0;
7494 }
7495 
7496 int netif_set_threaded(struct net_device *dev,
7497 		       enum netdev_napi_threaded threaded)
7498 {
7499 	struct napi_struct *napi;
7500 	int i, err = 0;
7501 
7502 	netdev_assert_locked_or_invisible(dev);
7503 
7504 	if (threaded) {
7505 		list_for_each_entry(napi, &dev->napi_list, dev_list) {
7506 			if (!napi->thread) {
7507 				err = napi_kthread_create(napi);
7508 				if (err) {
7509 					threaded = NETDEV_NAPI_THREADED_DISABLED;
7510 					break;
7511 				}
7512 			}
7513 		}
7514 	}
7515 
7516 	WRITE_ONCE(dev->threaded, threaded);
7517 
7518 	/* The error should not occur as the kthreads are already created. */
7519 	list_for_each_entry(napi, &dev->napi_list, dev_list)
7520 		WARN_ON_ONCE(napi_set_threaded(napi, threaded));
7521 
7522 	/* Override the config for all NAPIs even if currently not listed */
7523 	for (i = 0; i < dev->num_napi_configs; i++)
7524 		dev->napi_config[i].threaded = threaded;
7525 
7526 	return err;
7527 }
7528 
7529 /**
7530  * netif_threaded_enable() - enable threaded NAPIs
7531  * @dev: net_device instance
7532  *
7533  * Enable threaded mode for the NAPI instances of the device. This may be useful
7534  * for devices where multiple NAPI instances get scheduled by a single
7535  * interrupt. Threaded NAPI allows moving the NAPI processing to cores other
7536  * than the core where IRQ is mapped.
7537  *
7538  * This function should be called before @dev is registered.
7539  */
7540 void netif_threaded_enable(struct net_device *dev)
7541 {
7542 	WARN_ON_ONCE(netif_set_threaded(dev, NETDEV_NAPI_THREADED_ENABLED));
7543 }
7544 EXPORT_SYMBOL(netif_threaded_enable);
7545 
7546 /**
7547  * netif_queue_set_napi - Associate queue with the napi
7548  * @dev: device to which NAPI and queue belong
7549  * @queue_index: Index of queue
7550  * @type: queue type as RX or TX
7551  * @napi: NAPI context, pass NULL to clear previously set NAPI
7552  *
7553  * Set queue with its corresponding napi context. This should be done after
7554  * registering the NAPI handler for the queue-vector and the queues have been
7555  * mapped to the corresponding interrupt vector.
7556  */
7557 void netif_queue_set_napi(struct net_device *dev, unsigned int queue_index,
7558 			  enum netdev_queue_type type, struct napi_struct *napi)
7559 {
7560 	struct netdev_rx_queue *rxq;
7561 	struct netdev_queue *txq;
7562 
7563 	if (WARN_ON_ONCE(napi && !napi->dev))
7564 		return;
7565 	netdev_assert_locked_ops_compat_or_invisible(dev);
7566 
7567 	switch (type) {
7568 	case NETDEV_QUEUE_TYPE_RX:
7569 		rxq = __netif_get_rx_queue(dev, queue_index);
7570 		rxq->napi = napi;
7571 		return;
7572 	case NETDEV_QUEUE_TYPE_TX:
7573 		txq = netdev_get_tx_queue(dev, queue_index);
7574 		txq->napi = napi;
7575 		return;
7576 	default:
7577 		return;
7578 	}
7579 }
7580 EXPORT_SYMBOL(netif_queue_set_napi);
7581 
7582 static void
7583 netif_napi_irq_notify(struct irq_affinity_notify *notify,
7584 		      const cpumask_t *mask)
7585 {
7586 	struct napi_struct *napi =
7587 		container_of(notify, struct napi_struct, notify);
7588 #ifdef CONFIG_RFS_ACCEL
7589 	struct cpu_rmap *rmap = napi->dev->rx_cpu_rmap;
7590 	int err;
7591 #endif
7592 
7593 	if (napi->config && napi->dev->irq_affinity_auto)
7594 		cpumask_copy(&napi->config->affinity_mask, mask);
7595 
7596 #ifdef CONFIG_RFS_ACCEL
7597 	if (napi->dev->rx_cpu_rmap_auto) {
7598 		err = cpu_rmap_update(rmap, napi->napi_rmap_idx, mask);
7599 		if (err)
7600 			netdev_warn(napi->dev, "RMAP update failed (%d)\n",
7601 				    err);
7602 	}
7603 #endif
7604 }
7605 
7606 #ifdef CONFIG_RFS_ACCEL
7607 static void netif_napi_affinity_release(struct kref *ref)
7608 {
7609 	struct napi_struct *napi =
7610 		container_of(ref, struct napi_struct, notify.kref);
7611 	struct cpu_rmap *rmap = napi->dev->rx_cpu_rmap;
7612 
7613 	netdev_assert_locked(napi->dev);
7614 	WARN_ON(test_and_clear_bit(NAPI_STATE_HAS_NOTIFIER,
7615 				   &napi->state));
7616 
7617 	if (!napi->dev->rx_cpu_rmap_auto)
7618 		return;
7619 	rmap->obj[napi->napi_rmap_idx] = NULL;
7620 	napi->napi_rmap_idx = -1;
7621 	cpu_rmap_put(rmap);
7622 }
7623 
7624 int netif_enable_cpu_rmap(struct net_device *dev, unsigned int num_irqs)
7625 {
7626 	if (dev->rx_cpu_rmap_auto)
7627 		return 0;
7628 
7629 	dev->rx_cpu_rmap = alloc_irq_cpu_rmap(num_irqs);
7630 	if (!dev->rx_cpu_rmap)
7631 		return -ENOMEM;
7632 
7633 	dev->rx_cpu_rmap_auto = true;
7634 	return 0;
7635 }
7636 EXPORT_SYMBOL(netif_enable_cpu_rmap);
7637 
7638 static void netif_del_cpu_rmap(struct net_device *dev)
7639 {
7640 	struct cpu_rmap *rmap = dev->rx_cpu_rmap;
7641 
7642 	if (!dev->rx_cpu_rmap_auto)
7643 		return;
7644 
7645 	/* Free the rmap */
7646 	cpu_rmap_put(rmap);
7647 	dev->rx_cpu_rmap = NULL;
7648 	dev->rx_cpu_rmap_auto = false;
7649 }
7650 
7651 #else
7652 static void netif_napi_affinity_release(struct kref *ref)
7653 {
7654 }
7655 
7656 int netif_enable_cpu_rmap(struct net_device *dev, unsigned int num_irqs)
7657 {
7658 	return 0;
7659 }
7660 EXPORT_SYMBOL(netif_enable_cpu_rmap);
7661 
7662 static void netif_del_cpu_rmap(struct net_device *dev)
7663 {
7664 }
7665 #endif
7666 
7667 void netif_set_affinity_auto(struct net_device *dev)
7668 {
7669 	unsigned int i, maxqs, numa;
7670 
7671 	maxqs = max(dev->num_tx_queues, dev->num_rx_queues);
7672 	numa = dev_to_node(&dev->dev);
7673 
7674 	for (i = 0; i < maxqs; i++)
7675 		cpumask_set_cpu(cpumask_local_spread(i, numa),
7676 				&dev->napi_config[i].affinity_mask);
7677 
7678 	dev->irq_affinity_auto = true;
7679 }
7680 EXPORT_SYMBOL(netif_set_affinity_auto);
7681 
7682 void netif_napi_set_irq_locked(struct napi_struct *napi, int irq)
7683 {
7684 	int rc;
7685 
7686 	netdev_assert_locked_or_invisible(napi->dev);
7687 
7688 	if (napi->irq == irq)
7689 		return;
7690 
7691 	/* Remove existing resources */
7692 	if (test_and_clear_bit(NAPI_STATE_HAS_NOTIFIER, &napi->state))
7693 		irq_set_affinity_notifier(napi->irq, NULL);
7694 
7695 	napi->irq = irq;
7696 	if (irq < 0 ||
7697 	    (!napi->dev->rx_cpu_rmap_auto && !napi->dev->irq_affinity_auto))
7698 		return;
7699 
7700 	/* Abort for buggy drivers */
7701 	if (napi->dev->irq_affinity_auto && WARN_ON_ONCE(!napi->config))
7702 		return;
7703 
7704 #ifdef CONFIG_RFS_ACCEL
7705 	if (napi->dev->rx_cpu_rmap_auto) {
7706 		rc = cpu_rmap_add(napi->dev->rx_cpu_rmap, napi);
7707 		if (rc < 0)
7708 			return;
7709 
7710 		cpu_rmap_get(napi->dev->rx_cpu_rmap);
7711 		napi->napi_rmap_idx = rc;
7712 	}
7713 #endif
7714 
7715 	/* Use core IRQ notifier */
7716 	napi->notify.notify = netif_napi_irq_notify;
7717 	napi->notify.release = netif_napi_affinity_release;
7718 	rc = irq_set_affinity_notifier(irq, &napi->notify);
7719 	if (rc) {
7720 		netdev_warn(napi->dev, "Unable to set IRQ notifier (%d)\n",
7721 			    rc);
7722 		goto put_rmap;
7723 	}
7724 
7725 	set_bit(NAPI_STATE_HAS_NOTIFIER, &napi->state);
7726 	return;
7727 
7728 put_rmap:
7729 #ifdef CONFIG_RFS_ACCEL
7730 	if (napi->dev->rx_cpu_rmap_auto) {
7731 		napi->dev->rx_cpu_rmap->obj[napi->napi_rmap_idx] = NULL;
7732 		cpu_rmap_put(napi->dev->rx_cpu_rmap);
7733 		napi->napi_rmap_idx = -1;
7734 	}
7735 #endif
7736 	napi->notify.notify = NULL;
7737 	napi->notify.release = NULL;
7738 }
7739 EXPORT_SYMBOL(netif_napi_set_irq_locked);
7740 
7741 static void napi_restore_config(struct napi_struct *n)
7742 {
7743 	n->defer_hard_irqs = n->config->defer_hard_irqs;
7744 	n->gro_flush_timeout = n->config->gro_flush_timeout;
7745 	n->irq_suspend_timeout = n->config->irq_suspend_timeout;
7746 
7747 	if (n->dev->irq_affinity_auto &&
7748 	    test_bit(NAPI_STATE_HAS_NOTIFIER, &n->state))
7749 		irq_set_affinity(n->irq, &n->config->affinity_mask);
7750 
7751 	/* a NAPI ID might be stored in the config, if so use it. if not, use
7752 	 * napi_hash_add to generate one for us.
7753 	 */
7754 	if (n->config->napi_id) {
7755 		napi_hash_add_with_id(n, n->config->napi_id);
7756 	} else {
7757 		napi_hash_add(n);
7758 		n->config->napi_id = n->napi_id;
7759 	}
7760 
7761 	WARN_ON_ONCE(napi_set_threaded(n, n->config->threaded));
7762 }
7763 
7764 static void napi_save_config(struct napi_struct *n)
7765 {
7766 	n->config->defer_hard_irqs = n->defer_hard_irqs;
7767 	n->config->gro_flush_timeout = n->gro_flush_timeout;
7768 	n->config->irq_suspend_timeout = n->irq_suspend_timeout;
7769 	napi_hash_del(n);
7770 }
7771 
7772 /* Netlink wants the NAPI list to be sorted by ID, if adding a NAPI which will
7773  * inherit an existing ID try to insert it at the right position.
7774  */
7775 static void
7776 netif_napi_dev_list_add(struct net_device *dev, struct napi_struct *napi)
7777 {
7778 	unsigned int new_id, pos_id;
7779 	struct list_head *higher;
7780 	struct napi_struct *pos;
7781 
7782 	new_id = UINT_MAX;
7783 	if (napi->config && napi->config->napi_id)
7784 		new_id = napi->config->napi_id;
7785 
7786 	higher = &dev->napi_list;
7787 	list_for_each_entry(pos, &dev->napi_list, dev_list) {
7788 		if (napi_id_valid(pos->napi_id))
7789 			pos_id = pos->napi_id;
7790 		else if (pos->config)
7791 			pos_id = pos->config->napi_id;
7792 		else
7793 			pos_id = UINT_MAX;
7794 
7795 		if (pos_id <= new_id)
7796 			break;
7797 		higher = &pos->dev_list;
7798 	}
7799 	list_add_rcu(&napi->dev_list, higher); /* adds after higher */
7800 }
7801 
7802 /* Double check that napi_get_frags() allocates skbs with
7803  * skb->head being backed by slab, not a page fragment.
7804  * This is to make sure bug fixed in 3226b158e67c
7805  * ("net: avoid 32 x truesize under-estimation for tiny skbs")
7806  * does not accidentally come back.
7807  */
7808 static void napi_get_frags_check(struct napi_struct *napi)
7809 {
7810 	struct sk_buff *skb;
7811 
7812 	local_bh_disable();
7813 	skb = napi_get_frags(napi);
7814 	WARN_ON_ONCE(skb && skb->head_frag);
7815 	napi_free_frags(napi);
7816 	local_bh_enable();
7817 }
7818 
7819 void netif_napi_add_weight_locked(struct net_device *dev,
7820 				  struct napi_struct *napi,
7821 				  int (*poll)(struct napi_struct *, int),
7822 				  int weight)
7823 {
7824 	netdev_assert_locked(dev);
7825 	if (WARN_ON(test_and_set_bit(NAPI_STATE_LISTED, &napi->state)))
7826 		return;
7827 
7828 	INIT_LIST_HEAD(&napi->poll_list);
7829 	INIT_HLIST_NODE(&napi->napi_hash_node);
7830 	hrtimer_setup(&napi->timer, napi_watchdog, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED);
7831 	gro_init(&napi->gro);
7832 	napi->skb = NULL;
7833 	napi->poll = poll;
7834 	if (weight > NAPI_POLL_WEIGHT)
7835 		netdev_err_once(dev, "%s() called with weight %d\n", __func__,
7836 				weight);
7837 	napi->weight = weight;
7838 	napi->dev = dev;
7839 #ifdef CONFIG_NETPOLL
7840 	napi->poll_owner = -1;
7841 #endif
7842 	napi->list_owner = -1;
7843 	set_bit(NAPI_STATE_SCHED, &napi->state);
7844 	set_bit(NAPI_STATE_NPSVC, &napi->state);
7845 	set_bit(NAPI_STATE_DISABLED, &napi->state);
7846 	netif_napi_dev_list_add(dev, napi);
7847 
7848 	/* default settings from sysfs are applied to all NAPIs. any per-NAPI
7849 	 * configuration will be loaded in napi_enable
7850 	 */
7851 	napi_set_defer_hard_irqs(napi, READ_ONCE(dev->napi_defer_hard_irqs));
7852 	napi_set_gro_flush_timeout(napi, READ_ONCE(dev->gro_flush_timeout));
7853 
7854 	napi_get_frags_check(napi);
7855 	/* Create kthread for this napi if dev->threaded is set.
7856 	 * Clear dev->threaded if kthread creation failed so that
7857 	 * threaded mode will not be enabled in napi_enable().
7858 	 */
7859 	if (napi_get_threaded_config(dev, napi))
7860 		if (napi_kthread_create(napi))
7861 			dev->threaded = NETDEV_NAPI_THREADED_DISABLED;
7862 	netif_napi_set_irq_locked(napi, -1);
7863 }
7864 EXPORT_SYMBOL(netif_napi_add_weight_locked);
7865 
7866 void napi_disable_locked(struct napi_struct *n)
7867 {
7868 	unsigned long val, new;
7869 
7870 	might_sleep();
7871 	netdev_assert_locked(n->dev);
7872 
7873 	set_bit(NAPI_STATE_DISABLE, &n->state);
7874 
7875 	val = READ_ONCE(n->state);
7876 	do {
7877 		while (val & (NAPIF_STATE_SCHED | NAPIF_STATE_NPSVC)) {
7878 			usleep_range(20, 200);
7879 			val = READ_ONCE(n->state);
7880 		}
7881 
7882 		new = val | NAPIF_STATE_SCHED | NAPIF_STATE_NPSVC |
7883 		      NAPIF_STATE_DISABLED;
7884 		new &= ~(NAPIF_STATE_THREADED |
7885 			 NAPIF_STATE_THREADED_BUSY_POLL |
7886 			 NAPIF_STATE_PREFER_BUSY_POLL);
7887 	} while (!try_cmpxchg(&n->state, &val, new));
7888 
7889 	hrtimer_cancel(&n->timer);
7890 
7891 	if (n->config)
7892 		napi_save_config(n);
7893 	else
7894 		napi_hash_del(n);
7895 
7896 	clear_bit(NAPI_STATE_DISABLE, &n->state);
7897 }
7898 EXPORT_SYMBOL(napi_disable_locked);
7899 
7900 /**
7901  * napi_disable() - prevent NAPI from scheduling
7902  * @n: NAPI context
7903  *
7904  * Stop NAPI from being scheduled on this context.
7905  * Waits till any outstanding processing completes.
7906  * Takes netdev_lock() for associated net_device.
7907  */
7908 void napi_disable(struct napi_struct *n)
7909 {
7910 	netdev_lock(n->dev);
7911 	napi_disable_locked(n);
7912 	netdev_unlock(n->dev);
7913 }
7914 EXPORT_SYMBOL(napi_disable);
7915 
7916 void napi_enable_locked(struct napi_struct *n)
7917 {
7918 	unsigned long new, val = READ_ONCE(n->state);
7919 
7920 	if (n->config)
7921 		napi_restore_config(n);
7922 	else
7923 		napi_hash_add(n);
7924 
7925 	do {
7926 		BUG_ON(!test_bit(NAPI_STATE_SCHED, &val));
7927 
7928 		new = val & ~(NAPIF_STATE_SCHED | NAPIF_STATE_NPSVC |
7929 			      NAPIF_STATE_DISABLED);
7930 		if (n->dev->threaded && n->thread)
7931 			new |= NAPIF_STATE_THREADED;
7932 	} while (!try_cmpxchg(&n->state, &val, new));
7933 }
7934 EXPORT_SYMBOL(napi_enable_locked);
7935 
7936 /**
7937  * napi_enable() - enable NAPI scheduling
7938  * @n: NAPI context
7939  *
7940  * Enable scheduling of a NAPI instance.
7941  * Must be paired with napi_disable().
7942  * Takes netdev_lock() for associated net_device.
7943  */
7944 void napi_enable(struct napi_struct *n)
7945 {
7946 	netdev_lock(n->dev);
7947 	napi_enable_locked(n);
7948 	netdev_unlock(n->dev);
7949 }
7950 EXPORT_SYMBOL(napi_enable);
7951 
7952 /* Must be called in process context */
7953 void __netif_napi_del_locked(struct napi_struct *napi)
7954 {
7955 	netdev_assert_locked(napi->dev);
7956 
7957 	if (!test_and_clear_bit(NAPI_STATE_LISTED, &napi->state))
7958 		return;
7959 
7960 	/* Make sure NAPI is disabled (or was never enabled). */
7961 	WARN_ON(!test_bit(NAPI_STATE_SCHED, &napi->state));
7962 
7963 	if (test_and_clear_bit(NAPI_STATE_HAS_NOTIFIER, &napi->state))
7964 		irq_set_affinity_notifier(napi->irq, NULL);
7965 
7966 	if (napi->config) {
7967 		napi->index = -1;
7968 		napi->config = NULL;
7969 	}
7970 
7971 	list_del_rcu(&napi->dev_list);
7972 	napi_free_frags(napi);
7973 
7974 	gro_cleanup(&napi->gro);
7975 
7976 	if (napi->thread) {
7977 		kthread_stop(napi->thread);
7978 		napi->thread = NULL;
7979 	}
7980 }
7981 EXPORT_SYMBOL(__netif_napi_del_locked);
7982 
7983 static int __napi_poll(struct napi_struct *n, bool *repoll)
7984 {
7985 	int work, weight;
7986 
7987 	weight = n->weight;
7988 
7989 	/* This NAPI_STATE_SCHED test is for avoiding a race
7990 	 * with netpoll's poll_napi().  Only the entity which
7991 	 * obtains the lock and sees NAPI_STATE_SCHED set will
7992 	 * actually make the ->poll() call.  Therefore we avoid
7993 	 * accidentally calling ->poll() when NAPI is not scheduled.
7994 	 */
7995 	work = 0;
7996 	if (napi_is_scheduled(n)) {
7997 		work = n->poll(n, weight);
7998 		trace_napi_poll(n, work, weight);
7999 
8000 		xdp_do_check_flushed(n);
8001 	}
8002 
8003 	if (unlikely(work > weight))
8004 		netdev_err_once(n->dev, "NAPI poll function %pS returned %d, exceeding its budget of %d.\n",
8005 				n->poll, work, weight);
8006 
8007 	if (likely(work < weight))
8008 		return work;
8009 
8010 	/* Drivers must not modify the NAPI state if they
8011 	 * consume the entire weight.  In such cases this code
8012 	 * still "owns" the NAPI instance and therefore can
8013 	 * move the instance around on the list at-will.
8014 	 */
8015 	if (unlikely(napi_disable_pending(n))) {
8016 		napi_complete(n);
8017 		return work;
8018 	}
8019 
8020 	/* The NAPI context has more processing work, but busy-polling
8021 	 * is preferred. Exit early.
8022 	 */
8023 	if (napi_prefer_busy_poll(n)) {
8024 		if (napi_complete_done(n, work)) {
8025 			/* If timeout is not set, we need to make sure
8026 			 * that the NAPI is re-scheduled.
8027 			 */
8028 			napi_schedule(n);
8029 		}
8030 		return work;
8031 	}
8032 
8033 	/* Flush too old packets. If HZ < 1000, flush all packets */
8034 	gro_flush_normal(&n->gro, HZ >= 1000);
8035 
8036 	/* Some drivers may have called napi_schedule
8037 	 * prior to exhausting their budget.
8038 	 */
8039 	if (unlikely(!list_empty(&n->poll_list))) {
8040 		pr_warn_once("%s: Budget exhausted after napi rescheduled\n",
8041 			     n->dev ? n->dev->name : "backlog");
8042 		return work;
8043 	}
8044 
8045 	*repoll = true;
8046 
8047 	return work;
8048 }
8049 
8050 static int napi_poll(struct napi_struct *n, struct list_head *repoll)
8051 {
8052 	bool do_repoll = false;
8053 	void *have;
8054 	int work;
8055 
8056 	list_del_init(&n->poll_list);
8057 
8058 	have = netpoll_poll_lock(n);
8059 
8060 	work = __napi_poll(n, &do_repoll);
8061 
8062 	if (do_repoll) {
8063 #if defined(CONFIG_DEBUG_NET)
8064 		if (unlikely(!napi_is_scheduled(n)))
8065 			pr_crit("repoll requested for device %s %ps but napi is not scheduled.\n",
8066 				n->dev->name, n->poll);
8067 #endif
8068 		list_add_tail(&n->poll_list, repoll);
8069 	}
8070 	netpoll_poll_unlock(have);
8071 
8072 	return work;
8073 }
8074 
8075 static int napi_thread_wait(struct napi_struct *napi)
8076 {
8077 	set_current_state(TASK_INTERRUPTIBLE);
8078 
8079 	while (!kthread_should_stop()) {
8080 		/* Testing SCHED_THREADED bit here to make sure the current
8081 		 * kthread owns this napi and could poll on this napi.
8082 		 * Testing SCHED bit is not enough because SCHED bit might be
8083 		 * set by some other busy poll thread or by napi_disable().
8084 		 */
8085 		if (test_bit(NAPI_STATE_SCHED_THREADED, &napi->state)) {
8086 			WARN_ON(!list_empty(&napi->poll_list));
8087 			__set_current_state(TASK_RUNNING);
8088 			return 0;
8089 		}
8090 
8091 		schedule();
8092 		set_current_state(TASK_INTERRUPTIBLE);
8093 	}
8094 	__set_current_state(TASK_RUNNING);
8095 
8096 	return -1;
8097 }
8098 
8099 static void napi_threaded_poll_loop(struct napi_struct *napi,
8100 				    unsigned long *busy_poll_last_qs)
8101 {
8102 	unsigned long last_qs = busy_poll_last_qs ? *busy_poll_last_qs : jiffies;
8103 	struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
8104 	struct softnet_data *sd;
8105 
8106 	for (;;) {
8107 		bool repoll = false;
8108 		void *have;
8109 
8110 		local_bh_disable();
8111 		bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
8112 
8113 		sd = this_cpu_ptr(&softnet_data);
8114 		sd->in_napi_threaded_poll = true;
8115 
8116 		have = netpoll_poll_lock(napi);
8117 		__napi_poll(napi, &repoll);
8118 		netpoll_poll_unlock(have);
8119 
8120 		sd->in_napi_threaded_poll = false;
8121 		barrier();
8122 
8123 		if (sd_has_rps_ipi_waiting(sd)) {
8124 			local_irq_disable();
8125 			net_rps_action_and_irq_enable(sd);
8126 		}
8127 		skb_defer_free_flush();
8128 		bpf_net_ctx_clear(bpf_net_ctx);
8129 
8130 		/* When busy poll is enabled, the old packets are not flushed in
8131 		 * napi_complete_done. So flush them here.
8132 		 */
8133 		if (busy_poll_last_qs)
8134 			gro_flush_normal(&napi->gro, HZ >= 1000);
8135 		local_bh_enable();
8136 
8137 		/* Call cond_resched here to avoid watchdog warnings. */
8138 		if (repoll || busy_poll_last_qs) {
8139 			rcu_softirq_qs_periodic(last_qs);
8140 			cond_resched();
8141 		}
8142 
8143 		if (!repoll)
8144 			break;
8145 	}
8146 
8147 	if (busy_poll_last_qs)
8148 		*busy_poll_last_qs = last_qs;
8149 }
8150 
8151 static int napi_threaded_poll(void *data)
8152 {
8153 	struct napi_struct *napi = data;
8154 	unsigned long last_qs = jiffies;
8155 	bool want_busy_poll;
8156 	bool in_busy_poll;
8157 	unsigned long val;
8158 
8159 	while (!napi_thread_wait(napi)) {
8160 		val = READ_ONCE(napi->state);
8161 
8162 		want_busy_poll = val & NAPIF_STATE_THREADED_BUSY_POLL;
8163 		in_busy_poll = val & NAPIF_STATE_IN_BUSY_POLL;
8164 
8165 		if (unlikely(val & NAPIF_STATE_DISABLE))
8166 			want_busy_poll = false;
8167 
8168 		if (want_busy_poll != in_busy_poll)
8169 			assign_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state,
8170 				   want_busy_poll);
8171 
8172 		napi_threaded_poll_loop(napi, want_busy_poll ? &last_qs : NULL);
8173 	}
8174 
8175 	return 0;
8176 }
8177 
8178 static __latent_entropy void net_rx_action(void)
8179 {
8180 	struct softnet_data *sd = this_cpu_ptr(&softnet_data);
8181 	unsigned long time_limit = jiffies +
8182 		usecs_to_jiffies(READ_ONCE(net_hotdata.netdev_budget_usecs));
8183 	struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx;
8184 	int budget = READ_ONCE(net_hotdata.netdev_budget);
8185 	LIST_HEAD(list);
8186 	LIST_HEAD(repoll);
8187 
8188 	bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx);
8189 start:
8190 	sd->in_net_rx_action = true;
8191 	local_irq_disable();
8192 	list_splice_init(&sd->poll_list, &list);
8193 	local_irq_enable();
8194 
8195 	for (;;) {
8196 		struct napi_struct *n;
8197 
8198 		skb_defer_free_flush();
8199 
8200 		if (list_empty(&list)) {
8201 			if (list_empty(&repoll)) {
8202 				sd->in_net_rx_action = false;
8203 				barrier();
8204 				/* We need to check if ____napi_schedule()
8205 				 * had refilled poll_list while
8206 				 * sd->in_net_rx_action was true.
8207 				 */
8208 				if (!list_empty(&sd->poll_list))
8209 					goto start;
8210 				if (!sd_has_rps_ipi_waiting(sd))
8211 					goto end;
8212 			}
8213 			break;
8214 		}
8215 
8216 		n = list_first_entry(&list, struct napi_struct, poll_list);
8217 		budget -= napi_poll(n, &repoll);
8218 
8219 		/* If softirq window is exhausted then punt.
8220 		 * Allow this to run for 2 jiffies since which will allow
8221 		 * an average latency of 1.5/HZ.
8222 		 */
8223 		if (unlikely(budget <= 0 ||
8224 			     time_after_eq(jiffies, time_limit))) {
8225 			/* Pairs with READ_ONCE() in softnet_seq_show() */
8226 			WRITE_ONCE(sd->time_squeeze, sd->time_squeeze + 1);
8227 			break;
8228 		}
8229 	}
8230 
8231 	local_irq_disable();
8232 
8233 	list_splice_tail_init(&sd->poll_list, &list);
8234 	list_splice_tail(&repoll, &list);
8235 	list_splice(&list, &sd->poll_list);
8236 	if (!list_empty(&sd->poll_list))
8237 		__raise_softirq_irqoff(NET_RX_SOFTIRQ);
8238 	else
8239 		sd->in_net_rx_action = false;
8240 
8241 	net_rps_action_and_irq_enable(sd);
8242 end:
8243 	bpf_net_ctx_clear(bpf_net_ctx);
8244 }
8245 
8246 struct netdev_adjacent {
8247 	struct net_device *dev;
8248 	netdevice_tracker dev_tracker;
8249 
8250 	/* upper master flag, there can only be one master device per list */
8251 	bool master;
8252 
8253 	/* lookup ignore flag */
8254 	bool ignore;
8255 
8256 	/* counter for the number of times this device was added to us */
8257 	u16 ref_nr;
8258 
8259 	/* private field for the users */
8260 	void *private;
8261 
8262 	struct list_head list;
8263 	struct rcu_head rcu;
8264 };
8265 
8266 static struct netdev_adjacent *__netdev_find_adj(struct net_device *adj_dev,
8267 						 struct list_head *adj_list)
8268 {
8269 	struct netdev_adjacent *adj;
8270 
8271 	list_for_each_entry(adj, adj_list, list) {
8272 		if (adj->dev == adj_dev)
8273 			return adj;
8274 	}
8275 	return NULL;
8276 }
8277 
8278 static int ____netdev_has_upper_dev(struct net_device *upper_dev,
8279 				    struct netdev_nested_priv *priv)
8280 {
8281 	struct net_device *dev = (struct net_device *)priv->data;
8282 
8283 	return upper_dev == dev;
8284 }
8285 
8286 /**
8287  * netdev_has_upper_dev - Check if device is linked to an upper device
8288  * @dev: device
8289  * @upper_dev: upper device to check
8290  *
8291  * Find out if a device is linked to specified upper device and return true
8292  * in case it is. Note that this checks only immediate upper device,
8293  * not through a complete stack of devices. The caller must hold the RTNL lock.
8294  */
8295 bool netdev_has_upper_dev(struct net_device *dev,
8296 			  struct net_device *upper_dev)
8297 {
8298 	struct netdev_nested_priv priv = {
8299 		.data = (void *)upper_dev,
8300 	};
8301 
8302 	ASSERT_RTNL();
8303 
8304 	return netdev_walk_all_upper_dev_rcu(dev, ____netdev_has_upper_dev,
8305 					     &priv);
8306 }
8307 EXPORT_SYMBOL(netdev_has_upper_dev);
8308 
8309 /**
8310  * netdev_has_upper_dev_all_rcu - Check if device is linked to an upper device
8311  * @dev: device
8312  * @upper_dev: upper device to check
8313  *
8314  * Find out if a device is linked to specified upper device and return true
8315  * in case it is. Note that this checks the entire upper device chain.
8316  * The caller must hold rcu lock.
8317  */
8318 
8319 bool netdev_has_upper_dev_all_rcu(struct net_device *dev,
8320 				  struct net_device *upper_dev)
8321 {
8322 	struct netdev_nested_priv priv = {
8323 		.data = (void *)upper_dev,
8324 	};
8325 
8326 	return !!netdev_walk_all_upper_dev_rcu(dev, ____netdev_has_upper_dev,
8327 					       &priv);
8328 }
8329 EXPORT_SYMBOL(netdev_has_upper_dev_all_rcu);
8330 
8331 /**
8332  * netdev_has_any_upper_dev - Check if device is linked to some device
8333  * @dev: device
8334  *
8335  * Find out if a device is linked to an upper device and return true in case
8336  * it is. The caller must hold the RTNL lock.
8337  */
8338 bool netdev_has_any_upper_dev(struct net_device *dev)
8339 {
8340 	ASSERT_RTNL();
8341 
8342 	return !list_empty(&dev->adj_list.upper);
8343 }
8344 EXPORT_SYMBOL(netdev_has_any_upper_dev);
8345 
8346 /**
8347  * netdev_master_upper_dev_get - Get master upper device
8348  * @dev: device
8349  *
8350  * Find a master upper device and return pointer to it or NULL in case
8351  * it's not there. The caller must hold the RTNL lock.
8352  */
8353 struct net_device *netdev_master_upper_dev_get(struct net_device *dev)
8354 {
8355 	struct netdev_adjacent *upper;
8356 
8357 	ASSERT_RTNL();
8358 
8359 	if (list_empty(&dev->adj_list.upper))
8360 		return NULL;
8361 
8362 	upper = list_first_entry(&dev->adj_list.upper,
8363 				 struct netdev_adjacent, list);
8364 	if (likely(upper->master))
8365 		return upper->dev;
8366 	return NULL;
8367 }
8368 EXPORT_SYMBOL(netdev_master_upper_dev_get);
8369 
8370 static struct net_device *__netdev_master_upper_dev_get(struct net_device *dev)
8371 {
8372 	struct netdev_adjacent *upper;
8373 
8374 	ASSERT_RTNL();
8375 
8376 	if (list_empty(&dev->adj_list.upper))
8377 		return NULL;
8378 
8379 	upper = list_first_entry(&dev->adj_list.upper,
8380 				 struct netdev_adjacent, list);
8381 	if (likely(upper->master) && !upper->ignore)
8382 		return upper->dev;
8383 	return NULL;
8384 }
8385 
8386 /**
8387  * netdev_has_any_lower_dev - Check if device is linked to some device
8388  * @dev: device
8389  *
8390  * Find out if a device is linked to a lower device and return true in case
8391  * it is. The caller must hold the RTNL lock.
8392  */
8393 static bool netdev_has_any_lower_dev(struct net_device *dev)
8394 {
8395 	ASSERT_RTNL();
8396 
8397 	return !list_empty(&dev->adj_list.lower);
8398 }
8399 
8400 void *netdev_adjacent_get_private(struct list_head *adj_list)
8401 {
8402 	struct netdev_adjacent *adj;
8403 
8404 	adj = list_entry(adj_list, struct netdev_adjacent, list);
8405 
8406 	return adj->private;
8407 }
8408 EXPORT_SYMBOL(netdev_adjacent_get_private);
8409 
8410 /**
8411  * netdev_upper_get_next_dev_rcu - Get the next dev from upper list
8412  * @dev: device
8413  * @iter: list_head ** of the current position
8414  *
8415  * Gets the next device from the dev's upper list, starting from iter
8416  * position. The caller must hold RCU read lock.
8417  */
8418 struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
8419 						 struct list_head **iter)
8420 {
8421 	struct netdev_adjacent *upper;
8422 
8423 	WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held() &&
8424 		     !lockdep_rtnl_is_held());
8425 
8426 	upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
8427 
8428 	if (&upper->list == &dev->adj_list.upper)
8429 		return NULL;
8430 
8431 	*iter = &upper->list;
8432 
8433 	return upper->dev;
8434 }
8435 EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu);
8436 
8437 static struct net_device *__netdev_next_upper_dev(struct net_device *dev,
8438 						  struct list_head **iter,
8439 						  bool *ignore)
8440 {
8441 	struct netdev_adjacent *upper;
8442 
8443 	upper = list_entry((*iter)->next, struct netdev_adjacent, list);
8444 
8445 	if (&upper->list == &dev->adj_list.upper)
8446 		return NULL;
8447 
8448 	*iter = &upper->list;
8449 	*ignore = upper->ignore;
8450 
8451 	return upper->dev;
8452 }
8453 
8454 static struct net_device *netdev_next_upper_dev_rcu(struct net_device *dev,
8455 						    struct list_head **iter)
8456 {
8457 	struct netdev_adjacent *upper;
8458 
8459 	WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held());
8460 
8461 	upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
8462 
8463 	if (&upper->list == &dev->adj_list.upper)
8464 		return NULL;
8465 
8466 	*iter = &upper->list;
8467 
8468 	return upper->dev;
8469 }
8470 
8471 static int __netdev_walk_all_upper_dev(struct net_device *dev,
8472 				       int (*fn)(struct net_device *dev,
8473 					 struct netdev_nested_priv *priv),
8474 				       struct netdev_nested_priv *priv)
8475 {
8476 	struct net_device *udev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
8477 	struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
8478 	int ret, cur = 0;
8479 	bool ignore;
8480 
8481 	now = dev;
8482 	iter = &dev->adj_list.upper;
8483 
8484 	while (1) {
8485 		if (now != dev) {
8486 			ret = fn(now, priv);
8487 			if (ret)
8488 				return ret;
8489 		}
8490 
8491 		next = NULL;
8492 		while (1) {
8493 			udev = __netdev_next_upper_dev(now, &iter, &ignore);
8494 			if (!udev)
8495 				break;
8496 			if (ignore)
8497 				continue;
8498 
8499 			next = udev;
8500 			niter = &udev->adj_list.upper;
8501 			dev_stack[cur] = now;
8502 			iter_stack[cur++] = iter;
8503 			break;
8504 		}
8505 
8506 		if (!next) {
8507 			if (!cur)
8508 				return 0;
8509 			next = dev_stack[--cur];
8510 			niter = iter_stack[cur];
8511 		}
8512 
8513 		now = next;
8514 		iter = niter;
8515 	}
8516 
8517 	return 0;
8518 }
8519 
8520 int netdev_walk_all_upper_dev_rcu(struct net_device *dev,
8521 				  int (*fn)(struct net_device *dev,
8522 					    struct netdev_nested_priv *priv),
8523 				  struct netdev_nested_priv *priv)
8524 {
8525 	struct net_device *udev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
8526 	struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
8527 	int ret, cur = 0;
8528 
8529 	now = dev;
8530 	iter = &dev->adj_list.upper;
8531 
8532 	while (1) {
8533 		if (now != dev) {
8534 			ret = fn(now, priv);
8535 			if (ret)
8536 				return ret;
8537 		}
8538 
8539 		next = NULL;
8540 		while (1) {
8541 			udev = netdev_next_upper_dev_rcu(now, &iter);
8542 			if (!udev)
8543 				break;
8544 
8545 			next = udev;
8546 			niter = &udev->adj_list.upper;
8547 			dev_stack[cur] = now;
8548 			iter_stack[cur++] = iter;
8549 			break;
8550 		}
8551 
8552 		if (!next) {
8553 			if (!cur)
8554 				return 0;
8555 			next = dev_stack[--cur];
8556 			niter = iter_stack[cur];
8557 		}
8558 
8559 		now = next;
8560 		iter = niter;
8561 	}
8562 
8563 	return 0;
8564 }
8565 EXPORT_SYMBOL_GPL(netdev_walk_all_upper_dev_rcu);
8566 
8567 static bool __netdev_has_upper_dev(struct net_device *dev,
8568 				   struct net_device *upper_dev)
8569 {
8570 	struct netdev_nested_priv priv = {
8571 		.flags = 0,
8572 		.data = (void *)upper_dev,
8573 	};
8574 
8575 	ASSERT_RTNL();
8576 
8577 	return __netdev_walk_all_upper_dev(dev, ____netdev_has_upper_dev,
8578 					   &priv);
8579 }
8580 
8581 /**
8582  * netdev_lower_get_next_private - Get the next ->private from the
8583  *				   lower neighbour list
8584  * @dev: device
8585  * @iter: list_head ** of the current position
8586  *
8587  * Gets the next netdev_adjacent->private from the dev's lower neighbour
8588  * list, starting from iter position. The caller must hold either hold the
8589  * RTNL lock or its own locking that guarantees that the neighbour lower
8590  * list will remain unchanged.
8591  */
8592 void *netdev_lower_get_next_private(struct net_device *dev,
8593 				    struct list_head **iter)
8594 {
8595 	struct netdev_adjacent *lower;
8596 
8597 	lower = list_entry(*iter, struct netdev_adjacent, list);
8598 
8599 	if (&lower->list == &dev->adj_list.lower)
8600 		return NULL;
8601 
8602 	*iter = lower->list.next;
8603 
8604 	return lower->private;
8605 }
8606 EXPORT_SYMBOL(netdev_lower_get_next_private);
8607 
8608 /**
8609  * netdev_lower_get_next_private_rcu - Get the next ->private from the
8610  *				       lower neighbour list, RCU
8611  *				       variant
8612  * @dev: device
8613  * @iter: list_head ** of the current position
8614  *
8615  * Gets the next netdev_adjacent->private from the dev's lower neighbour
8616  * list, starting from iter position. The caller must hold RCU read lock.
8617  */
8618 void *netdev_lower_get_next_private_rcu(struct net_device *dev,
8619 					struct list_head **iter)
8620 {
8621 	struct netdev_adjacent *lower;
8622 
8623 	WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held());
8624 
8625 	lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
8626 
8627 	if (&lower->list == &dev->adj_list.lower)
8628 		return NULL;
8629 
8630 	*iter = &lower->list;
8631 
8632 	return lower->private;
8633 }
8634 EXPORT_SYMBOL(netdev_lower_get_next_private_rcu);
8635 
8636 /**
8637  * netdev_lower_get_next - Get the next device from the lower neighbour
8638  *                         list
8639  * @dev: device
8640  * @iter: list_head ** of the current position
8641  *
8642  * Gets the next netdev_adjacent from the dev's lower neighbour
8643  * list, starting from iter position. The caller must hold RTNL lock or
8644  * its own locking that guarantees that the neighbour lower
8645  * list will remain unchanged.
8646  */
8647 void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter)
8648 {
8649 	struct netdev_adjacent *lower;
8650 
8651 	lower = list_entry(*iter, struct netdev_adjacent, list);
8652 
8653 	if (&lower->list == &dev->adj_list.lower)
8654 		return NULL;
8655 
8656 	*iter = lower->list.next;
8657 
8658 	return lower->dev;
8659 }
8660 EXPORT_SYMBOL(netdev_lower_get_next);
8661 
8662 static struct net_device *netdev_next_lower_dev(struct net_device *dev,
8663 						struct list_head **iter)
8664 {
8665 	struct netdev_adjacent *lower;
8666 
8667 	lower = list_entry((*iter)->next, struct netdev_adjacent, list);
8668 
8669 	if (&lower->list == &dev->adj_list.lower)
8670 		return NULL;
8671 
8672 	*iter = &lower->list;
8673 
8674 	return lower->dev;
8675 }
8676 
8677 static struct net_device *__netdev_next_lower_dev(struct net_device *dev,
8678 						  struct list_head **iter,
8679 						  bool *ignore)
8680 {
8681 	struct netdev_adjacent *lower;
8682 
8683 	lower = list_entry((*iter)->next, struct netdev_adjacent, list);
8684 
8685 	if (&lower->list == &dev->adj_list.lower)
8686 		return NULL;
8687 
8688 	*iter = &lower->list;
8689 	*ignore = lower->ignore;
8690 
8691 	return lower->dev;
8692 }
8693 
8694 int netdev_walk_all_lower_dev(struct net_device *dev,
8695 			      int (*fn)(struct net_device *dev,
8696 					struct netdev_nested_priv *priv),
8697 			      struct netdev_nested_priv *priv)
8698 {
8699 	struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
8700 	struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
8701 	int ret, cur = 0;
8702 
8703 	now = dev;
8704 	iter = &dev->adj_list.lower;
8705 
8706 	while (1) {
8707 		if (now != dev) {
8708 			ret = fn(now, priv);
8709 			if (ret)
8710 				return ret;
8711 		}
8712 
8713 		next = NULL;
8714 		while (1) {
8715 			ldev = netdev_next_lower_dev(now, &iter);
8716 			if (!ldev)
8717 				break;
8718 
8719 			next = ldev;
8720 			niter = &ldev->adj_list.lower;
8721 			dev_stack[cur] = now;
8722 			iter_stack[cur++] = iter;
8723 			break;
8724 		}
8725 
8726 		if (!next) {
8727 			if (!cur)
8728 				return 0;
8729 			next = dev_stack[--cur];
8730 			niter = iter_stack[cur];
8731 		}
8732 
8733 		now = next;
8734 		iter = niter;
8735 	}
8736 
8737 	return 0;
8738 }
8739 EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev);
8740 
8741 static int __netdev_walk_all_lower_dev(struct net_device *dev,
8742 				       int (*fn)(struct net_device *dev,
8743 					 struct netdev_nested_priv *priv),
8744 				       struct netdev_nested_priv *priv)
8745 {
8746 	struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
8747 	struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
8748 	int ret, cur = 0;
8749 	bool ignore;
8750 
8751 	now = dev;
8752 	iter = &dev->adj_list.lower;
8753 
8754 	while (1) {
8755 		if (now != dev) {
8756 			ret = fn(now, priv);
8757 			if (ret)
8758 				return ret;
8759 		}
8760 
8761 		next = NULL;
8762 		while (1) {
8763 			ldev = __netdev_next_lower_dev(now, &iter, &ignore);
8764 			if (!ldev)
8765 				break;
8766 			if (ignore)
8767 				continue;
8768 
8769 			next = ldev;
8770 			niter = &ldev->adj_list.lower;
8771 			dev_stack[cur] = now;
8772 			iter_stack[cur++] = iter;
8773 			break;
8774 		}
8775 
8776 		if (!next) {
8777 			if (!cur)
8778 				return 0;
8779 			next = dev_stack[--cur];
8780 			niter = iter_stack[cur];
8781 		}
8782 
8783 		now = next;
8784 		iter = niter;
8785 	}
8786 
8787 	return 0;
8788 }
8789 
8790 struct net_device *netdev_next_lower_dev_rcu(struct net_device *dev,
8791 					     struct list_head **iter)
8792 {
8793 	struct netdev_adjacent *lower;
8794 
8795 	lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list);
8796 	if (&lower->list == &dev->adj_list.lower)
8797 		return NULL;
8798 
8799 	*iter = &lower->list;
8800 
8801 	return lower->dev;
8802 }
8803 EXPORT_SYMBOL(netdev_next_lower_dev_rcu);
8804 
8805 static u8 __netdev_upper_depth(struct net_device *dev)
8806 {
8807 	struct net_device *udev;
8808 	struct list_head *iter;
8809 	u8 max_depth = 0;
8810 	bool ignore;
8811 
8812 	for (iter = &dev->adj_list.upper,
8813 	     udev = __netdev_next_upper_dev(dev, &iter, &ignore);
8814 	     udev;
8815 	     udev = __netdev_next_upper_dev(dev, &iter, &ignore)) {
8816 		if (ignore)
8817 			continue;
8818 		if (max_depth < udev->upper_level)
8819 			max_depth = udev->upper_level;
8820 	}
8821 
8822 	return max_depth;
8823 }
8824 
8825 static u8 __netdev_lower_depth(struct net_device *dev)
8826 {
8827 	struct net_device *ldev;
8828 	struct list_head *iter;
8829 	u8 max_depth = 0;
8830 	bool ignore;
8831 
8832 	for (iter = &dev->adj_list.lower,
8833 	     ldev = __netdev_next_lower_dev(dev, &iter, &ignore);
8834 	     ldev;
8835 	     ldev = __netdev_next_lower_dev(dev, &iter, &ignore)) {
8836 		if (ignore)
8837 			continue;
8838 		if (max_depth < ldev->lower_level)
8839 			max_depth = ldev->lower_level;
8840 	}
8841 
8842 	return max_depth;
8843 }
8844 
8845 static int __netdev_update_upper_level(struct net_device *dev,
8846 				       struct netdev_nested_priv *__unused)
8847 {
8848 	dev->upper_level = __netdev_upper_depth(dev) + 1;
8849 	return 0;
8850 }
8851 
8852 #ifdef CONFIG_LOCKDEP
8853 static LIST_HEAD(net_unlink_list);
8854 
8855 static void net_unlink_todo(struct net_device *dev)
8856 {
8857 	if (list_empty(&dev->unlink_list))
8858 		list_add_tail(&dev->unlink_list, &net_unlink_list);
8859 }
8860 #endif
8861 
8862 static int __netdev_update_lower_level(struct net_device *dev,
8863 				       struct netdev_nested_priv *priv)
8864 {
8865 	dev->lower_level = __netdev_lower_depth(dev) + 1;
8866 
8867 #ifdef CONFIG_LOCKDEP
8868 	if (!priv)
8869 		return 0;
8870 
8871 	if (priv->flags & NESTED_SYNC_IMM)
8872 		dev->nested_level = dev->lower_level - 1;
8873 	if (priv->flags & NESTED_SYNC_TODO)
8874 		net_unlink_todo(dev);
8875 #endif
8876 	return 0;
8877 }
8878 
8879 int netdev_walk_all_lower_dev_rcu(struct net_device *dev,
8880 				  int (*fn)(struct net_device *dev,
8881 					    struct netdev_nested_priv *priv),
8882 				  struct netdev_nested_priv *priv)
8883 {
8884 	struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1];
8885 	struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1];
8886 	int ret, cur = 0;
8887 
8888 	now = dev;
8889 	iter = &dev->adj_list.lower;
8890 
8891 	while (1) {
8892 		if (now != dev) {
8893 			ret = fn(now, priv);
8894 			if (ret)
8895 				return ret;
8896 		}
8897 
8898 		next = NULL;
8899 		while (1) {
8900 			ldev = netdev_next_lower_dev_rcu(now, &iter);
8901 			if (!ldev)
8902 				break;
8903 
8904 			next = ldev;
8905 			niter = &ldev->adj_list.lower;
8906 			dev_stack[cur] = now;
8907 			iter_stack[cur++] = iter;
8908 			break;
8909 		}
8910 
8911 		if (!next) {
8912 			if (!cur)
8913 				return 0;
8914 			next = dev_stack[--cur];
8915 			niter = iter_stack[cur];
8916 		}
8917 
8918 		now = next;
8919 		iter = niter;
8920 	}
8921 
8922 	return 0;
8923 }
8924 EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev_rcu);
8925 
8926 /**
8927  * netdev_lower_get_first_private_rcu - Get the first ->private from the
8928  *				       lower neighbour list, RCU
8929  *				       variant
8930  * @dev: device
8931  *
8932  * Gets the first netdev_adjacent->private from the dev's lower neighbour
8933  * list. The caller must hold RCU read lock.
8934  */
8935 void *netdev_lower_get_first_private_rcu(struct net_device *dev)
8936 {
8937 	struct netdev_adjacent *lower;
8938 
8939 	lower = list_first_or_null_rcu(&dev->adj_list.lower,
8940 			struct netdev_adjacent, list);
8941 	if (lower)
8942 		return lower->private;
8943 	return NULL;
8944 }
8945 EXPORT_SYMBOL(netdev_lower_get_first_private_rcu);
8946 
8947 /**
8948  * netdev_master_upper_dev_get_rcu - Get master upper device
8949  * @dev: device
8950  *
8951  * Find a master upper device and return pointer to it or NULL in case
8952  * it's not there. The caller must hold the RCU read lock.
8953  */
8954 struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev)
8955 {
8956 	struct netdev_adjacent *upper;
8957 
8958 	upper = list_first_or_null_rcu(&dev->adj_list.upper,
8959 				       struct netdev_adjacent, list);
8960 	if (upper && likely(upper->master))
8961 		return upper->dev;
8962 	return NULL;
8963 }
8964 EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu);
8965 
8966 static int netdev_adjacent_sysfs_add(struct net_device *dev,
8967 			      struct net_device *adj_dev,
8968 			      struct list_head *dev_list)
8969 {
8970 	char linkname[IFNAMSIZ+7];
8971 
8972 	sprintf(linkname, dev_list == &dev->adj_list.upper ?
8973 		"upper_%s" : "lower_%s", adj_dev->name);
8974 	return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj),
8975 				 linkname);
8976 }
8977 static void netdev_adjacent_sysfs_del(struct net_device *dev,
8978 			       char *name,
8979 			       struct list_head *dev_list)
8980 {
8981 	char linkname[IFNAMSIZ+7];
8982 
8983 	sprintf(linkname, dev_list == &dev->adj_list.upper ?
8984 		"upper_%s" : "lower_%s", name);
8985 	sysfs_remove_link(&(dev->dev.kobj), linkname);
8986 }
8987 
8988 static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev,
8989 						 struct net_device *adj_dev,
8990 						 struct list_head *dev_list)
8991 {
8992 	return (dev_list == &dev->adj_list.upper ||
8993 		dev_list == &dev->adj_list.lower) &&
8994 		net_eq(dev_net(dev), dev_net(adj_dev));
8995 }
8996 
8997 static int __netdev_adjacent_dev_insert(struct net_device *dev,
8998 					struct net_device *adj_dev,
8999 					struct list_head *dev_list,
9000 					void *private, bool master)
9001 {
9002 	struct netdev_adjacent *adj;
9003 	int ret;
9004 
9005 	adj = __netdev_find_adj(adj_dev, dev_list);
9006 
9007 	if (adj) {
9008 		adj->ref_nr += 1;
9009 		pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d\n",
9010 			 dev->name, adj_dev->name, adj->ref_nr);
9011 
9012 		return 0;
9013 	}
9014 
9015 	adj = kmalloc_obj(*adj);
9016 	if (!adj)
9017 		return -ENOMEM;
9018 
9019 	adj->dev = adj_dev;
9020 	adj->master = master;
9021 	adj->ref_nr = 1;
9022 	adj->private = private;
9023 	adj->ignore = false;
9024 	netdev_hold(adj_dev, &adj->dev_tracker, GFP_KERNEL);
9025 
9026 	pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d; dev_hold on %s\n",
9027 		 dev->name, adj_dev->name, adj->ref_nr, adj_dev->name);
9028 
9029 	if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) {
9030 		ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list);
9031 		if (ret)
9032 			goto free_adj;
9033 	}
9034 
9035 	/* Ensure that master link is always the first item in list. */
9036 	if (master) {
9037 		ret = sysfs_create_link(&(dev->dev.kobj),
9038 					&(adj_dev->dev.kobj), "master");
9039 		if (ret)
9040 			goto remove_symlinks;
9041 
9042 		list_add_rcu(&adj->list, dev_list);
9043 	} else {
9044 		list_add_tail_rcu(&adj->list, dev_list);
9045 	}
9046 
9047 	return 0;
9048 
9049 remove_symlinks:
9050 	if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
9051 		netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
9052 free_adj:
9053 	netdev_put(adj_dev, &adj->dev_tracker);
9054 	kfree(adj);
9055 
9056 	return ret;
9057 }
9058 
9059 static void __netdev_adjacent_dev_remove(struct net_device *dev,
9060 					 struct net_device *adj_dev,
9061 					 u16 ref_nr,
9062 					 struct list_head *dev_list)
9063 {
9064 	struct netdev_adjacent *adj;
9065 
9066 	pr_debug("Remove adjacency: dev %s adj_dev %s ref_nr %d\n",
9067 		 dev->name, adj_dev->name, ref_nr);
9068 
9069 	adj = __netdev_find_adj(adj_dev, dev_list);
9070 
9071 	if (!adj) {
9072 		pr_err("Adjacency does not exist for device %s from %s\n",
9073 		       dev->name, adj_dev->name);
9074 		WARN_ON(1);
9075 		return;
9076 	}
9077 
9078 	if (adj->ref_nr > ref_nr) {
9079 		pr_debug("adjacency: %s to %s ref_nr - %d = %d\n",
9080 			 dev->name, adj_dev->name, ref_nr,
9081 			 adj->ref_nr - ref_nr);
9082 		adj->ref_nr -= ref_nr;
9083 		return;
9084 	}
9085 
9086 	if (adj->master)
9087 		sysfs_remove_link(&(dev->dev.kobj), "master");
9088 
9089 	if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list))
9090 		netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list);
9091 
9092 	list_del_rcu(&adj->list);
9093 	pr_debug("adjacency: dev_put for %s, because link removed from %s to %s\n",
9094 		 adj_dev->name, dev->name, adj_dev->name);
9095 	netdev_put(adj_dev, &adj->dev_tracker);
9096 	kfree_rcu(adj, rcu);
9097 }
9098 
9099 static int __netdev_adjacent_dev_link_lists(struct net_device *dev,
9100 					    struct net_device *upper_dev,
9101 					    struct list_head *up_list,
9102 					    struct list_head *down_list,
9103 					    void *private, bool master)
9104 {
9105 	int ret;
9106 
9107 	ret = __netdev_adjacent_dev_insert(dev, upper_dev, up_list,
9108 					   private, master);
9109 	if (ret)
9110 		return ret;
9111 
9112 	ret = __netdev_adjacent_dev_insert(upper_dev, dev, down_list,
9113 					   private, false);
9114 	if (ret) {
9115 		__netdev_adjacent_dev_remove(dev, upper_dev, 1, up_list);
9116 		return ret;
9117 	}
9118 
9119 	return 0;
9120 }
9121 
9122 static void __netdev_adjacent_dev_unlink_lists(struct net_device *dev,
9123 					       struct net_device *upper_dev,
9124 					       u16 ref_nr,
9125 					       struct list_head *up_list,
9126 					       struct list_head *down_list)
9127 {
9128 	__netdev_adjacent_dev_remove(dev, upper_dev, ref_nr, up_list);
9129 	__netdev_adjacent_dev_remove(upper_dev, dev, ref_nr, down_list);
9130 }
9131 
9132 static int __netdev_adjacent_dev_link_neighbour(struct net_device *dev,
9133 						struct net_device *upper_dev,
9134 						void *private, bool master)
9135 {
9136 	return __netdev_adjacent_dev_link_lists(dev, upper_dev,
9137 						&dev->adj_list.upper,
9138 						&upper_dev->adj_list.lower,
9139 						private, master);
9140 }
9141 
9142 static void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev,
9143 						   struct net_device *upper_dev)
9144 {
9145 	__netdev_adjacent_dev_unlink_lists(dev, upper_dev, 1,
9146 					   &dev->adj_list.upper,
9147 					   &upper_dev->adj_list.lower);
9148 }
9149 
9150 static int __netdev_upper_dev_link(struct net_device *dev,
9151 				   struct net_device *upper_dev, bool master,
9152 				   void *upper_priv, void *upper_info,
9153 				   struct netdev_nested_priv *priv,
9154 				   struct netlink_ext_ack *extack)
9155 {
9156 	struct netdev_notifier_changeupper_info changeupper_info = {
9157 		.info = {
9158 			.dev = dev,
9159 			.extack = extack,
9160 		},
9161 		.upper_dev = upper_dev,
9162 		.master = master,
9163 		.linking = true,
9164 		.upper_info = upper_info,
9165 	};
9166 	struct net_device *master_dev;
9167 	int ret = 0;
9168 
9169 	ASSERT_RTNL();
9170 
9171 	if (dev == upper_dev)
9172 		return -EBUSY;
9173 
9174 	/* To prevent loops, check if dev is not upper device to upper_dev. */
9175 	if (__netdev_has_upper_dev(upper_dev, dev))
9176 		return -EBUSY;
9177 
9178 	if ((dev->lower_level + upper_dev->upper_level) > MAX_NEST_DEV)
9179 		return -EMLINK;
9180 
9181 	if (!master) {
9182 		if (__netdev_has_upper_dev(dev, upper_dev))
9183 			return -EEXIST;
9184 	} else {
9185 		master_dev = __netdev_master_upper_dev_get(dev);
9186 		if (master_dev)
9187 			return master_dev == upper_dev ? -EEXIST : -EBUSY;
9188 	}
9189 
9190 	ret = call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER,
9191 					    &changeupper_info.info);
9192 	ret = notifier_to_errno(ret);
9193 	if (ret)
9194 		return ret;
9195 
9196 	ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, upper_priv,
9197 						   master);
9198 	if (ret)
9199 		return ret;
9200 
9201 	ret = call_netdevice_notifiers_info(NETDEV_CHANGEUPPER,
9202 					    &changeupper_info.info);
9203 	ret = notifier_to_errno(ret);
9204 	if (ret)
9205 		goto rollback;
9206 
9207 	__netdev_update_upper_level(dev, NULL);
9208 	__netdev_walk_all_lower_dev(dev, __netdev_update_upper_level, NULL);
9209 
9210 	__netdev_update_lower_level(upper_dev, priv);
9211 	__netdev_walk_all_upper_dev(upper_dev, __netdev_update_lower_level,
9212 				    priv);
9213 
9214 	return 0;
9215 
9216 rollback:
9217 	__netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
9218 
9219 	return ret;
9220 }
9221 
9222 /**
9223  * netdev_upper_dev_link - Add a link to the upper device
9224  * @dev: device
9225  * @upper_dev: new upper device
9226  * @extack: netlink extended ack
9227  *
9228  * Adds a link to device which is upper to this one. The caller must hold
9229  * the RTNL lock. On a failure a negative errno code is returned.
9230  * On success the reference counts are adjusted and the function
9231  * returns zero.
9232  */
9233 int netdev_upper_dev_link(struct net_device *dev,
9234 			  struct net_device *upper_dev,
9235 			  struct netlink_ext_ack *extack)
9236 {
9237 	struct netdev_nested_priv priv = {
9238 		.flags = NESTED_SYNC_IMM | NESTED_SYNC_TODO,
9239 		.data = NULL,
9240 	};
9241 
9242 	return __netdev_upper_dev_link(dev, upper_dev, false,
9243 				       NULL, NULL, &priv, extack);
9244 }
9245 EXPORT_SYMBOL(netdev_upper_dev_link);
9246 
9247 /**
9248  * netdev_master_upper_dev_link - Add a master link to the upper device
9249  * @dev: device
9250  * @upper_dev: new upper device
9251  * @upper_priv: upper device private
9252  * @upper_info: upper info to be passed down via notifier
9253  * @extack: netlink extended ack
9254  *
9255  * Adds a link to device which is upper to this one. In this case, only
9256  * one master upper device can be linked, although other non-master devices
9257  * might be linked as well. The caller must hold the RTNL lock.
9258  * On a failure a negative errno code is returned. On success the reference
9259  * counts are adjusted and the function returns zero.
9260  */
9261 int netdev_master_upper_dev_link(struct net_device *dev,
9262 				 struct net_device *upper_dev,
9263 				 void *upper_priv, void *upper_info,
9264 				 struct netlink_ext_ack *extack)
9265 {
9266 	struct netdev_nested_priv priv = {
9267 		.flags = NESTED_SYNC_IMM | NESTED_SYNC_TODO,
9268 		.data = NULL,
9269 	};
9270 
9271 	return __netdev_upper_dev_link(dev, upper_dev, true,
9272 				       upper_priv, upper_info, &priv, extack);
9273 }
9274 EXPORT_SYMBOL(netdev_master_upper_dev_link);
9275 
9276 static void __netdev_upper_dev_unlink(struct net_device *dev,
9277 				      struct net_device *upper_dev,
9278 				      struct netdev_nested_priv *priv)
9279 {
9280 	struct netdev_notifier_changeupper_info changeupper_info = {
9281 		.info = {
9282 			.dev = dev,
9283 		},
9284 		.upper_dev = upper_dev,
9285 		.linking = false,
9286 	};
9287 
9288 	ASSERT_RTNL();
9289 
9290 	changeupper_info.master = netdev_master_upper_dev_get(dev) == upper_dev;
9291 
9292 	call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER,
9293 				      &changeupper_info.info);
9294 
9295 	__netdev_adjacent_dev_unlink_neighbour(dev, upper_dev);
9296 
9297 	call_netdevice_notifiers_info(NETDEV_CHANGEUPPER,
9298 				      &changeupper_info.info);
9299 
9300 	__netdev_update_upper_level(dev, NULL);
9301 	__netdev_walk_all_lower_dev(dev, __netdev_update_upper_level, NULL);
9302 
9303 	__netdev_update_lower_level(upper_dev, priv);
9304 	__netdev_walk_all_upper_dev(upper_dev, __netdev_update_lower_level,
9305 				    priv);
9306 }
9307 
9308 /**
9309  * netdev_upper_dev_unlink - Removes a link to upper device
9310  * @dev: device
9311  * @upper_dev: new upper device
9312  *
9313  * Removes a link to device which is upper to this one. The caller must hold
9314  * the RTNL lock.
9315  */
9316 void netdev_upper_dev_unlink(struct net_device *dev,
9317 			     struct net_device *upper_dev)
9318 {
9319 	struct netdev_nested_priv priv = {
9320 		.flags = NESTED_SYNC_TODO,
9321 		.data = NULL,
9322 	};
9323 
9324 	__netdev_upper_dev_unlink(dev, upper_dev, &priv);
9325 }
9326 EXPORT_SYMBOL(netdev_upper_dev_unlink);
9327 
9328 static void __netdev_adjacent_dev_set(struct net_device *upper_dev,
9329 				      struct net_device *lower_dev,
9330 				      bool val)
9331 {
9332 	struct netdev_adjacent *adj;
9333 
9334 	adj = __netdev_find_adj(lower_dev, &upper_dev->adj_list.lower);
9335 	if (adj)
9336 		adj->ignore = val;
9337 
9338 	adj = __netdev_find_adj(upper_dev, &lower_dev->adj_list.upper);
9339 	if (adj)
9340 		adj->ignore = val;
9341 }
9342 
9343 static void netdev_adjacent_dev_disable(struct net_device *upper_dev,
9344 					struct net_device *lower_dev)
9345 {
9346 	__netdev_adjacent_dev_set(upper_dev, lower_dev, true);
9347 }
9348 
9349 static void netdev_adjacent_dev_enable(struct net_device *upper_dev,
9350 				       struct net_device *lower_dev)
9351 {
9352 	__netdev_adjacent_dev_set(upper_dev, lower_dev, false);
9353 }
9354 
9355 int netdev_adjacent_change_prepare(struct net_device *old_dev,
9356 				   struct net_device *new_dev,
9357 				   struct net_device *dev,
9358 				   struct netlink_ext_ack *extack)
9359 {
9360 	struct netdev_nested_priv priv = {
9361 		.flags = 0,
9362 		.data = NULL,
9363 	};
9364 	int err;
9365 
9366 	if (!new_dev)
9367 		return 0;
9368 
9369 	if (old_dev && new_dev != old_dev)
9370 		netdev_adjacent_dev_disable(dev, old_dev);
9371 	err = __netdev_upper_dev_link(new_dev, dev, false, NULL, NULL, &priv,
9372 				      extack);
9373 	if (err) {
9374 		if (old_dev && new_dev != old_dev)
9375 			netdev_adjacent_dev_enable(dev, old_dev);
9376 		return err;
9377 	}
9378 
9379 	return 0;
9380 }
9381 EXPORT_SYMBOL(netdev_adjacent_change_prepare);
9382 
9383 void netdev_adjacent_change_commit(struct net_device *old_dev,
9384 				   struct net_device *new_dev,
9385 				   struct net_device *dev)
9386 {
9387 	struct netdev_nested_priv priv = {
9388 		.flags = NESTED_SYNC_IMM | NESTED_SYNC_TODO,
9389 		.data = NULL,
9390 	};
9391 
9392 	if (!new_dev || !old_dev)
9393 		return;
9394 
9395 	if (new_dev == old_dev)
9396 		return;
9397 
9398 	netdev_adjacent_dev_enable(dev, old_dev);
9399 	__netdev_upper_dev_unlink(old_dev, dev, &priv);
9400 }
9401 EXPORT_SYMBOL(netdev_adjacent_change_commit);
9402 
9403 void netdev_adjacent_change_abort(struct net_device *old_dev,
9404 				  struct net_device *new_dev,
9405 				  struct net_device *dev)
9406 {
9407 	struct netdev_nested_priv priv = {
9408 		.flags = 0,
9409 		.data = NULL,
9410 	};
9411 
9412 	if (!new_dev)
9413 		return;
9414 
9415 	if (old_dev && new_dev != old_dev)
9416 		netdev_adjacent_dev_enable(dev, old_dev);
9417 
9418 	__netdev_upper_dev_unlink(new_dev, dev, &priv);
9419 }
9420 EXPORT_SYMBOL(netdev_adjacent_change_abort);
9421 
9422 /**
9423  * netdev_bonding_info_change - Dispatch event about slave change
9424  * @dev: device
9425  * @bonding_info: info to dispatch
9426  *
9427  * Send NETDEV_BONDING_INFO to netdev notifiers with info.
9428  * The caller must hold the RTNL lock.
9429  */
9430 void netdev_bonding_info_change(struct net_device *dev,
9431 				struct netdev_bonding_info *bonding_info)
9432 {
9433 	struct netdev_notifier_bonding_info info = {
9434 		.info.dev = dev,
9435 	};
9436 
9437 	memcpy(&info.bonding_info, bonding_info,
9438 	       sizeof(struct netdev_bonding_info));
9439 	call_netdevice_notifiers_info(NETDEV_BONDING_INFO,
9440 				      &info.info);
9441 }
9442 EXPORT_SYMBOL(netdev_bonding_info_change);
9443 
9444 static int netdev_offload_xstats_enable_l3(struct net_device *dev,
9445 					   struct netlink_ext_ack *extack)
9446 {
9447 	struct netdev_notifier_offload_xstats_info info = {
9448 		.info.dev = dev,
9449 		.info.extack = extack,
9450 		.type = NETDEV_OFFLOAD_XSTATS_TYPE_L3,
9451 	};
9452 	int err;
9453 	int rc;
9454 
9455 	dev->offload_xstats_l3 = kzalloc_obj(*dev->offload_xstats_l3);
9456 	if (!dev->offload_xstats_l3)
9457 		return -ENOMEM;
9458 
9459 	rc = call_netdevice_notifiers_info_robust(NETDEV_OFFLOAD_XSTATS_ENABLE,
9460 						  NETDEV_OFFLOAD_XSTATS_DISABLE,
9461 						  &info.info);
9462 	err = notifier_to_errno(rc);
9463 	if (err)
9464 		goto free_stats;
9465 
9466 	return 0;
9467 
9468 free_stats:
9469 	kfree(dev->offload_xstats_l3);
9470 	dev->offload_xstats_l3 = NULL;
9471 	return err;
9472 }
9473 
9474 int netdev_offload_xstats_enable(struct net_device *dev,
9475 				 enum netdev_offload_xstats_type type,
9476 				 struct netlink_ext_ack *extack)
9477 {
9478 	ASSERT_RTNL();
9479 
9480 	if (netdev_offload_xstats_enabled(dev, type))
9481 		return -EALREADY;
9482 
9483 	switch (type) {
9484 	case NETDEV_OFFLOAD_XSTATS_TYPE_L3:
9485 		return netdev_offload_xstats_enable_l3(dev, extack);
9486 	}
9487 
9488 	WARN_ON(1);
9489 	return -EINVAL;
9490 }
9491 EXPORT_SYMBOL(netdev_offload_xstats_enable);
9492 
9493 static void netdev_offload_xstats_disable_l3(struct net_device *dev)
9494 {
9495 	struct netdev_notifier_offload_xstats_info info = {
9496 		.info.dev = dev,
9497 		.type = NETDEV_OFFLOAD_XSTATS_TYPE_L3,
9498 	};
9499 
9500 	call_netdevice_notifiers_info(NETDEV_OFFLOAD_XSTATS_DISABLE,
9501 				      &info.info);
9502 	kfree(dev->offload_xstats_l3);
9503 	dev->offload_xstats_l3 = NULL;
9504 }
9505 
9506 int netdev_offload_xstats_disable(struct net_device *dev,
9507 				  enum netdev_offload_xstats_type type)
9508 {
9509 	ASSERT_RTNL();
9510 
9511 	if (!netdev_offload_xstats_enabled(dev, type))
9512 		return -EALREADY;
9513 
9514 	switch (type) {
9515 	case NETDEV_OFFLOAD_XSTATS_TYPE_L3:
9516 		netdev_offload_xstats_disable_l3(dev);
9517 		return 0;
9518 	}
9519 
9520 	WARN_ON(1);
9521 	return -EINVAL;
9522 }
9523 EXPORT_SYMBOL(netdev_offload_xstats_disable);
9524 
9525 static void netdev_offload_xstats_disable_all(struct net_device *dev)
9526 {
9527 	netdev_offload_xstats_disable(dev, NETDEV_OFFLOAD_XSTATS_TYPE_L3);
9528 }
9529 
9530 static struct rtnl_hw_stats64 *
9531 netdev_offload_xstats_get_ptr(const struct net_device *dev,
9532 			      enum netdev_offload_xstats_type type)
9533 {
9534 	switch (type) {
9535 	case NETDEV_OFFLOAD_XSTATS_TYPE_L3:
9536 		return dev->offload_xstats_l3;
9537 	}
9538 
9539 	WARN_ON(1);
9540 	return NULL;
9541 }
9542 
9543 bool netdev_offload_xstats_enabled(const struct net_device *dev,
9544 				   enum netdev_offload_xstats_type type)
9545 {
9546 	ASSERT_RTNL();
9547 
9548 	return netdev_offload_xstats_get_ptr(dev, type);
9549 }
9550 EXPORT_SYMBOL(netdev_offload_xstats_enabled);
9551 
9552 struct netdev_notifier_offload_xstats_ru {
9553 	bool used;
9554 };
9555 
9556 struct netdev_notifier_offload_xstats_rd {
9557 	struct rtnl_hw_stats64 stats;
9558 	bool used;
9559 };
9560 
9561 static void netdev_hw_stats64_add(struct rtnl_hw_stats64 *dest,
9562 				  const struct rtnl_hw_stats64 *src)
9563 {
9564 	dest->rx_packets	  += src->rx_packets;
9565 	dest->tx_packets	  += src->tx_packets;
9566 	dest->rx_bytes		  += src->rx_bytes;
9567 	dest->tx_bytes		  += src->tx_bytes;
9568 	dest->rx_errors		  += src->rx_errors;
9569 	dest->tx_errors		  += src->tx_errors;
9570 	dest->rx_dropped	  += src->rx_dropped;
9571 	dest->tx_dropped	  += src->tx_dropped;
9572 	dest->multicast		  += src->multicast;
9573 }
9574 
9575 static int netdev_offload_xstats_get_used(struct net_device *dev,
9576 					  enum netdev_offload_xstats_type type,
9577 					  bool *p_used,
9578 					  struct netlink_ext_ack *extack)
9579 {
9580 	struct netdev_notifier_offload_xstats_ru report_used = {};
9581 	struct netdev_notifier_offload_xstats_info info = {
9582 		.info.dev = dev,
9583 		.info.extack = extack,
9584 		.type = type,
9585 		.report_used = &report_used,
9586 	};
9587 	int rc;
9588 
9589 	WARN_ON(!netdev_offload_xstats_enabled(dev, type));
9590 	rc = call_netdevice_notifiers_info(NETDEV_OFFLOAD_XSTATS_REPORT_USED,
9591 					   &info.info);
9592 	*p_used = report_used.used;
9593 	return notifier_to_errno(rc);
9594 }
9595 
9596 static int netdev_offload_xstats_get_stats(struct net_device *dev,
9597 					   enum netdev_offload_xstats_type type,
9598 					   struct rtnl_hw_stats64 *p_stats,
9599 					   bool *p_used,
9600 					   struct netlink_ext_ack *extack)
9601 {
9602 	struct netdev_notifier_offload_xstats_rd report_delta = {};
9603 	struct netdev_notifier_offload_xstats_info info = {
9604 		.info.dev = dev,
9605 		.info.extack = extack,
9606 		.type = type,
9607 		.report_delta = &report_delta,
9608 	};
9609 	struct rtnl_hw_stats64 *stats;
9610 	int rc;
9611 
9612 	stats = netdev_offload_xstats_get_ptr(dev, type);
9613 	if (WARN_ON(!stats))
9614 		return -EINVAL;
9615 
9616 	rc = call_netdevice_notifiers_info(NETDEV_OFFLOAD_XSTATS_REPORT_DELTA,
9617 					   &info.info);
9618 
9619 	/* Cache whatever we got, even if there was an error, otherwise the
9620 	 * successful stats retrievals would get lost.
9621 	 */
9622 	netdev_hw_stats64_add(stats, &report_delta.stats);
9623 
9624 	if (p_stats)
9625 		*p_stats = *stats;
9626 	*p_used = report_delta.used;
9627 
9628 	return notifier_to_errno(rc);
9629 }
9630 
9631 int netdev_offload_xstats_get(struct net_device *dev,
9632 			      enum netdev_offload_xstats_type type,
9633 			      struct rtnl_hw_stats64 *p_stats, bool *p_used,
9634 			      struct netlink_ext_ack *extack)
9635 {
9636 	ASSERT_RTNL();
9637 
9638 	if (p_stats)
9639 		return netdev_offload_xstats_get_stats(dev, type, p_stats,
9640 						       p_used, extack);
9641 	else
9642 		return netdev_offload_xstats_get_used(dev, type, p_used,
9643 						      extack);
9644 }
9645 EXPORT_SYMBOL(netdev_offload_xstats_get);
9646 
9647 void
9648 netdev_offload_xstats_report_delta(struct netdev_notifier_offload_xstats_rd *report_delta,
9649 				   const struct rtnl_hw_stats64 *stats)
9650 {
9651 	report_delta->used = true;
9652 	netdev_hw_stats64_add(&report_delta->stats, stats);
9653 }
9654 EXPORT_SYMBOL(netdev_offload_xstats_report_delta);
9655 
9656 void
9657 netdev_offload_xstats_report_used(struct netdev_notifier_offload_xstats_ru *report_used)
9658 {
9659 	report_used->used = true;
9660 }
9661 EXPORT_SYMBOL(netdev_offload_xstats_report_used);
9662 
9663 void netdev_offload_xstats_push_delta(struct net_device *dev,
9664 				      enum netdev_offload_xstats_type type,
9665 				      const struct rtnl_hw_stats64 *p_stats)
9666 {
9667 	struct rtnl_hw_stats64 *stats;
9668 
9669 	ASSERT_RTNL();
9670 
9671 	stats = netdev_offload_xstats_get_ptr(dev, type);
9672 	if (WARN_ON(!stats))
9673 		return;
9674 
9675 	netdev_hw_stats64_add(stats, p_stats);
9676 }
9677 EXPORT_SYMBOL(netdev_offload_xstats_push_delta);
9678 
9679 /**
9680  * netdev_get_xmit_slave - Get the xmit slave of master device
9681  * @dev: device
9682  * @skb: The packet
9683  * @all_slaves: assume all the slaves are active
9684  *
9685  * The reference counters are not incremented so the caller must be
9686  * careful with locks. The caller must hold RCU lock.
9687  * %NULL is returned if no slave is found.
9688  */
9689 
9690 struct net_device *netdev_get_xmit_slave(struct net_device *dev,
9691 					 struct sk_buff *skb,
9692 					 bool all_slaves)
9693 {
9694 	const struct net_device_ops *ops = dev->netdev_ops;
9695 
9696 	if (!ops->ndo_get_xmit_slave)
9697 		return NULL;
9698 	return ops->ndo_get_xmit_slave(dev, skb, all_slaves);
9699 }
9700 EXPORT_SYMBOL(netdev_get_xmit_slave);
9701 
9702 static struct net_device *netdev_sk_get_lower_dev(struct net_device *dev,
9703 						  struct sock *sk)
9704 {
9705 	const struct net_device_ops *ops = dev->netdev_ops;
9706 
9707 	if (!ops->ndo_sk_get_lower_dev)
9708 		return NULL;
9709 	return ops->ndo_sk_get_lower_dev(dev, sk);
9710 }
9711 
9712 /**
9713  * netdev_sk_get_lowest_dev - Get the lowest device in chain given device and socket
9714  * @dev: device
9715  * @sk: the socket
9716  *
9717  * %NULL is returned if no lower device is found.
9718  */
9719 
9720 struct net_device *netdev_sk_get_lowest_dev(struct net_device *dev,
9721 					    struct sock *sk)
9722 {
9723 	struct net_device *lower;
9724 
9725 	lower = netdev_sk_get_lower_dev(dev, sk);
9726 	while (lower) {
9727 		dev = lower;
9728 		lower = netdev_sk_get_lower_dev(dev, sk);
9729 	}
9730 
9731 	return dev;
9732 }
9733 EXPORT_SYMBOL(netdev_sk_get_lowest_dev);
9734 
9735 static void netdev_adjacent_add_links(struct net_device *dev)
9736 {
9737 	struct netdev_adjacent *iter;
9738 
9739 	struct net *net = dev_net(dev);
9740 
9741 	list_for_each_entry(iter, &dev->adj_list.upper, list) {
9742 		if (!net_eq(net, dev_net(iter->dev)))
9743 			continue;
9744 		netdev_adjacent_sysfs_add(iter->dev, dev,
9745 					  &iter->dev->adj_list.lower);
9746 		netdev_adjacent_sysfs_add(dev, iter->dev,
9747 					  &dev->adj_list.upper);
9748 	}
9749 
9750 	list_for_each_entry(iter, &dev->adj_list.lower, list) {
9751 		if (!net_eq(net, dev_net(iter->dev)))
9752 			continue;
9753 		netdev_adjacent_sysfs_add(iter->dev, dev,
9754 					  &iter->dev->adj_list.upper);
9755 		netdev_adjacent_sysfs_add(dev, iter->dev,
9756 					  &dev->adj_list.lower);
9757 	}
9758 }
9759 
9760 static void netdev_adjacent_del_links(struct net_device *dev)
9761 {
9762 	struct netdev_adjacent *iter;
9763 
9764 	struct net *net = dev_net(dev);
9765 
9766 	list_for_each_entry(iter, &dev->adj_list.upper, list) {
9767 		if (!net_eq(net, dev_net(iter->dev)))
9768 			continue;
9769 		netdev_adjacent_sysfs_del(iter->dev, dev->name,
9770 					  &iter->dev->adj_list.lower);
9771 		netdev_adjacent_sysfs_del(dev, iter->dev->name,
9772 					  &dev->adj_list.upper);
9773 	}
9774 
9775 	list_for_each_entry(iter, &dev->adj_list.lower, list) {
9776 		if (!net_eq(net, dev_net(iter->dev)))
9777 			continue;
9778 		netdev_adjacent_sysfs_del(iter->dev, dev->name,
9779 					  &iter->dev->adj_list.upper);
9780 		netdev_adjacent_sysfs_del(dev, iter->dev->name,
9781 					  &dev->adj_list.lower);
9782 	}
9783 }
9784 
9785 void netdev_adjacent_rename_links(struct net_device *dev, char *oldname)
9786 {
9787 	struct netdev_adjacent *iter;
9788 
9789 	struct net *net = dev_net(dev);
9790 
9791 	list_for_each_entry(iter, &dev->adj_list.upper, list) {
9792 		if (!net_eq(net, dev_net(iter->dev)))
9793 			continue;
9794 		netdev_adjacent_sysfs_del(iter->dev, oldname,
9795 					  &iter->dev->adj_list.lower);
9796 		netdev_adjacent_sysfs_add(iter->dev, dev,
9797 					  &iter->dev->adj_list.lower);
9798 	}
9799 
9800 	list_for_each_entry(iter, &dev->adj_list.lower, list) {
9801 		if (!net_eq(net, dev_net(iter->dev)))
9802 			continue;
9803 		netdev_adjacent_sysfs_del(iter->dev, oldname,
9804 					  &iter->dev->adj_list.upper);
9805 		netdev_adjacent_sysfs_add(iter->dev, dev,
9806 					  &iter->dev->adj_list.upper);
9807 	}
9808 }
9809 
9810 void *netdev_lower_dev_get_private(struct net_device *dev,
9811 				   struct net_device *lower_dev)
9812 {
9813 	struct netdev_adjacent *lower;
9814 
9815 	if (!lower_dev)
9816 		return NULL;
9817 	lower = __netdev_find_adj(lower_dev, &dev->adj_list.lower);
9818 	if (!lower)
9819 		return NULL;
9820 
9821 	return lower->private;
9822 }
9823 EXPORT_SYMBOL(netdev_lower_dev_get_private);
9824 
9825 
9826 /**
9827  * netdev_lower_state_changed - Dispatch event about lower device state change
9828  * @lower_dev: device
9829  * @lower_state_info: state to dispatch
9830  *
9831  * Send NETDEV_CHANGELOWERSTATE to netdev notifiers with info.
9832  * The caller must hold the RTNL lock.
9833  */
9834 void netdev_lower_state_changed(struct net_device *lower_dev,
9835 				void *lower_state_info)
9836 {
9837 	struct netdev_notifier_changelowerstate_info changelowerstate_info = {
9838 		.info.dev = lower_dev,
9839 	};
9840 
9841 	ASSERT_RTNL();
9842 	changelowerstate_info.lower_state_info = lower_state_info;
9843 	call_netdevice_notifiers_info(NETDEV_CHANGELOWERSTATE,
9844 				      &changelowerstate_info.info);
9845 }
9846 EXPORT_SYMBOL(netdev_lower_state_changed);
9847 
9848 static void dev_change_rx_flags(struct net_device *dev, int flags)
9849 {
9850 	const struct net_device_ops *ops = dev->netdev_ops;
9851 
9852 	if (ops->ndo_change_rx_flags)
9853 		ops->ndo_change_rx_flags(dev, flags);
9854 }
9855 
9856 int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify)
9857 {
9858 	unsigned int old_flags = dev->flags;
9859 	unsigned int promiscuity, flags;
9860 	kuid_t uid;
9861 	kgid_t gid;
9862 
9863 	netdev_assert_locked_ops_compat(dev);
9864 
9865 	promiscuity = dev->promiscuity + inc;
9866 	if (promiscuity == 0) {
9867 		/*
9868 		 * Avoid overflow.
9869 		 * If inc causes overflow, untouch promisc and return error.
9870 		 */
9871 		if (unlikely(inc > 0)) {
9872 			netdev_warn(dev, "promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n");
9873 			return -EOVERFLOW;
9874 		}
9875 		flags = old_flags & ~IFF_PROMISC;
9876 	} else {
9877 		flags = old_flags | IFF_PROMISC;
9878 	}
9879 	WRITE_ONCE(dev->promiscuity, promiscuity);
9880 	if (flags != old_flags) {
9881 		WRITE_ONCE(dev->flags, flags);
9882 		netdev_info(dev, "%s promiscuous mode\n",
9883 			    dev->flags & IFF_PROMISC ? "entered" : "left");
9884 		if (audit_enabled) {
9885 			current_uid_gid(&uid, &gid);
9886 			audit_log(audit_context(), GFP_ATOMIC,
9887 				  AUDIT_ANOM_PROMISCUOUS,
9888 				  "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u",
9889 				  dev->name, (dev->flags & IFF_PROMISC),
9890 				  (old_flags & IFF_PROMISC),
9891 				  from_kuid(&init_user_ns, audit_get_loginuid(current)),
9892 				  from_kuid(&init_user_ns, uid),
9893 				  from_kgid(&init_user_ns, gid),
9894 				  audit_get_sessionid(current));
9895 		}
9896 
9897 		dev_change_rx_flags(dev, IFF_PROMISC);
9898 	}
9899 	if (notify)
9900 		__dev_notify_flags(dev, old_flags, IFF_PROMISC, 0, NULL);
9901 	return 0;
9902 }
9903 
9904 int netif_set_promiscuity(struct net_device *dev, int inc)
9905 {
9906 	unsigned int old_flags = dev->flags;
9907 	int err;
9908 
9909 	err = __dev_set_promiscuity(dev, inc, true);
9910 	if (err < 0)
9911 		return err;
9912 	if (dev->flags != old_flags)
9913 		dev_set_rx_mode(dev);
9914 	return err;
9915 }
9916 
9917 int netif_set_allmulti(struct net_device *dev, int inc, bool notify)
9918 {
9919 	unsigned int old_flags = dev->flags, old_gflags = dev->gflags;
9920 	unsigned int allmulti, flags;
9921 
9922 	netdev_assert_locked_ops_compat(dev);
9923 
9924 	allmulti = dev->allmulti + inc;
9925 	if (allmulti == 0) {
9926 		/*
9927 		 * Avoid overflow.
9928 		 * If inc causes overflow, untouch allmulti and return error.
9929 		 */
9930 		if (unlikely(inc > 0)) {
9931 			netdev_warn(dev, "allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n");
9932 			return -EOVERFLOW;
9933 		}
9934 		flags = old_flags & ~IFF_ALLMULTI;
9935 	} else {
9936 		flags = old_flags | IFF_ALLMULTI;
9937 	}
9938 	WRITE_ONCE(dev->allmulti, allmulti);
9939 	if (flags != old_flags) {
9940 		WRITE_ONCE(dev->flags, flags);
9941 		netdev_info(dev, "%s allmulticast mode\n",
9942 			    dev->flags & IFF_ALLMULTI ? "entered" : "left");
9943 		dev_change_rx_flags(dev, IFF_ALLMULTI);
9944 		dev_set_rx_mode(dev);
9945 		if (notify)
9946 			__dev_notify_flags(dev, old_flags,
9947 					   dev->gflags ^ old_gflags, 0, NULL);
9948 	}
9949 	return 0;
9950 }
9951 
9952 
9953 /**
9954  * netif_get_flags() - get flags reported to userspace
9955  * @dev: device
9956  *
9957  * Get the combination of flag bits exported through APIs to userspace.
9958  */
9959 unsigned int netif_get_flags(const struct net_device *dev)
9960 {
9961 	unsigned int flags;
9962 
9963 	flags = (READ_ONCE(dev->flags) & ~(IFF_PROMISC |
9964 				IFF_ALLMULTI |
9965 				IFF_RUNNING |
9966 				IFF_LOWER_UP |
9967 				IFF_DORMANT)) |
9968 		(READ_ONCE(dev->gflags) & (IFF_PROMISC |
9969 				IFF_ALLMULTI));
9970 
9971 	if (netif_running(dev)) {
9972 		if (netif_oper_up(dev))
9973 			flags |= IFF_RUNNING;
9974 		if (netif_carrier_ok(dev))
9975 			flags |= IFF_LOWER_UP;
9976 		if (netif_dormant(dev))
9977 			flags |= IFF_DORMANT;
9978 	}
9979 
9980 	return flags;
9981 }
9982 EXPORT_SYMBOL(netif_get_flags);
9983 
9984 int __dev_change_flags(struct net_device *dev, unsigned int flags,
9985 		       struct netlink_ext_ack *extack)
9986 {
9987 	unsigned int old_flags = dev->flags;
9988 	int ret;
9989 
9990 	netdev_assert_locked_ops_compat(dev);
9991 
9992 	/*
9993 	 *	Set the flags on our device.
9994 	 */
9995 
9996 	dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP |
9997 			       IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL |
9998 			       IFF_AUTOMEDIA)) |
9999 		     (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC |
10000 				    IFF_ALLMULTI));
10001 
10002 	/*
10003 	 *	Load in the correct multicast list now the flags have changed.
10004 	 */
10005 
10006 	if ((old_flags ^ flags) & IFF_MULTICAST)
10007 		dev_change_rx_flags(dev, IFF_MULTICAST);
10008 
10009 	dev_set_rx_mode(dev);
10010 
10011 	/*
10012 	 *	Have we downed the interface. We handle IFF_UP ourselves
10013 	 *	according to user attempts to set it, rather than blindly
10014 	 *	setting it.
10015 	 */
10016 
10017 	ret = 0;
10018 	if ((old_flags ^ flags) & IFF_UP) {
10019 		if (old_flags & IFF_UP)
10020 			__dev_close(dev);
10021 		else
10022 			ret = __dev_open(dev, extack);
10023 	}
10024 
10025 	if ((flags ^ dev->gflags) & IFF_PROMISC) {
10026 		int inc = (flags & IFF_PROMISC) ? 1 : -1;
10027 		old_flags = dev->flags;
10028 
10029 		dev->gflags ^= IFF_PROMISC;
10030 
10031 		if (__dev_set_promiscuity(dev, inc, false) >= 0)
10032 			if (dev->flags != old_flags)
10033 				dev_set_rx_mode(dev);
10034 	}
10035 
10036 	/* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI
10037 	 * is important. Some (broken) drivers set IFF_PROMISC, when
10038 	 * IFF_ALLMULTI is requested not asking us and not reporting.
10039 	 */
10040 	if ((flags ^ dev->gflags) & IFF_ALLMULTI) {
10041 		int inc = (flags & IFF_ALLMULTI) ? 1 : -1;
10042 
10043 		dev->gflags ^= IFF_ALLMULTI;
10044 		netif_set_allmulti(dev, inc, false);
10045 	}
10046 
10047 	return ret;
10048 }
10049 
10050 void __dev_notify_flags(struct net_device *dev, unsigned int old_flags,
10051 			unsigned int gchanges, u32 portid,
10052 			const struct nlmsghdr *nlh)
10053 {
10054 	unsigned int changes = dev->flags ^ old_flags;
10055 
10056 	netdev_assert_locked_ops_compat(dev);
10057 
10058 	if (gchanges)
10059 		rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC, portid, nlh);
10060 
10061 	if (changes & IFF_UP) {
10062 		if (dev->flags & IFF_UP)
10063 			call_netdevice_notifiers(NETDEV_UP, dev);
10064 		else
10065 			call_netdevice_notifiers(NETDEV_DOWN, dev);
10066 	}
10067 
10068 	if (dev->flags & IFF_UP &&
10069 	    (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE))) {
10070 		struct netdev_notifier_change_info change_info = {
10071 			.info = {
10072 				.dev = dev,
10073 			},
10074 			.flags_changed = changes,
10075 		};
10076 
10077 		call_netdevice_notifiers_info(NETDEV_CHANGE, &change_info.info);
10078 	}
10079 }
10080 
10081 int netif_change_flags(struct net_device *dev, unsigned int flags,
10082 		       struct netlink_ext_ack *extack)
10083 {
10084 	int ret;
10085 	unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags;
10086 
10087 	ret = __dev_change_flags(dev, flags, extack);
10088 	if (ret < 0)
10089 		return ret;
10090 
10091 	changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags);
10092 	__dev_notify_flags(dev, old_flags, changes, 0, NULL);
10093 	return ret;
10094 }
10095 EXPORT_SYMBOL(netif_change_flags);
10096 
10097 int __netif_set_mtu(struct net_device *dev, int new_mtu)
10098 {
10099 	const struct net_device_ops *ops = dev->netdev_ops;
10100 
10101 	if (ops->ndo_change_mtu)
10102 		return ops->ndo_change_mtu(dev, new_mtu);
10103 
10104 	/* Pairs with all the lockless reads of dev->mtu in the stack */
10105 	WRITE_ONCE(dev->mtu, new_mtu);
10106 	return 0;
10107 }
10108 EXPORT_SYMBOL_NS_GPL(__netif_set_mtu, "NETDEV_INTERNAL");
10109 
10110 int dev_validate_mtu(struct net_device *dev, int new_mtu,
10111 		     struct netlink_ext_ack *extack)
10112 {
10113 	/* MTU must be positive, and in range */
10114 	if (new_mtu < 0 || new_mtu < dev->min_mtu) {
10115 		NL_SET_ERR_MSG(extack, "mtu less than device minimum");
10116 		return -EINVAL;
10117 	}
10118 
10119 	if (dev->max_mtu > 0 && new_mtu > dev->max_mtu) {
10120 		NL_SET_ERR_MSG(extack, "mtu greater than device maximum");
10121 		return -EINVAL;
10122 	}
10123 	return 0;
10124 }
10125 
10126 /**
10127  * netif_set_mtu_ext() - Change maximum transfer unit
10128  * @dev: device
10129  * @new_mtu: new transfer unit
10130  * @extack: netlink extended ack
10131  *
10132  * Change the maximum transfer size of the network device.
10133  *
10134  * Return: 0 on success, -errno on failure.
10135  */
10136 int netif_set_mtu_ext(struct net_device *dev, int new_mtu,
10137 		      struct netlink_ext_ack *extack)
10138 {
10139 	int err, orig_mtu;
10140 
10141 	netdev_assert_locked_ops_compat(dev);
10142 
10143 	if (new_mtu == dev->mtu)
10144 		return 0;
10145 
10146 	err = dev_validate_mtu(dev, new_mtu, extack);
10147 	if (err)
10148 		return err;
10149 
10150 	if (!netif_device_present(dev))
10151 		return -ENODEV;
10152 
10153 	err = call_netdevice_notifiers(NETDEV_PRECHANGEMTU, dev);
10154 	err = notifier_to_errno(err);
10155 	if (err)
10156 		return err;
10157 
10158 	orig_mtu = dev->mtu;
10159 	err = __netif_set_mtu(dev, new_mtu);
10160 
10161 	if (!err) {
10162 		err = call_netdevice_notifiers_mtu(NETDEV_CHANGEMTU, dev,
10163 						   orig_mtu);
10164 		err = notifier_to_errno(err);
10165 		if (err) {
10166 			/* setting mtu back and notifying everyone again,
10167 			 * so that they have a chance to revert changes.
10168 			 */
10169 			__netif_set_mtu(dev, orig_mtu);
10170 			call_netdevice_notifiers_mtu(NETDEV_CHANGEMTU, dev,
10171 						     new_mtu);
10172 		}
10173 	}
10174 	return err;
10175 }
10176 
10177 int netif_set_mtu(struct net_device *dev, int new_mtu)
10178 {
10179 	struct netlink_ext_ack extack;
10180 	int err;
10181 
10182 	memset(&extack, 0, sizeof(extack));
10183 	err = netif_set_mtu_ext(dev, new_mtu, &extack);
10184 	if (err && extack._msg)
10185 		net_err_ratelimited("%s: %s\n", dev->name, extack._msg);
10186 	return err;
10187 }
10188 EXPORT_SYMBOL(netif_set_mtu);
10189 
10190 int netif_change_tx_queue_len(struct net_device *dev, unsigned long new_len)
10191 {
10192 	unsigned int orig_len = dev->tx_queue_len;
10193 	int res;
10194 
10195 	if (new_len > S16_MAX)
10196 		return -ERANGE;
10197 
10198 	if (new_len != orig_len) {
10199 		WRITE_ONCE(dev->tx_queue_len, new_len);
10200 		res = call_netdevice_notifiers(NETDEV_CHANGE_TX_QUEUE_LEN, dev);
10201 		res = notifier_to_errno(res);
10202 		if (res)
10203 			goto err_rollback;
10204 		res = dev_qdisc_change_tx_queue_len(dev);
10205 		if (res)
10206 			goto err_rollback;
10207 	}
10208 
10209 	return 0;
10210 
10211 err_rollback:
10212 	netdev_err(dev, "refused to change device tx_queue_len\n");
10213 	WRITE_ONCE(dev->tx_queue_len, orig_len);
10214 	return res;
10215 }
10216 
10217 void netif_set_group(struct net_device *dev, int new_group)
10218 {
10219 	dev->group = new_group;
10220 }
10221 
10222 /**
10223  * netif_pre_changeaddr_notify() - Call NETDEV_PRE_CHANGEADDR.
10224  * @dev: device
10225  * @addr: new address
10226  * @extack: netlink extended ack
10227  *
10228  * Return: 0 on success, -errno on failure.
10229  */
10230 int netif_pre_changeaddr_notify(struct net_device *dev, const char *addr,
10231 				struct netlink_ext_ack *extack)
10232 {
10233 	struct netdev_notifier_pre_changeaddr_info info = {
10234 		.info.dev = dev,
10235 		.info.extack = extack,
10236 		.dev_addr = addr,
10237 	};
10238 	int rc;
10239 
10240 	rc = call_netdevice_notifiers_info(NETDEV_PRE_CHANGEADDR, &info.info);
10241 	return notifier_to_errno(rc);
10242 }
10243 EXPORT_SYMBOL_NS_GPL(netif_pre_changeaddr_notify, "NETDEV_INTERNAL");
10244 
10245 int netif_set_mac_address(struct net_device *dev, struct sockaddr_storage *ss,
10246 			  struct netlink_ext_ack *extack)
10247 {
10248 	const struct net_device_ops *ops = dev->netdev_ops;
10249 	int err;
10250 
10251 	if (!ops->ndo_set_mac_address)
10252 		return -EOPNOTSUPP;
10253 	if (ss->ss_family != dev->type)
10254 		return -EINVAL;
10255 	if (!netif_device_present(dev))
10256 		return -ENODEV;
10257 	err = netif_pre_changeaddr_notify(dev, ss->__data, extack);
10258 	if (err)
10259 		return err;
10260 	if (memcmp(dev->dev_addr, ss->__data, dev->addr_len)) {
10261 		err = ops->ndo_set_mac_address(dev, ss);
10262 		if (err)
10263 			return err;
10264 	}
10265 	dev->addr_assign_type = NET_ADDR_SET;
10266 	call_netdevice_notifiers(NETDEV_CHANGEADDR, dev);
10267 	add_device_randomness(dev->dev_addr, dev->addr_len);
10268 	return 0;
10269 }
10270 
10271 DECLARE_RWSEM(dev_addr_sem);
10272 
10273 /* "sa" is a true struct sockaddr with limited "sa_data" member. */
10274 int netif_get_mac_address(struct sockaddr *sa, struct net *net, char *dev_name)
10275 {
10276 	size_t size = sizeof(sa->sa_data);
10277 	struct net_device *dev;
10278 	int ret = 0;
10279 
10280 	down_read(&dev_addr_sem);
10281 	rcu_read_lock();
10282 
10283 	dev = dev_get_by_name_rcu(net, dev_name);
10284 	if (!dev) {
10285 		ret = -ENODEV;
10286 		goto unlock;
10287 	}
10288 	if (!dev->addr_len)
10289 		memset(sa->sa_data, 0, size);
10290 	else
10291 		memcpy(sa->sa_data, dev->dev_addr,
10292 		       min_t(size_t, size, dev->addr_len));
10293 	sa->sa_family = dev->type;
10294 
10295 unlock:
10296 	rcu_read_unlock();
10297 	up_read(&dev_addr_sem);
10298 	return ret;
10299 }
10300 EXPORT_SYMBOL_NS_GPL(netif_get_mac_address, "NETDEV_INTERNAL");
10301 
10302 int netif_change_carrier(struct net_device *dev, bool new_carrier)
10303 {
10304 	const struct net_device_ops *ops = dev->netdev_ops;
10305 
10306 	if (!ops->ndo_change_carrier)
10307 		return -EOPNOTSUPP;
10308 	if (!netif_device_present(dev))
10309 		return -ENODEV;
10310 	return ops->ndo_change_carrier(dev, new_carrier);
10311 }
10312 
10313 /**
10314  *	dev_get_phys_port_id - Get device physical port ID
10315  *	@dev: device
10316  *	@ppid: port ID
10317  *
10318  *	Get device physical port ID
10319  */
10320 int dev_get_phys_port_id(struct net_device *dev,
10321 			 struct netdev_phys_item_id *ppid)
10322 {
10323 	const struct net_device_ops *ops = dev->netdev_ops;
10324 
10325 	if (!ops->ndo_get_phys_port_id)
10326 		return -EOPNOTSUPP;
10327 	return ops->ndo_get_phys_port_id(dev, ppid);
10328 }
10329 
10330 /**
10331  *	dev_get_phys_port_name - Get device physical port name
10332  *	@dev: device
10333  *	@name: port name
10334  *	@len: limit of bytes to copy to name
10335  *
10336  *	Get device physical port name
10337  */
10338 int dev_get_phys_port_name(struct net_device *dev,
10339 			   char *name, size_t len)
10340 {
10341 	const struct net_device_ops *ops = dev->netdev_ops;
10342 	int err;
10343 
10344 	if (ops->ndo_get_phys_port_name) {
10345 		err = ops->ndo_get_phys_port_name(dev, name, len);
10346 		if (err != -EOPNOTSUPP)
10347 			return err;
10348 	}
10349 	return devlink_compat_phys_port_name_get(dev, name, len);
10350 }
10351 
10352 /**
10353  * netif_get_port_parent_id() - Get the device's port parent identifier
10354  * @dev: network device
10355  * @ppid: pointer to a storage for the port's parent identifier
10356  * @recurse: allow/disallow recursion to lower devices
10357  *
10358  * Get the devices's port parent identifier.
10359  *
10360  * Return: 0 on success, -errno on failure.
10361  */
10362 int netif_get_port_parent_id(struct net_device *dev,
10363 			     struct netdev_phys_item_id *ppid, bool recurse)
10364 {
10365 	const struct net_device_ops *ops = dev->netdev_ops;
10366 	struct netdev_phys_item_id first = { };
10367 	struct net_device *lower_dev;
10368 	struct list_head *iter;
10369 	int err;
10370 
10371 	if (ops->ndo_get_port_parent_id) {
10372 		err = ops->ndo_get_port_parent_id(dev, ppid);
10373 		if (err != -EOPNOTSUPP)
10374 			return err;
10375 	}
10376 
10377 	err = devlink_compat_switch_id_get(dev, ppid);
10378 	if (!recurse || err != -EOPNOTSUPP)
10379 		return err;
10380 
10381 	netdev_for_each_lower_dev(dev, lower_dev, iter) {
10382 		err = netif_get_port_parent_id(lower_dev, ppid, true);
10383 		if (err)
10384 			break;
10385 		if (!first.id_len)
10386 			first = *ppid;
10387 		else if (memcmp(&first, ppid, sizeof(*ppid)))
10388 			return -EOPNOTSUPP;
10389 	}
10390 
10391 	return err;
10392 }
10393 EXPORT_SYMBOL(netif_get_port_parent_id);
10394 
10395 /**
10396  *	netdev_port_same_parent_id - Indicate if two network devices have
10397  *	the same port parent identifier
10398  *	@a: first network device
10399  *	@b: second network device
10400  */
10401 bool netdev_port_same_parent_id(struct net_device *a, struct net_device *b)
10402 {
10403 	struct netdev_phys_item_id a_id = { };
10404 	struct netdev_phys_item_id b_id = { };
10405 
10406 	if (netif_get_port_parent_id(a, &a_id, true) ||
10407 	    netif_get_port_parent_id(b, &b_id, true))
10408 		return false;
10409 
10410 	return netdev_phys_item_id_same(&a_id, &b_id);
10411 }
10412 EXPORT_SYMBOL(netdev_port_same_parent_id);
10413 
10414 static struct net_device *dev_get_iflink_dev(struct net_device *dev)
10415 {
10416 	struct net *net;
10417 
10418 	ASSERT_RTNL();
10419 
10420 	if (!dev->netdev_ops->ndo_get_iflink || !dev->rtnl_link_ops ||
10421 	    !dev->rtnl_link_ops->get_link_net)
10422 		return dev;
10423 
10424 	net = dev->rtnl_link_ops->get_link_net(dev);
10425 	return __dev_get_by_index(net, dev_get_iflink(dev));
10426 }
10427 
10428 int netif_change_proto_down(struct net_device *dev, bool proto_down)
10429 {
10430 	struct net_device *iflink_dev;
10431 
10432 	if (!dev->change_proto_down)
10433 		return -EOPNOTSUPP;
10434 	if (!netif_device_present(dev))
10435 		return -ENODEV;
10436 	iflink_dev = dev_get_iflink_dev(dev);
10437 	if (!iflink_dev)
10438 		return -ENODEV;
10439 	WRITE_ONCE(dev->proto_down, proto_down);
10440 	if (proto_down)
10441 		netif_carrier_off(dev);
10442 	else if (dev == iflink_dev || netif_carrier_ok(iflink_dev))
10443 		netif_carrier_on(dev);
10444 	return 0;
10445 }
10446 
10447 /**
10448  *	netdev_change_proto_down_reason_locked - proto down reason
10449  *
10450  *	@dev: device
10451  *	@mask: proto down mask
10452  *	@value: proto down value
10453  */
10454 void netdev_change_proto_down_reason_locked(struct net_device *dev,
10455 					    unsigned long mask, u32 value)
10456 {
10457 	u32 proto_down_reason;
10458 	int b;
10459 
10460 	if (!mask) {
10461 		proto_down_reason = value;
10462 	} else {
10463 		proto_down_reason = dev->proto_down_reason;
10464 		for_each_set_bit(b, &mask, 32) {
10465 			if (value & (1 << b))
10466 				proto_down_reason |= BIT(b);
10467 			else
10468 				proto_down_reason &= ~BIT(b);
10469 		}
10470 	}
10471 	WRITE_ONCE(dev->proto_down_reason, proto_down_reason);
10472 }
10473 
10474 struct bpf_xdp_link {
10475 	struct bpf_link link;
10476 	struct net_device *dev; /* protected by rtnl_lock, no refcnt held */
10477 	int flags;
10478 };
10479 
10480 static enum bpf_xdp_mode dev_xdp_mode(struct net_device *dev, u32 flags)
10481 {
10482 	if (flags & XDP_FLAGS_HW_MODE)
10483 		return XDP_MODE_HW;
10484 	if (flags & XDP_FLAGS_DRV_MODE)
10485 		return XDP_MODE_DRV;
10486 	if (flags & XDP_FLAGS_SKB_MODE)
10487 		return XDP_MODE_SKB;
10488 	return dev->netdev_ops->ndo_bpf ? XDP_MODE_DRV : XDP_MODE_SKB;
10489 }
10490 
10491 static bpf_op_t dev_xdp_bpf_op(struct net_device *dev, enum bpf_xdp_mode mode)
10492 {
10493 	switch (mode) {
10494 	case XDP_MODE_SKB:
10495 		return generic_xdp_install;
10496 	case XDP_MODE_DRV:
10497 	case XDP_MODE_HW:
10498 		return dev->netdev_ops->ndo_bpf;
10499 	default:
10500 		return NULL;
10501 	}
10502 }
10503 
10504 static struct bpf_xdp_link *dev_xdp_link(struct net_device *dev,
10505 					 enum bpf_xdp_mode mode)
10506 {
10507 	return dev->xdp_state[mode].link;
10508 }
10509 
10510 static struct bpf_prog *dev_xdp_prog(struct net_device *dev,
10511 				     enum bpf_xdp_mode mode)
10512 {
10513 	struct bpf_xdp_link *link = dev_xdp_link(dev, mode);
10514 
10515 	if (link)
10516 		return link->link.prog;
10517 	return dev->xdp_state[mode].prog;
10518 }
10519 
10520 u8 dev_xdp_prog_count(struct net_device *dev)
10521 {
10522 	u8 count = 0;
10523 	int i;
10524 
10525 	for (i = 0; i < __MAX_XDP_MODE; i++)
10526 		if (dev->xdp_state[i].prog || dev->xdp_state[i].link)
10527 			count++;
10528 	return count;
10529 }
10530 EXPORT_SYMBOL_GPL(dev_xdp_prog_count);
10531 
10532 u8 dev_xdp_sb_prog_count(struct net_device *dev)
10533 {
10534 	struct bpf_prog *prog;
10535 	u8 count = 0;
10536 	int i;
10537 
10538 	for (i = 0; i < __MAX_XDP_MODE; i++) {
10539 		prog = dev_xdp_prog(dev, i);
10540 		if (prog && !prog->aux->xdp_has_frags)
10541 			count++;
10542 	}
10543 	return count;
10544 }
10545 
10546 int netif_xdp_propagate(struct net_device *dev, struct netdev_bpf *bpf)
10547 {
10548 	if (!dev->netdev_ops->ndo_bpf)
10549 		return -EOPNOTSUPP;
10550 
10551 	if (dev->cfg->hds_config == ETHTOOL_TCP_DATA_SPLIT_ENABLED &&
10552 	    bpf->command == XDP_SETUP_PROG &&
10553 	    bpf->prog && !bpf->prog->aux->xdp_has_frags) {
10554 		NL_SET_ERR_MSG(bpf->extack,
10555 			       "unable to propagate XDP to device using tcp-data-split");
10556 		return -EBUSY;
10557 	}
10558 
10559 	if (dev_get_min_mp_channel_count(dev)) {
10560 		NL_SET_ERR_MSG(bpf->extack, "unable to propagate XDP to device using memory provider");
10561 		return -EBUSY;
10562 	}
10563 
10564 	return dev->netdev_ops->ndo_bpf(dev, bpf);
10565 }
10566 EXPORT_SYMBOL_GPL(netif_xdp_propagate);
10567 
10568 u32 dev_xdp_prog_id(struct net_device *dev, enum bpf_xdp_mode mode)
10569 {
10570 	struct bpf_prog *prog = dev_xdp_prog(dev, mode);
10571 
10572 	return prog ? prog->aux->id : 0;
10573 }
10574 
10575 static void dev_xdp_set_link(struct net_device *dev, enum bpf_xdp_mode mode,
10576 			     struct bpf_xdp_link *link)
10577 {
10578 	dev->xdp_state[mode].link = link;
10579 	dev->xdp_state[mode].prog = NULL;
10580 }
10581 
10582 static void dev_xdp_set_prog(struct net_device *dev, enum bpf_xdp_mode mode,
10583 			     struct bpf_prog *prog)
10584 {
10585 	dev->xdp_state[mode].link = NULL;
10586 	dev->xdp_state[mode].prog = prog;
10587 }
10588 
10589 static int dev_xdp_install(struct net_device *dev, enum bpf_xdp_mode mode,
10590 			   bpf_op_t bpf_op, struct netlink_ext_ack *extack,
10591 			   u32 flags, struct bpf_prog *prog)
10592 {
10593 	struct netdev_bpf xdp;
10594 	int err;
10595 
10596 	netdev_assert_locked_ops_compat(dev);
10597 
10598 	if (prog) {
10599 		enum bpf_xdp_mode other_mode = mode == XDP_MODE_SKB
10600 					       ? XDP_MODE_DRV : XDP_MODE_SKB;
10601 		bool offload = mode == XDP_MODE_HW;
10602 
10603 		if (!offload && dev_xdp_prog(dev, other_mode)) {
10604 			NL_SET_ERR_MSG(extack, "Native and generic XDP can't be active at the same time");
10605 			return -EEXIST;
10606 		}
10607 		if (!offload && bpf_prog_is_offloaded(prog->aux)) {
10608 			NL_SET_ERR_MSG(extack, "Using offloaded program without HW_MODE flag is not supported");
10609 			return -EINVAL;
10610 		}
10611 		if (bpf_prog_is_dev_bound(prog->aux) && !bpf_offload_dev_match(prog, dev)) {
10612 			NL_SET_ERR_MSG(extack, "Program bound to different device");
10613 			return -EINVAL;
10614 		}
10615 		if (bpf_prog_is_dev_bound(prog->aux) && mode == XDP_MODE_SKB) {
10616 			NL_SET_ERR_MSG(extack, "Can't attach device-bound programs in generic mode");
10617 			return -EINVAL;
10618 		}
10619 		if (prog->expected_attach_type == BPF_XDP_DEVMAP) {
10620 			NL_SET_ERR_MSG(extack, "BPF_XDP_DEVMAP programs can not be attached to a device");
10621 			return -EINVAL;
10622 		}
10623 		if (prog->expected_attach_type == BPF_XDP_CPUMAP) {
10624 			NL_SET_ERR_MSG(extack, "BPF_XDP_CPUMAP programs can not be attached to a device");
10625 			return -EINVAL;
10626 		}
10627 	}
10628 
10629 	if (dev->cfg->hds_config == ETHTOOL_TCP_DATA_SPLIT_ENABLED &&
10630 	    prog && !prog->aux->xdp_has_frags) {
10631 		NL_SET_ERR_MSG(extack, "unable to install XDP to device using tcp-data-split");
10632 		return -EBUSY;
10633 	}
10634 
10635 	if (dev_get_min_mp_channel_count(dev)) {
10636 		NL_SET_ERR_MSG(extack, "unable to install XDP to device using memory provider");
10637 		return -EBUSY;
10638 	}
10639 
10640 	memset(&xdp, 0, sizeof(xdp));
10641 	xdp.command = mode == XDP_MODE_HW ? XDP_SETUP_PROG_HW : XDP_SETUP_PROG;
10642 	xdp.extack = extack;
10643 	xdp.flags = flags;
10644 	xdp.prog = prog;
10645 
10646 	/* Drivers assume refcnt is already incremented (i.e, prog pointer is
10647 	 * "moved" into driver), so they don't increment it on their own, but
10648 	 * they do decrement refcnt when program is detached or replaced.
10649 	 * Given net_device also owns link/prog, we need to bump refcnt here
10650 	 * to prevent drivers from underflowing it.
10651 	 */
10652 	if (prog)
10653 		bpf_prog_inc(prog);
10654 	err = bpf_op(dev, &xdp);
10655 	if (err) {
10656 		if (prog)
10657 			bpf_prog_put(prog);
10658 		return err;
10659 	}
10660 
10661 	if (mode != XDP_MODE_HW)
10662 		bpf_prog_change_xdp(dev_xdp_prog(dev, mode), prog);
10663 
10664 	return 0;
10665 }
10666 
10667 static void dev_xdp_uninstall(struct net_device *dev)
10668 {
10669 	struct bpf_xdp_link *link;
10670 	struct bpf_prog *prog;
10671 	enum bpf_xdp_mode mode;
10672 	bpf_op_t bpf_op;
10673 
10674 	ASSERT_RTNL();
10675 
10676 	for (mode = XDP_MODE_SKB; mode < __MAX_XDP_MODE; mode++) {
10677 		prog = dev_xdp_prog(dev, mode);
10678 		if (!prog)
10679 			continue;
10680 
10681 		bpf_op = dev_xdp_bpf_op(dev, mode);
10682 		if (!bpf_op)
10683 			continue;
10684 
10685 		WARN_ON(dev_xdp_install(dev, mode, bpf_op, NULL, 0, NULL));
10686 
10687 		/* auto-detach link from net device */
10688 		link = dev_xdp_link(dev, mode);
10689 		if (link)
10690 			link->dev = NULL;
10691 		else
10692 			bpf_prog_put(prog);
10693 
10694 		dev_xdp_set_link(dev, mode, NULL);
10695 	}
10696 }
10697 
10698 static int dev_xdp_attach(struct net_device *dev, struct netlink_ext_ack *extack,
10699 			  struct bpf_xdp_link *link, struct bpf_prog *new_prog,
10700 			  struct bpf_prog *old_prog, u32 flags)
10701 {
10702 	unsigned int num_modes = hweight32(flags & XDP_FLAGS_MODES);
10703 	struct bpf_prog *cur_prog;
10704 	struct net_device *upper;
10705 	struct list_head *iter;
10706 	enum bpf_xdp_mode mode;
10707 	bpf_op_t bpf_op;
10708 	int err;
10709 
10710 	ASSERT_RTNL();
10711 
10712 	/* either link or prog attachment, never both */
10713 	if (link && (new_prog || old_prog))
10714 		return -EINVAL;
10715 	/* link supports only XDP mode flags */
10716 	if (link && (flags & ~XDP_FLAGS_MODES)) {
10717 		NL_SET_ERR_MSG(extack, "Invalid XDP flags for BPF link attachment");
10718 		return -EINVAL;
10719 	}
10720 	/* just one XDP mode bit should be set, zero defaults to drv/skb mode */
10721 	if (num_modes > 1) {
10722 		NL_SET_ERR_MSG(extack, "Only one XDP mode flag can be set");
10723 		return -EINVAL;
10724 	}
10725 	/* avoid ambiguity if offload + drv/skb mode progs are both loaded */
10726 	if (!num_modes && dev_xdp_prog_count(dev) > 1) {
10727 		NL_SET_ERR_MSG(extack,
10728 			       "More than one program loaded, unset mode is ambiguous");
10729 		return -EINVAL;
10730 	}
10731 	/* old_prog != NULL implies XDP_FLAGS_REPLACE is set */
10732 	if (old_prog && !(flags & XDP_FLAGS_REPLACE)) {
10733 		NL_SET_ERR_MSG(extack, "XDP_FLAGS_REPLACE is not specified");
10734 		return -EINVAL;
10735 	}
10736 
10737 	mode = dev_xdp_mode(dev, flags);
10738 	/* can't replace attached link */
10739 	if (dev_xdp_link(dev, mode)) {
10740 		NL_SET_ERR_MSG(extack, "Can't replace active BPF XDP link");
10741 		return -EBUSY;
10742 	}
10743 
10744 	/* don't allow if an upper device already has a program */
10745 	netdev_for_each_upper_dev_rcu(dev, upper, iter) {
10746 		if (dev_xdp_prog_count(upper) > 0) {
10747 			NL_SET_ERR_MSG(extack, "Cannot attach when an upper device already has a program");
10748 			return -EEXIST;
10749 		}
10750 	}
10751 
10752 	cur_prog = dev_xdp_prog(dev, mode);
10753 	/* can't replace attached prog with link */
10754 	if (link && cur_prog) {
10755 		NL_SET_ERR_MSG(extack, "Can't replace active XDP program with BPF link");
10756 		return -EBUSY;
10757 	}
10758 	if ((flags & XDP_FLAGS_REPLACE) && cur_prog != old_prog) {
10759 		NL_SET_ERR_MSG(extack, "Active program does not match expected");
10760 		return -EEXIST;
10761 	}
10762 
10763 	/* put effective new program into new_prog */
10764 	if (link)
10765 		new_prog = link->link.prog;
10766 
10767 	if (new_prog) {
10768 		if ((flags & XDP_FLAGS_UPDATE_IF_NOEXIST) && cur_prog) {
10769 			NL_SET_ERR_MSG(extack, "XDP program already attached");
10770 			return -EBUSY;
10771 		}
10772 	}
10773 
10774 	/* don't call drivers if the effective program didn't change */
10775 	if (new_prog != cur_prog) {
10776 		bpf_op = dev_xdp_bpf_op(dev, mode);
10777 		if (!bpf_op) {
10778 			NL_SET_ERR_MSG(extack, "Underlying driver does not support XDP in native mode");
10779 			return -EOPNOTSUPP;
10780 		}
10781 
10782 		err = dev_xdp_install(dev, mode, bpf_op, extack, flags, new_prog);
10783 		if (err)
10784 			return err;
10785 	}
10786 
10787 	if (link)
10788 		dev_xdp_set_link(dev, mode, link);
10789 	else
10790 		dev_xdp_set_prog(dev, mode, new_prog);
10791 	if (cur_prog)
10792 		bpf_prog_put(cur_prog);
10793 
10794 	return 0;
10795 }
10796 
10797 static int dev_xdp_attach_link(struct net_device *dev,
10798 			       struct netlink_ext_ack *extack,
10799 			       struct bpf_xdp_link *link)
10800 {
10801 	return dev_xdp_attach(dev, extack, link, NULL, NULL, link->flags);
10802 }
10803 
10804 static int dev_xdp_detach_link(struct net_device *dev,
10805 			       struct netlink_ext_ack *extack,
10806 			       struct bpf_xdp_link *link)
10807 {
10808 	enum bpf_xdp_mode mode;
10809 	bpf_op_t bpf_op;
10810 
10811 	ASSERT_RTNL();
10812 
10813 	mode = dev_xdp_mode(dev, link->flags);
10814 	if (dev_xdp_link(dev, mode) != link)
10815 		return -EINVAL;
10816 
10817 	bpf_op = dev_xdp_bpf_op(dev, mode);
10818 	WARN_ON(dev_xdp_install(dev, mode, bpf_op, NULL, 0, NULL));
10819 	dev_xdp_set_link(dev, mode, NULL);
10820 	return 0;
10821 }
10822 
10823 static void bpf_xdp_link_release(struct bpf_link *link)
10824 {
10825 	struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
10826 
10827 	rtnl_lock();
10828 
10829 	/* if racing with net_device's tear down, xdp_link->dev might be
10830 	 * already NULL, in which case link was already auto-detached
10831 	 */
10832 	if (xdp_link->dev) {
10833 		netdev_lock_ops(xdp_link->dev);
10834 		WARN_ON(dev_xdp_detach_link(xdp_link->dev, NULL, xdp_link));
10835 		netdev_unlock_ops(xdp_link->dev);
10836 		xdp_link->dev = NULL;
10837 	}
10838 
10839 	rtnl_unlock();
10840 }
10841 
10842 static int bpf_xdp_link_detach(struct bpf_link *link)
10843 {
10844 	bpf_xdp_link_release(link);
10845 	return 0;
10846 }
10847 
10848 static void bpf_xdp_link_dealloc(struct bpf_link *link)
10849 {
10850 	struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
10851 
10852 	kfree(xdp_link);
10853 }
10854 
10855 static void bpf_xdp_link_show_fdinfo(const struct bpf_link *link,
10856 				     struct seq_file *seq)
10857 {
10858 	struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
10859 	u32 ifindex = 0;
10860 
10861 	rtnl_lock();
10862 	if (xdp_link->dev)
10863 		ifindex = xdp_link->dev->ifindex;
10864 	rtnl_unlock();
10865 
10866 	seq_printf(seq, "ifindex:\t%u\n", ifindex);
10867 }
10868 
10869 static int bpf_xdp_link_fill_link_info(const struct bpf_link *link,
10870 				       struct bpf_link_info *info)
10871 {
10872 	struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
10873 	u32 ifindex = 0;
10874 
10875 	rtnl_lock();
10876 	if (xdp_link->dev)
10877 		ifindex = xdp_link->dev->ifindex;
10878 	rtnl_unlock();
10879 
10880 	info->xdp.ifindex = ifindex;
10881 	return 0;
10882 }
10883 
10884 static int bpf_xdp_link_update(struct bpf_link *link, struct bpf_prog *new_prog,
10885 			       struct bpf_prog *old_prog)
10886 {
10887 	struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link);
10888 	enum bpf_xdp_mode mode;
10889 	bpf_op_t bpf_op;
10890 	int err = 0;
10891 
10892 	rtnl_lock();
10893 
10894 	/* link might have been auto-released already, so fail */
10895 	if (!xdp_link->dev) {
10896 		err = -ENOLINK;
10897 		goto out_unlock;
10898 	}
10899 
10900 	if (old_prog && link->prog != old_prog) {
10901 		err = -EPERM;
10902 		goto out_unlock;
10903 	}
10904 	old_prog = link->prog;
10905 	if (old_prog->type != new_prog->type ||
10906 	    old_prog->expected_attach_type != new_prog->expected_attach_type) {
10907 		err = -EINVAL;
10908 		goto out_unlock;
10909 	}
10910 
10911 	if (old_prog == new_prog) {
10912 		/* no-op, don't disturb drivers */
10913 		bpf_prog_put(new_prog);
10914 		goto out_unlock;
10915 	}
10916 
10917 	netdev_lock_ops(xdp_link->dev);
10918 	mode = dev_xdp_mode(xdp_link->dev, xdp_link->flags);
10919 	bpf_op = dev_xdp_bpf_op(xdp_link->dev, mode);
10920 	err = dev_xdp_install(xdp_link->dev, mode, bpf_op, NULL,
10921 			      xdp_link->flags, new_prog);
10922 	if (!err)
10923 		old_prog = xchg(&link->prog, new_prog);
10924 	netdev_unlock_ops(xdp_link->dev);
10925 	if (err)
10926 		goto out_unlock;
10927 
10928 	bpf_prog_put(old_prog);
10929 
10930 out_unlock:
10931 	rtnl_unlock();
10932 	return err;
10933 }
10934 
10935 static const struct bpf_link_ops bpf_xdp_link_lops = {
10936 	.release = bpf_xdp_link_release,
10937 	.dealloc = bpf_xdp_link_dealloc,
10938 	.detach = bpf_xdp_link_detach,
10939 	.show_fdinfo = bpf_xdp_link_show_fdinfo,
10940 	.fill_link_info = bpf_xdp_link_fill_link_info,
10941 	.update_prog = bpf_xdp_link_update,
10942 };
10943 
10944 int bpf_xdp_link_attach(const union bpf_attr *attr, struct bpf_prog *prog)
10945 {
10946 	struct net *net = current->nsproxy->net_ns;
10947 	struct bpf_link_primer link_primer;
10948 	struct netlink_ext_ack extack = {};
10949 	struct bpf_xdp_link *link;
10950 	struct net_device *dev;
10951 	int err, fd;
10952 
10953 	rtnl_lock();
10954 	dev = dev_get_by_index(net, attr->link_create.target_ifindex);
10955 	if (!dev) {
10956 		rtnl_unlock();
10957 		return -EINVAL;
10958 	}
10959 
10960 	link = kzalloc_obj(*link, GFP_USER);
10961 	if (!link) {
10962 		err = -ENOMEM;
10963 		goto unlock;
10964 	}
10965 
10966 	bpf_link_init(&link->link, BPF_LINK_TYPE_XDP, &bpf_xdp_link_lops, prog,
10967 		      attr->link_create.attach_type);
10968 	link->dev = dev;
10969 	link->flags = attr->link_create.flags;
10970 
10971 	err = bpf_link_prime(&link->link, &link_primer);
10972 	if (err) {
10973 		kfree(link);
10974 		goto unlock;
10975 	}
10976 
10977 	netdev_lock_ops(dev);
10978 	err = dev_xdp_attach_link(dev, &extack, link);
10979 	netdev_unlock_ops(dev);
10980 	rtnl_unlock();
10981 
10982 	if (err) {
10983 		link->dev = NULL;
10984 		bpf_link_cleanup(&link_primer);
10985 		trace_bpf_xdp_link_attach_failed(extack._msg);
10986 		goto out_put_dev;
10987 	}
10988 
10989 	fd = bpf_link_settle(&link_primer);
10990 	/* link itself doesn't hold dev's refcnt to not complicate shutdown */
10991 	dev_put(dev);
10992 	return fd;
10993 
10994 unlock:
10995 	rtnl_unlock();
10996 
10997 out_put_dev:
10998 	dev_put(dev);
10999 	return err;
11000 }
11001 
11002 /**
11003  *	dev_change_xdp_fd - set or clear a bpf program for a device rx path
11004  *	@dev: device
11005  *	@extack: netlink extended ack
11006  *	@fd: new program fd or negative value to clear
11007  *	@expected_fd: old program fd that userspace expects to replace or clear
11008  *	@flags: xdp-related flags
11009  *
11010  *	Set or clear a bpf program for a device
11011  */
11012 int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack,
11013 		      int fd, int expected_fd, u32 flags)
11014 {
11015 	enum bpf_xdp_mode mode = dev_xdp_mode(dev, flags);
11016 	struct bpf_prog *new_prog = NULL, *old_prog = NULL;
11017 	int err;
11018 
11019 	ASSERT_RTNL();
11020 
11021 	if (fd >= 0) {
11022 		new_prog = bpf_prog_get_type_dev(fd, BPF_PROG_TYPE_XDP,
11023 						 mode != XDP_MODE_SKB);
11024 		if (IS_ERR(new_prog))
11025 			return PTR_ERR(new_prog);
11026 	}
11027 
11028 	if (expected_fd >= 0) {
11029 		old_prog = bpf_prog_get_type_dev(expected_fd, BPF_PROG_TYPE_XDP,
11030 						 mode != XDP_MODE_SKB);
11031 		if (IS_ERR(old_prog)) {
11032 			err = PTR_ERR(old_prog);
11033 			old_prog = NULL;
11034 			goto err_out;
11035 		}
11036 	}
11037 
11038 	err = dev_xdp_attach(dev, extack, NULL, new_prog, old_prog, flags);
11039 
11040 err_out:
11041 	if (err && new_prog)
11042 		bpf_prog_put(new_prog);
11043 	if (old_prog)
11044 		bpf_prog_put(old_prog);
11045 	return err;
11046 }
11047 
11048 u32 dev_get_min_mp_channel_count(const struct net_device *dev)
11049 {
11050 	int i;
11051 
11052 	netdev_assert_locked_ops_compat(dev);
11053 
11054 	for (i = dev->real_num_rx_queues - 1; i >= 0; i--)
11055 		if (dev->_rx[i].mp_params.mp_priv)
11056 			/* The channel count is the idx plus 1. */
11057 			return i + 1;
11058 
11059 	return 0;
11060 }
11061 
11062 /**
11063  * dev_index_reserve() - allocate an ifindex in a namespace
11064  * @net: the applicable net namespace
11065  * @ifindex: requested ifindex, pass %0 to get one allocated
11066  *
11067  * Allocate a ifindex for a new device. Caller must either use the ifindex
11068  * to store the device (via list_netdevice()) or call dev_index_release()
11069  * to give the index up.
11070  *
11071  * Return: a suitable unique value for a new device interface number or -errno.
11072  */
11073 static int dev_index_reserve(struct net *net, u32 ifindex)
11074 {
11075 	int err;
11076 
11077 	if (ifindex > INT_MAX) {
11078 		DEBUG_NET_WARN_ON_ONCE(1);
11079 		return -EINVAL;
11080 	}
11081 
11082 	if (!ifindex)
11083 		err = xa_alloc_cyclic(&net->dev_by_index, &ifindex, NULL,
11084 				      xa_limit_31b, &net->ifindex, GFP_KERNEL);
11085 	else
11086 		err = xa_insert(&net->dev_by_index, ifindex, NULL, GFP_KERNEL);
11087 	if (err < 0)
11088 		return err;
11089 
11090 	return ifindex;
11091 }
11092 
11093 static void dev_index_release(struct net *net, int ifindex)
11094 {
11095 	/* Expect only unused indexes, unlist_netdevice() removes the used */
11096 	WARN_ON(xa_erase(&net->dev_by_index, ifindex));
11097 }
11098 
11099 static bool from_cleanup_net(void)
11100 {
11101 #ifdef CONFIG_NET_NS
11102 	return current == READ_ONCE(cleanup_net_task);
11103 #else
11104 	return false;
11105 #endif
11106 }
11107 
11108 /* Delayed registration/unregisteration */
11109 LIST_HEAD(net_todo_list);
11110 DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq);
11111 atomic_t dev_unreg_count = ATOMIC_INIT(0);
11112 
11113 static void net_set_todo(struct net_device *dev)
11114 {
11115 	list_add_tail(&dev->todo_list, &net_todo_list);
11116 	atomic_inc(&dev_net(dev)->dev_unreg_pending);
11117 }
11118 
11119 /* Paired with net_set_todo(). @net must not be used after this call. */
11120 static void net_unreg_done(struct net *net)
11121 {
11122 	if (atomic_dec_and_test(&net->dev_unreg_pending))
11123 		wake_up(&netdev_unregistering_wq);
11124 }
11125 
11126 static netdev_features_t netdev_sync_upper_features(struct net_device *lower,
11127 	struct net_device *upper, netdev_features_t features)
11128 {
11129 	netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
11130 	netdev_features_t feature;
11131 	int feature_bit;
11132 
11133 	for_each_netdev_feature(upper_disables, feature_bit) {
11134 		feature = __NETIF_F_BIT(feature_bit);
11135 		if (!(upper->wanted_features & feature)
11136 		    && (features & feature)) {
11137 			netdev_dbg(lower, "Dropping feature %pNF, upper dev %s has it off.\n",
11138 				   &feature, upper->name);
11139 			features &= ~feature;
11140 		}
11141 	}
11142 
11143 	return features;
11144 }
11145 
11146 static void netdev_sync_lower_features(struct net_device *upper,
11147 	struct net_device *lower, netdev_features_t features)
11148 {
11149 	netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES;
11150 	netdev_features_t feature;
11151 	int feature_bit;
11152 
11153 	for_each_netdev_feature(upper_disables, feature_bit) {
11154 		feature = __NETIF_F_BIT(feature_bit);
11155 		if (!(features & feature) && (lower->features & feature)) {
11156 			netdev_dbg(upper, "Disabling feature %pNF on lower dev %s.\n",
11157 				   &feature, lower->name);
11158 			netdev_lock_ops(lower);
11159 			lower->wanted_features &= ~feature;
11160 			__netdev_update_features(lower);
11161 
11162 			if (unlikely(lower->features & feature))
11163 				netdev_WARN(upper, "failed to disable %pNF on %s!\n",
11164 					    &feature, lower->name);
11165 			else
11166 				netdev_features_change(lower);
11167 			netdev_unlock_ops(lower);
11168 		}
11169 	}
11170 }
11171 
11172 static bool netdev_has_ip_or_hw_csum(netdev_features_t features)
11173 {
11174 	netdev_features_t ip_csum_mask = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
11175 	bool ip_csum = (features & ip_csum_mask) == ip_csum_mask;
11176 	bool hw_csum = features & NETIF_F_HW_CSUM;
11177 
11178 	return ip_csum || hw_csum;
11179 }
11180 
11181 static netdev_features_t netdev_fix_features(struct net_device *dev,
11182 	netdev_features_t features)
11183 {
11184 	/* Fix illegal checksum combinations */
11185 	if ((features & NETIF_F_HW_CSUM) &&
11186 	    (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) {
11187 		netdev_warn(dev, "mixed HW and IP checksum settings.\n");
11188 		features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
11189 	}
11190 
11191 	/* TSO requires that SG is present as well. */
11192 	if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) {
11193 		netdev_dbg(dev, "Dropping TSO features since no SG feature.\n");
11194 		features &= ~NETIF_F_ALL_TSO;
11195 	}
11196 
11197 	if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) &&
11198 					!(features & NETIF_F_IP_CSUM)) {
11199 		netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n");
11200 		features &= ~NETIF_F_TSO;
11201 		features &= ~NETIF_F_TSO_ECN;
11202 	}
11203 
11204 	if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) &&
11205 					 !(features & NETIF_F_IPV6_CSUM)) {
11206 		netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n");
11207 		features &= ~NETIF_F_TSO6;
11208 	}
11209 
11210 	/* TSO with IPv4 ID mangling requires IPv4 TSO be enabled */
11211 	if ((features & NETIF_F_TSO_MANGLEID) && !(features & NETIF_F_TSO))
11212 		features &= ~NETIF_F_TSO_MANGLEID;
11213 
11214 	/* TSO ECN requires that TSO is present as well. */
11215 	if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN)
11216 		features &= ~NETIF_F_TSO_ECN;
11217 
11218 	/* Software GSO depends on SG. */
11219 	if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) {
11220 		netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n");
11221 		features &= ~NETIF_F_GSO;
11222 	}
11223 
11224 	/* GSO partial features require GSO partial be set */
11225 	if ((features & dev->gso_partial_features) &&
11226 	    !(features & NETIF_F_GSO_PARTIAL)) {
11227 		netdev_dbg(dev,
11228 			   "Dropping partially supported GSO features since no GSO partial.\n");
11229 		features &= ~dev->gso_partial_features;
11230 	}
11231 
11232 	if (!(features & NETIF_F_RXCSUM)) {
11233 		/* NETIF_F_GRO_HW implies doing RXCSUM since every packet
11234 		 * successfully merged by hardware must also have the
11235 		 * checksum verified by hardware.  If the user does not
11236 		 * want to enable RXCSUM, logically, we should disable GRO_HW.
11237 		 */
11238 		if (features & NETIF_F_GRO_HW) {
11239 			netdev_dbg(dev, "Dropping NETIF_F_GRO_HW since no RXCSUM feature.\n");
11240 			features &= ~NETIF_F_GRO_HW;
11241 		}
11242 	}
11243 
11244 	/* LRO/HW-GRO features cannot be combined with RX-FCS */
11245 	if (features & NETIF_F_RXFCS) {
11246 		if (features & NETIF_F_LRO) {
11247 			netdev_dbg(dev, "Dropping LRO feature since RX-FCS is requested.\n");
11248 			features &= ~NETIF_F_LRO;
11249 		}
11250 
11251 		if (features & NETIF_F_GRO_HW) {
11252 			netdev_dbg(dev, "Dropping HW-GRO feature since RX-FCS is requested.\n");
11253 			features &= ~NETIF_F_GRO_HW;
11254 		}
11255 	}
11256 
11257 	if ((features & NETIF_F_GRO_HW) && (features & NETIF_F_LRO)) {
11258 		netdev_dbg(dev, "Dropping LRO feature since HW-GRO is requested.\n");
11259 		features &= ~NETIF_F_LRO;
11260 	}
11261 
11262 	if ((features & NETIF_F_HW_TLS_TX) && !netdev_has_ip_or_hw_csum(features)) {
11263 		netdev_dbg(dev, "Dropping TLS TX HW offload feature since no CSUM feature.\n");
11264 		features &= ~NETIF_F_HW_TLS_TX;
11265 	}
11266 
11267 	if ((features & NETIF_F_HW_TLS_RX) && !(features & NETIF_F_RXCSUM)) {
11268 		netdev_dbg(dev, "Dropping TLS RX HW offload feature since no RXCSUM feature.\n");
11269 		features &= ~NETIF_F_HW_TLS_RX;
11270 	}
11271 
11272 	if ((features & NETIF_F_GSO_UDP_L4) && !netdev_has_ip_or_hw_csum(features)) {
11273 		netdev_dbg(dev, "Dropping USO feature since no CSUM feature.\n");
11274 		features &= ~NETIF_F_GSO_UDP_L4;
11275 	}
11276 
11277 	return features;
11278 }
11279 
11280 int __netdev_update_features(struct net_device *dev)
11281 {
11282 	struct net_device *upper, *lower;
11283 	netdev_features_t features;
11284 	struct list_head *iter;
11285 	int err = -1;
11286 
11287 	ASSERT_RTNL();
11288 	netdev_assert_locked_ops_compat(dev);
11289 
11290 	features = netdev_get_wanted_features(dev);
11291 
11292 	if (dev->netdev_ops->ndo_fix_features)
11293 		features = dev->netdev_ops->ndo_fix_features(dev, features);
11294 
11295 	/* driver might be less strict about feature dependencies */
11296 	features = netdev_fix_features(dev, features);
11297 
11298 	/* some features can't be enabled if they're off on an upper device */
11299 	netdev_for_each_upper_dev_rcu(dev, upper, iter)
11300 		features = netdev_sync_upper_features(dev, upper, features);
11301 
11302 	if (dev->features == features)
11303 		goto sync_lower;
11304 
11305 	netdev_dbg(dev, "Features changed: %pNF -> %pNF\n",
11306 		&dev->features, &features);
11307 
11308 	if (dev->netdev_ops->ndo_set_features)
11309 		err = dev->netdev_ops->ndo_set_features(dev, features);
11310 	else
11311 		err = 0;
11312 
11313 	if (unlikely(err < 0)) {
11314 		netdev_err(dev,
11315 			"set_features() failed (%d); wanted %pNF, left %pNF\n",
11316 			err, &features, &dev->features);
11317 		/* return non-0 since some features might have changed and
11318 		 * it's better to fire a spurious notification than miss it
11319 		 */
11320 		return -1;
11321 	}
11322 
11323 sync_lower:
11324 	/* some features must be disabled on lower devices when disabled
11325 	 * on an upper device (think: bonding master or bridge)
11326 	 */
11327 	netdev_for_each_lower_dev(dev, lower, iter)
11328 		netdev_sync_lower_features(dev, lower, features);
11329 
11330 	if (!err) {
11331 		netdev_features_t diff = features ^ dev->features;
11332 
11333 		if (diff & NETIF_F_RX_UDP_TUNNEL_PORT) {
11334 			/* udp_tunnel_{get,drop}_rx_info both need
11335 			 * NETIF_F_RX_UDP_TUNNEL_PORT enabled on the
11336 			 * device, or they won't do anything.
11337 			 * Thus we need to update dev->features
11338 			 * *before* calling udp_tunnel_get_rx_info,
11339 			 * but *after* calling udp_tunnel_drop_rx_info.
11340 			 */
11341 			udp_tunnel_nic_lock(dev);
11342 			if (features & NETIF_F_RX_UDP_TUNNEL_PORT) {
11343 				dev->features = features;
11344 				udp_tunnel_get_rx_info(dev);
11345 			} else {
11346 				udp_tunnel_drop_rx_info(dev);
11347 			}
11348 			udp_tunnel_nic_unlock(dev);
11349 		}
11350 
11351 		if (diff & NETIF_F_HW_VLAN_CTAG_FILTER) {
11352 			if (features & NETIF_F_HW_VLAN_CTAG_FILTER) {
11353 				dev->features = features;
11354 				err |= vlan_get_rx_ctag_filter_info(dev);
11355 			} else {
11356 				vlan_drop_rx_ctag_filter_info(dev);
11357 			}
11358 		}
11359 
11360 		if (diff & NETIF_F_HW_VLAN_STAG_FILTER) {
11361 			if (features & NETIF_F_HW_VLAN_STAG_FILTER) {
11362 				dev->features = features;
11363 				err |= vlan_get_rx_stag_filter_info(dev);
11364 			} else {
11365 				vlan_drop_rx_stag_filter_info(dev);
11366 			}
11367 		}
11368 
11369 		dev->features = features;
11370 	}
11371 
11372 	return err < 0 ? 0 : 1;
11373 }
11374 
11375 /**
11376  *	netdev_update_features - recalculate device features
11377  *	@dev: the device to check
11378  *
11379  *	Recalculate dev->features set and send notifications if it
11380  *	has changed. Should be called after driver or hardware dependent
11381  *	conditions might have changed that influence the features.
11382  */
11383 void netdev_update_features(struct net_device *dev)
11384 {
11385 	if (__netdev_update_features(dev))
11386 		netdev_features_change(dev);
11387 }
11388 EXPORT_SYMBOL(netdev_update_features);
11389 
11390 /**
11391  *	netdev_change_features - recalculate device features
11392  *	@dev: the device to check
11393  *
11394  *	Recalculate dev->features set and send notifications even
11395  *	if they have not changed. Should be called instead of
11396  *	netdev_update_features() if also dev->vlan_features might
11397  *	have changed to allow the changes to be propagated to stacked
11398  *	VLAN devices.
11399  */
11400 void netdev_change_features(struct net_device *dev)
11401 {
11402 	__netdev_update_features(dev);
11403 	netdev_features_change(dev);
11404 }
11405 EXPORT_SYMBOL(netdev_change_features);
11406 
11407 /**
11408  *	netif_stacked_transfer_operstate -	transfer operstate
11409  *	@rootdev: the root or lower level device to transfer state from
11410  *	@dev: the device to transfer operstate to
11411  *
11412  *	Transfer operational state from root to device. This is normally
11413  *	called when a stacking relationship exists between the root
11414  *	device and the device(a leaf device).
11415  */
11416 void netif_stacked_transfer_operstate(const struct net_device *rootdev,
11417 					struct net_device *dev)
11418 {
11419 	if (rootdev->operstate == IF_OPER_DORMANT)
11420 		netif_dormant_on(dev);
11421 	else
11422 		netif_dormant_off(dev);
11423 
11424 	if (rootdev->operstate == IF_OPER_TESTING)
11425 		netif_testing_on(dev);
11426 	else
11427 		netif_testing_off(dev);
11428 
11429 	if (netif_carrier_ok(rootdev))
11430 		netif_carrier_on(dev);
11431 	else
11432 		netif_carrier_off(dev);
11433 }
11434 EXPORT_SYMBOL(netif_stacked_transfer_operstate);
11435 
11436 static int netif_alloc_rx_queues(struct net_device *dev)
11437 {
11438 	unsigned int i, count = dev->num_rx_queues;
11439 	struct netdev_rx_queue *rx;
11440 	size_t sz = count * sizeof(*rx);
11441 	int err = 0;
11442 
11443 	BUG_ON(count < 1);
11444 
11445 	rx = kvzalloc(sz, GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL);
11446 	if (!rx)
11447 		return -ENOMEM;
11448 
11449 	dev->_rx = rx;
11450 
11451 	for (i = 0; i < count; i++) {
11452 		rx[i].dev = dev;
11453 
11454 		/* XDP RX-queue setup */
11455 		err = xdp_rxq_info_reg(&rx[i].xdp_rxq, dev, i, 0);
11456 		if (err < 0)
11457 			goto err_rxq_info;
11458 	}
11459 	return 0;
11460 
11461 err_rxq_info:
11462 	/* Rollback successful reg's and free other resources */
11463 	while (i--)
11464 		xdp_rxq_info_unreg(&rx[i].xdp_rxq);
11465 	kvfree(dev->_rx);
11466 	dev->_rx = NULL;
11467 	return err;
11468 }
11469 
11470 static void netif_free_rx_queues(struct net_device *dev)
11471 {
11472 	unsigned int i, count = dev->num_rx_queues;
11473 
11474 	/* netif_alloc_rx_queues alloc failed, resources have been unreg'ed */
11475 	if (!dev->_rx)
11476 		return;
11477 
11478 	for (i = 0; i < count; i++)
11479 		xdp_rxq_info_unreg(&dev->_rx[i].xdp_rxq);
11480 
11481 	kvfree(dev->_rx);
11482 }
11483 
11484 static void netdev_init_one_queue(struct net_device *dev,
11485 				  struct netdev_queue *queue, void *_unused)
11486 {
11487 	/* Initialize queue lock */
11488 	spin_lock_init(&queue->_xmit_lock);
11489 	netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type);
11490 	queue->xmit_lock_owner = -1;
11491 	netdev_queue_numa_node_write(queue, NUMA_NO_NODE);
11492 	queue->dev = dev;
11493 #ifdef CONFIG_BQL
11494 	dql_init(&queue->dql, HZ);
11495 #endif
11496 }
11497 
11498 static void netif_free_tx_queues(struct net_device *dev)
11499 {
11500 	kvfree(dev->_tx);
11501 }
11502 
11503 static int netif_alloc_netdev_queues(struct net_device *dev)
11504 {
11505 	unsigned int count = dev->num_tx_queues;
11506 	struct netdev_queue *tx;
11507 	size_t sz = count * sizeof(*tx);
11508 
11509 	if (count < 1 || count > 0xffff)
11510 		return -EINVAL;
11511 
11512 	tx = kvzalloc(sz, GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL);
11513 	if (!tx)
11514 		return -ENOMEM;
11515 
11516 	dev->_tx = tx;
11517 
11518 	netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL);
11519 	spin_lock_init(&dev->tx_global_lock);
11520 	spin_lock_init(&dev->watchdog_lock);
11521 	dev->watchdog_ref_held = false;
11522 	return 0;
11523 }
11524 
11525 void netif_tx_stop_all_queues(struct net_device *dev)
11526 {
11527 	unsigned int i;
11528 
11529 	for (i = 0; i < dev->num_tx_queues; i++) {
11530 		struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
11531 
11532 		netif_tx_stop_queue(txq);
11533 	}
11534 }
11535 EXPORT_SYMBOL(netif_tx_stop_all_queues);
11536 
11537 static int netdev_do_alloc_pcpu_stats(struct net_device *dev)
11538 {
11539 	void __percpu *v;
11540 
11541 	/* Drivers implementing ndo_get_peer_dev must support tstat
11542 	 * accounting, so that skb_do_redirect() can bump the dev's
11543 	 * RX stats upon network namespace switch.
11544 	 */
11545 	if (dev->netdev_ops->ndo_get_peer_dev &&
11546 	    dev->pcpu_stat_type != NETDEV_PCPU_STAT_TSTATS)
11547 		return -EOPNOTSUPP;
11548 
11549 	switch (dev->pcpu_stat_type) {
11550 	case NETDEV_PCPU_STAT_NONE:
11551 		return 0;
11552 	case NETDEV_PCPU_STAT_LSTATS:
11553 		v = dev->lstats = netdev_alloc_pcpu_stats(struct pcpu_lstats);
11554 		break;
11555 	case NETDEV_PCPU_STAT_TSTATS:
11556 		v = dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats);
11557 		break;
11558 	case NETDEV_PCPU_STAT_DSTATS:
11559 		v = dev->dstats = netdev_alloc_pcpu_stats(struct pcpu_dstats);
11560 		break;
11561 	default:
11562 		return -EINVAL;
11563 	}
11564 
11565 	return v ? 0 : -ENOMEM;
11566 }
11567 
11568 static void netdev_do_free_pcpu_stats(struct net_device *dev)
11569 {
11570 	switch (dev->pcpu_stat_type) {
11571 	case NETDEV_PCPU_STAT_NONE:
11572 		return;
11573 	case NETDEV_PCPU_STAT_LSTATS:
11574 		free_percpu(dev->lstats);
11575 		break;
11576 	case NETDEV_PCPU_STAT_TSTATS:
11577 		free_percpu(dev->tstats);
11578 		break;
11579 	case NETDEV_PCPU_STAT_DSTATS:
11580 		free_percpu(dev->dstats);
11581 		break;
11582 	}
11583 }
11584 
11585 static void netdev_free_phy_link_topology(struct net_device *dev)
11586 {
11587 	struct phy_link_topology *topo = dev->link_topo;
11588 
11589 	if (IS_ENABLED(CONFIG_PHYLIB) && topo) {
11590 		xa_destroy(&topo->phys);
11591 		xa_destroy(&topo->ports);
11592 		kfree(topo);
11593 		dev->link_topo = NULL;
11594 	}
11595 }
11596 
11597 static int netdev_check_ops(struct net_device *dev)
11598 {
11599 	const struct net_device_ops *ops = dev->netdev_ops;
11600 
11601 	if (((dev->hw_features | dev->features) &
11602 	     NETIF_F_HW_VLAN_CTAG_FILTER) &&
11603 	    (!ops->ndo_vlan_rx_add_vid || !ops->ndo_vlan_rx_kill_vid)) {
11604 		netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n");
11605 		return -EINVAL;
11606 	}
11607 
11608 	if (!ops->ndo_hwtstamp_get != !ops->ndo_hwtstamp_set) {
11609 		netdev_WARN(dev, "driver implements only one hwtstamp NDO\n");
11610 		return -EINVAL;
11611 	}
11612 
11613 	if (netdev_need_ops_lock(dev) && ops->ndo_set_rx_mode &&
11614 	    !ops->ndo_set_rx_mode_async) {
11615 		netdev_WARN(dev, "ops-locked drivers should use ndo_set_rx_mode_async\n");
11616 		return -EINVAL;
11617 	}
11618 
11619 	return 0;
11620 }
11621 
11622 /**
11623  * register_netdevice() - register a network device
11624  * @dev: device to register
11625  *
11626  * Take a prepared network device structure and make it externally accessible.
11627  * A %NETDEV_REGISTER message is sent to the netdev notifier chain.
11628  * Callers must hold the rtnl lock - you may want register_netdev()
11629  * instead of this.
11630  */
11631 int register_netdevice(struct net_device *dev)
11632 {
11633 	int ret;
11634 	struct net *net = dev_net(dev);
11635 
11636 	BUILD_BUG_ON(sizeof(netdev_features_t) * BITS_PER_BYTE <
11637 		     NETDEV_FEATURE_COUNT);
11638 	BUG_ON(dev_boot_phase);
11639 	ASSERT_RTNL();
11640 
11641 	might_sleep();
11642 
11643 	/* When net_device's are persistent, this will be fatal. */
11644 	BUG_ON(dev->reg_state != NETREG_UNINITIALIZED);
11645 	BUG_ON(!net);
11646 
11647 	ret = ethtool_check_ops(dev->ethtool_ops);
11648 	if (ret)
11649 		return ret;
11650 
11651 	/* rss ctx ID 0 is reserved for the default context, start from 1 */
11652 	xa_init_flags(&dev->ethtool->rss_ctx, XA_FLAGS_ALLOC1);
11653 	mutex_init(&dev->ethtool->rss_lock);
11654 
11655 	spin_lock_init(&dev->addr_list_lock);
11656 	netdev_set_addr_lockdep_class(dev);
11657 
11658 	ret = dev_get_valid_name(net, dev, dev->name);
11659 	if (ret < 0)
11660 		goto out;
11661 
11662 	ret = -ENOMEM;
11663 	dev->name_node = netdev_name_node_head_alloc(dev);
11664 	if (!dev->name_node)
11665 		goto out;
11666 
11667 	/* Init, if this function is available */
11668 	if (dev->netdev_ops->ndo_init) {
11669 		ret = dev->netdev_ops->ndo_init(dev);
11670 		if (ret) {
11671 			if (ret > 0)
11672 				ret = -EIO;
11673 			goto err_free_name;
11674 		}
11675 	}
11676 
11677 	ret = netdev_check_ops(dev);
11678 	if (ret)
11679 		goto err_uninit;
11680 
11681 	ret = netdev_do_alloc_pcpu_stats(dev);
11682 	if (ret)
11683 		goto err_uninit;
11684 
11685 	ret = dev_index_reserve(net, dev->ifindex);
11686 	if (ret < 0)
11687 		goto err_free_pcpu;
11688 	dev->ifindex = ret;
11689 
11690 	/* Transfer changeable features to wanted_features and enable
11691 	 * software offloads (GSO and GRO).
11692 	 */
11693 	dev->hw_features |= (NETIF_F_SOFT_FEATURES | NETIF_F_SOFT_FEATURES_OFF);
11694 	dev->features |= NETIF_F_SOFT_FEATURES;
11695 
11696 	if (dev->udp_tunnel_nic_info) {
11697 		dev->features |= NETIF_F_RX_UDP_TUNNEL_PORT;
11698 		dev->hw_features |= NETIF_F_RX_UDP_TUNNEL_PORT;
11699 	}
11700 
11701 	dev->wanted_features = dev->features & dev->hw_features;
11702 
11703 	if (!(dev->flags & IFF_LOOPBACK))
11704 		dev->hw_features |= NETIF_F_NOCACHE_COPY;
11705 
11706 	/* If IPv4 TCP segmentation offload is supported we should also
11707 	 * allow the device to enable segmenting the frame with the option
11708 	 * of ignoring a static IP ID value.  This doesn't enable the
11709 	 * feature itself but allows the user to enable it later.
11710 	 */
11711 	if (dev->hw_features & NETIF_F_TSO)
11712 		dev->hw_features |= NETIF_F_TSO_MANGLEID;
11713 	if (dev->vlan_features & NETIF_F_TSO)
11714 		dev->vlan_features |= NETIF_F_TSO_MANGLEID;
11715 	if (dev->mpls_features & NETIF_F_TSO)
11716 		dev->mpls_features |= NETIF_F_TSO_MANGLEID;
11717 	if (dev->hw_enc_features & NETIF_F_TSO)
11718 		dev->hw_enc_features |= NETIF_F_TSO_MANGLEID;
11719 
11720 	/* TSO_MANGLEID belongs in mangleid_features by definition */
11721 	dev->mangleid_features |= NETIF_F_TSO_MANGLEID;
11722 
11723 	/* Make NETIF_F_HIGHDMA inheritable to VLAN devices.
11724 	 */
11725 	dev->vlan_features |= NETIF_F_HIGHDMA;
11726 
11727 	/* Make NETIF_F_SG inheritable to tunnel devices.
11728 	 */
11729 	dev->hw_enc_features |= NETIF_F_SG | NETIF_F_GSO_PARTIAL;
11730 
11731 	/* Make NETIF_F_SG inheritable to MPLS.
11732 	 */
11733 	dev->mpls_features |= NETIF_F_SG;
11734 
11735 	ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev);
11736 	ret = notifier_to_errno(ret);
11737 	if (ret)
11738 		goto err_ifindex_release;
11739 
11740 	ret = netdev_register_kobject(dev);
11741 
11742 	netdev_lock(dev);
11743 	WRITE_ONCE(dev->reg_state, ret ? NETREG_UNREGISTERED : NETREG_REGISTERED);
11744 	netdev_unlock(dev);
11745 
11746 	if (ret)
11747 		goto err_uninit_notify;
11748 
11749 	netdev_lock_ops(dev);
11750 	__netdev_update_features(dev);
11751 	netdev_unlock_ops(dev);
11752 
11753 	/*
11754 	 *	Default initial state at registry is that the
11755 	 *	device is present.
11756 	 */
11757 
11758 	set_bit(__LINK_STATE_PRESENT, &dev->state);
11759 
11760 	linkwatch_init_dev(dev);
11761 
11762 	dev_init_scheduler(dev);
11763 
11764 	netdev_hold(dev, &dev->dev_registered_tracker, GFP_KERNEL);
11765 	list_netdevice(dev);
11766 
11767 	add_device_randomness(dev->dev_addr, dev->addr_len);
11768 
11769 	/* If the device has permanent device address, driver should
11770 	 * set dev_addr and also addr_assign_type should be set to
11771 	 * NET_ADDR_PERM (default value).
11772 	 */
11773 	if (dev->addr_assign_type == NET_ADDR_PERM)
11774 		memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len);
11775 
11776 	/* Notify protocols, that a new device appeared. */
11777 	netdev_lock_ops(dev);
11778 	ret = call_netdevice_notifiers(NETDEV_REGISTER, dev);
11779 	netdev_unlock_ops(dev);
11780 	ret = notifier_to_errno(ret);
11781 	if (ret) {
11782 		/* Expect explicit free_netdev() on failure */
11783 		dev->needs_free_netdev = false;
11784 		unregister_netdevice_queue(dev, NULL);
11785 		goto out;
11786 	}
11787 	/*
11788 	 *	Prevent userspace races by waiting until the network
11789 	 *	device is fully setup before sending notifications.
11790 	 */
11791 	netdev_uevent_add(dev);
11792 	if (!(dev->rtnl_link_ops && dev->rtnl_link_initializing))
11793 		rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL, 0, NULL);
11794 
11795 out:
11796 	return ret;
11797 
11798 err_uninit_notify:
11799 	call_netdevice_notifiers(NETDEV_PRE_UNINIT, dev);
11800 err_ifindex_release:
11801 	dev_index_release(net, dev->ifindex);
11802 err_free_pcpu:
11803 	netdev_do_free_pcpu_stats(dev);
11804 err_uninit:
11805 	if (dev->netdev_ops->ndo_uninit)
11806 		dev->netdev_ops->ndo_uninit(dev);
11807 	if (dev->priv_destructor)
11808 		dev->priv_destructor(dev);
11809 err_free_name:
11810 	netdev_name_node_free(dev->name_node);
11811 	goto out;
11812 }
11813 EXPORT_SYMBOL(register_netdevice);
11814 
11815 /* Initialize the core of a dummy net device.
11816  * The setup steps dummy netdevs need which normal netdevs get by going
11817  * through register_netdevice().
11818  */
11819 static void init_dummy_netdev(struct net_device *dev)
11820 {
11821 	/* make sure we BUG if trying to hit standard
11822 	 * register/unregister code path
11823 	 */
11824 	dev->reg_state = NETREG_DUMMY;
11825 
11826 	/* a dummy interface is started by default */
11827 	set_bit(__LINK_STATE_PRESENT, &dev->state);
11828 	set_bit(__LINK_STATE_START, &dev->state);
11829 
11830 	/* Note : We dont allocate pcpu_refcnt for dummy devices,
11831 	 * because users of this 'device' dont need to change
11832 	 * its refcount.
11833 	 */
11834 }
11835 
11836 /**
11837  *	register_netdev	- register a network device
11838  *	@dev: device to register
11839  *
11840  *	Take a completed network device structure and add it to the kernel
11841  *	interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier
11842  *	chain. 0 is returned on success. A negative errno code is returned
11843  *	on a failure to set up the device, or if the name is a duplicate.
11844  *
11845  *	This is a wrapper around register_netdevice that takes the rtnl semaphore
11846  *	and expands the device name if you passed a format string to
11847  *	alloc_netdev.
11848  */
11849 int register_netdev(struct net_device *dev)
11850 {
11851 	struct net *net = dev_net(dev);
11852 	int err;
11853 
11854 	if (rtnl_net_lock_killable(net))
11855 		return -EINTR;
11856 
11857 	err = register_netdevice(dev);
11858 
11859 	rtnl_net_unlock(net);
11860 
11861 	return err;
11862 }
11863 EXPORT_SYMBOL(register_netdev);
11864 
11865 int netdev_refcnt_read(const struct net_device *dev)
11866 {
11867 #ifdef CONFIG_PCPU_DEV_REFCNT
11868 	int i, refcnt = 0;
11869 
11870 	for_each_possible_cpu(i)
11871 		refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i);
11872 	return refcnt;
11873 #else
11874 	return refcount_read(&dev->dev_refcnt);
11875 #endif
11876 }
11877 EXPORT_SYMBOL(netdev_refcnt_read);
11878 
11879 int netdev_unregister_timeout_secs __read_mostly = 10;
11880 
11881 #define WAIT_REFS_MIN_MSECS 1
11882 #define WAIT_REFS_MAX_MSECS 250
11883 /**
11884  * netdev_wait_allrefs_any - wait until all references are gone.
11885  * @list: list of net_devices to wait on
11886  *
11887  * This is called when unregistering network devices.
11888  *
11889  * Any protocol or device that holds a reference should register
11890  * for netdevice notification, and cleanup and put back the
11891  * reference if they receive an UNREGISTER event.
11892  * We can get stuck here if buggy protocols don't correctly
11893  * call dev_put.
11894  */
11895 static struct net_device *netdev_wait_allrefs_any(struct list_head *list)
11896 {
11897 	unsigned long rebroadcast_time, warning_time;
11898 	struct net_device *dev;
11899 	int wait = 0;
11900 
11901 	rebroadcast_time = warning_time = jiffies;
11902 
11903 	list_for_each_entry(dev, list, todo_list)
11904 		if (netdev_refcnt_read(dev) == 1)
11905 			return dev;
11906 
11907 	while (true) {
11908 		if (time_after(jiffies, rebroadcast_time + 1 * HZ)) {
11909 			rtnl_lock();
11910 
11911 			/* Rebroadcast unregister notification */
11912 			list_for_each_entry(dev, list, todo_list) {
11913 				struct net *net = dev_net(dev);
11914 
11915 				__rtnl_net_lock(net);
11916 				call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
11917 				__rtnl_net_unlock(net);
11918 			}
11919 
11920 			__rtnl_unlock();
11921 			rcu_barrier();
11922 			rtnl_lock();
11923 
11924 			list_for_each_entry(dev, list, todo_list)
11925 				if (test_bit(__LINK_STATE_LINKWATCH_PENDING,
11926 					     &dev->state)) {
11927 					/* We must not have linkwatch events
11928 					 * pending on unregister. If this
11929 					 * happens, we simply run the queue
11930 					 * unscheduled, resulting in a noop
11931 					 * for this device.
11932 					 */
11933 					linkwatch_run_queue();
11934 					break;
11935 				}
11936 
11937 			__rtnl_unlock();
11938 
11939 			rebroadcast_time = jiffies;
11940 		}
11941 
11942 		rcu_barrier();
11943 
11944 		if (!wait) {
11945 			wait = WAIT_REFS_MIN_MSECS;
11946 		} else {
11947 			msleep(wait);
11948 			wait = min(wait << 1, WAIT_REFS_MAX_MSECS);
11949 		}
11950 
11951 		list_for_each_entry(dev, list, todo_list)
11952 			if (netdev_refcnt_read(dev) == 1)
11953 				return dev;
11954 
11955 		if (time_after(jiffies, warning_time +
11956 			       READ_ONCE(netdev_unregister_timeout_secs) * HZ)) {
11957 			list_for_each_entry(dev, list, todo_list) {
11958 				pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n",
11959 					 dev->name, netdev_refcnt_read(dev));
11960 				ref_tracker_dir_print(&dev->refcnt_tracker, 10);
11961 			}
11962 
11963 			warning_time = jiffies;
11964 		}
11965 	}
11966 }
11967 
11968 /* The sequence is:
11969  *
11970  *	rtnl_lock();
11971  *	...
11972  *	register_netdevice(x1);
11973  *	register_netdevice(x2);
11974  *	...
11975  *	unregister_netdevice(y1);
11976  *	unregister_netdevice(y2);
11977  *      ...
11978  *	rtnl_unlock();
11979  *	free_netdev(y1);
11980  *	free_netdev(y2);
11981  *
11982  * We are invoked by rtnl_unlock().
11983  * This allows us to deal with problems:
11984  * 1) We can delete sysfs objects which invoke hotplug
11985  *    without deadlocking with linkwatch via keventd.
11986  * 2) Since we run with the RTNL semaphore not held, we can sleep
11987  *    safely in order to wait for the netdev refcnt to drop to zero.
11988  *
11989  * We must not return until all unregister events added during
11990  * the interval the lock was held have been completed.
11991  */
11992 void netdev_run_todo(void)
11993 {
11994 	struct net_device *dev, *tmp;
11995 	struct list_head list;
11996 	int cnt;
11997 #ifdef CONFIG_LOCKDEP
11998 	struct list_head unlink_list;
11999 
12000 	list_replace_init(&net_unlink_list, &unlink_list);
12001 
12002 	while (!list_empty(&unlink_list)) {
12003 		dev = list_first_entry(&unlink_list, struct net_device,
12004 				       unlink_list);
12005 		list_del_init(&dev->unlink_list);
12006 		dev->nested_level = dev->lower_level - 1;
12007 	}
12008 #endif
12009 
12010 	/* Snapshot list, allow later requests */
12011 	list_replace_init(&net_todo_list, &list);
12012 
12013 	__rtnl_unlock();
12014 
12015 	/* Wait for rcu callbacks to finish before next phase */
12016 	if (!list_empty(&list))
12017 		rcu_barrier();
12018 
12019 	list_for_each_entry_safe(dev, tmp, &list, todo_list) {
12020 		if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) {
12021 			netdev_WARN(dev, "run_todo but not unregistering\n");
12022 			list_del(&dev->todo_list);
12023 			net_unreg_done(dev_net(dev));
12024 			continue;
12025 		}
12026 
12027 		netdev_lock(dev);
12028 		WRITE_ONCE(dev->reg_state, NETREG_UNREGISTERED);
12029 		netdev_unlock(dev);
12030 		linkwatch_sync_dev(dev);
12031 	}
12032 
12033 	if (!list_empty(&list)) {
12034 		rt_flush_dev(NULL);
12035 		rt6_uncached_list_flush_dev(NULL);
12036 	}
12037 
12038 	cnt = 0;
12039 	while (!list_empty(&list)) {
12040 		struct net *net;
12041 
12042 		dev = netdev_wait_allrefs_any(&list);
12043 		list_del(&dev->todo_list);
12044 		net = dev_net(dev);
12045 
12046 		/* paranoia */
12047 		BUG_ON(netdev_refcnt_read(dev) != 1);
12048 		BUG_ON(!list_empty(&dev->ptype_all));
12049 		BUG_ON(!list_empty(&dev->ptype_specific));
12050 		WARN_ON(rcu_access_pointer(dev->ip_ptr));
12051 		WARN_ON(rcu_access_pointer(dev->ip6_ptr));
12052 
12053 		netdev_name_node_alt_flush(dev);
12054 		netdev_name_node_free(dev->name_node);
12055 		netdev_do_free_pcpu_stats(dev);
12056 		if (dev->priv_destructor)
12057 			dev->priv_destructor(dev);
12058 		if (dev->needs_free_netdev)
12059 			free_netdev(dev);
12060 
12061 		cnt++;
12062 
12063 		/* Free network device */
12064 		kobject_put(&dev->dev.kobj);
12065 
12066 		net_unreg_done(net);
12067 	}
12068 	if (cnt && atomic_sub_and_test(cnt, &dev_unreg_count))
12069 		wake_up(&netdev_unregistering_wq);
12070 }
12071 
12072 /* Collate per-cpu network dstats statistics
12073  *
12074  * Read per-cpu network statistics from dev->dstats and populate the related
12075  * fields in @s.
12076  */
12077 static void dev_fetch_dstats(struct rtnl_link_stats64 *s,
12078 			     const struct pcpu_dstats __percpu *dstats)
12079 {
12080 	int cpu;
12081 
12082 	for_each_possible_cpu(cpu) {
12083 		u64 rx_packets, rx_bytes, rx_drops;
12084 		u64 tx_packets, tx_bytes, tx_drops;
12085 		const struct pcpu_dstats *stats;
12086 		unsigned int start;
12087 
12088 		stats = per_cpu_ptr(dstats, cpu);
12089 		do {
12090 			start = u64_stats_fetch_begin(&stats->syncp);
12091 			rx_packets = u64_stats_read(&stats->rx_packets);
12092 			rx_bytes   = u64_stats_read(&stats->rx_bytes);
12093 			rx_drops   = u64_stats_read(&stats->rx_drops);
12094 			tx_packets = u64_stats_read(&stats->tx_packets);
12095 			tx_bytes   = u64_stats_read(&stats->tx_bytes);
12096 			tx_drops   = u64_stats_read(&stats->tx_drops);
12097 		} while (u64_stats_fetch_retry(&stats->syncp, start));
12098 
12099 		s->rx_packets += rx_packets;
12100 		s->rx_bytes   += rx_bytes;
12101 		s->rx_dropped += rx_drops;
12102 		s->tx_packets += tx_packets;
12103 		s->tx_bytes   += tx_bytes;
12104 		s->tx_dropped += tx_drops;
12105 	}
12106 }
12107 
12108 /* ndo_get_stats64 implementation for dtstats-based accounting.
12109  *
12110  * Populate @s from dev->stats and dev->dstats. This is used internally by the
12111  * core for NETDEV_PCPU_STAT_DSTAT-type stats collection.
12112  */
12113 static void dev_get_dstats64(const struct net_device *dev,
12114 			     struct rtnl_link_stats64 *s)
12115 {
12116 	netdev_stats_to_stats64(s, &dev->stats);
12117 	dev_fetch_dstats(s, dev->dstats);
12118 }
12119 
12120 /* Convert net_device_stats to rtnl_link_stats64. rtnl_link_stats64 has
12121  * all the same fields in the same order as net_device_stats, with only
12122  * the type differing, but rtnl_link_stats64 may have additional fields
12123  * at the end for newer counters.
12124  */
12125 void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
12126 			     const struct net_device_stats *netdev_stats)
12127 {
12128 	size_t i, n = sizeof(*netdev_stats) / sizeof(atomic_long_t);
12129 	const atomic_long_t *src = (atomic_long_t *)netdev_stats;
12130 	u64 *dst = (u64 *)stats64;
12131 
12132 	BUILD_BUG_ON(n > sizeof(*stats64) / sizeof(u64));
12133 	for (i = 0; i < n; i++)
12134 		dst[i] = (unsigned long)atomic_long_read(&src[i]);
12135 	/* zero out counters that only exist in rtnl_link_stats64 */
12136 	memset((char *)stats64 + n * sizeof(u64), 0,
12137 	       sizeof(*stats64) - n * sizeof(u64));
12138 }
12139 EXPORT_SYMBOL(netdev_stats_to_stats64);
12140 
12141 static __cold struct net_device_core_stats __percpu *netdev_core_stats_alloc(
12142 		struct net_device *dev)
12143 {
12144 	struct net_device_core_stats __percpu *p;
12145 
12146 	p = alloc_percpu_gfp(struct net_device_core_stats,
12147 			     GFP_ATOMIC | __GFP_NOWARN);
12148 
12149 	if (p && cmpxchg(&dev->core_stats, NULL, p))
12150 		free_percpu(p);
12151 
12152 	/* This READ_ONCE() pairs with the cmpxchg() above */
12153 	return READ_ONCE(dev->core_stats);
12154 }
12155 
12156 noinline void netdev_core_stats_inc(struct net_device *dev, u32 offset)
12157 {
12158 	/* This READ_ONCE() pairs with the write in netdev_core_stats_alloc() */
12159 	struct net_device_core_stats __percpu *p = READ_ONCE(dev->core_stats);
12160 	unsigned long __percpu *field;
12161 
12162 	if (unlikely(!p)) {
12163 		p = netdev_core_stats_alloc(dev);
12164 		if (!p)
12165 			return;
12166 	}
12167 
12168 	field = (unsigned long __percpu *)((void __percpu *)p + offset);
12169 	this_cpu_inc(*field);
12170 }
12171 EXPORT_SYMBOL_GPL(netdev_core_stats_inc);
12172 
12173 /**
12174  *	dev_get_stats	- get network device statistics
12175  *	@dev: device to get statistics from
12176  *	@storage: place to store stats
12177  *
12178  *	Get network statistics from device. Return @storage.
12179  *	The device driver may provide its own method by setting
12180  *	dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats;
12181  *	otherwise the internal statistics structure is used.
12182  */
12183 struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
12184 					struct rtnl_link_stats64 *storage)
12185 {
12186 	const struct net_device_ops *ops = dev->netdev_ops;
12187 	const struct net_device_core_stats __percpu *p;
12188 
12189 	/*
12190 	 * IPv{4,6} and udp tunnels share common stat helpers and use
12191 	 * different stat type (NETDEV_PCPU_STAT_TSTATS vs
12192 	 * NETDEV_PCPU_STAT_DSTATS). Ensure the accounting is consistent.
12193 	 */
12194 	BUILD_BUG_ON(offsetof(struct pcpu_sw_netstats, rx_bytes) !=
12195 		     offsetof(struct pcpu_dstats, rx_bytes));
12196 	BUILD_BUG_ON(offsetof(struct pcpu_sw_netstats, rx_packets) !=
12197 		     offsetof(struct pcpu_dstats, rx_packets));
12198 	BUILD_BUG_ON(offsetof(struct pcpu_sw_netstats, tx_bytes) !=
12199 		     offsetof(struct pcpu_dstats, tx_bytes));
12200 	BUILD_BUG_ON(offsetof(struct pcpu_sw_netstats, tx_packets) !=
12201 		     offsetof(struct pcpu_dstats, tx_packets));
12202 
12203 	if (ops->ndo_get_stats64) {
12204 		memset(storage, 0, sizeof(*storage));
12205 		ops->ndo_get_stats64(dev, storage);
12206 	} else if (ops->ndo_get_stats) {
12207 		netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev));
12208 	} else if (dev->pcpu_stat_type == NETDEV_PCPU_STAT_TSTATS) {
12209 		dev_get_tstats64(dev, storage);
12210 	} else if (dev->pcpu_stat_type == NETDEV_PCPU_STAT_DSTATS) {
12211 		dev_get_dstats64(dev, storage);
12212 	} else {
12213 		netdev_stats_to_stats64(storage, &dev->stats);
12214 	}
12215 
12216 	/* This READ_ONCE() pairs with the write in netdev_core_stats_alloc() */
12217 	p = READ_ONCE(dev->core_stats);
12218 	if (p) {
12219 		const struct net_device_core_stats *core_stats;
12220 		int i;
12221 
12222 		for_each_possible_cpu(i) {
12223 			core_stats = per_cpu_ptr(p, i);
12224 			storage->rx_dropped += READ_ONCE(core_stats->rx_dropped);
12225 			storage->tx_dropped += READ_ONCE(core_stats->tx_dropped);
12226 			storage->rx_nohandler += READ_ONCE(core_stats->rx_nohandler);
12227 			storage->rx_otherhost_dropped += READ_ONCE(core_stats->rx_otherhost_dropped);
12228 		}
12229 	}
12230 	return storage;
12231 }
12232 EXPORT_SYMBOL(dev_get_stats);
12233 
12234 /**
12235  *	dev_fetch_sw_netstats - get per-cpu network device statistics
12236  *	@s: place to store stats
12237  *	@netstats: per-cpu network stats to read from
12238  *
12239  *	Read per-cpu network statistics and populate the related fields in @s.
12240  */
12241 void dev_fetch_sw_netstats(struct rtnl_link_stats64 *s,
12242 			   const struct pcpu_sw_netstats __percpu *netstats)
12243 {
12244 	int cpu;
12245 
12246 	for_each_possible_cpu(cpu) {
12247 		u64 rx_packets, rx_bytes, tx_packets, tx_bytes;
12248 		const struct pcpu_sw_netstats *stats;
12249 		unsigned int start;
12250 
12251 		stats = per_cpu_ptr(netstats, cpu);
12252 		do {
12253 			start = u64_stats_fetch_begin(&stats->syncp);
12254 			rx_packets = u64_stats_read(&stats->rx_packets);
12255 			rx_bytes   = u64_stats_read(&stats->rx_bytes);
12256 			tx_packets = u64_stats_read(&stats->tx_packets);
12257 			tx_bytes   = u64_stats_read(&stats->tx_bytes);
12258 		} while (u64_stats_fetch_retry(&stats->syncp, start));
12259 
12260 		s->rx_packets += rx_packets;
12261 		s->rx_bytes   += rx_bytes;
12262 		s->tx_packets += tx_packets;
12263 		s->tx_bytes   += tx_bytes;
12264 	}
12265 }
12266 EXPORT_SYMBOL_GPL(dev_fetch_sw_netstats);
12267 
12268 /**
12269  *	dev_get_tstats64 - ndo_get_stats64 implementation
12270  *	@dev: device to get statistics from
12271  *	@s: place to store stats
12272  *
12273  *	Populate @s from dev->stats and dev->tstats. Can be used as
12274  *	ndo_get_stats64() callback.
12275  */
12276 void dev_get_tstats64(struct net_device *dev, struct rtnl_link_stats64 *s)
12277 {
12278 	netdev_stats_to_stats64(s, &dev->stats);
12279 	dev_fetch_sw_netstats(s, dev->tstats);
12280 }
12281 EXPORT_SYMBOL_GPL(dev_get_tstats64);
12282 
12283 struct netdev_queue *dev_ingress_queue_create(struct net_device *dev)
12284 {
12285 	struct netdev_queue *queue = dev_ingress_queue(dev);
12286 
12287 #ifdef CONFIG_NET_CLS_ACT
12288 	if (queue)
12289 		return queue;
12290 	queue = kzalloc_obj(*queue);
12291 	if (!queue)
12292 		return NULL;
12293 	netdev_init_one_queue(dev, queue, NULL);
12294 	RCU_INIT_POINTER(queue->qdisc, &noop_qdisc);
12295 	RCU_INIT_POINTER(queue->qdisc_sleeping, &noop_qdisc);
12296 	rcu_assign_pointer(dev->ingress_queue, queue);
12297 #endif
12298 	return queue;
12299 }
12300 
12301 static const struct ethtool_ops default_ethtool_ops;
12302 
12303 void netdev_set_default_ethtool_ops(struct net_device *dev,
12304 				    const struct ethtool_ops *ops)
12305 {
12306 	if (dev->ethtool_ops == &default_ethtool_ops)
12307 		dev->ethtool_ops = ops;
12308 }
12309 EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops);
12310 
12311 /**
12312  * netdev_sw_irq_coalesce_default_on() - enable SW IRQ coalescing by default
12313  * @dev: netdev to enable the IRQ coalescing on
12314  *
12315  * Sets a conservative default for SW IRQ coalescing. Users can use
12316  * sysfs attributes to override the default values.
12317  */
12318 void netdev_sw_irq_coalesce_default_on(struct net_device *dev)
12319 {
12320 	WARN_ON(dev->reg_state == NETREG_REGISTERED);
12321 
12322 	if (!IS_ENABLED(CONFIG_PREEMPT_RT)) {
12323 		netdev_set_gro_flush_timeout(dev, 20000);
12324 		netdev_set_defer_hard_irqs(dev, 1);
12325 	}
12326 }
12327 EXPORT_SYMBOL_GPL(netdev_sw_irq_coalesce_default_on);
12328 
12329 /**
12330  * alloc_netdev_mqs - allocate network device
12331  * @sizeof_priv: size of private data to allocate space for
12332  * @name: device name format string
12333  * @name_assign_type: origin of device name
12334  * @setup: callback to initialize device
12335  * @txqs: the number of TX subqueues to allocate
12336  * @rxqs: the number of RX subqueues to allocate
12337  *
12338  * Allocates a struct net_device with private data area for driver use
12339  * and performs basic initialization.  Also allocates subqueue structs
12340  * for each queue on the device.
12341  */
12342 struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
12343 		unsigned char name_assign_type,
12344 		void (*setup)(struct net_device *),
12345 		unsigned int txqs, unsigned int rxqs)
12346 {
12347 	struct net_device *dev;
12348 	size_t napi_config_sz;
12349 	unsigned int maxqs;
12350 
12351 	BUG_ON(strlen(name) >= sizeof(dev->name));
12352 
12353 	if (txqs < 1) {
12354 		pr_err("alloc_netdev: Unable to allocate device with zero queues\n");
12355 		return NULL;
12356 	}
12357 
12358 	if (rxqs < 1) {
12359 		pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n");
12360 		return NULL;
12361 	}
12362 
12363 	maxqs = max(txqs, rxqs);
12364 
12365 	dev = kvzalloc_flex(*dev, priv, sizeof_priv,
12366 			    GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL);
12367 	if (!dev)
12368 		return NULL;
12369 
12370 	dev->priv_len = sizeof_priv;
12371 
12372 	ref_tracker_dir_init(&dev->refcnt_tracker, 128, "netdev");
12373 #ifdef CONFIG_PCPU_DEV_REFCNT
12374 	dev->pcpu_refcnt = alloc_percpu(int);
12375 	if (!dev->pcpu_refcnt)
12376 		goto free_dev;
12377 	__dev_hold(dev);
12378 #else
12379 	refcount_set(&dev->dev_refcnt, 1);
12380 #endif
12381 
12382 	if (dev_addr_init(dev))
12383 		goto free_pcpu;
12384 
12385 	dev_mc_init(dev);
12386 	dev_uc_init(dev);
12387 
12388 	dev_net_set(dev, &init_net);
12389 
12390 	dev->gso_max_size = GSO_LEGACY_MAX_SIZE;
12391 	dev->xdp_zc_max_segs = 1;
12392 	dev->gso_max_segs = GSO_MAX_SEGS;
12393 	dev->gro_max_size = GRO_LEGACY_MAX_SIZE;
12394 	dev->gso_ipv4_max_size = GSO_LEGACY_MAX_SIZE;
12395 	dev->gro_ipv4_max_size = GRO_LEGACY_MAX_SIZE;
12396 	dev->tso_max_size = TSO_LEGACY_MAX_SIZE;
12397 	dev->tso_max_segs = TSO_MAX_SEGS;
12398 	dev->upper_level = 1;
12399 	dev->lower_level = 1;
12400 #ifdef CONFIG_LOCKDEP
12401 	dev->nested_level = 0;
12402 	INIT_LIST_HEAD(&dev->unlink_list);
12403 #endif
12404 
12405 	INIT_LIST_HEAD(&dev->napi_list);
12406 	INIT_LIST_HEAD(&dev->unreg_list);
12407 #ifdef CONFIG_DEBUG_NET_SMALL_RTNL
12408 	INIT_LIST_HEAD(&dev->unreg_list_net);
12409 #endif
12410 	INIT_LIST_HEAD(&dev->close_list);
12411 	INIT_LIST_HEAD(&dev->link_watch_list);
12412 	INIT_LIST_HEAD(&dev->adj_list.upper);
12413 	INIT_LIST_HEAD(&dev->adj_list.lower);
12414 	INIT_LIST_HEAD(&dev->ptype_all);
12415 	INIT_LIST_HEAD(&dev->ptype_specific);
12416 	INIT_LIST_HEAD(&dev->net_notifier_list);
12417 	INIT_LIST_HEAD(&dev->work_node);
12418 #ifdef CONFIG_NET_SCHED
12419 	hash_init(dev->qdisc_hash);
12420 #endif
12421 
12422 	mutex_init(&dev->lock);
12423 	/* see also netdev_set_instance_lock_class() */
12424 	lock_set_cmp_fn(&dev->lock, netdev_lock_cmp_fn, NULL);
12425 	netif_rx_mode_init(dev);
12426 
12427 	dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM;
12428 	setup(dev);
12429 
12430 	if (!dev->tx_queue_len) {
12431 		dev->priv_flags |= IFF_NO_QUEUE;
12432 		dev->tx_queue_len = DEFAULT_TX_QUEUE_LEN;
12433 	}
12434 
12435 	dev->num_tx_queues = txqs;
12436 	dev->real_num_tx_queues = txqs;
12437 	if (netif_alloc_netdev_queues(dev))
12438 		goto free_all;
12439 
12440 	dev->num_rx_queues = rxqs;
12441 	dev->real_num_rx_queues = rxqs;
12442 	if (netif_alloc_rx_queues(dev))
12443 		goto free_all;
12444 	dev->ethtool = kzalloc_obj(*dev->ethtool, GFP_KERNEL_ACCOUNT);
12445 	if (!dev->ethtool)
12446 		goto free_all;
12447 
12448 	if (netdev_alloc_config(dev))
12449 		goto free_all;
12450 
12451 	dev->num_napi_configs = maxqs;
12452 	napi_config_sz = array_size(maxqs, sizeof(*dev->napi_config));
12453 	dev->napi_config = kvzalloc(napi_config_sz, GFP_KERNEL_ACCOUNT);
12454 	if (!dev->napi_config)
12455 		goto free_all;
12456 
12457 	strscpy(dev->name, name);
12458 	dev->name_assign_type = name_assign_type;
12459 	dev->group = INIT_NETDEV_GROUP;
12460 	if (!dev->ethtool_ops)
12461 		dev->ethtool_ops = &default_ethtool_ops;
12462 
12463 	nf_hook_netdev_init(dev);
12464 
12465 	return dev;
12466 
12467 free_all:
12468 	free_netdev(dev);
12469 	return NULL;
12470 
12471 free_pcpu:
12472 #ifdef CONFIG_PCPU_DEV_REFCNT
12473 	free_percpu(dev->pcpu_refcnt);
12474 free_dev:
12475 #endif
12476 	ref_tracker_dir_exit(&dev->refcnt_tracker);
12477 	kvfree(dev);
12478 	return NULL;
12479 }
12480 EXPORT_SYMBOL(alloc_netdev_mqs);
12481 
12482 static void netdev_napi_exit(struct net_device *dev)
12483 {
12484 	if (!list_empty(&dev->napi_list)) {
12485 		struct napi_struct *p, *n;
12486 
12487 		netdev_lock(dev);
12488 		list_for_each_entry_safe(p, n, &dev->napi_list, dev_list)
12489 			__netif_napi_del_locked(p);
12490 		netdev_unlock(dev);
12491 
12492 		synchronize_net();
12493 	}
12494 
12495 	kvfree(dev->napi_config);
12496 }
12497 
12498 /**
12499  * free_netdev - free network device
12500  * @dev: device
12501  *
12502  * This function does the last stage of destroying an allocated device
12503  * interface. The reference to the device object is released. If this
12504  * is the last reference then it will be freed.Must be called in process
12505  * context.
12506  */
12507 void free_netdev(struct net_device *dev)
12508 {
12509 	might_sleep();
12510 
12511 	/* When called immediately after register_netdevice() failed the unwind
12512 	 * handling may still be dismantling the device. Handle that case by
12513 	 * deferring the free.
12514 	 */
12515 	if (dev->reg_state == NETREG_UNREGISTERING) {
12516 		ASSERT_RTNL();
12517 		dev->needs_free_netdev = true;
12518 		return;
12519 	}
12520 
12521 	netdev_free_config(dev);
12522 	kfree(dev->ethtool);
12523 	netif_free_tx_queues(dev);
12524 	netif_free_rx_queues(dev);
12525 
12526 	kfree(rcu_dereference_protected(dev->ingress_queue, 1));
12527 
12528 	__hw_addr_flush(&dev->rx_mode_addr_cache);
12529 
12530 	/* Flush device addresses */
12531 	dev_addr_flush(dev);
12532 
12533 	netdev_napi_exit(dev);
12534 
12535 	netif_del_cpu_rmap(dev);
12536 
12537 	ref_tracker_dir_exit(&dev->refcnt_tracker);
12538 #ifdef CONFIG_PCPU_DEV_REFCNT
12539 	free_percpu(dev->pcpu_refcnt);
12540 	dev->pcpu_refcnt = NULL;
12541 #endif
12542 	free_percpu(dev->core_stats);
12543 	dev->core_stats = NULL;
12544 	free_percpu(dev->xdp_bulkq);
12545 	dev->xdp_bulkq = NULL;
12546 
12547 	netdev_free_phy_link_topology(dev);
12548 
12549 	mutex_destroy(&dev->lock);
12550 
12551 	/*  Compatibility with error handling in drivers */
12552 	if (dev->reg_state == NETREG_UNINITIALIZED ||
12553 	    dev->reg_state == NETREG_DUMMY) {
12554 		kvfree(dev);
12555 		return;
12556 	}
12557 
12558 	BUG_ON(dev->reg_state != NETREG_UNREGISTERED);
12559 	WRITE_ONCE(dev->reg_state, NETREG_RELEASED);
12560 
12561 	/* will free via device release */
12562 	put_device(&dev->dev);
12563 }
12564 EXPORT_SYMBOL(free_netdev);
12565 
12566 /**
12567  * alloc_netdev_dummy - Allocate and initialize a dummy net device.
12568  * @sizeof_priv: size of private data to allocate space for
12569  *
12570  * Return: the allocated net_device on success, NULL otherwise
12571  */
12572 struct net_device *alloc_netdev_dummy(int sizeof_priv)
12573 {
12574 	return alloc_netdev(sizeof_priv, "dummy#", NET_NAME_UNKNOWN,
12575 			    init_dummy_netdev);
12576 }
12577 EXPORT_SYMBOL_GPL(alloc_netdev_dummy);
12578 
12579 /**
12580  *	synchronize_net -  Synchronize with packet receive processing
12581  *
12582  *	Wait for packets currently being received to be done.
12583  *	Does not block later packets from starting.
12584  */
12585 void synchronize_net(void)
12586 {
12587 	might_sleep();
12588 	if (from_cleanup_net() || rtnl_is_locked())
12589 		synchronize_rcu_expedited();
12590 	else
12591 		synchronize_rcu();
12592 }
12593 EXPORT_SYMBOL(synchronize_net);
12594 
12595 static void netdev_rss_contexts_free(struct net_device *dev)
12596 {
12597 	struct ethtool_rxfh_context *ctx;
12598 	unsigned long context;
12599 
12600 	mutex_lock(&dev->ethtool->rss_lock);
12601 	xa_for_each(&dev->ethtool->rss_ctx, context, ctx) {
12602 		xa_erase(&dev->ethtool->rss_ctx, context);
12603 		dev->ethtool_ops->remove_rxfh_context(dev, ctx, context, NULL);
12604 		kfree(ctx);
12605 	}
12606 	xa_destroy(&dev->ethtool->rss_ctx);
12607 	mutex_unlock(&dev->ethtool->rss_lock);
12608 }
12609 
12610 /**
12611  *	unregister_netdevice_queue - remove device from the kernel
12612  *	@dev: device
12613  *	@head: list
12614  *
12615  *	This function shuts down a device interface and removes it
12616  *	from the kernel tables.
12617  *	If head not NULL, device is queued to be unregistered later.
12618  *
12619  *	Callers must hold the rtnl semaphore.  You may want
12620  *	unregister_netdev() instead of this.
12621  */
12622 
12623 void unregister_netdevice_queue(struct net_device *dev, struct list_head *head)
12624 {
12625 	ASSERT_RTNL();
12626 
12627 #ifdef CONFIG_DEBUG_NET_SMALL_RTNL
12628 	DEBUG_NET_WARN_ON_ONCE(!list_empty(&dev->unreg_list_net));
12629 #endif
12630 
12631 	if (head) {
12632 		list_move_tail(&dev->unreg_list, head);
12633 	} else {
12634 		LIST_HEAD(single);
12635 
12636 		list_add(&dev->unreg_list, &single);
12637 		unregister_netdevice_many(&single);
12638 	}
12639 }
12640 EXPORT_SYMBOL(unregister_netdevice_queue);
12641 
12642 static void dev_memory_provider_uninstall(struct net_device *dev)
12643 {
12644 	unsigned int i;
12645 
12646 	for (i = 0; i < dev->real_num_rx_queues; i++) {
12647 		struct netdev_rx_queue *rxq = &dev->_rx[i];
12648 
12649 		__netif_mp_uninstall_rxq(rxq, &rxq->mp_params);
12650 	}
12651 }
12652 
12653 /* devices must be UP and netdev_lock()'d */
12654 static void netif_close_many_and_unlock(struct list_head *close_head)
12655 {
12656 	struct net_device *dev, *tmp;
12657 
12658 	netif_close_many(close_head, false);
12659 
12660 	/* ... now unlock them */
12661 	list_for_each_entry_safe(dev, tmp, close_head, close_list) {
12662 		netdev_unlock(dev);
12663 		list_del_init(&dev->close_list);
12664 	}
12665 }
12666 
12667 /* Handle one class of ops-locked devices. Since close requires the lock
12668  * we need to be careful about which classes we allow to nest.
12669  */
12670 static void netdev_lock_ops_close_many(struct list_head *head,
12671 				       struct list_head *close_head,
12672 				       bool leasing)
12673 {
12674 	struct net_device *dev;
12675 
12676 	list_for_each_entry(dev, head, unreg_list) {
12677 		if (!(dev->flags & IFF_UP) || !netdev_need_ops_lock(dev) ||
12678 		    netdev_can_create_queue(dev, NULL) != leasing)
12679 			continue;
12680 		list_add_tail(&dev->close_list, close_head);
12681 		netdev_lock(dev);
12682 
12683 #ifdef CONFIG_LOCKDEP
12684 		/* We can only track up to MAX_LOCK_DEPTH locks per task.
12685 		 * Reserve half the available slots for additional locks
12686 		 * possibly taken by notifiers and (soft)irqs.
12687 		 */
12688 		if (lockdep_depth(current) > MAX_LOCK_DEPTH / 2)
12689 			netif_close_many_and_unlock(close_head);
12690 #endif
12691 	}
12692 }
12693 
12694 bool unregister_netdevice_queued(const struct net_device *dev)
12695 {
12696 	ASSERT_RTNL();
12697 	return !list_empty(&dev->unreg_list);
12698 }
12699 
12700 void unregister_netdevice_many_notify(struct list_head *head,
12701 				      u32 portid, const struct nlmsghdr *nlh)
12702 {
12703 	struct net_device *dev, *tmp;
12704 	LIST_HEAD(close_head);
12705 	int cnt = 0;
12706 
12707 	BUG_ON(dev_boot_phase);
12708 	ASSERT_RTNL();
12709 
12710 	if (list_empty(head))
12711 		return;
12712 
12713 	list_for_each_entry_safe(dev, tmp, head, unreg_list) {
12714 		/* Some devices call without registering
12715 		 * for initialization unwind. Remove those
12716 		 * devices and proceed with the remaining.
12717 		 */
12718 		if (dev->reg_state == NETREG_UNINITIALIZED) {
12719 			pr_debug("unregister_netdevice: device %s/%p never was registered\n",
12720 				 dev->name, dev);
12721 
12722 			WARN_ON(1);
12723 			list_del(&dev->unreg_list);
12724 			continue;
12725 		}
12726 		dev->dismantle = true;
12727 		BUG_ON(dev->reg_state != NETREG_REGISTERED);
12728 	}
12729 
12730 	/* If device is running, close it first. Start with ops locked... */
12731 	netdev_lock_ops_close_many(head, &close_head, true); /* queue leasing */
12732 	netdev_lock_ops_close_many(head, &close_head, false); /* the rest */
12733 	netif_close_many_and_unlock(&close_head);
12734 	/* ... now go over the rest. */
12735 	list_for_each_entry(dev, head, unreg_list) {
12736 		if (!netdev_need_ops_lock(dev))
12737 			list_add_tail(&dev->close_list, &close_head);
12738 	}
12739 	netif_close_many(&close_head, true);
12740 
12741 	list_for_each_entry(dev, head, unreg_list) {
12742 		/* And unlink it from device chain. */
12743 		unlist_netdevice(dev);
12744 		netdev_lock(dev);
12745 		WRITE_ONCE(dev->reg_state, NETREG_UNREGISTERING);
12746 		netdev_unlock(dev);
12747 	}
12748 	flush_all_backlogs();
12749 
12750 	synchronize_net();
12751 
12752 	list_for_each_entry(dev, head, unreg_list) {
12753 		struct sk_buff *skb = NULL;
12754 
12755 		/* Shutdown queueing discipline. */
12756 		netdev_lock_ops(dev);
12757 		dev_shutdown(dev);
12758 		dev_tcx_uninstall(dev);
12759 		dev_xdp_uninstall(dev);
12760 		dev_memory_provider_uninstall(dev);
12761 		netdev_work_cancel_all(dev);
12762 		netdev_unlock_ops(dev);
12763 		bpf_dev_bound_netdev_unregister(dev);
12764 
12765 		netdev_offload_xstats_disable_all(dev);
12766 
12767 		/* Notify protocols, that we are about to destroy
12768 		 * this device. They should clean all the things.
12769 		 */
12770 		call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
12771 
12772 		if (!(dev->rtnl_link_ops && dev->rtnl_link_initializing))
12773 			skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U, 0,
12774 						     GFP_KERNEL, NULL, 0,
12775 						     portid, nlh);
12776 
12777 		/*
12778 		 *	Flush the unicast and multicast chains
12779 		 */
12780 		dev_uc_flush(dev);
12781 		dev_mc_flush(dev);
12782 
12783 
12784 		netdev_rss_contexts_free(dev);
12785 
12786 		call_netdevice_notifiers(NETDEV_PRE_UNINIT, dev);
12787 
12788 		if (dev->netdev_ops->ndo_uninit)
12789 			dev->netdev_ops->ndo_uninit(dev);
12790 
12791 		mutex_destroy(&dev->ethtool->rss_lock);
12792 
12793 		net_shaper_flush_netdev(dev);
12794 
12795 		if (skb)
12796 			rtmsg_ifinfo_send(skb, dev, GFP_KERNEL, portid, nlh);
12797 
12798 		/* Notifier chain MUST detach us all upper devices. */
12799 		WARN_ON(netdev_has_any_upper_dev(dev));
12800 		WARN_ON(netdev_has_any_lower_dev(dev));
12801 
12802 		/* Remove entries from kobject tree */
12803 		netdev_unregister_kobject(dev);
12804 #ifdef CONFIG_XPS
12805 		/* Remove XPS queueing entries */
12806 		netif_reset_xps_queues_gt(dev, 0);
12807 #endif
12808 	}
12809 
12810 	synchronize_net();
12811 
12812 	list_for_each_entry(dev, head, unreg_list) {
12813 #ifdef CONFIG_DEBUG_NET_SMALL_RTNL
12814 		struct net *net = dev_net(dev);
12815 
12816 		/* spin_lock() can be moved outside of the loop
12817 		 * once the per-netns RTNL conversion completes.
12818 		 */
12819 		spin_lock(&net->dev_unreg_lock);
12820 		list_del(&dev->unreg_list_net);
12821 		spin_unlock(&net->dev_unreg_lock);
12822 #endif
12823 		netdev_put(dev, &dev->dev_registered_tracker);
12824 		net_set_todo(dev);
12825 		cnt++;
12826 	}
12827 	atomic_add(cnt, &dev_unreg_count);
12828 
12829 	list_del(head);
12830 }
12831 
12832 /**
12833  *	unregister_netdevice_many - unregister many devices
12834  *	@head: list of devices
12835  *
12836  *  Note: As most callers use a stack allocated list_head,
12837  *  we force a list_del() to make sure stack won't be corrupted later.
12838  */
12839 void unregister_netdevice_many(struct list_head *head)
12840 {
12841 	unregister_netdevice_many_notify(head, 0, NULL);
12842 }
12843 EXPORT_SYMBOL(unregister_netdevice_many);
12844 
12845 #ifdef CONFIG_DEBUG_NET_SMALL_RTNL
12846 void unregister_netdevice_queue_net(struct net *net, struct net_device *dev,
12847 				    struct list_head *head)
12848 {
12849 	netdev_lock(dev);
12850 
12851 	if (net_eq(dev_net(dev), net)) {
12852 		netdev_unlock(dev);
12853 		unregister_netdevice_queue(dev, head);
12854 		return;
12855 	}
12856 
12857 	net = dev_net(dev);
12858 
12859 	spin_lock(&net->dev_unreg_lock);
12860 
12861 	DEBUG_NET_WARN_ON_ONCE(!list_empty(&dev->unreg_list));
12862 	DEBUG_NET_WARN_ON_ONCE(!list_empty(&dev->unreg_list_net));
12863 
12864 	list_add_tail(&dev->unreg_list_net, &net->dev_unreg_head);
12865 	rtnl_net_queue_work(net);
12866 
12867 	spin_unlock(&net->dev_unreg_lock);
12868 
12869 	netdev_unlock(dev);
12870 }
12871 EXPORT_SYMBOL(unregister_netdevice_queue_net);
12872 
12873 void unregister_netdevice_queue_many_net(struct net *net, struct list_head *head)
12874 {
12875 	struct net_device *dev, *tmp;
12876 
12877 	spin_lock(&net->dev_unreg_lock);
12878 	list_for_each_entry_safe(dev, tmp, head, unreg_list) {
12879 		/* Once all cross-netns unregister_netdevice_queue() is
12880 		 * converted to _net() (or for debugging), remove this check.
12881 		 */
12882 		if (!net_eq(dev_net(dev), net))
12883 			continue;
12884 
12885 		DEBUG_NET_WARN_ONCE(!net_eq(dev_net(dev), net),
12886 				    "%s was unregistered from a different netns.\n",
12887 				    dev->name);
12888 
12889 		list_del_init(&dev->unreg_list);
12890 		list_move_tail(&dev->unreg_list_net, &net->dev_unreg_head);
12891 	}
12892 	spin_unlock(&net->dev_unreg_lock);
12893 }
12894 
12895 static void unregister_netdevice_move_net(struct net *net_old,
12896 					  struct net *net,
12897 					  struct net_device *dev)
12898 {
12899 	if (net_old > net) {
12900 		spin_lock(&net->dev_unreg_lock);
12901 		spin_lock_nested(&net_old->dev_unreg_lock, SINGLE_DEPTH_NESTING);
12902 	} else {
12903 		spin_lock(&net_old->dev_unreg_lock);
12904 		spin_lock_nested(&net->dev_unreg_lock, SINGLE_DEPTH_NESTING);
12905 	}
12906 
12907 	if (!list_empty(&dev->unreg_list_net)) {
12908 		list_del(&dev->unreg_list_net);
12909 		list_add_tail(&dev->unreg_list_net, &net->dev_unreg_head);
12910 	}
12911 
12912 	spin_unlock(&net_old->dev_unreg_lock);
12913 	spin_unlock(&net->dev_unreg_lock);
12914 }
12915 
12916 void unregister_netdevice_many_net(struct net *net)
12917 {
12918 	struct net_device *dev, *tmp;
12919 	LIST_HEAD(unreg_head_net);
12920 	LIST_HEAD(unreg_head);
12921 
12922 	spin_lock(&net->dev_unreg_lock);
12923 	list_splice_init(&net->dev_unreg_head, &unreg_head_net);
12924 	spin_unlock(&net->dev_unreg_lock);
12925 
12926 	list_for_each_entry_safe(dev, tmp, &unreg_head_net, unreg_list_net) {
12927 		list_del_init(&dev->unreg_list_net);
12928 		list_add_tail(&dev->unreg_list, &unreg_head);
12929 	}
12930 
12931 	unregister_netdevice_many(&unreg_head);
12932 }
12933 #endif
12934 
12935 /**
12936  *	unregister_netdev - remove device from the kernel
12937  *	@dev: device
12938  *
12939  *	This function shuts down a device interface and removes it
12940  *	from the kernel tables.
12941  *
12942  *	This is just a wrapper for unregister_netdevice that takes
12943  *	the rtnl semaphore.  In general you want to use this and not
12944  *	unregister_netdevice.
12945  */
12946 void unregister_netdev(struct net_device *dev)
12947 {
12948 	rtnl_net_dev_lock(dev);
12949 	unregister_netdevice(dev);
12950 	rtnl_net_dev_unlock(dev);
12951 }
12952 EXPORT_SYMBOL(unregister_netdev);
12953 
12954 int __dev_change_net_namespace(struct net_device *dev, struct net *net,
12955 			       const char *pat, int new_ifindex,
12956 			       struct netlink_ext_ack *extack)
12957 {
12958 	struct netdev_name_node *name_node, *tmp;
12959 	struct net *net_old = dev_net(dev);
12960 	char new_name[IFNAMSIZ] = {};
12961 	int err, new_nsid;
12962 
12963 	ASSERT_RTNL();
12964 
12965 	/* Don't allow namespace local devices to be moved. */
12966 	err = -EINVAL;
12967 	if (dev->netns_immutable) {
12968 		NL_SET_ERR_MSG(extack, "The interface netns is immutable");
12969 		goto out;
12970 	}
12971 
12972 	/* Ensure the device has been registered */
12973 	if (dev->reg_state != NETREG_REGISTERED) {
12974 		NL_SET_ERR_MSG(extack, "The interface isn't registered");
12975 		goto out;
12976 	}
12977 
12978 	/* Get out if there is nothing todo */
12979 	err = 0;
12980 	if (net_eq(net_old, net))
12981 		goto out;
12982 
12983 	/* Pick the destination device name, and ensure
12984 	 * we can use it in the destination network namespace.
12985 	 */
12986 	err = -EEXIST;
12987 	if (netdev_name_in_use(net, dev->name)) {
12988 		/* We get here if we can't use the current device name */
12989 		if (!pat) {
12990 			NL_SET_ERR_MSG(extack,
12991 				       "An interface with the same name exists in the target netns");
12992 			goto out;
12993 		}
12994 		err = dev_prep_valid_name(net, dev, pat, new_name, EEXIST);
12995 		if (err < 0) {
12996 			NL_SET_ERR_MSG_FMT(extack,
12997 					   "Unable to use '%s' for the new interface name in the target netns",
12998 					   pat);
12999 			goto out;
13000 		}
13001 	}
13002 	/* Check that none of the altnames conflicts. */
13003 	err = -EEXIST;
13004 	netdev_for_each_altname_safe(dev, name_node, tmp) {
13005 		if (!netdev_name_in_use(net, name_node->name))
13006 			continue;
13007 
13008 		if (!check_net(net_old)) {
13009 			__netdev_name_node_alt_destroy(name_node);
13010 			continue;
13011 		}
13012 
13013 		NL_SET_ERR_MSG_FMT(extack,
13014 				   "An interface with the altname %s exists in the target netns",
13015 				   name_node->name);
13016 		goto out;
13017 	}
13018 
13019 	/* Check that new_ifindex isn't used yet. */
13020 	if (new_ifindex) {
13021 		err = dev_index_reserve(net, new_ifindex);
13022 		if (err < 0) {
13023 			NL_SET_ERR_MSG_FMT(extack,
13024 					   "The ifindex %d is not available in the target netns",
13025 					   new_ifindex);
13026 			goto out;
13027 		}
13028 	} else {
13029 		/* If there is an ifindex conflict assign a new one */
13030 		err = dev_index_reserve(net, dev->ifindex);
13031 		if (err == -EBUSY)
13032 			err = dev_index_reserve(net, 0);
13033 		if (err < 0) {
13034 			NL_SET_ERR_MSG(extack,
13035 				       "Unable to allocate a new ifindex in the target netns");
13036 			goto out;
13037 		}
13038 		new_ifindex = err;
13039 	}
13040 
13041 	/*
13042 	 * And now a mini version of register_netdevice unregister_netdevice.
13043 	 */
13044 
13045 	netdev_lock_ops(dev);
13046 	/* If device is running close it first. */
13047 	netif_close(dev);
13048 	/* And unlink it from device chain */
13049 	unlist_netdevice(dev);
13050 
13051 	if (!netdev_need_ops_lock(dev))
13052 		netdev_lock(dev);
13053 	dev->moving_ns = true;
13054 	netdev_unlock(dev);
13055 
13056 	synchronize_net();
13057 
13058 	/* Shutdown queueing discipline. */
13059 	netdev_lock_ops(dev);
13060 	dev_shutdown(dev);
13061 	netdev_unlock_ops(dev);
13062 
13063 	/* Notify protocols, that we are about to destroy
13064 	 * this device. They should clean all the things.
13065 	 *
13066 	 * Note that dev->reg_state stays at NETREG_REGISTERED.
13067 	 * This is wanted because this way 8021q and macvlan know
13068 	 * the device is just moving and can keep their slaves up.
13069 	 */
13070 	call_netdevice_notifiers(NETDEV_UNREGISTER, dev);
13071 	rcu_barrier();
13072 
13073 	new_nsid = peernet2id_alloc(dev_net(dev), net, GFP_KERNEL);
13074 
13075 	rtmsg_ifinfo_newnet(RTM_DELLINK, dev, ~0U, GFP_KERNEL, &new_nsid,
13076 			    new_ifindex);
13077 
13078 	/*
13079 	 *	Flush the unicast and multicast chains
13080 	 */
13081 	dev_uc_flush(dev);
13082 	dev_mc_flush(dev);
13083 
13084 	/* Send a netdev-removed uevent to the old namespace */
13085 	kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE);
13086 	netdev_adjacent_del_links(dev);
13087 
13088 	/* Move per-net netdevice notifiers that are following the netdevice */
13089 	move_netdevice_notifiers_dev_net(dev, net);
13090 
13091 	/* Actually switch the network namespace */
13092 	netdev_lock(dev);
13093 	dev_net_set(dev, net);
13094 	netdev_unlock(dev);
13095 	dev->ifindex = new_ifindex;
13096 
13097 #ifdef CONFIG_DEBUG_NET_SMALL_RTNL
13098 	unregister_netdevice_move_net(net_old, net, dev);
13099 #endif
13100 
13101 	if (new_name[0]) {
13102 		/* Rename the netdev to prepared name */
13103 		write_seqlock_bh(&netdev_rename_lock);
13104 		strscpy(dev->name, new_name, IFNAMSIZ);
13105 		write_sequnlock_bh(&netdev_rename_lock);
13106 	}
13107 
13108 	/* Fixup kobjects */
13109 	dev_set_uevent_suppress(&dev->dev, 1);
13110 	err = device_rename(&dev->dev, dev->name);
13111 	dev_set_uevent_suppress(&dev->dev, 0);
13112 	WARN_ON(err);
13113 
13114 	/* Send a netdev-add uevent to the new namespace */
13115 	kobject_uevent(&dev->dev.kobj, KOBJ_ADD);
13116 	netdev_adjacent_add_links(dev);
13117 
13118 	/* Adapt owner in case owning user namespace of target network
13119 	 * namespace is different from the original one.
13120 	 */
13121 	err = netdev_change_owner(dev, net_old, net);
13122 	WARN_ON(err);
13123 
13124 	netdev_lock(dev);
13125 	dev->moving_ns = false;
13126 	if (!netdev_need_ops_lock(dev))
13127 		netdev_unlock(dev);
13128 
13129 	/* Add the device back in the hashes */
13130 	list_netdevice(dev);
13131 	/* Notify protocols, that a new device appeared. */
13132 	call_netdevice_notifiers(NETDEV_REGISTER, dev);
13133 	netdev_unlock_ops(dev);
13134 
13135 	/*
13136 	 *	Prevent userspace races by waiting until the network
13137 	 *	device is fully setup before sending notifications.
13138 	 */
13139 	rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL, 0, NULL);
13140 
13141 	synchronize_net();
13142 	err = 0;
13143 out:
13144 	return err;
13145 }
13146 
13147 static int dev_cpu_dead(unsigned int oldcpu)
13148 {
13149 	struct sk_buff **list_skb;
13150 	struct sk_buff *skb;
13151 	unsigned int cpu;
13152 	int node;
13153 	struct softnet_data *sd, *oldsd, *remsd = NULL;
13154 
13155 	local_irq_disable();
13156 	cpu = smp_processor_id();
13157 	sd = &per_cpu(softnet_data, cpu);
13158 	oldsd = &per_cpu(softnet_data, oldcpu);
13159 
13160 	/* Find end of our completion_queue. */
13161 	list_skb = &sd->completion_queue;
13162 	while (*list_skb)
13163 		list_skb = &(*list_skb)->next;
13164 	/* Append completion queue from offline CPU. */
13165 	*list_skb = oldsd->completion_queue;
13166 	oldsd->completion_queue = NULL;
13167 
13168 	/* Append output queue from offline CPU. */
13169 	if (oldsd->output_queue) {
13170 		*sd->output_queue_tailp = oldsd->output_queue;
13171 		sd->output_queue_tailp = oldsd->output_queue_tailp;
13172 		oldsd->output_queue = NULL;
13173 		oldsd->output_queue_tailp = &oldsd->output_queue;
13174 	}
13175 	/* Append NAPI poll list from offline CPU, with one exception :
13176 	 * process_backlog() must be called by cpu owning percpu backlog.
13177 	 * We properly handle process_queue & input_pkt_queue later.
13178 	 */
13179 	while (!list_empty(&oldsd->poll_list)) {
13180 		struct napi_struct *napi = list_first_entry(&oldsd->poll_list,
13181 							    struct napi_struct,
13182 							    poll_list);
13183 
13184 		list_del_init(&napi->poll_list);
13185 		if (napi->poll == process_backlog)
13186 			napi->state &= NAPIF_STATE_THREADED;
13187 		else
13188 			____napi_schedule(sd, napi);
13189 	}
13190 
13191 	raise_softirq_irqoff(NET_TX_SOFTIRQ);
13192 	local_irq_enable();
13193 
13194 	if (!use_backlog_threads()) {
13195 #ifdef CONFIG_RPS
13196 		remsd = oldsd->rps_ipi_list;
13197 		oldsd->rps_ipi_list = NULL;
13198 #endif
13199 		/* send out pending IPI's on offline CPU */
13200 		net_rps_send_ipi(remsd);
13201 	}
13202 
13203 	/* Process offline CPU's input_pkt_queue */
13204 	while ((skb = __skb_dequeue(&oldsd->process_queue))) {
13205 		netif_rx(skb);
13206 		rps_input_queue_head_incr(oldsd);
13207 	}
13208 	while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) {
13209 		netif_rx(skb);
13210 		rps_input_queue_head_incr(oldsd);
13211 	}
13212 
13213 	for_each_node(node)
13214 		skb_defer_node_flush(per_cpu_ptr(net_hotdata.skb_defer_nodes,
13215 						 oldcpu) + node);
13216 	node = cpu_to_node(oldcpu);
13217 	if (node_possible(node) &&
13218 	    !cpumask_intersects(cpumask_of_node(node), cpu_online_mask)) {
13219 		for_each_possible_cpu(cpu)
13220 			skb_defer_node_flush(per_cpu_ptr(net_hotdata.skb_defer_nodes,
13221 							 cpu) + node);
13222 	}
13223 
13224 	return 0;
13225 }
13226 
13227 /**
13228  *	netdev_increment_features - increment feature set by one
13229  *	@all: current feature set
13230  *	@one: new feature set
13231  *	@mask: mask feature set
13232  *
13233  *	Computes a new feature set after adding a device with feature set
13234  *	@one to the master device with current feature set @all.  Will not
13235  *	enable anything that is off in @mask. Returns the new feature set.
13236  */
13237 netdev_features_t netdev_increment_features(netdev_features_t all,
13238 	netdev_features_t one, netdev_features_t mask)
13239 {
13240 	if (mask & NETIF_F_HW_CSUM)
13241 		mask |= NETIF_F_CSUM_MASK;
13242 	mask |= NETIF_F_VLAN_CHALLENGED;
13243 
13244 	all |= one & (NETIF_F_ONE_FOR_ALL | NETIF_F_CSUM_MASK) & mask;
13245 	all &= one | ~NETIF_F_ALL_FOR_ALL;
13246 
13247 	/* If one device supports hw checksumming, set for all. */
13248 	if (all & NETIF_F_HW_CSUM)
13249 		all &= ~(NETIF_F_CSUM_MASK & ~NETIF_F_HW_CSUM);
13250 
13251 	return all;
13252 }
13253 EXPORT_SYMBOL(netdev_increment_features);
13254 
13255 /**
13256  *	netdev_compute_master_upper_features - compute feature from lowers
13257  *	@dev: the upper device
13258  *	@update_header: whether to update upper device's header_len/headroom/tailroom
13259  *
13260  *	Recompute the upper device's feature based on all lower devices.
13261  */
13262 void netdev_compute_master_upper_features(struct net_device *dev, bool update_header)
13263 {
13264 	unsigned int dst_release_flag = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM;
13265 	netdev_features_t gso_partial_features = MASTER_UPPER_DEV_GSO_PARTIAL_FEATURES;
13266 	netdev_features_t xfrm_features = MASTER_UPPER_DEV_XFRM_FEATURES;
13267 	netdev_features_t mpls_features = MASTER_UPPER_DEV_MPLS_FEATURES;
13268 	netdev_features_t vlan_features = MASTER_UPPER_DEV_VLAN_FEATURES;
13269 	netdev_features_t enc_features = MASTER_UPPER_DEV_ENC_FEATURES;
13270 	unsigned short max_header_len = ETH_HLEN;
13271 	unsigned int tso_max_size = TSO_MAX_SIZE;
13272 	unsigned short max_headroom = 0;
13273 	unsigned short max_tailroom = 0;
13274 	u16 tso_max_segs = TSO_MAX_SEGS;
13275 	struct net_device *lower_dev;
13276 	struct list_head *iter;
13277 
13278 	mpls_features = netdev_base_features(mpls_features);
13279 	vlan_features = netdev_base_features(vlan_features);
13280 	enc_features = netdev_base_features(enc_features);
13281 
13282 	netdev_for_each_lower_dev(dev, lower_dev, iter) {
13283 		gso_partial_features = netdev_increment_features(gso_partial_features,
13284 								 lower_dev->gso_partial_features,
13285 								 MASTER_UPPER_DEV_GSO_PARTIAL_FEATURES);
13286 
13287 		vlan_features = netdev_increment_features(vlan_features,
13288 							  lower_dev->vlan_features,
13289 							  MASTER_UPPER_DEV_VLAN_FEATURES);
13290 
13291 		enc_features = netdev_increment_features(enc_features,
13292 							 lower_dev->hw_enc_features,
13293 							 MASTER_UPPER_DEV_ENC_FEATURES);
13294 
13295 		if (IS_ENABLED(CONFIG_XFRM_OFFLOAD))
13296 			xfrm_features = netdev_increment_features(xfrm_features,
13297 								  lower_dev->hw_enc_features,
13298 								  MASTER_UPPER_DEV_XFRM_FEATURES);
13299 
13300 		mpls_features = netdev_increment_features(mpls_features,
13301 							  lower_dev->mpls_features,
13302 							  MASTER_UPPER_DEV_MPLS_FEATURES);
13303 
13304 		dst_release_flag &= lower_dev->priv_flags;
13305 
13306 		if (update_header) {
13307 			max_header_len = max(max_header_len, lower_dev->hard_header_len);
13308 			max_headroom = max(max_headroom, lower_dev->needed_headroom);
13309 			max_tailroom = max(max_tailroom, lower_dev->needed_tailroom);
13310 		}
13311 
13312 		tso_max_size = min(tso_max_size, lower_dev->tso_max_size);
13313 		tso_max_segs = min(tso_max_segs, lower_dev->tso_max_segs);
13314 	}
13315 
13316 	dev->gso_partial_features = gso_partial_features;
13317 	dev->vlan_features = vlan_features;
13318 	dev->hw_enc_features = enc_features | NETIF_F_GSO_ENCAP_ALL |
13319 			       NETIF_F_HW_VLAN_CTAG_TX |
13320 			       NETIF_F_HW_VLAN_STAG_TX;
13321 	if (IS_ENABLED(CONFIG_XFRM_OFFLOAD))
13322 		dev->hw_enc_features |= xfrm_features;
13323 	dev->mpls_features = mpls_features;
13324 
13325 	dev->priv_flags &= ~IFF_XMIT_DST_RELEASE;
13326 	if ((dev->priv_flags & IFF_XMIT_DST_RELEASE_PERM) &&
13327 	    dst_release_flag == (IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM))
13328 		dev->priv_flags |= IFF_XMIT_DST_RELEASE;
13329 
13330 	if (update_header) {
13331 		dev->hard_header_len = max_header_len;
13332 		dev->needed_headroom = max_headroom;
13333 		dev->needed_tailroom = max_tailroom;
13334 	}
13335 
13336 	netif_set_tso_max_segs(dev, tso_max_segs);
13337 	netif_set_tso_max_size(dev, tso_max_size);
13338 
13339 	netdev_change_features(dev);
13340 }
13341 EXPORT_SYMBOL(netdev_compute_master_upper_features);
13342 
13343 static struct hlist_head * __net_init netdev_create_hash(void)
13344 {
13345 	int i;
13346 	struct hlist_head *hash;
13347 
13348 	hash = kmalloc_objs(*hash, NETDEV_HASHENTRIES);
13349 	if (hash != NULL)
13350 		for (i = 0; i < NETDEV_HASHENTRIES; i++)
13351 			INIT_HLIST_HEAD(&hash[i]);
13352 
13353 	return hash;
13354 }
13355 
13356 /* Initialize per network namespace state */
13357 static int __net_init netdev_init(struct net *net)
13358 {
13359 	BUILD_BUG_ON(GRO_HASH_BUCKETS >
13360 		     BITS_PER_BYTE * sizeof_field(struct gro_node, bitmask));
13361 
13362 	INIT_LIST_HEAD(&net->dev_base_head);
13363 
13364 	net->dev_name_head = netdev_create_hash();
13365 	if (net->dev_name_head == NULL)
13366 		goto err_name;
13367 
13368 	net->dev_index_head = netdev_create_hash();
13369 	if (net->dev_index_head == NULL)
13370 		goto err_idx;
13371 
13372 	xa_init_flags(&net->dev_by_index, XA_FLAGS_ALLOC1);
13373 
13374 	RAW_INIT_NOTIFIER_HEAD(&net->netdev_chain);
13375 
13376 	return 0;
13377 
13378 err_idx:
13379 	kfree(net->dev_name_head);
13380 err_name:
13381 	return -ENOMEM;
13382 }
13383 
13384 /**
13385  *	netdev_drivername - network driver for the device
13386  *	@dev: network device
13387  *
13388  *	Determine network driver for device.
13389  */
13390 const char *netdev_drivername(const struct net_device *dev)
13391 {
13392 	const struct device_driver *driver;
13393 	const struct device *parent;
13394 	const char *empty = "";
13395 
13396 	parent = dev->dev.parent;
13397 	if (!parent)
13398 		return empty;
13399 
13400 	driver = parent->driver;
13401 	if (driver && driver->name)
13402 		return driver->name;
13403 	return empty;
13404 }
13405 
13406 static void __netdev_printk(const char *level, const struct net_device *dev,
13407 			    struct va_format *vaf)
13408 {
13409 	if (dev && dev->dev.parent) {
13410 		dev_printk_emit(level[1] - '0',
13411 				dev->dev.parent,
13412 				"%s %s %s%s: %pV",
13413 				dev_driver_string(dev->dev.parent),
13414 				dev_name(dev->dev.parent),
13415 				netdev_name(dev), netdev_reg_state(dev),
13416 				vaf);
13417 	} else if (dev) {
13418 		printk("%s%s%s: %pV",
13419 		       level, netdev_name(dev), netdev_reg_state(dev), vaf);
13420 	} else {
13421 		printk("%s(NULL net_device): %pV", level, vaf);
13422 	}
13423 }
13424 
13425 void netdev_printk(const char *level, const struct net_device *dev,
13426 		   const char *format, ...)
13427 {
13428 	struct va_format vaf;
13429 	va_list args;
13430 
13431 	va_start(args, format);
13432 
13433 	vaf.fmt = format;
13434 	vaf.va = &args;
13435 
13436 	__netdev_printk(level, dev, &vaf);
13437 
13438 	va_end(args);
13439 }
13440 EXPORT_SYMBOL(netdev_printk);
13441 
13442 #define define_netdev_printk_level(func, level)			\
13443 void func(const struct net_device *dev, const char *fmt, ...)	\
13444 {								\
13445 	struct va_format vaf;					\
13446 	va_list args;						\
13447 								\
13448 	va_start(args, fmt);					\
13449 								\
13450 	vaf.fmt = fmt;						\
13451 	vaf.va = &args;						\
13452 								\
13453 	__netdev_printk(level, dev, &vaf);			\
13454 								\
13455 	va_end(args);						\
13456 }								\
13457 EXPORT_SYMBOL(func);
13458 
13459 define_netdev_printk_level(netdev_emerg, KERN_EMERG);
13460 define_netdev_printk_level(netdev_alert, KERN_ALERT);
13461 define_netdev_printk_level(netdev_crit, KERN_CRIT);
13462 define_netdev_printk_level(netdev_err, KERN_ERR);
13463 define_netdev_printk_level(netdev_warn, KERN_WARNING);
13464 define_netdev_printk_level(netdev_notice, KERN_NOTICE);
13465 define_netdev_printk_level(netdev_info, KERN_INFO);
13466 
13467 static void __net_exit netdev_exit(struct net *net)
13468 {
13469 	kfree(net->dev_name_head);
13470 	kfree(net->dev_index_head);
13471 	xa_destroy(&net->dev_by_index);
13472 	if (net != &init_net)
13473 		WARN_ON_ONCE(!list_empty(&net->dev_base_head));
13474 }
13475 
13476 static struct pernet_operations __net_initdata netdev_net_ops = {
13477 	.init = netdev_init,
13478 	.exit = netdev_exit,
13479 };
13480 
13481 static void __net_exit default_device_exit_net(struct net *net)
13482 {
13483 	struct net_device *dev, *aux;
13484 	/*
13485 	 * Push all migratable network devices back to the
13486 	 * initial network namespace
13487 	 */
13488 
13489 	for_each_netdev_safe(net, dev, aux) {
13490 		int err;
13491 		char fb_name[IFNAMSIZ];
13492 
13493 		/* Ignore unmoveable devices (i.e. loopback) */
13494 		if (dev->netns_immutable)
13495 			continue;
13496 
13497 		/* Leave virtual devices for the generic cleanup */
13498 		if (dev->rtnl_link_ops && !dev->rtnl_link_ops->netns_refund)
13499 			continue;
13500 
13501 		/* Push remaining network devices to init_net */
13502 		snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex);
13503 		if (netdev_name_in_use(&init_net, fb_name))
13504 			snprintf(fb_name, IFNAMSIZ, "dev%%d");
13505 
13506 		err = dev_change_net_namespace(dev, &init_net, fb_name);
13507 		if (err) {
13508 			pr_emerg("%s: failed to move %s to init_net: %d\n",
13509 				 __func__, dev->name, err);
13510 			BUG();
13511 		}
13512 	}
13513 }
13514 
13515 static bool __net_exit netns_dev_unreg_pending(const struct list_head *net_list)
13516 {
13517 	struct net *net;
13518 
13519 	list_for_each_entry(net, net_list, exit_list) {
13520 		if (atomic_read(&net->dev_unreg_pending))
13521 			return true;
13522 	}
13523 	return false;
13524 }
13525 
13526 static void __net_exit default_device_exit_batch(struct list_head *net_list)
13527 {
13528 	/* At exit all network devices most be removed from a network
13529 	 * namespace.  Do this in the reverse order of registration.
13530 	 * Do this across as many network namespaces as possible to
13531 	 * improve batching efficiency.
13532 	 */
13533 	struct net_device *dev;
13534 	struct net *net;
13535 	LIST_HEAD(dev_kill_list);
13536 
13537 	rtnl_lock();
13538 
13539 	__rtnl_net_lock(&init_net);
13540 
13541 	list_for_each_entry(net, net_list, exit_list) {
13542 		__rtnl_net_lock(net);
13543 		default_device_exit_net(net);
13544 		__rtnl_net_unlock(net);
13545 
13546 		cond_resched();
13547 	}
13548 
13549 	__rtnl_net_unlock(&init_net);
13550 
13551 	list_for_each_entry(net, net_list, exit_list) {
13552 		__rtnl_net_lock(net);
13553 
13554 		for_each_netdev_reverse(net, dev) {
13555 			if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink)
13556 				dev->rtnl_link_ops->dellink(dev, &dev_kill_list);
13557 			else
13558 				unregister_netdevice_queue(dev, &dev_kill_list);
13559 		}
13560 
13561 		unregister_netdevice_queue_many_net(net, &dev_kill_list);
13562 		__rtnl_net_unlock(net);
13563 	}
13564 	unregister_netdevice_many(&dev_kill_list);
13565 	rtnl_unlock();
13566 
13567 	/* Devices of these netns unregistered from another context
13568 	 * (e.g. a veth whose peer was deleted) can still be waiting
13569 	 * in netdev_run_todo() for their last references, while
13570 	 * NETDEV_UNREGISTER is rebroadcast for them. Wait for them
13571 	 * before pernet exit handlers tear down the state used by
13572 	 * netdevice notifiers, and before the netns is freed.
13573 	 * No new unregistration can start: all devices are unlisted.
13574 	 */
13575 	wait_event(netdev_unregistering_wq, !netns_dev_unreg_pending(net_list));
13576 
13577 	rtnl_net_flush_workqueue();
13578 }
13579 
13580 static struct pernet_operations __net_initdata default_device_ops = {
13581 	.exit_batch = default_device_exit_batch,
13582 };
13583 
13584 static void __init net_dev_struct_check(void)
13585 {
13586 	/* TX read-mostly hotpath */
13587 	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, priv_flags_fast);
13588 	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, netdev_ops);
13589 	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, header_ops);
13590 	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, _tx);
13591 	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, real_num_tx_queues);
13592 	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, gso_max_size);
13593 	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, gso_ipv4_max_size);
13594 	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, gso_max_segs);
13595 	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, gso_partial_features);
13596 	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, num_tc);
13597 	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, mtu);
13598 	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, needed_headroom);
13599 	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, tc_to_txq);
13600 #ifdef CONFIG_XPS
13601 	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, xps_maps);
13602 #endif
13603 #ifdef CONFIG_NETFILTER_EGRESS
13604 	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, nf_hooks_egress);
13605 #endif
13606 #ifdef CONFIG_NET_XGRESS
13607 	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, tcx_egress);
13608 #endif
13609 	CACHELINE_ASSERT_GROUP_SIZE(struct net_device, net_device_read_tx, 160);
13610 
13611 	/* TXRX read-mostly hotpath */
13612 	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, lstats);
13613 	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, state);
13614 	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, flags);
13615 	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, hard_header_len);
13616 	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, features);
13617 	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, ip6_ptr);
13618 	CACHELINE_ASSERT_GROUP_SIZE(struct net_device, net_device_read_txrx, 46);
13619 
13620 	/* RX read-mostly hotpath */
13621 	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, ptype_specific);
13622 	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, ifindex);
13623 	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, real_num_rx_queues);
13624 	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, _rx);
13625 	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, gro_max_size);
13626 	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, gro_ipv4_max_size);
13627 	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, rx_handler);
13628 	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, rx_handler_data);
13629 	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, nd_net);
13630 #ifdef CONFIG_NETPOLL
13631 	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, npinfo);
13632 #endif
13633 #ifdef CONFIG_NET_XGRESS
13634 	CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, tcx_ingress);
13635 #endif
13636 	CACHELINE_ASSERT_GROUP_SIZE(struct net_device, net_device_read_rx, 92);
13637 }
13638 
13639 /*
13640  *	Initialize the DEV module. At boot time this walks the device list and
13641  *	unhooks any devices that fail to initialise (normally hardware not
13642  *	present) and leaves us with a valid list of present and active devices.
13643  *
13644  */
13645 
13646 /* We allocate 256 pages for each CPU if PAGE_SHIFT is 12 */
13647 #define SYSTEM_PERCPU_PAGE_POOL_SIZE	((1 << 20) / PAGE_SIZE)
13648 
13649 static int net_page_pool_create(int cpuid)
13650 {
13651 #if IS_ENABLED(CONFIG_PAGE_POOL)
13652 	struct page_pool_params page_pool_params = {
13653 		.pool_size = SYSTEM_PERCPU_PAGE_POOL_SIZE,
13654 		.flags = PP_FLAG_SYSTEM_POOL,
13655 		.nid = cpu_to_mem(cpuid),
13656 	};
13657 	struct page_pool *pp_ptr;
13658 	int err;
13659 
13660 	pp_ptr = page_pool_create_percpu(&page_pool_params, cpuid);
13661 	if (IS_ERR(pp_ptr))
13662 		return -ENOMEM;
13663 
13664 	err = xdp_reg_page_pool(pp_ptr);
13665 	if (err) {
13666 		page_pool_destroy(pp_ptr);
13667 		return err;
13668 	}
13669 
13670 	per_cpu(system_page_pool.pool, cpuid) = pp_ptr;
13671 #endif
13672 	return 0;
13673 }
13674 
13675 static int backlog_napi_should_run(unsigned int cpu)
13676 {
13677 	struct softnet_data *sd = per_cpu_ptr(&softnet_data, cpu);
13678 	struct napi_struct *napi = &sd->backlog;
13679 
13680 	return test_bit(NAPI_STATE_SCHED_THREADED, &napi->state);
13681 }
13682 
13683 static void run_backlog_napi(unsigned int cpu)
13684 {
13685 	struct softnet_data *sd = per_cpu_ptr(&softnet_data, cpu);
13686 
13687 	napi_threaded_poll_loop(&sd->backlog, NULL);
13688 }
13689 
13690 static void backlog_napi_setup(unsigned int cpu)
13691 {
13692 	struct softnet_data *sd = per_cpu_ptr(&softnet_data, cpu);
13693 	struct napi_struct *napi = &sd->backlog;
13694 
13695 	napi->thread = this_cpu_read(backlog_napi);
13696 	set_bit(NAPI_STATE_THREADED, &napi->state);
13697 }
13698 
13699 static struct smp_hotplug_thread backlog_threads = {
13700 	.store			= &backlog_napi,
13701 	.thread_should_run	= backlog_napi_should_run,
13702 	.thread_fn		= run_backlog_napi,
13703 	.thread_comm		= "backlog_napi/%u",
13704 	.setup			= backlog_napi_setup,
13705 };
13706 
13707 /*
13708  *       This is called single threaded during boot, so no need
13709  *       to take the rtnl semaphore.
13710  */
13711 static int __init net_dev_init(void)
13712 {
13713 	int i, rc = -ENOMEM;
13714 
13715 	BUG_ON(!dev_boot_phase);
13716 
13717 	net_dev_struct_check();
13718 
13719 	if (dev_proc_init())
13720 		goto out;
13721 
13722 	if (netdev_kobject_init())
13723 		goto out;
13724 
13725 	for (i = 0; i < PTYPE_HASH_SIZE; i++)
13726 		INIT_LIST_HEAD(&ptype_base[i]);
13727 
13728 	if (register_pernet_subsys(&netdev_net_ops))
13729 		goto out;
13730 
13731 	/*
13732 	 *	Initialise the packet receive queues.
13733 	 */
13734 
13735 	flush_backlogs_fallback = flush_backlogs_alloc();
13736 	if (!flush_backlogs_fallback)
13737 		goto out;
13738 
13739 	for_each_possible_cpu(i) {
13740 		struct softnet_data *sd = &per_cpu(softnet_data, i);
13741 
13742 		skb_queue_head_init(&sd->input_pkt_queue);
13743 		skb_queue_head_init(&sd->process_queue);
13744 #ifdef CONFIG_XFRM_OFFLOAD
13745 		skb_queue_head_init(&sd->xfrm_backlog);
13746 #endif
13747 		INIT_LIST_HEAD(&sd->poll_list);
13748 		sd->output_queue_tailp = &sd->output_queue;
13749 #ifdef CONFIG_RPS
13750 		INIT_CSD(&sd->csd, rps_trigger_softirq, sd);
13751 		sd->cpu = i;
13752 #endif
13753 		INIT_CSD(&sd->defer_csd, trigger_rx_softirq, sd);
13754 
13755 		gro_init(&sd->backlog.gro);
13756 		sd->backlog.poll = process_backlog;
13757 		sd->backlog.weight = weight_p;
13758 		INIT_LIST_HEAD(&sd->backlog.poll_list);
13759 
13760 		if (net_page_pool_create(i))
13761 			goto out;
13762 	}
13763 	net_hotdata.skb_defer_nodes =
13764 		 __alloc_percpu(sizeof(struct skb_defer_node) * nr_node_ids,
13765 				__alignof__(struct skb_defer_node));
13766 	if (!net_hotdata.skb_defer_nodes)
13767 		goto out;
13768 	if (use_backlog_threads())
13769 		smpboot_register_percpu_thread(&backlog_threads);
13770 
13771 	dev_boot_phase = 0;
13772 
13773 	/* The loopback device is special if any other network devices
13774 	 * is present in a network namespace the loopback device must
13775 	 * be present. Since we now dynamically allocate and free the
13776 	 * loopback device ensure this invariant is maintained by
13777 	 * keeping the loopback device as the first device on the
13778 	 * list of network devices.  Ensuring the loopback devices
13779 	 * is the first device that appears and the last network device
13780 	 * that disappears.
13781 	 */
13782 	if (register_pernet_device(&loopback_net_ops))
13783 		goto out;
13784 
13785 	if (register_pernet_device(&default_device_ops))
13786 		goto out;
13787 
13788 	open_softirq(NET_TX_SOFTIRQ, net_tx_action);
13789 	open_softirq(NET_RX_SOFTIRQ, net_rx_action);
13790 
13791 	rc = cpuhp_setup_state_nocalls(CPUHP_NET_DEV_DEAD, "net/dev:dead",
13792 				       NULL, dev_cpu_dead);
13793 	WARN_ON(rc < 0);
13794 	rc = 0;
13795 
13796 	/* avoid static key IPIs to isolated CPUs */
13797 	if (housekeeping_enabled(HK_TYPE_MISC))
13798 		net_enable_timestamp();
13799 out:
13800 	if (rc < 0) {
13801 		for_each_possible_cpu(i) {
13802 			struct page_pool *pp_ptr;
13803 
13804 			pp_ptr = per_cpu(system_page_pool.pool, i);
13805 			if (!pp_ptr)
13806 				continue;
13807 
13808 			xdp_unreg_page_pool(pp_ptr);
13809 			page_pool_destroy(pp_ptr);
13810 			per_cpu(system_page_pool.pool, i) = NULL;
13811 		}
13812 	}
13813 
13814 	return rc;
13815 }
13816 
13817 subsys_initcall(net_dev_init);
13818