xref: /linux/net/core/netpoll.c (revision 00a6d7b6762c27d441e9ac8faff36384bc0fc180)
1 /*
2  * Common framework for low-level network console, dump, and debugger code
3  *
4  * Sep 8 2003  Matt Mackall <mpm@selenic.com>
5  *
6  * based on the netconsole code from:
7  *
8  * Copyright (C) 2001  Ingo Molnar <mingo@redhat.com>
9  * Copyright (C) 2002  Red Hat, Inc.
10  */
11 
12 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
13 
14 #include <linux/moduleparam.h>
15 #include <linux/kernel.h>
16 #include <linux/netdevice.h>
17 #include <linux/etherdevice.h>
18 #include <linux/string.h>
19 #include <linux/if_arp.h>
20 #include <linux/inetdevice.h>
21 #include <linux/inet.h>
22 #include <linux/interrupt.h>
23 #include <linux/netpoll.h>
24 #include <linux/sched.h>
25 #include <linux/delay.h>
26 #include <linux/rcupdate.h>
27 #include <linux/workqueue.h>
28 #include <linux/slab.h>
29 #include <linux/export.h>
30 #include <linux/if_vlan.h>
31 #include <net/tcp.h>
32 #include <net/udp.h>
33 #include <net/addrconf.h>
34 #include <net/ndisc.h>
35 #include <net/ip6_checksum.h>
36 #include <asm/unaligned.h>
37 #include <trace/events/napi.h>
38 
39 /*
40  * We maintain a small pool of fully-sized skbs, to make sure the
41  * message gets out even in extreme OOM situations.
42  */
43 
44 #define MAX_UDP_CHUNK 1460
45 #define MAX_SKBS 32
46 
47 static struct sk_buff_head skb_pool;
48 
49 DEFINE_STATIC_SRCU(netpoll_srcu);
50 
51 #define USEC_PER_POLL	50
52 
53 #define MAX_SKB_SIZE							\
54 	(sizeof(struct ethhdr) +					\
55 	 sizeof(struct iphdr) +						\
56 	 sizeof(struct udphdr) +					\
57 	 MAX_UDP_CHUNK)
58 
59 static void zap_completion_queue(void);
60 static void netpoll_async_cleanup(struct work_struct *work);
61 
62 static unsigned int carrier_timeout = 4;
63 module_param(carrier_timeout, uint, 0644);
64 
65 #define np_info(np, fmt, ...)				\
66 	pr_info("%s: " fmt, np->name, ##__VA_ARGS__)
67 #define np_err(np, fmt, ...)				\
68 	pr_err("%s: " fmt, np->name, ##__VA_ARGS__)
69 #define np_notice(np, fmt, ...)				\
70 	pr_notice("%s: " fmt, np->name, ##__VA_ARGS__)
71 
72 static int netpoll_start_xmit(struct sk_buff *skb, struct net_device *dev,
73 			      struct netdev_queue *txq)
74 {
75 	const struct net_device_ops *ops = dev->netdev_ops;
76 	int status = NETDEV_TX_OK;
77 	netdev_features_t features;
78 
79 	features = netif_skb_features(skb);
80 
81 	if (vlan_tx_tag_present(skb) &&
82 	    !vlan_hw_offload_capable(features, skb->vlan_proto)) {
83 		skb = __vlan_put_tag(skb, skb->vlan_proto,
84 				     vlan_tx_tag_get(skb));
85 		if (unlikely(!skb)) {
86 			/* This is actually a packet drop, but we
87 			 * don't want the code that calls this
88 			 * function to try and operate on a NULL skb.
89 			 */
90 			goto out;
91 		}
92 		skb->vlan_tci = 0;
93 	}
94 
95 	status = ops->ndo_start_xmit(skb, dev);
96 	if (status == NETDEV_TX_OK)
97 		txq_trans_update(txq);
98 
99 out:
100 	return status;
101 }
102 
103 static void queue_process(struct work_struct *work)
104 {
105 	struct netpoll_info *npinfo =
106 		container_of(work, struct netpoll_info, tx_work.work);
107 	struct sk_buff *skb;
108 	unsigned long flags;
109 
110 	while ((skb = skb_dequeue(&npinfo->txq))) {
111 		struct net_device *dev = skb->dev;
112 		struct netdev_queue *txq;
113 
114 		if (!netif_device_present(dev) || !netif_running(dev)) {
115 			kfree_skb(skb);
116 			continue;
117 		}
118 
119 		txq = netdev_get_tx_queue(dev, skb_get_queue_mapping(skb));
120 
121 		local_irq_save(flags);
122 		HARD_TX_LOCK(dev, txq, smp_processor_id());
123 		if (netif_xmit_frozen_or_stopped(txq) ||
124 		    netpoll_start_xmit(skb, dev, txq) != NETDEV_TX_OK) {
125 			skb_queue_head(&npinfo->txq, skb);
126 			HARD_TX_UNLOCK(dev, txq);
127 			local_irq_restore(flags);
128 
129 			schedule_delayed_work(&npinfo->tx_work, HZ/10);
130 			return;
131 		}
132 		HARD_TX_UNLOCK(dev, txq);
133 		local_irq_restore(flags);
134 	}
135 }
136 
137 /*
138  * Check whether delayed processing was scheduled for our NIC. If so,
139  * we attempt to grab the poll lock and use ->poll() to pump the card.
140  * If this fails, either we've recursed in ->poll() or it's already
141  * running on another CPU.
142  *
143  * Note: we don't mask interrupts with this lock because we're using
144  * trylock here and interrupts are already disabled in the softirq
145  * case. Further, we test the poll_owner to avoid recursion on UP
146  * systems where the lock doesn't exist.
147  */
148 static int poll_one_napi(struct napi_struct *napi, int budget)
149 {
150 	int work;
151 
152 	/* net_rx_action's ->poll() invocations and our's are
153 	 * synchronized by this test which is only made while
154 	 * holding the napi->poll_lock.
155 	 */
156 	if (!test_bit(NAPI_STATE_SCHED, &napi->state))
157 		return budget;
158 
159 	set_bit(NAPI_STATE_NPSVC, &napi->state);
160 
161 	work = napi->poll(napi, budget);
162 	WARN_ONCE(work > budget, "%pF exceeded budget in poll\n", napi->poll);
163 	trace_napi_poll(napi);
164 
165 	clear_bit(NAPI_STATE_NPSVC, &napi->state);
166 
167 	return budget - work;
168 }
169 
170 static void poll_napi(struct net_device *dev, int budget)
171 {
172 	struct napi_struct *napi;
173 
174 	list_for_each_entry(napi, &dev->napi_list, dev_list) {
175 		if (napi->poll_owner != smp_processor_id() &&
176 		    spin_trylock(&napi->poll_lock)) {
177 			budget = poll_one_napi(napi, budget);
178 			spin_unlock(&napi->poll_lock);
179 		}
180 	}
181 }
182 
183 static void netpoll_poll_dev(struct net_device *dev)
184 {
185 	const struct net_device_ops *ops;
186 	struct netpoll_info *ni = rcu_dereference_bh(dev->npinfo);
187 	int budget = 0;
188 
189 	/* Don't do any rx activity if the dev_lock mutex is held
190 	 * the dev_open/close paths use this to block netpoll activity
191 	 * while changing device state
192 	 */
193 	if (down_trylock(&ni->dev_lock))
194 		return;
195 
196 	if (!netif_running(dev)) {
197 		up(&ni->dev_lock);
198 		return;
199 	}
200 
201 	ops = dev->netdev_ops;
202 	if (!ops->ndo_poll_controller) {
203 		up(&ni->dev_lock);
204 		return;
205 	}
206 
207 	/* Process pending work on NIC */
208 	ops->ndo_poll_controller(dev);
209 
210 	poll_napi(dev, budget);
211 
212 	up(&ni->dev_lock);
213 
214 	zap_completion_queue();
215 }
216 
217 void netpoll_poll_disable(struct net_device *dev)
218 {
219 	struct netpoll_info *ni;
220 	int idx;
221 	might_sleep();
222 	idx = srcu_read_lock(&netpoll_srcu);
223 	ni = srcu_dereference(dev->npinfo, &netpoll_srcu);
224 	if (ni)
225 		down(&ni->dev_lock);
226 	srcu_read_unlock(&netpoll_srcu, idx);
227 }
228 EXPORT_SYMBOL(netpoll_poll_disable);
229 
230 void netpoll_poll_enable(struct net_device *dev)
231 {
232 	struct netpoll_info *ni;
233 	rcu_read_lock();
234 	ni = rcu_dereference(dev->npinfo);
235 	if (ni)
236 		up(&ni->dev_lock);
237 	rcu_read_unlock();
238 }
239 EXPORT_SYMBOL(netpoll_poll_enable);
240 
241 static void refill_skbs(void)
242 {
243 	struct sk_buff *skb;
244 	unsigned long flags;
245 
246 	spin_lock_irqsave(&skb_pool.lock, flags);
247 	while (skb_pool.qlen < MAX_SKBS) {
248 		skb = alloc_skb(MAX_SKB_SIZE, GFP_ATOMIC);
249 		if (!skb)
250 			break;
251 
252 		__skb_queue_tail(&skb_pool, skb);
253 	}
254 	spin_unlock_irqrestore(&skb_pool.lock, flags);
255 }
256 
257 static void zap_completion_queue(void)
258 {
259 	unsigned long flags;
260 	struct softnet_data *sd = &get_cpu_var(softnet_data);
261 
262 	if (sd->completion_queue) {
263 		struct sk_buff *clist;
264 
265 		local_irq_save(flags);
266 		clist = sd->completion_queue;
267 		sd->completion_queue = NULL;
268 		local_irq_restore(flags);
269 
270 		while (clist != NULL) {
271 			struct sk_buff *skb = clist;
272 			clist = clist->next;
273 			if (!skb_irq_freeable(skb)) {
274 				atomic_inc(&skb->users);
275 				dev_kfree_skb_any(skb); /* put this one back */
276 			} else {
277 				__kfree_skb(skb);
278 			}
279 		}
280 	}
281 
282 	put_cpu_var(softnet_data);
283 }
284 
285 static struct sk_buff *find_skb(struct netpoll *np, int len, int reserve)
286 {
287 	int count = 0;
288 	struct sk_buff *skb;
289 
290 	zap_completion_queue();
291 	refill_skbs();
292 repeat:
293 
294 	skb = alloc_skb(len, GFP_ATOMIC);
295 	if (!skb)
296 		skb = skb_dequeue(&skb_pool);
297 
298 	if (!skb) {
299 		if (++count < 10) {
300 			netpoll_poll_dev(np->dev);
301 			goto repeat;
302 		}
303 		return NULL;
304 	}
305 
306 	atomic_set(&skb->users, 1);
307 	skb_reserve(skb, reserve);
308 	return skb;
309 }
310 
311 static int netpoll_owner_active(struct net_device *dev)
312 {
313 	struct napi_struct *napi;
314 
315 	list_for_each_entry(napi, &dev->napi_list, dev_list) {
316 		if (napi->poll_owner == smp_processor_id())
317 			return 1;
318 	}
319 	return 0;
320 }
321 
322 /* call with IRQ disabled */
323 void netpoll_send_skb_on_dev(struct netpoll *np, struct sk_buff *skb,
324 			     struct net_device *dev)
325 {
326 	int status = NETDEV_TX_BUSY;
327 	unsigned long tries;
328 	/* It is up to the caller to keep npinfo alive. */
329 	struct netpoll_info *npinfo;
330 
331 	WARN_ON_ONCE(!irqs_disabled());
332 
333 	npinfo = rcu_dereference_bh(np->dev->npinfo);
334 	if (!npinfo || !netif_running(dev) || !netif_device_present(dev)) {
335 		dev_kfree_skb_irq(skb);
336 		return;
337 	}
338 
339 	/* don't get messages out of order, and no recursion */
340 	if (skb_queue_len(&npinfo->txq) == 0 && !netpoll_owner_active(dev)) {
341 		struct netdev_queue *txq;
342 
343 		txq = netdev_pick_tx(dev, skb, NULL);
344 
345 		/* try until next clock tick */
346 		for (tries = jiffies_to_usecs(1)/USEC_PER_POLL;
347 		     tries > 0; --tries) {
348 			if (HARD_TX_TRYLOCK(dev, txq)) {
349 				if (!netif_xmit_stopped(txq))
350 					status = netpoll_start_xmit(skb, dev, txq);
351 
352 				HARD_TX_UNLOCK(dev, txq);
353 
354 				if (status == NETDEV_TX_OK)
355 					break;
356 
357 			}
358 
359 			/* tickle device maybe there is some cleanup */
360 			netpoll_poll_dev(np->dev);
361 
362 			udelay(USEC_PER_POLL);
363 		}
364 
365 		WARN_ONCE(!irqs_disabled(),
366 			"netpoll_send_skb_on_dev(): %s enabled interrupts in poll (%pF)\n",
367 			dev->name, dev->netdev_ops->ndo_start_xmit);
368 
369 	}
370 
371 	if (status != NETDEV_TX_OK) {
372 		skb_queue_tail(&npinfo->txq, skb);
373 		schedule_delayed_work(&npinfo->tx_work,0);
374 	}
375 }
376 EXPORT_SYMBOL(netpoll_send_skb_on_dev);
377 
378 void netpoll_send_udp(struct netpoll *np, const char *msg, int len)
379 {
380 	int total_len, ip_len, udp_len;
381 	struct sk_buff *skb;
382 	struct udphdr *udph;
383 	struct iphdr *iph;
384 	struct ethhdr *eth;
385 	static atomic_t ip_ident;
386 	struct ipv6hdr *ip6h;
387 
388 	udp_len = len + sizeof(*udph);
389 	if (np->ipv6)
390 		ip_len = udp_len + sizeof(*ip6h);
391 	else
392 		ip_len = udp_len + sizeof(*iph);
393 
394 	total_len = ip_len + LL_RESERVED_SPACE(np->dev);
395 
396 	skb = find_skb(np, total_len + np->dev->needed_tailroom,
397 		       total_len - len);
398 	if (!skb)
399 		return;
400 
401 	skb_copy_to_linear_data(skb, msg, len);
402 	skb_put(skb, len);
403 
404 	skb_push(skb, sizeof(*udph));
405 	skb_reset_transport_header(skb);
406 	udph = udp_hdr(skb);
407 	udph->source = htons(np->local_port);
408 	udph->dest = htons(np->remote_port);
409 	udph->len = htons(udp_len);
410 
411 	if (np->ipv6) {
412 		udph->check = 0;
413 		udph->check = csum_ipv6_magic(&np->local_ip.in6,
414 					      &np->remote_ip.in6,
415 					      udp_len, IPPROTO_UDP,
416 					      csum_partial(udph, udp_len, 0));
417 		if (udph->check == 0)
418 			udph->check = CSUM_MANGLED_0;
419 
420 		skb_push(skb, sizeof(*ip6h));
421 		skb_reset_network_header(skb);
422 		ip6h = ipv6_hdr(skb);
423 
424 		/* ip6h->version = 6; ip6h->priority = 0; */
425 		put_unaligned(0x60, (unsigned char *)ip6h);
426 		ip6h->flow_lbl[0] = 0;
427 		ip6h->flow_lbl[1] = 0;
428 		ip6h->flow_lbl[2] = 0;
429 
430 		ip6h->payload_len = htons(sizeof(struct udphdr) + len);
431 		ip6h->nexthdr = IPPROTO_UDP;
432 		ip6h->hop_limit = 32;
433 		ip6h->saddr = np->local_ip.in6;
434 		ip6h->daddr = np->remote_ip.in6;
435 
436 		eth = (struct ethhdr *) skb_push(skb, ETH_HLEN);
437 		skb_reset_mac_header(skb);
438 		skb->protocol = eth->h_proto = htons(ETH_P_IPV6);
439 	} else {
440 		udph->check = 0;
441 		udph->check = csum_tcpudp_magic(np->local_ip.ip,
442 						np->remote_ip.ip,
443 						udp_len, IPPROTO_UDP,
444 						csum_partial(udph, udp_len, 0));
445 		if (udph->check == 0)
446 			udph->check = CSUM_MANGLED_0;
447 
448 		skb_push(skb, sizeof(*iph));
449 		skb_reset_network_header(skb);
450 		iph = ip_hdr(skb);
451 
452 		/* iph->version = 4; iph->ihl = 5; */
453 		put_unaligned(0x45, (unsigned char *)iph);
454 		iph->tos      = 0;
455 		put_unaligned(htons(ip_len), &(iph->tot_len));
456 		iph->id       = htons(atomic_inc_return(&ip_ident));
457 		iph->frag_off = 0;
458 		iph->ttl      = 64;
459 		iph->protocol = IPPROTO_UDP;
460 		iph->check    = 0;
461 		put_unaligned(np->local_ip.ip, &(iph->saddr));
462 		put_unaligned(np->remote_ip.ip, &(iph->daddr));
463 		iph->check    = ip_fast_csum((unsigned char *)iph, iph->ihl);
464 
465 		eth = (struct ethhdr *) skb_push(skb, ETH_HLEN);
466 		skb_reset_mac_header(skb);
467 		skb->protocol = eth->h_proto = htons(ETH_P_IP);
468 	}
469 
470 	ether_addr_copy(eth->h_source, np->dev->dev_addr);
471 	ether_addr_copy(eth->h_dest, np->remote_mac);
472 
473 	skb->dev = np->dev;
474 
475 	netpoll_send_skb(np, skb);
476 }
477 EXPORT_SYMBOL(netpoll_send_udp);
478 
479 void netpoll_print_options(struct netpoll *np)
480 {
481 	np_info(np, "local port %d\n", np->local_port);
482 	if (np->ipv6)
483 		np_info(np, "local IPv6 address %pI6c\n", &np->local_ip.in6);
484 	else
485 		np_info(np, "local IPv4 address %pI4\n", &np->local_ip.ip);
486 	np_info(np, "interface '%s'\n", np->dev_name);
487 	np_info(np, "remote port %d\n", np->remote_port);
488 	if (np->ipv6)
489 		np_info(np, "remote IPv6 address %pI6c\n", &np->remote_ip.in6);
490 	else
491 		np_info(np, "remote IPv4 address %pI4\n", &np->remote_ip.ip);
492 	np_info(np, "remote ethernet address %pM\n", np->remote_mac);
493 }
494 EXPORT_SYMBOL(netpoll_print_options);
495 
496 static int netpoll_parse_ip_addr(const char *str, union inet_addr *addr)
497 {
498 	const char *end;
499 
500 	if (!strchr(str, ':') &&
501 	    in4_pton(str, -1, (void *)addr, -1, &end) > 0) {
502 		if (!*end)
503 			return 0;
504 	}
505 	if (in6_pton(str, -1, addr->in6.s6_addr, -1, &end) > 0) {
506 #if IS_ENABLED(CONFIG_IPV6)
507 		if (!*end)
508 			return 1;
509 #else
510 		return -1;
511 #endif
512 	}
513 	return -1;
514 }
515 
516 int netpoll_parse_options(struct netpoll *np, char *opt)
517 {
518 	char *cur=opt, *delim;
519 	int ipv6;
520 	bool ipversion_set = false;
521 
522 	if (*cur != '@') {
523 		if ((delim = strchr(cur, '@')) == NULL)
524 			goto parse_failed;
525 		*delim = 0;
526 		if (kstrtou16(cur, 10, &np->local_port))
527 			goto parse_failed;
528 		cur = delim;
529 	}
530 	cur++;
531 
532 	if (*cur != '/') {
533 		ipversion_set = true;
534 		if ((delim = strchr(cur, '/')) == NULL)
535 			goto parse_failed;
536 		*delim = 0;
537 		ipv6 = netpoll_parse_ip_addr(cur, &np->local_ip);
538 		if (ipv6 < 0)
539 			goto parse_failed;
540 		else
541 			np->ipv6 = (bool)ipv6;
542 		cur = delim;
543 	}
544 	cur++;
545 
546 	if (*cur != ',') {
547 		/* parse out dev name */
548 		if ((delim = strchr(cur, ',')) == NULL)
549 			goto parse_failed;
550 		*delim = 0;
551 		strlcpy(np->dev_name, cur, sizeof(np->dev_name));
552 		cur = delim;
553 	}
554 	cur++;
555 
556 	if (*cur != '@') {
557 		/* dst port */
558 		if ((delim = strchr(cur, '@')) == NULL)
559 			goto parse_failed;
560 		*delim = 0;
561 		if (*cur == ' ' || *cur == '\t')
562 			np_info(np, "warning: whitespace is not allowed\n");
563 		if (kstrtou16(cur, 10, &np->remote_port))
564 			goto parse_failed;
565 		cur = delim;
566 	}
567 	cur++;
568 
569 	/* dst ip */
570 	if ((delim = strchr(cur, '/')) == NULL)
571 		goto parse_failed;
572 	*delim = 0;
573 	ipv6 = netpoll_parse_ip_addr(cur, &np->remote_ip);
574 	if (ipv6 < 0)
575 		goto parse_failed;
576 	else if (ipversion_set && np->ipv6 != (bool)ipv6)
577 		goto parse_failed;
578 	else
579 		np->ipv6 = (bool)ipv6;
580 	cur = delim + 1;
581 
582 	if (*cur != 0) {
583 		/* MAC address */
584 		if (!mac_pton(cur, np->remote_mac))
585 			goto parse_failed;
586 	}
587 
588 	netpoll_print_options(np);
589 
590 	return 0;
591 
592  parse_failed:
593 	np_info(np, "couldn't parse config at '%s'!\n", cur);
594 	return -1;
595 }
596 EXPORT_SYMBOL(netpoll_parse_options);
597 
598 int __netpoll_setup(struct netpoll *np, struct net_device *ndev)
599 {
600 	struct netpoll_info *npinfo;
601 	const struct net_device_ops *ops;
602 	int err;
603 
604 	np->dev = ndev;
605 	strlcpy(np->dev_name, ndev->name, IFNAMSIZ);
606 	INIT_WORK(&np->cleanup_work, netpoll_async_cleanup);
607 
608 	if ((ndev->priv_flags & IFF_DISABLE_NETPOLL) ||
609 	    !ndev->netdev_ops->ndo_poll_controller) {
610 		np_err(np, "%s doesn't support polling, aborting\n",
611 		       np->dev_name);
612 		err = -ENOTSUPP;
613 		goto out;
614 	}
615 
616 	if (!ndev->npinfo) {
617 		npinfo = kmalloc(sizeof(*npinfo), GFP_KERNEL);
618 		if (!npinfo) {
619 			err = -ENOMEM;
620 			goto out;
621 		}
622 
623 		sema_init(&npinfo->dev_lock, 1);
624 		skb_queue_head_init(&npinfo->txq);
625 		INIT_DELAYED_WORK(&npinfo->tx_work, queue_process);
626 
627 		atomic_set(&npinfo->refcnt, 1);
628 
629 		ops = np->dev->netdev_ops;
630 		if (ops->ndo_netpoll_setup) {
631 			err = ops->ndo_netpoll_setup(ndev, npinfo);
632 			if (err)
633 				goto free_npinfo;
634 		}
635 	} else {
636 		npinfo = rtnl_dereference(ndev->npinfo);
637 		atomic_inc(&npinfo->refcnt);
638 	}
639 
640 	npinfo->netpoll = np;
641 
642 	/* last thing to do is link it to the net device structure */
643 	rcu_assign_pointer(ndev->npinfo, npinfo);
644 
645 	return 0;
646 
647 free_npinfo:
648 	kfree(npinfo);
649 out:
650 	return err;
651 }
652 EXPORT_SYMBOL_GPL(__netpoll_setup);
653 
654 int netpoll_setup(struct netpoll *np)
655 {
656 	struct net_device *ndev = NULL;
657 	struct in_device *in_dev;
658 	int err;
659 
660 	rtnl_lock();
661 	if (np->dev_name) {
662 		struct net *net = current->nsproxy->net_ns;
663 		ndev = __dev_get_by_name(net, np->dev_name);
664 	}
665 	if (!ndev) {
666 		np_err(np, "%s doesn't exist, aborting\n", np->dev_name);
667 		err = -ENODEV;
668 		goto unlock;
669 	}
670 	dev_hold(ndev);
671 
672 	if (netdev_master_upper_dev_get(ndev)) {
673 		np_err(np, "%s is a slave device, aborting\n", np->dev_name);
674 		err = -EBUSY;
675 		goto put;
676 	}
677 
678 	if (!netif_running(ndev)) {
679 		unsigned long atmost, atleast;
680 
681 		np_info(np, "device %s not up yet, forcing it\n", np->dev_name);
682 
683 		err = dev_open(ndev);
684 
685 		if (err) {
686 			np_err(np, "failed to open %s\n", ndev->name);
687 			goto put;
688 		}
689 
690 		rtnl_unlock();
691 		atleast = jiffies + HZ/10;
692 		atmost = jiffies + carrier_timeout * HZ;
693 		while (!netif_carrier_ok(ndev)) {
694 			if (time_after(jiffies, atmost)) {
695 				np_notice(np, "timeout waiting for carrier\n");
696 				break;
697 			}
698 			msleep(1);
699 		}
700 
701 		/* If carrier appears to come up instantly, we don't
702 		 * trust it and pause so that we don't pump all our
703 		 * queued console messages into the bitbucket.
704 		 */
705 
706 		if (time_before(jiffies, atleast)) {
707 			np_notice(np, "carrier detect appears untrustworthy, waiting 4 seconds\n");
708 			msleep(4000);
709 		}
710 		rtnl_lock();
711 	}
712 
713 	if (!np->local_ip.ip) {
714 		if (!np->ipv6) {
715 			in_dev = __in_dev_get_rtnl(ndev);
716 
717 			if (!in_dev || !in_dev->ifa_list) {
718 				np_err(np, "no IP address for %s, aborting\n",
719 				       np->dev_name);
720 				err = -EDESTADDRREQ;
721 				goto put;
722 			}
723 
724 			np->local_ip.ip = in_dev->ifa_list->ifa_local;
725 			np_info(np, "local IP %pI4\n", &np->local_ip.ip);
726 		} else {
727 #if IS_ENABLED(CONFIG_IPV6)
728 			struct inet6_dev *idev;
729 
730 			err = -EDESTADDRREQ;
731 			idev = __in6_dev_get(ndev);
732 			if (idev) {
733 				struct inet6_ifaddr *ifp;
734 
735 				read_lock_bh(&idev->lock);
736 				list_for_each_entry(ifp, &idev->addr_list, if_list) {
737 					if (ipv6_addr_type(&ifp->addr) & IPV6_ADDR_LINKLOCAL)
738 						continue;
739 					np->local_ip.in6 = ifp->addr;
740 					err = 0;
741 					break;
742 				}
743 				read_unlock_bh(&idev->lock);
744 			}
745 			if (err) {
746 				np_err(np, "no IPv6 address for %s, aborting\n",
747 				       np->dev_name);
748 				goto put;
749 			} else
750 				np_info(np, "local IPv6 %pI6c\n", &np->local_ip.in6);
751 #else
752 			np_err(np, "IPv6 is not supported %s, aborting\n",
753 			       np->dev_name);
754 			err = -EINVAL;
755 			goto put;
756 #endif
757 		}
758 	}
759 
760 	/* fill up the skb queue */
761 	refill_skbs();
762 
763 	err = __netpoll_setup(np, ndev);
764 	if (err)
765 		goto put;
766 
767 	rtnl_unlock();
768 	return 0;
769 
770 put:
771 	dev_put(ndev);
772 unlock:
773 	rtnl_unlock();
774 	return err;
775 }
776 EXPORT_SYMBOL(netpoll_setup);
777 
778 static int __init netpoll_init(void)
779 {
780 	skb_queue_head_init(&skb_pool);
781 	return 0;
782 }
783 core_initcall(netpoll_init);
784 
785 static void rcu_cleanup_netpoll_info(struct rcu_head *rcu_head)
786 {
787 	struct netpoll_info *npinfo =
788 			container_of(rcu_head, struct netpoll_info, rcu);
789 
790 	skb_queue_purge(&npinfo->txq);
791 
792 	/* we can't call cancel_delayed_work_sync here, as we are in softirq */
793 	cancel_delayed_work(&npinfo->tx_work);
794 
795 	/* clean after last, unfinished work */
796 	__skb_queue_purge(&npinfo->txq);
797 	/* now cancel it again */
798 	cancel_delayed_work(&npinfo->tx_work);
799 	kfree(npinfo);
800 }
801 
802 void __netpoll_cleanup(struct netpoll *np)
803 {
804 	struct netpoll_info *npinfo;
805 
806 	/* rtnl_dereference would be preferable here but
807 	 * rcu_cleanup_netpoll path can put us in here safely without
808 	 * holding the rtnl, so plain rcu_dereference it is
809 	 */
810 	npinfo = rtnl_dereference(np->dev->npinfo);
811 	if (!npinfo)
812 		return;
813 
814 	synchronize_srcu(&netpoll_srcu);
815 
816 	if (atomic_dec_and_test(&npinfo->refcnt)) {
817 		const struct net_device_ops *ops;
818 
819 		ops = np->dev->netdev_ops;
820 		if (ops->ndo_netpoll_cleanup)
821 			ops->ndo_netpoll_cleanup(np->dev);
822 
823 		RCU_INIT_POINTER(np->dev->npinfo, NULL);
824 		call_rcu_bh(&npinfo->rcu, rcu_cleanup_netpoll_info);
825 	}
826 }
827 EXPORT_SYMBOL_GPL(__netpoll_cleanup);
828 
829 static void netpoll_async_cleanup(struct work_struct *work)
830 {
831 	struct netpoll *np = container_of(work, struct netpoll, cleanup_work);
832 
833 	rtnl_lock();
834 	__netpoll_cleanup(np);
835 	rtnl_unlock();
836 	kfree(np);
837 }
838 
839 void __netpoll_free_async(struct netpoll *np)
840 {
841 	schedule_work(&np->cleanup_work);
842 }
843 EXPORT_SYMBOL_GPL(__netpoll_free_async);
844 
845 void netpoll_cleanup(struct netpoll *np)
846 {
847 	rtnl_lock();
848 	if (!np->dev)
849 		goto out;
850 	__netpoll_cleanup(np);
851 	dev_put(np->dev);
852 	np->dev = NULL;
853 out:
854 	rtnl_unlock();
855 }
856 EXPORT_SYMBOL(netpoll_cleanup);
857