1457c8996SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
21da177e4SLinus Torvalds /*
31da177e4SLinus Torvalds * Common framework for low-level network console, dump, and debugger code
41da177e4SLinus Torvalds *
51da177e4SLinus Torvalds * Sep 8 2003 Matt Mackall <mpm@selenic.com>
61da177e4SLinus Torvalds *
71da177e4SLinus Torvalds * based on the netconsole code from:
81da177e4SLinus Torvalds *
91da177e4SLinus Torvalds * Copyright (C) 2001 Ingo Molnar <mingo@redhat.com>
101da177e4SLinus Torvalds * Copyright (C) 2002 Red Hat, Inc.
111da177e4SLinus Torvalds */
121da177e4SLinus Torvalds
13e6ec2693SJoe Perches #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
14e6ec2693SJoe Perches
15bff38771SAnton Vorontsov #include <linux/moduleparam.h>
164cd5773aSAndy Shevchenko #include <linux/kernel.h>
171da177e4SLinus Torvalds #include <linux/netdevice.h>
181da177e4SLinus Torvalds #include <linux/etherdevice.h>
191da177e4SLinus Torvalds #include <linux/string.h>
2014c85021SArnaldo Carvalho de Melo #include <linux/if_arp.h>
211da177e4SLinus Torvalds #include <linux/inetdevice.h>
221da177e4SLinus Torvalds #include <linux/inet.h>
231da177e4SLinus Torvalds #include <linux/interrupt.h>
241da177e4SLinus Torvalds #include <linux/netpoll.h>
251da177e4SLinus Torvalds #include <linux/sched.h>
261da177e4SLinus Torvalds #include <linux/delay.h>
271da177e4SLinus Torvalds #include <linux/rcupdate.h>
281da177e4SLinus Torvalds #include <linux/workqueue.h>
295a0e3ad6STejun Heo #include <linux/slab.h>
30bc3b2d7fSPaul Gortmaker #include <linux/export.h>
31689971b4SAmerigo Wang #include <linux/if_vlan.h>
321da177e4SLinus Torvalds #include <net/tcp.h>
331da177e4SLinus Torvalds #include <net/udp.h>
34b3d936f3SCong Wang #include <net/addrconf.h>
35b3d936f3SCong Wang #include <net/ndisc.h>
36b3d936f3SCong Wang #include <net/ip6_checksum.h>
371da177e4SLinus Torvalds #include <asm/unaligned.h>
389cbc1cb8SDavid S. Miller #include <trace/events/napi.h>
39b0f6c9acSWander Lairson Costa #include <linux/kconfig.h>
401da177e4SLinus Torvalds
411da177e4SLinus Torvalds /*
421da177e4SLinus Torvalds * We maintain a small pool of fully-sized skbs, to make sure the
431da177e4SLinus Torvalds * message gets out even in extreme OOM situations.
441da177e4SLinus Torvalds */
451da177e4SLinus Torvalds
461da177e4SLinus Torvalds #define MAX_UDP_CHUNK 1460
471da177e4SLinus Torvalds #define MAX_SKBS 32
481da177e4SLinus Torvalds
49a1bcfacdSStephen Hemminger static struct sk_buff_head skb_pool;
501da177e4SLinus Torvalds
512bdfe0baSStephen Hemminger #define USEC_PER_POLL 50
521da177e4SLinus Torvalds
531da177e4SLinus Torvalds #define MAX_SKB_SIZE \
546f706245SJoe Perches (sizeof(struct ethhdr) + \
556f706245SJoe Perches sizeof(struct iphdr) + \
566f706245SJoe Perches sizeof(struct udphdr) + \
576f706245SJoe Perches MAX_UDP_CHUNK)
581da177e4SLinus Torvalds
593578b0c8SDavid S. Miller static void zap_completion_queue(void);
601da177e4SLinus Torvalds
61bff38771SAnton Vorontsov static unsigned int carrier_timeout = 4;
62bff38771SAnton Vorontsov module_param(carrier_timeout, uint, 0644);
63bff38771SAnton Vorontsov
64e6ec2693SJoe Perches #define np_info(np, fmt, ...) \
65e6ec2693SJoe Perches pr_info("%s: " fmt, np->name, ##__VA_ARGS__)
66e6ec2693SJoe Perches #define np_err(np, fmt, ...) \
67e6ec2693SJoe Perches pr_err("%s: " fmt, np->name, ##__VA_ARGS__)
68e6ec2693SJoe Perches #define np_notice(np, fmt, ...) \
69e6ec2693SJoe Perches pr_notice("%s: " fmt, np->name, ##__VA_ARGS__)
70e6ec2693SJoe Perches
netpoll_start_xmit(struct sk_buff * skb,struct net_device * dev,struct netdev_queue * txq)71a54776f2SYunjian Wang static netdev_tx_t netpoll_start_xmit(struct sk_buff *skb,
72a54776f2SYunjian Wang struct net_device *dev,
73944e2948SEric W. Biederman struct netdev_queue *txq)
74944e2948SEric W. Biederman {
75a54776f2SYunjian Wang netdev_tx_t status = NETDEV_TX_OK;
76944e2948SEric W. Biederman netdev_features_t features;
77944e2948SEric W. Biederman
78944e2948SEric W. Biederman features = netif_skb_features(skb);
79944e2948SEric W. Biederman
80df8a39deSJiri Pirko if (skb_vlan_tag_present(skb) &&
81944e2948SEric W. Biederman !vlan_hw_offload_capable(features, skb->vlan_proto)) {
825968250cSJiri Pirko skb = __vlan_hwaccel_push_inside(skb);
83944e2948SEric W. Biederman if (unlikely(!skb)) {
84944e2948SEric W. Biederman /* This is actually a packet drop, but we
85944e2948SEric W. Biederman * don't want the code that calls this
86944e2948SEric W. Biederman * function to try and operate on a NULL skb.
87944e2948SEric W. Biederman */
88944e2948SEric W. Biederman goto out;
89944e2948SEric W. Biederman }
90944e2948SEric W. Biederman }
91944e2948SEric W. Biederman
92fa2dbdc2SDavid S. Miller status = netdev_start_xmit(skb, dev, txq, false);
93944e2948SEric W. Biederman
94944e2948SEric W. Biederman out:
95944e2948SEric W. Biederman return status;
96944e2948SEric W. Biederman }
97944e2948SEric W. Biederman
queue_process(struct work_struct * work)98c4028958SDavid Howells static void queue_process(struct work_struct *work)
991da177e4SLinus Torvalds {
1004c1ac1b4SDavid Howells struct netpoll_info *npinfo =
1014c1ac1b4SDavid Howells container_of(work, struct netpoll_info, tx_work.work);
1021da177e4SLinus Torvalds struct sk_buff *skb;
1033640543dSIngo Molnar unsigned long flags;
1041da177e4SLinus Torvalds
1056c43ff18SStephen Hemminger while ((skb = skb_dequeue(&npinfo->txq))) {
1066c43ff18SStephen Hemminger struct net_device *dev = skb->dev;
107fd2ea0a7SDavid S. Miller struct netdev_queue *txq;
108c70b17b7STushar Dave unsigned int q_index;
1091da177e4SLinus Torvalds
1106c43ff18SStephen Hemminger if (!netif_device_present(dev) || !netif_running(dev)) {
111080b3c19SEric W. Biederman kfree_skb(skb);
1126c43ff18SStephen Hemminger continue;
1136c43ff18SStephen Hemminger }
1146c43ff18SStephen Hemminger
1153640543dSIngo Molnar local_irq_save(flags);
116c70b17b7STushar Dave /* check if skb->queue_mapping is still valid */
117c70b17b7STushar Dave q_index = skb_get_queue_mapping(skb);
118c70b17b7STushar Dave if (unlikely(q_index >= dev->real_num_tx_queues)) {
119c70b17b7STushar Dave q_index = q_index % dev->real_num_tx_queues;
120c70b17b7STushar Dave skb_set_queue_mapping(skb, q_index);
121c70b17b7STushar Dave }
122c70b17b7STushar Dave txq = netdev_get_tx_queue(dev, q_index);
1235efeac44SEric W. Biederman HARD_TX_LOCK(dev, txq, smp_processor_id());
12473466498STom Herbert if (netif_xmit_frozen_or_stopped(txq) ||
1252c1644cfSFeng Sun !dev_xmit_complete(netpoll_start_xmit(skb, dev, txq))) {
1266c43ff18SStephen Hemminger skb_queue_head(&npinfo->txq, skb);
1275efeac44SEric W. Biederman HARD_TX_UNLOCK(dev, txq);
1283640543dSIngo Molnar local_irq_restore(flags);
1296c43ff18SStephen Hemminger
1306c43ff18SStephen Hemminger schedule_delayed_work(&npinfo->tx_work, HZ/10);
1311da177e4SLinus Torvalds return;
1321da177e4SLinus Torvalds }
1335efeac44SEric W. Biederman HARD_TX_UNLOCK(dev, txq);
1343640543dSIngo Molnar local_irq_restore(flags);
1356c43ff18SStephen Hemminger }
1361da177e4SLinus Torvalds }
1371da177e4SLinus Torvalds
netif_local_xmit_active(struct net_device * dev)138275b471eSJakub Kicinski static int netif_local_xmit_active(struct net_device *dev)
139275b471eSJakub Kicinski {
140275b471eSJakub Kicinski int i;
141275b471eSJakub Kicinski
142275b471eSJakub Kicinski for (i = 0; i < dev->num_tx_queues; i++) {
143275b471eSJakub Kicinski struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
144275b471eSJakub Kicinski
145275b471eSJakub Kicinski if (READ_ONCE(txq->xmit_lock_owner) == smp_processor_id())
146275b471eSJakub Kicinski return 1;
147275b471eSJakub Kicinski }
148275b471eSJakub Kicinski
149275b471eSJakub Kicinski return 0;
150275b471eSJakub Kicinski }
151275b471eSJakub Kicinski
poll_one_napi(struct napi_struct * napi)152822d54b9SAlexander Duyck static void poll_one_napi(struct napi_struct *napi)
1530a7606c1SDavid S. Miller {
154c24498c6SEric Dumazet int work;
1550a7606c1SDavid S. Miller
1562d8bff12SNeil Horman /* If we set this bit but see that it has already been set,
1572d8bff12SNeil Horman * that indicates that napi has been disabled and we need
1582d8bff12SNeil Horman * to abort this operation
1592d8bff12SNeil Horman */
1602d8bff12SNeil Horman if (test_and_set_bit(NAPI_STATE_NPSVC, &napi->state))
161822d54b9SAlexander Duyck return;
1620a7606c1SDavid S. Miller
163a8c924e9SSimon Horman /* We explicitly pass the polling call a budget of 0 to
164822d54b9SAlexander Duyck * indicate that we are clearing the Tx path only.
165822d54b9SAlexander Duyck */
166822d54b9SAlexander Duyck work = napi->poll(napi, 0);
167d75f773cSSakari Ailus WARN_ONCE(work, "%pS exceeded budget in poll\n", napi->poll);
1681db19db7SJesper Dangaard Brouer trace_napi_poll(napi, work, 0);
1690a7606c1SDavid S. Miller
1707b363e44SNeil Horman clear_bit(NAPI_STATE_NPSVC, &napi->state);
1710a7606c1SDavid S. Miller }
1720a7606c1SDavid S. Miller
poll_napi(struct net_device * dev)173822d54b9SAlexander Duyck static void poll_napi(struct net_device *dev)
1741da177e4SLinus Torvalds {
175bea3348eSStephen Hemminger struct napi_struct *napi;
17689c4b442SEric Dumazet int cpu = smp_processor_id();
1771da177e4SLinus Torvalds
17896e97bc0SJakub Kicinski list_for_each_entry_rcu(napi, &dev->napi_list, dev_list) {
17989c4b442SEric Dumazet if (cmpxchg(&napi->poll_owner, -1, cpu) == -1) {
180822d54b9SAlexander Duyck poll_one_napi(napi);
18189c4b442SEric Dumazet smp_store_release(&napi->poll_owner, -1);
182bea3348eSStephen Hemminger }
1831da177e4SLinus Torvalds }
1841da177e4SLinus Torvalds }
1851da177e4SLinus Torvalds
netpoll_poll_dev(struct net_device * dev)186ac3d9dd0SEric Dumazet void netpoll_poll_dev(struct net_device *dev)
1871da177e4SLinus Torvalds {
1882899656bSAmerigo Wang struct netpoll_info *ni = rcu_dereference_bh(dev->npinfo);
189ac3d9dd0SEric Dumazet const struct net_device_ops *ops;
1905106930bSStephen Hemminger
191ca99ca14SNeil Horman /* Don't do any rx activity if the dev_lock mutex is held
192ca99ca14SNeil Horman * the dev_open/close paths use this to block netpoll activity
193ca99ca14SNeil Horman * while changing device state
194ca99ca14SNeil Horman */
195ac3d9dd0SEric Dumazet if (!ni || down_trylock(&ni->dev_lock))
196ca99ca14SNeil Horman return;
197ca99ca14SNeil Horman
198275b471eSJakub Kicinski /* Some drivers will take the same locks in poll and xmit,
199275b471eSJakub Kicinski * we can't poll if local CPU is already in xmit.
200275b471eSJakub Kicinski */
201275b471eSJakub Kicinski if (!netif_running(dev) || netif_local_xmit_active(dev)) {
202bd7c4b60SNeil Horman up(&ni->dev_lock);
2035e392739SPavel Emelyanov return;
204959d5fdeSNeil Horman }
2055e392739SPavel Emelyanov
2065e392739SPavel Emelyanov ops = dev->netdev_ops;
207ac3d9dd0SEric Dumazet if (ops->ndo_poll_controller)
208d314774cSStephen Hemminger ops->ndo_poll_controller(dev);
2091da177e4SLinus Torvalds
210822d54b9SAlexander Duyck poll_napi(dev);
2115106930bSStephen Hemminger
212bd7c4b60SNeil Horman up(&ni->dev_lock);
213ca99ca14SNeil Horman
2143578b0c8SDavid S. Miller zap_completion_queue();
2151da177e4SLinus Torvalds }
216ac3d9dd0SEric Dumazet EXPORT_SYMBOL(netpoll_poll_dev);
2171da177e4SLinus Torvalds
netpoll_poll_disable(struct net_device * dev)21866b5552fSEric W. Biederman void netpoll_poll_disable(struct net_device *dev)
219ca99ca14SNeil Horman {
220ca99ca14SNeil Horman struct netpoll_info *ni;
221*9a95eedcSEric Dumazet
222ca99ca14SNeil Horman might_sleep();
223*9a95eedcSEric Dumazet ni = rtnl_dereference(dev->npinfo);
224ca99ca14SNeil Horman if (ni)
225bd7c4b60SNeil Horman down(&ni->dev_lock);
226ca99ca14SNeil Horman }
227ca99ca14SNeil Horman
netpoll_poll_enable(struct net_device * dev)22866b5552fSEric W. Biederman void netpoll_poll_enable(struct net_device *dev)
229ca99ca14SNeil Horman {
230ca99ca14SNeil Horman struct netpoll_info *ni;
231*9a95eedcSEric Dumazet
232*9a95eedcSEric Dumazet ni = rtnl_dereference(dev->npinfo);
233ca99ca14SNeil Horman if (ni)
234bd7c4b60SNeil Horman up(&ni->dev_lock);
235ca99ca14SNeil Horman }
236ca99ca14SNeil Horman
refill_skbs(void)2371da177e4SLinus Torvalds static void refill_skbs(void)
2381da177e4SLinus Torvalds {
2391da177e4SLinus Torvalds struct sk_buff *skb;
2401da177e4SLinus Torvalds unsigned long flags;
2411da177e4SLinus Torvalds
242a1bcfacdSStephen Hemminger spin_lock_irqsave(&skb_pool.lock, flags);
243a1bcfacdSStephen Hemminger while (skb_pool.qlen < MAX_SKBS) {
2441da177e4SLinus Torvalds skb = alloc_skb(MAX_SKB_SIZE, GFP_ATOMIC);
2451da177e4SLinus Torvalds if (!skb)
2461da177e4SLinus Torvalds break;
2471da177e4SLinus Torvalds
248a1bcfacdSStephen Hemminger __skb_queue_tail(&skb_pool, skb);
2491da177e4SLinus Torvalds }
250a1bcfacdSStephen Hemminger spin_unlock_irqrestore(&skb_pool.lock, flags);
2511da177e4SLinus Torvalds }
2521da177e4SLinus Torvalds
zap_completion_queue(void)2533578b0c8SDavid S. Miller static void zap_completion_queue(void)
2543578b0c8SDavid S. Miller {
2553578b0c8SDavid S. Miller unsigned long flags;
2563578b0c8SDavid S. Miller struct softnet_data *sd = &get_cpu_var(softnet_data);
2573578b0c8SDavid S. Miller
2583578b0c8SDavid S. Miller if (sd->completion_queue) {
2593578b0c8SDavid S. Miller struct sk_buff *clist;
2603578b0c8SDavid S. Miller
2613578b0c8SDavid S. Miller local_irq_save(flags);
2623578b0c8SDavid S. Miller clist = sd->completion_queue;
2633578b0c8SDavid S. Miller sd->completion_queue = NULL;
2643578b0c8SDavid S. Miller local_irq_restore(flags);
2653578b0c8SDavid S. Miller
2663578b0c8SDavid S. Miller while (clist != NULL) {
2673578b0c8SDavid S. Miller struct sk_buff *skb = clist;
2683578b0c8SDavid S. Miller clist = clist->next;
269b1586f09SEric W. Biederman if (!skb_irq_freeable(skb)) {
270230cd127SWANG Cong refcount_set(&skb->users, 1);
2713578b0c8SDavid S. Miller dev_kfree_skb_any(skb); /* put this one back */
2723578b0c8SDavid S. Miller } else {
2733578b0c8SDavid S. Miller __kfree_skb(skb);
2743578b0c8SDavid S. Miller }
2753578b0c8SDavid S. Miller }
2763578b0c8SDavid S. Miller }
2773578b0c8SDavid S. Miller
2783578b0c8SDavid S. Miller put_cpu_var(softnet_data);
2793578b0c8SDavid S. Miller }
2803578b0c8SDavid S. Miller
find_skb(struct netpoll * np,int len,int reserve)2811da177e4SLinus Torvalds static struct sk_buff *find_skb(struct netpoll *np, int len, int reserve)
2821da177e4SLinus Torvalds {
283a1bcfacdSStephen Hemminger int count = 0;
284a1bcfacdSStephen Hemminger struct sk_buff *skb;
2851da177e4SLinus Torvalds
2863578b0c8SDavid S. Miller zap_completion_queue();
2871da177e4SLinus Torvalds refill_skbs();
288a1bcfacdSStephen Hemminger repeat:
2891da177e4SLinus Torvalds
2901da177e4SLinus Torvalds skb = alloc_skb(len, GFP_ATOMIC);
291a1bcfacdSStephen Hemminger if (!skb)
292a1bcfacdSStephen Hemminger skb = skb_dequeue(&skb_pool);
2931da177e4SLinus Torvalds
2941da177e4SLinus Torvalds if (!skb) {
295a1bcfacdSStephen Hemminger if (++count < 10) {
2962a49e001SJoe Perches netpoll_poll_dev(np->dev);
2971da177e4SLinus Torvalds goto repeat;
2981da177e4SLinus Torvalds }
299a1bcfacdSStephen Hemminger return NULL;
300a1bcfacdSStephen Hemminger }
3011da177e4SLinus Torvalds
30263354797SReshetova, Elena refcount_set(&skb->users, 1);
3031da177e4SLinus Torvalds skb_reserve(skb, reserve);
3041da177e4SLinus Torvalds return skb;
3051da177e4SLinus Torvalds }
3061da177e4SLinus Torvalds
netpoll_owner_active(struct net_device * dev)307bea3348eSStephen Hemminger static int netpoll_owner_active(struct net_device *dev)
308bea3348eSStephen Hemminger {
309bea3348eSStephen Hemminger struct napi_struct *napi;
310bea3348eSStephen Hemminger
3115251ef82SJakub Kicinski list_for_each_entry_rcu(napi, &dev->napi_list, dev_list) {
312c2e6a872SBreno Leitao if (READ_ONCE(napi->poll_owner) == smp_processor_id())
313bea3348eSStephen Hemminger return 1;
314bea3348eSStephen Hemminger }
315bea3348eSStephen Hemminger return 0;
316bea3348eSStephen Hemminger }
317bea3348eSStephen Hemminger
3182899656bSAmerigo Wang /* call with IRQ disabled */
__netpoll_send_skb(struct netpoll * np,struct sk_buff * skb)3191ddabdfaSEric Dumazet static netdev_tx_t __netpoll_send_skb(struct netpoll *np, struct sk_buff *skb)
3201da177e4SLinus Torvalds {
321a54776f2SYunjian Wang netdev_tx_t status = NETDEV_TX_BUSY;
322307f660dSEric Dumazet struct net_device *dev;
3232bdfe0baSStephen Hemminger unsigned long tries;
324de85d99eSHerbert Xu /* It is up to the caller to keep npinfo alive. */
3252899656bSAmerigo Wang struct netpoll_info *npinfo;
3261da177e4SLinus Torvalds
327af073393SFrederic Weisbecker lockdep_assert_irqs_disabled();
3282899656bSAmerigo Wang
329307f660dSEric Dumazet dev = np->dev;
330307f660dSEric Dumazet npinfo = rcu_dereference_bh(dev->npinfo);
331307f660dSEric Dumazet
3322bdfe0baSStephen Hemminger if (!npinfo || !netif_running(dev) || !netif_device_present(dev)) {
333080b3c19SEric W. Biederman dev_kfree_skb_irq(skb);
3341ddabdfaSEric Dumazet return NET_XMIT_DROP;
3351da177e4SLinus Torvalds }
3361da177e4SLinus Torvalds
3372bdfe0baSStephen Hemminger /* don't get messages out of order, and no recursion */
338bea3348eSStephen Hemminger if (skb_queue_len(&npinfo->txq) == 0 && !netpoll_owner_active(dev)) {
339fd2ea0a7SDavid S. Miller struct netdev_queue *txq;
340a49f99ffSAndrew Morton
3414bd97d51SPaolo Abeni txq = netdev_core_pick_tx(dev, skb, NULL);
342fd2ea0a7SDavid S. Miller
3432bdfe0baSStephen Hemminger /* try until next clock tick */
344e37b8d93SAndrew Morton for (tries = jiffies_to_usecs(1)/USEC_PER_POLL;
345e37b8d93SAndrew Morton tries > 0; --tries) {
3465efeac44SEric W. Biederman if (HARD_TX_TRYLOCK(dev, txq)) {
347944e2948SEric W. Biederman if (!netif_xmit_stopped(txq))
348944e2948SEric W. Biederman status = netpoll_start_xmit(skb, dev, txq);
349689971b4SAmerigo Wang
3505efeac44SEric W. Biederman HARD_TX_UNLOCK(dev, txq);
351f0d3459dSMatt Mackall
3522c1644cfSFeng Sun if (dev_xmit_complete(status))
3532bdfe0baSStephen Hemminger break;
3541da177e4SLinus Torvalds
3550db3dc73SStephen Hemminger }
3560db3dc73SStephen Hemminger
3572bdfe0baSStephen Hemminger /* tickle device maybe there is some cleanup */
3582a49e001SJoe Perches netpoll_poll_dev(np->dev);
3592bdfe0baSStephen Hemminger
3602bdfe0baSStephen Hemminger udelay(USEC_PER_POLL);
3612bdfe0baSStephen Hemminger }
36279b1bee8SDongdong Deng
36379b1bee8SDongdong Deng WARN_ONCE(!irqs_disabled(),
364d75f773cSSakari Ailus "netpoll_send_skb_on_dev(): %s enabled interrupts in poll (%pS)\n",
365944e2948SEric W. Biederman dev->name, dev->netdev_ops->ndo_start_xmit);
36679b1bee8SDongdong Deng
367e37b8d93SAndrew Morton }
3682bdfe0baSStephen Hemminger
3692c1644cfSFeng Sun if (!dev_xmit_complete(status)) {
3705de4a473SStephen Hemminger skb_queue_tail(&npinfo->txq, skb);
3714c1ac1b4SDavid Howells schedule_delayed_work(&npinfo->tx_work,0);
3721da177e4SLinus Torvalds }
3731ddabdfaSEric Dumazet return NETDEV_TX_OK;
3741da177e4SLinus Torvalds }
375fb1eee47SEric Dumazet
netpoll_send_skb(struct netpoll * np,struct sk_buff * skb)3761ddabdfaSEric Dumazet netdev_tx_t netpoll_send_skb(struct netpoll *np, struct sk_buff *skb)
377fb1eee47SEric Dumazet {
378fb1eee47SEric Dumazet unsigned long flags;
3791ddabdfaSEric Dumazet netdev_tx_t ret;
380fb1eee47SEric Dumazet
381f78ed220SEric Dumazet if (unlikely(!np)) {
382f78ed220SEric Dumazet dev_kfree_skb_irq(skb);
383f78ed220SEric Dumazet ret = NET_XMIT_DROP;
384f78ed220SEric Dumazet } else {
385fb1eee47SEric Dumazet local_irq_save(flags);
3861ddabdfaSEric Dumazet ret = __netpoll_send_skb(np, skb);
387fb1eee47SEric Dumazet local_irq_restore(flags);
388f78ed220SEric Dumazet }
3891ddabdfaSEric Dumazet return ret;
390fb1eee47SEric Dumazet }
391fb1eee47SEric Dumazet EXPORT_SYMBOL(netpoll_send_skb);
3921da177e4SLinus Torvalds
netpoll_send_udp(struct netpoll * np,const char * msg,int len)3931da177e4SLinus Torvalds void netpoll_send_udp(struct netpoll *np, const char *msg, int len)
3941da177e4SLinus Torvalds {
395954fba02SEric Dumazet int total_len, ip_len, udp_len;
3961da177e4SLinus Torvalds struct sk_buff *skb;
3971da177e4SLinus Torvalds struct udphdr *udph;
3981da177e4SLinus Torvalds struct iphdr *iph;
3991da177e4SLinus Torvalds struct ethhdr *eth;
400ee130409SEric Dumazet static atomic_t ip_ident;
401b3d936f3SCong Wang struct ipv6hdr *ip6h;
4021da177e4SLinus Torvalds
403b0f6c9acSWander Lairson Costa if (!IS_ENABLED(CONFIG_PREEMPT_RT))
404c9fd56b3SNikolay Aleksandrov WARN_ON_ONCE(!irqs_disabled());
405c9fd56b3SNikolay Aleksandrov
4061da177e4SLinus Torvalds udp_len = len + sizeof(*udph);
407b3d936f3SCong Wang if (np->ipv6)
408b3d936f3SCong Wang ip_len = udp_len + sizeof(*ip6h);
409b3d936f3SCong Wang else
410954fba02SEric Dumazet ip_len = udp_len + sizeof(*iph);
411b7394d24SCong Wang
412954fba02SEric Dumazet total_len = ip_len + LL_RESERVED_SPACE(np->dev);
4131da177e4SLinus Torvalds
414954fba02SEric Dumazet skb = find_skb(np, total_len + np->dev->needed_tailroom,
415954fba02SEric Dumazet total_len - len);
4161da177e4SLinus Torvalds if (!skb)
4171da177e4SLinus Torvalds return;
4181da177e4SLinus Torvalds
41927d7ff46SArnaldo Carvalho de Melo skb_copy_to_linear_data(skb, msg, len);
420954fba02SEric Dumazet skb_put(skb, len);
4211da177e4SLinus Torvalds
4224bedb452SArnaldo Carvalho de Melo skb_push(skb, sizeof(*udph));
4234bedb452SArnaldo Carvalho de Melo skb_reset_transport_header(skb);
4244bedb452SArnaldo Carvalho de Melo udph = udp_hdr(skb);
4251da177e4SLinus Torvalds udph->source = htons(np->local_port);
4261da177e4SLinus Torvalds udph->dest = htons(np->remote_port);
4271da177e4SLinus Torvalds udph->len = htons(udp_len);
428b7394d24SCong Wang
429b3d936f3SCong Wang if (np->ipv6) {
430b3d936f3SCong Wang udph->check = 0;
431b3d936f3SCong Wang udph->check = csum_ipv6_magic(&np->local_ip.in6,
432b3d936f3SCong Wang &np->remote_ip.in6,
433b3d936f3SCong Wang udp_len, IPPROTO_UDP,
434b3d936f3SCong Wang csum_partial(udph, udp_len, 0));
435b3d936f3SCong Wang if (udph->check == 0)
436b3d936f3SCong Wang udph->check = CSUM_MANGLED_0;
437b3d936f3SCong Wang
438b3d936f3SCong Wang skb_push(skb, sizeof(*ip6h));
439b3d936f3SCong Wang skb_reset_network_header(skb);
440b3d936f3SCong Wang ip6h = ipv6_hdr(skb);
441b3d936f3SCong Wang
442b3d936f3SCong Wang /* ip6h->version = 6; ip6h->priority = 0; */
443e3e22076SArnd Bergmann *(unsigned char *)ip6h = 0x60;
444b3d936f3SCong Wang ip6h->flow_lbl[0] = 0;
445b3d936f3SCong Wang ip6h->flow_lbl[1] = 0;
446b3d936f3SCong Wang ip6h->flow_lbl[2] = 0;
447b3d936f3SCong Wang
448b3d936f3SCong Wang ip6h->payload_len = htons(sizeof(struct udphdr) + len);
449b3d936f3SCong Wang ip6h->nexthdr = IPPROTO_UDP;
450b3d936f3SCong Wang ip6h->hop_limit = 32;
451b3d936f3SCong Wang ip6h->saddr = np->local_ip.in6;
452b3d936f3SCong Wang ip6h->daddr = np->remote_ip.in6;
453b3d936f3SCong Wang
454d58ff351SJohannes Berg eth = skb_push(skb, ETH_HLEN);
455b3d936f3SCong Wang skb_reset_mac_header(skb);
456b3d936f3SCong Wang skb->protocol = eth->h_proto = htons(ETH_P_IPV6);
457b3d936f3SCong Wang } else {
4581da177e4SLinus Torvalds udph->check = 0;
459b7394d24SCong Wang udph->check = csum_tcpudp_magic(np->local_ip.ip,
460b7394d24SCong Wang np->remote_ip.ip,
4618e365eecSChris Lalancette udp_len, IPPROTO_UDP,
46207f0757aSJoe Perches csum_partial(udph, udp_len, 0));
4638e365eecSChris Lalancette if (udph->check == 0)
4645e57dff2SAl Viro udph->check = CSUM_MANGLED_0;
4651da177e4SLinus Torvalds
466e2d1bca7SArnaldo Carvalho de Melo skb_push(skb, sizeof(*iph));
467e2d1bca7SArnaldo Carvalho de Melo skb_reset_network_header(skb);
468eddc9ec5SArnaldo Carvalho de Melo iph = ip_hdr(skb);
4691da177e4SLinus Torvalds
4701da177e4SLinus Torvalds /* iph->version = 4; iph->ihl = 5; */
471e3e22076SArnd Bergmann *(unsigned char *)iph = 0x45;
4721da177e4SLinus Torvalds iph->tos = 0;
4731da177e4SLinus Torvalds put_unaligned(htons(ip_len), &(iph->tot_len));
474ee130409SEric Dumazet iph->id = htons(atomic_inc_return(&ip_ident));
4751da177e4SLinus Torvalds iph->frag_off = 0;
4761da177e4SLinus Torvalds iph->ttl = 64;
4771da177e4SLinus Torvalds iph->protocol = IPPROTO_UDP;
4781da177e4SLinus Torvalds iph->check = 0;
479b7394d24SCong Wang put_unaligned(np->local_ip.ip, &(iph->saddr));
480b7394d24SCong Wang put_unaligned(np->remote_ip.ip, &(iph->daddr));
4811da177e4SLinus Torvalds iph->check = ip_fast_csum((unsigned char *)iph, iph->ihl);
4821da177e4SLinus Torvalds
483d58ff351SJohannes Berg eth = skb_push(skb, ETH_HLEN);
484459a98edSArnaldo Carvalho de Melo skb_reset_mac_header(skb);
485206daaf7SStephen Hemminger skb->protocol = eth->h_proto = htons(ETH_P_IP);
486b7394d24SCong Wang }
487b7394d24SCong Wang
488c62326abSJoe Perches ether_addr_copy(eth->h_source, np->dev->dev_addr);
489c62326abSJoe Perches ether_addr_copy(eth->h_dest, np->remote_mac);
4901da177e4SLinus Torvalds
4911da177e4SLinus Torvalds skb->dev = np->dev;
4921da177e4SLinus Torvalds
4931da177e4SLinus Torvalds netpoll_send_skb(np, skb);
4941da177e4SLinus Torvalds }
4959e34a5b5SEric Dumazet EXPORT_SYMBOL(netpoll_send_udp);
4961da177e4SLinus Torvalds
netpoll_print_options(struct netpoll * np)4970bcc1816SSatyam Sharma void netpoll_print_options(struct netpoll *np)
4980bcc1816SSatyam Sharma {
499e6ec2693SJoe Perches np_info(np, "local port %d\n", np->local_port);
500b3d936f3SCong Wang if (np->ipv6)
501b3d936f3SCong Wang np_info(np, "local IPv6 address %pI6c\n", &np->local_ip.in6);
502b3d936f3SCong Wang else
503b7394d24SCong Wang np_info(np, "local IPv4 address %pI4\n", &np->local_ip.ip);
504e6ec2693SJoe Perches np_info(np, "interface '%s'\n", np->dev_name);
505e6ec2693SJoe Perches np_info(np, "remote port %d\n", np->remote_port);
506b3d936f3SCong Wang if (np->ipv6)
507b3d936f3SCong Wang np_info(np, "remote IPv6 address %pI6c\n", &np->remote_ip.in6);
508b3d936f3SCong Wang else
509b7394d24SCong Wang np_info(np, "remote IPv4 address %pI4\n", &np->remote_ip.ip);
510e6ec2693SJoe Perches np_info(np, "remote ethernet address %pM\n", np->remote_mac);
5110bcc1816SSatyam Sharma }
5129e34a5b5SEric Dumazet EXPORT_SYMBOL(netpoll_print_options);
5130bcc1816SSatyam Sharma
netpoll_parse_ip_addr(const char * str,union inet_addr * addr)514b7394d24SCong Wang static int netpoll_parse_ip_addr(const char *str, union inet_addr *addr)
515b7394d24SCong Wang {
516b7394d24SCong Wang const char *end;
517b7394d24SCong Wang
518b7394d24SCong Wang if (!strchr(str, ':') &&
519b7394d24SCong Wang in4_pton(str, -1, (void *)addr, -1, &end) > 0) {
520b7394d24SCong Wang if (!*end)
521b7394d24SCong Wang return 0;
522b7394d24SCong Wang }
523b7394d24SCong Wang if (in6_pton(str, -1, addr->in6.s6_addr, -1, &end) > 0) {
524b7394d24SCong Wang #if IS_ENABLED(CONFIG_IPV6)
525b7394d24SCong Wang if (!*end)
526b7394d24SCong Wang return 1;
527b7394d24SCong Wang #else
528b7394d24SCong Wang return -1;
529b7394d24SCong Wang #endif
530b7394d24SCong Wang }
531b7394d24SCong Wang return -1;
532b7394d24SCong Wang }
533b7394d24SCong Wang
netpoll_parse_options(struct netpoll * np,char * opt)5341da177e4SLinus Torvalds int netpoll_parse_options(struct netpoll *np, char *opt)
5351da177e4SLinus Torvalds {
5361da177e4SLinus Torvalds char *cur=opt, *delim;
537b7394d24SCong Wang int ipv6;
53800fe11b3SSabrina Dubroca bool ipversion_set = false;
5391da177e4SLinus Torvalds
5401da177e4SLinus Torvalds if (*cur != '@') {
5411da177e4SLinus Torvalds if ((delim = strchr(cur, '@')) == NULL)
5421da177e4SLinus Torvalds goto parse_failed;
5431da177e4SLinus Torvalds *delim = 0;
5444b5511ebSAbhijit Pawar if (kstrtou16(cur, 10, &np->local_port))
5454b5511ebSAbhijit Pawar goto parse_failed;
5461da177e4SLinus Torvalds cur = delim;
5471da177e4SLinus Torvalds }
5481da177e4SLinus Torvalds cur++;
5491da177e4SLinus Torvalds
5501da177e4SLinus Torvalds if (*cur != '/') {
55100fe11b3SSabrina Dubroca ipversion_set = true;
5521da177e4SLinus Torvalds if ((delim = strchr(cur, '/')) == NULL)
5531da177e4SLinus Torvalds goto parse_failed;
5541da177e4SLinus Torvalds *delim = 0;
555b7394d24SCong Wang ipv6 = netpoll_parse_ip_addr(cur, &np->local_ip);
556b7394d24SCong Wang if (ipv6 < 0)
557b7394d24SCong Wang goto parse_failed;
558b7394d24SCong Wang else
559b7394d24SCong Wang np->ipv6 = (bool)ipv6;
5601da177e4SLinus Torvalds cur = delim;
5611da177e4SLinus Torvalds }
5621da177e4SLinus Torvalds cur++;
5631da177e4SLinus Torvalds
5641da177e4SLinus Torvalds if (*cur != ',') {
5651da177e4SLinus Torvalds /* parse out dev name */
5661da177e4SLinus Torvalds if ((delim = strchr(cur, ',')) == NULL)
5671da177e4SLinus Torvalds goto parse_failed;
5681da177e4SLinus Torvalds *delim = 0;
56970986397SWolfram Sang strscpy(np->dev_name, cur, sizeof(np->dev_name));
5701da177e4SLinus Torvalds cur = delim;
5711da177e4SLinus Torvalds }
5721da177e4SLinus Torvalds cur++;
5731da177e4SLinus Torvalds
5741da177e4SLinus Torvalds if (*cur != '@') {
5751da177e4SLinus Torvalds /* dst port */
5761da177e4SLinus Torvalds if ((delim = strchr(cur, '@')) == NULL)
5771da177e4SLinus Torvalds goto parse_failed;
5781da177e4SLinus Torvalds *delim = 0;
5795fc05f87SAmerigo Wang if (*cur == ' ' || *cur == '\t')
580e6ec2693SJoe Perches np_info(np, "warning: whitespace is not allowed\n");
5814b5511ebSAbhijit Pawar if (kstrtou16(cur, 10, &np->remote_port))
5824b5511ebSAbhijit Pawar goto parse_failed;
5831da177e4SLinus Torvalds cur = delim;
5841da177e4SLinus Torvalds }
5851da177e4SLinus Torvalds cur++;
5861da177e4SLinus Torvalds
5871da177e4SLinus Torvalds /* dst ip */
5881da177e4SLinus Torvalds if ((delim = strchr(cur, '/')) == NULL)
5891da177e4SLinus Torvalds goto parse_failed;
5901da177e4SLinus Torvalds *delim = 0;
591b7394d24SCong Wang ipv6 = netpoll_parse_ip_addr(cur, &np->remote_ip);
592b7394d24SCong Wang if (ipv6 < 0)
593b7394d24SCong Wang goto parse_failed;
59400fe11b3SSabrina Dubroca else if (ipversion_set && np->ipv6 != (bool)ipv6)
595b7394d24SCong Wang goto parse_failed;
596b7394d24SCong Wang else
597b7394d24SCong Wang np->ipv6 = (bool)ipv6;
5981da177e4SLinus Torvalds cur = delim + 1;
5991da177e4SLinus Torvalds
600c68b9070SDavid S. Miller if (*cur != 0) {
6011da177e4SLinus Torvalds /* MAC address */
6024940fc88SAlexey Dobriyan if (!mac_pton(cur, np->remote_mac))
6031da177e4SLinus Torvalds goto parse_failed;
6041da177e4SLinus Torvalds }
6051da177e4SLinus Torvalds
6060bcc1816SSatyam Sharma netpoll_print_options(np);
6071da177e4SLinus Torvalds
6081da177e4SLinus Torvalds return 0;
6091da177e4SLinus Torvalds
6101da177e4SLinus Torvalds parse_failed:
611e6ec2693SJoe Perches np_info(np, "couldn't parse config at '%s'!\n", cur);
6121da177e4SLinus Torvalds return -1;
6131da177e4SLinus Torvalds }
6149e34a5b5SEric Dumazet EXPORT_SYMBOL(netpoll_parse_options);
6151da177e4SLinus Torvalds
__netpoll_setup(struct netpoll * np,struct net_device * ndev)616a8779ec1SEric W. Biederman int __netpoll_setup(struct netpoll *np, struct net_device *ndev)
6178fdd95ecSHerbert Xu {
6188fdd95ecSHerbert Xu struct netpoll_info *npinfo;
6198fdd95ecSHerbert Xu const struct net_device_ops *ops;
6208fdd95ecSHerbert Xu int err;
6218fdd95ecSHerbert Xu
622ac3d9dd0SEric Dumazet if (ndev->priv_flags & IFF_DISABLE_NETPOLL) {
623e6ec2693SJoe Perches np_err(np, "%s doesn't support polling, aborting\n",
624ae5a0456SBreno Leitao ndev->name);
6258fdd95ecSHerbert Xu err = -ENOTSUPP;
6268fdd95ecSHerbert Xu goto out;
6278fdd95ecSHerbert Xu }
6288fdd95ecSHerbert Xu
6298fdd95ecSHerbert Xu if (!ndev->npinfo) {
630a8779ec1SEric W. Biederman npinfo = kmalloc(sizeof(*npinfo), GFP_KERNEL);
6318fdd95ecSHerbert Xu if (!npinfo) {
6328fdd95ecSHerbert Xu err = -ENOMEM;
6338fdd95ecSHerbert Xu goto out;
6348fdd95ecSHerbert Xu }
6358fdd95ecSHerbert Xu
636bd7c4b60SNeil Horman sema_init(&npinfo->dev_lock, 1);
6378fdd95ecSHerbert Xu skb_queue_head_init(&npinfo->txq);
6388fdd95ecSHerbert Xu INIT_DELAYED_WORK(&npinfo->tx_work, queue_process);
6398fdd95ecSHerbert Xu
640433cea4dSReshetova, Elena refcount_set(&npinfo->refcnt, 1);
6418fdd95ecSHerbert Xu
642ae5a0456SBreno Leitao ops = ndev->netdev_ops;
6438fdd95ecSHerbert Xu if (ops->ndo_netpoll_setup) {
644a8779ec1SEric W. Biederman err = ops->ndo_netpoll_setup(ndev, npinfo);
6458fdd95ecSHerbert Xu if (err)
6468fdd95ecSHerbert Xu goto free_npinfo;
6478fdd95ecSHerbert Xu }
6488fdd95ecSHerbert Xu } else {
6490790bbb6SNeil Horman npinfo = rtnl_dereference(ndev->npinfo);
650433cea4dSReshetova, Elena refcount_inc(&npinfo->refcnt);
6518fdd95ecSHerbert Xu }
6528fdd95ecSHerbert Xu
653ae5a0456SBreno Leitao np->dev = ndev;
654ae5a0456SBreno Leitao strscpy(np->dev_name, ndev->name, IFNAMSIZ);
6558fdd95ecSHerbert Xu npinfo->netpoll = np;
6568fdd95ecSHerbert Xu
6578fdd95ecSHerbert Xu /* last thing to do is link it to the net device structure */
658cf778b00SEric Dumazet rcu_assign_pointer(ndev->npinfo, npinfo);
6598fdd95ecSHerbert Xu
6608fdd95ecSHerbert Xu return 0;
6618fdd95ecSHerbert Xu
6628fdd95ecSHerbert Xu free_npinfo:
6638fdd95ecSHerbert Xu kfree(npinfo);
6648fdd95ecSHerbert Xu out:
6658fdd95ecSHerbert Xu return err;
6668fdd95ecSHerbert Xu }
6678fdd95ecSHerbert Xu EXPORT_SYMBOL_GPL(__netpoll_setup);
6688fdd95ecSHerbert Xu
netpoll_setup(struct netpoll * np)6691da177e4SLinus Torvalds int netpoll_setup(struct netpoll *np)
6701da177e4SLinus Torvalds {
671ea92000dSVladimir Oltean struct net_device *ndev = NULL;
672ae5a0456SBreno Leitao bool ip_overwritten = false;
6731da177e4SLinus Torvalds struct in_device *in_dev;
674b41848b6SStephen Hemminger int err;
6751da177e4SLinus Torvalds
676f92d3180SCong Wang rtnl_lock();
677ea92000dSVladimir Oltean if (np->dev_name[0]) {
678ea92000dSVladimir Oltean struct net *net = current->nsproxy->net_ns;
679556e6256SCong Wang ndev = __dev_get_by_name(net, np->dev_name);
680ea92000dSVladimir Oltean }
6811da177e4SLinus Torvalds if (!ndev) {
682e6ec2693SJoe Perches np_err(np, "%s doesn't exist, aborting\n", np->dev_name);
683f92d3180SCong Wang err = -ENODEV;
684f92d3180SCong Wang goto unlock;
6851da177e4SLinus Torvalds }
68648eed027SJakub Kicinski netdev_hold(ndev, &np->dev_tracker, GFP_KERNEL);
6871da177e4SLinus Torvalds
68849bd8fb0SJiri Pirko if (netdev_master_upper_dev_get(ndev)) {
689e6ec2693SJoe Perches np_err(np, "%s is a slave device, aborting\n", np->dev_name);
69083fe32deSDan Carpenter err = -EBUSY;
69183fe32deSDan Carpenter goto put;
6920c1ad04aSWANG Cong }
6930c1ad04aSWANG Cong
6941da177e4SLinus Torvalds if (!netif_running(ndev)) {
695d8afe2f8SBreno Leitao unsigned long atmost;
6961da177e4SLinus Torvalds
697e6ec2693SJoe Perches np_info(np, "device %s not up yet, forcing it\n", np->dev_name);
6981da177e4SLinus Torvalds
69900f54e68SPetr Machata err = dev_open(ndev, NULL);
700b41848b6SStephen Hemminger
701b41848b6SStephen Hemminger if (err) {
702e6ec2693SJoe Perches np_err(np, "failed to open %s\n", ndev->name);
703dbaa1541SHerbert Xu goto put;
7041da177e4SLinus Torvalds }
7051da177e4SLinus Torvalds
706f92d3180SCong Wang rtnl_unlock();
707bff38771SAnton Vorontsov atmost = jiffies + carrier_timeout * HZ;
7081da177e4SLinus Torvalds while (!netif_carrier_ok(ndev)) {
7091da177e4SLinus Torvalds if (time_after(jiffies, atmost)) {
710e6ec2693SJoe Perches np_notice(np, "timeout waiting for carrier\n");
7111da177e4SLinus Torvalds break;
7121da177e4SLinus Torvalds }
7131b614fb9SAnton Vorontsov msleep(1);
7141da177e4SLinus Torvalds }
7151da177e4SLinus Torvalds
716f92d3180SCong Wang rtnl_lock();
7171da177e4SLinus Torvalds }
7181da177e4SLinus Torvalds
719b7394d24SCong Wang if (!np->local_ip.ip) {
720b7394d24SCong Wang if (!np->ipv6) {
7212638eb8bSFlorian Westphal const struct in_ifaddr *ifa;
722b7394d24SCong Wang
7232638eb8bSFlorian Westphal in_dev = __in_dev_get_rtnl(ndev);
7242638eb8bSFlorian Westphal if (!in_dev)
7252638eb8bSFlorian Westphal goto put_noaddr;
7262638eb8bSFlorian Westphal
7272638eb8bSFlorian Westphal ifa = rtnl_dereference(in_dev->ifa_list);
7282638eb8bSFlorian Westphal if (!ifa) {
7292638eb8bSFlorian Westphal put_noaddr:
730e6ec2693SJoe Perches np_err(np, "no IP address for %s, aborting\n",
731e6ec2693SJoe Perches np->dev_name);
732b41848b6SStephen Hemminger err = -EDESTADDRREQ;
733dbaa1541SHerbert Xu goto put;
7341da177e4SLinus Torvalds }
7351da177e4SLinus Torvalds
7362638eb8bSFlorian Westphal np->local_ip.ip = ifa->ifa_local;
737ae5a0456SBreno Leitao ip_overwritten = true;
738b7394d24SCong Wang np_info(np, "local IP %pI4\n", &np->local_ip.ip);
739b3d936f3SCong Wang } else {
740b3d936f3SCong Wang #if IS_ENABLED(CONFIG_IPV6)
741b3d936f3SCong Wang struct inet6_dev *idev;
742b3d936f3SCong Wang
743b3d936f3SCong Wang err = -EDESTADDRREQ;
744b3d936f3SCong Wang idev = __in6_dev_get(ndev);
745b3d936f3SCong Wang if (idev) {
746b3d936f3SCong Wang struct inet6_ifaddr *ifp;
747b3d936f3SCong Wang
748b3d936f3SCong Wang read_lock_bh(&idev->lock);
749b3d936f3SCong Wang list_for_each_entry(ifp, &idev->addr_list, if_list) {
750d016b4a3SMatwey V. Kornilov if (!!(ipv6_addr_type(&ifp->addr) & IPV6_ADDR_LINKLOCAL) !=
751d016b4a3SMatwey V. Kornilov !!(ipv6_addr_type(&np->remote_ip.in6) & IPV6_ADDR_LINKLOCAL))
752b3d936f3SCong Wang continue;
753b3d936f3SCong Wang np->local_ip.in6 = ifp->addr;
754ae5a0456SBreno Leitao ip_overwritten = true;
755b3d936f3SCong Wang err = 0;
756b3d936f3SCong Wang break;
757b3d936f3SCong Wang }
758b3d936f3SCong Wang read_unlock_bh(&idev->lock);
759b3d936f3SCong Wang }
760b3d936f3SCong Wang if (err) {
761b3d936f3SCong Wang np_err(np, "no IPv6 address for %s, aborting\n",
762b3d936f3SCong Wang np->dev_name);
763b3d936f3SCong Wang goto put;
764b3d936f3SCong Wang } else
765b3d936f3SCong Wang np_info(np, "local IPv6 %pI6c\n", &np->local_ip.in6);
766b3d936f3SCong Wang #else
767b3d936f3SCong Wang np_err(np, "IPv6 is not supported %s, aborting\n",
768b3d936f3SCong Wang np->dev_name);
769e39363a9SCong Wang err = -EINVAL;
770b3d936f3SCong Wang goto put;
771b3d936f3SCong Wang #endif
772b7394d24SCong Wang }
7731da177e4SLinus Torvalds }
7741da177e4SLinus Torvalds
775dbaa1541SHerbert Xu /* fill up the skb queue */
776dbaa1541SHerbert Xu refill_skbs();
777dbaa1541SHerbert Xu
778a8779ec1SEric W. Biederman err = __netpoll_setup(np, ndev);
7798fdd95ecSHerbert Xu if (err)
7808fdd95ecSHerbert Xu goto put;
781f92d3180SCong Wang rtnl_unlock();
7821da177e4SLinus Torvalds return 0;
7831da177e4SLinus Torvalds
78421edbb22SJiri Slaby put:
785ae5a0456SBreno Leitao DEBUG_NET_WARN_ON_ONCE(np->dev);
786ae5a0456SBreno Leitao if (ip_overwritten)
787ae5a0456SBreno Leitao memset(&np->local_ip, 0, sizeof(np->local_ip));
78848eed027SJakub Kicinski netdev_put(ndev, &np->dev_tracker);
789f92d3180SCong Wang unlock:
790f92d3180SCong Wang rtnl_unlock();
791b41848b6SStephen Hemminger return err;
7921da177e4SLinus Torvalds }
7939e34a5b5SEric Dumazet EXPORT_SYMBOL(netpoll_setup);
7941da177e4SLinus Torvalds
netpoll_init(void)795c68b9070SDavid S. Miller static int __init netpoll_init(void)
796c68b9070SDavid S. Miller {
797a1bcfacdSStephen Hemminger skb_queue_head_init(&skb_pool);
798a1bcfacdSStephen Hemminger return 0;
799a1bcfacdSStephen Hemminger }
800a1bcfacdSStephen Hemminger core_initcall(netpoll_init);
801a1bcfacdSStephen Hemminger
rcu_cleanup_netpoll_info(struct rcu_head * rcu_head)80238e6bc18SAmerigo Wang static void rcu_cleanup_netpoll_info(struct rcu_head *rcu_head)
80338e6bc18SAmerigo Wang {
80438e6bc18SAmerigo Wang struct netpoll_info *npinfo =
80538e6bc18SAmerigo Wang container_of(rcu_head, struct netpoll_info, rcu);
80638e6bc18SAmerigo Wang
80738e6bc18SAmerigo Wang skb_queue_purge(&npinfo->txq);
80838e6bc18SAmerigo Wang
80938e6bc18SAmerigo Wang /* we can't call cancel_delayed_work_sync here, as we are in softirq */
81038e6bc18SAmerigo Wang cancel_delayed_work(&npinfo->tx_work);
81138e6bc18SAmerigo Wang
81238e6bc18SAmerigo Wang /* clean after last, unfinished work */
81338e6bc18SAmerigo Wang __skb_queue_purge(&npinfo->txq);
81438e6bc18SAmerigo Wang /* now cancel it again */
81538e6bc18SAmerigo Wang cancel_delayed_work(&npinfo->tx_work);
81638e6bc18SAmerigo Wang kfree(npinfo);
81738e6bc18SAmerigo Wang }
81838e6bc18SAmerigo Wang
__netpoll_cleanup(struct netpoll * np)8198fdd95ecSHerbert Xu void __netpoll_cleanup(struct netpoll *np)
8201da177e4SLinus Torvalds {
821fbeec2e1SJeff Moyer struct netpoll_info *npinfo;
822fbeec2e1SJeff Moyer
8230790bbb6SNeil Horman npinfo = rtnl_dereference(np->dev->npinfo);
8248fdd95ecSHerbert Xu if (!npinfo)
825dbaa1541SHerbert Xu return;
826dbaa1541SHerbert Xu
827433cea4dSReshetova, Elena if (refcount_dec_and_test(&npinfo->refcnt)) {
8280e34e931SWANG Cong const struct net_device_ops *ops;
829de85d99eSHerbert Xu
830de85d99eSHerbert Xu ops = np->dev->netdev_ops;
831de85d99eSHerbert Xu if (ops->ndo_netpoll_cleanup)
832de85d99eSHerbert Xu ops->ndo_netpoll_cleanup(np->dev);
833de85d99eSHerbert Xu
834fcb144b5SMonam Agarwal RCU_INIT_POINTER(np->dev->npinfo, NULL);
8355da54c18SPaul E. McKenney call_rcu(&npinfo->rcu, rcu_cleanup_netpoll_info);
836efa95b01Sdavid decotigny } else
837efa95b01Sdavid decotigny RCU_INIT_POINTER(np->dev->npinfo, NULL);
8388fdd95ecSHerbert Xu }
8398fdd95ecSHerbert Xu EXPORT_SYMBOL_GPL(__netpoll_cleanup);
8408fdd95ecSHerbert Xu
__netpoll_free(struct netpoll * np)841c9fbd71fSDebabrata Banerjee void __netpoll_free(struct netpoll *np)
84238e6bc18SAmerigo Wang {
843c9fbd71fSDebabrata Banerjee ASSERT_RTNL();
84438e6bc18SAmerigo Wang
845c9fbd71fSDebabrata Banerjee /* Wait for transmitting packets to finish before freeing. */
8465da54c18SPaul E. McKenney synchronize_rcu();
84738e6bc18SAmerigo Wang __netpoll_cleanup(np);
84838e6bc18SAmerigo Wang kfree(np);
84938e6bc18SAmerigo Wang }
850c9fbd71fSDebabrata Banerjee EXPORT_SYMBOL_GPL(__netpoll_free);
85138e6bc18SAmerigo Wang
do_netpoll_cleanup(struct netpoll * np)8521ef33652SBreno Leitao void do_netpoll_cleanup(struct netpoll *np)
8531ef33652SBreno Leitao {
8541ef33652SBreno Leitao __netpoll_cleanup(np);
8551ef33652SBreno Leitao netdev_put(np->dev, &np->dev_tracker);
8561ef33652SBreno Leitao np->dev = NULL;
8571ef33652SBreno Leitao }
8581ef33652SBreno Leitao EXPORT_SYMBOL(do_netpoll_cleanup);
8591ef33652SBreno Leitao
netpoll_cleanup(struct netpoll * np)8608fdd95ecSHerbert Xu void netpoll_cleanup(struct netpoll *np)
8618fdd95ecSHerbert Xu {
8628fdd95ecSHerbert Xu rtnl_lock();
863d0fe8c88SNikolay Aleksandrov if (!np->dev)
864d0fe8c88SNikolay Aleksandrov goto out;
8651ef33652SBreno Leitao do_netpoll_cleanup(np);
866d0fe8c88SNikolay Aleksandrov out:
867d0fe8c88SNikolay Aleksandrov rtnl_unlock();
8681da177e4SLinus Torvalds }
8699e34a5b5SEric Dumazet EXPORT_SYMBOL(netpoll_cleanup);
870