1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Common framework for low-level network console, dump, and debugger code 4 * 5 * Sep 8 2003 Matt Mackall <mpm@selenic.com> 6 * 7 * based on the netconsole code from: 8 * 9 * Copyright (C) 2001 Ingo Molnar <mingo@redhat.com> 10 * Copyright (C) 2002 Red Hat, Inc. 11 */ 12 13 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 14 15 #include <linux/moduleparam.h> 16 #include <linux/kernel.h> 17 #include <linux/netdevice.h> 18 #include <linux/etherdevice.h> 19 #include <linux/string.h> 20 #include <linux/if_arp.h> 21 #include <linux/inetdevice.h> 22 #include <linux/inet.h> 23 #include <linux/interrupt.h> 24 #include <linux/netpoll.h> 25 #include <linux/sched.h> 26 #include <linux/delay.h> 27 #include <linux/rcupdate.h> 28 #include <linux/workqueue.h> 29 #include <linux/slab.h> 30 #include <linux/export.h> 31 #include <linux/if_vlan.h> 32 #include <linux/udp.h> 33 #include <net/tcp.h> 34 #include <net/addrconf.h> 35 #include <net/ndisc.h> 36 #include <trace/events/napi.h> 37 #include <linux/kconfig.h> 38 39 #define USEC_PER_POLL 50 40 41 /* 42 * carrier_timeout is netconsole-specific and only kept here to preserve the 43 * netpoll.carrier_timeout module-parameter ABI. Its value is exposed to 44 * netconsole through netpoll_get_carrier_timeout(). 45 */ 46 static unsigned int carrier_timeout = 4; 47 module_param(carrier_timeout, uint, 0644); 48 49 unsigned int netpoll_get_carrier_timeout(void) 50 { 51 return carrier_timeout; 52 } 53 EXPORT_SYMBOL_GPL(netpoll_get_carrier_timeout); 54 55 static netdev_tx_t netpoll_start_xmit(struct sk_buff *skb, 56 struct net_device *dev, 57 struct netdev_queue *txq) 58 { 59 netdev_tx_t status = NETDEV_TX_OK; 60 netdev_features_t features; 61 62 features = netif_skb_features(skb); 63 64 if (skb_vlan_tag_present(skb) && 65 !vlan_hw_offload_capable(features, skb->vlan_proto)) { 66 skb = __vlan_hwaccel_push_inside(skb); 67 if (unlikely(!skb)) { 68 /* This is actually a packet drop, but we 69 * don't want the code that calls this 70 * function to try and operate on a NULL skb. 71 */ 72 goto out; 73 } 74 } 75 76 status = netdev_start_xmit(skb, dev, txq, false); 77 78 out: 79 return status; 80 } 81 82 static void queue_process(struct work_struct *work) 83 { 84 struct netpoll_info *npinfo = 85 container_of(work, struct netpoll_info, tx_work.work); 86 struct sk_buff *skb; 87 unsigned long flags; 88 89 while ((skb = skb_dequeue(&npinfo->txq))) { 90 struct net_device *dev = skb->dev; 91 struct netdev_queue *txq; 92 unsigned int q_index; 93 94 if (!netif_device_present(dev) || !netif_running(dev)) { 95 kfree_skb(skb); 96 continue; 97 } 98 99 local_irq_save(flags); 100 /* check if skb->queue_mapping is still valid */ 101 q_index = skb_get_queue_mapping(skb); 102 if (unlikely(q_index >= dev->real_num_tx_queues)) { 103 q_index = q_index % dev->real_num_tx_queues; 104 skb_set_queue_mapping(skb, q_index); 105 } 106 txq = netdev_get_tx_queue(dev, q_index); 107 HARD_TX_LOCK(dev, txq, smp_processor_id()); 108 if (netif_xmit_frozen_or_stopped(txq) || 109 !dev_xmit_complete(netpoll_start_xmit(skb, dev, txq))) { 110 skb_queue_head(&npinfo->txq, skb); 111 HARD_TX_UNLOCK(dev, txq); 112 local_irq_restore(flags); 113 114 schedule_delayed_work(&npinfo->tx_work, HZ/10); 115 return; 116 } 117 HARD_TX_UNLOCK(dev, txq); 118 local_irq_restore(flags); 119 } 120 } 121 122 static int netif_local_xmit_active(struct net_device *dev) 123 { 124 int i; 125 126 for (i = 0; i < dev->num_tx_queues; i++) { 127 struct netdev_queue *txq = netdev_get_tx_queue(dev, i); 128 129 if (netif_tx_owned(txq, smp_processor_id())) 130 return 1; 131 } 132 133 return 0; 134 } 135 136 static void poll_one_napi(struct napi_struct *napi) 137 { 138 int work; 139 140 /* If we set this bit but see that it has already been set, 141 * that indicates that napi has been disabled and we need 142 * to abort this operation 143 */ 144 if (test_and_set_bit(NAPI_STATE_NPSVC, &napi->state)) 145 return; 146 147 /* We explicitly pass the polling call a budget of 0 to 148 * indicate that we are clearing the Tx path only. 149 */ 150 work = napi->poll(napi, 0); 151 WARN_ONCE(work, "%pS exceeded budget in poll\n", napi->poll); 152 trace_napi_poll(napi, work, 0); 153 154 clear_bit(NAPI_STATE_NPSVC, &napi->state); 155 } 156 157 static void poll_napi(struct net_device *dev) 158 { 159 struct napi_struct *napi; 160 int cpu = smp_processor_id(); 161 162 list_for_each_entry_rcu(napi, &dev->napi_list, dev_list) { 163 if (cmpxchg(&napi->poll_owner, -1, cpu) == -1) { 164 poll_one_napi(napi); 165 smp_store_release(&napi->poll_owner, -1); 166 } 167 } 168 } 169 170 void netpoll_poll_dev(struct net_device *dev) 171 { 172 struct netpoll_info *ni = rcu_dereference_bh(dev->npinfo); 173 const struct net_device_ops *ops; 174 175 /* Don't do any rx activity if the dev_lock mutex is held 176 * the dev_open/close paths use this to block netpoll activity 177 * while changing device state 178 */ 179 if (!ni || down_trylock(&ni->dev_lock)) 180 return; 181 182 /* Some drivers will take the same locks in poll and xmit, 183 * we can't poll if local CPU is already in xmit. 184 */ 185 if (!netif_running(dev) || netif_local_xmit_active(dev)) { 186 up(&ni->dev_lock); 187 return; 188 } 189 190 ops = dev->netdev_ops; 191 if (ops->ndo_poll_controller) 192 ops->ndo_poll_controller(dev); 193 194 poll_napi(dev); 195 196 up(&ni->dev_lock); 197 198 netpoll_zap_completion_queue(); 199 } 200 EXPORT_SYMBOL(netpoll_poll_dev); 201 202 void netpoll_poll_disable(struct net_device *dev) 203 { 204 struct netpoll_info *ni; 205 206 might_sleep(); 207 ni = rtnl_dereference(dev->npinfo); 208 if (ni) 209 down(&ni->dev_lock); 210 } 211 212 void netpoll_poll_enable(struct net_device *dev) 213 { 214 struct netpoll_info *ni; 215 216 ni = rtnl_dereference(dev->npinfo); 217 if (ni) 218 up(&ni->dev_lock); 219 } 220 221 void netpoll_zap_completion_queue(void) 222 { 223 unsigned long flags; 224 struct softnet_data *sd = &get_cpu_var(softnet_data); 225 226 if (sd->completion_queue) { 227 struct sk_buff *clist; 228 229 local_irq_save(flags); 230 clist = sd->completion_queue; 231 sd->completion_queue = NULL; 232 local_irq_restore(flags); 233 234 while (clist != NULL) { 235 struct sk_buff *skb = clist; 236 clist = clist->next; 237 if (!skb_irq_freeable(skb)) { 238 refcount_set(&skb->users, 1); 239 dev_kfree_skb_any(skb); /* put this one back */ 240 } else { 241 __kfree_skb(skb); 242 } 243 } 244 } 245 246 put_cpu_var(softnet_data); 247 } 248 EXPORT_SYMBOL_NS_GPL(netpoll_zap_completion_queue, "NETDEV_INTERNAL"); 249 250 static int netpoll_owner_active(struct net_device *dev) 251 { 252 struct napi_struct *napi; 253 254 list_for_each_entry_rcu(napi, &dev->napi_list, dev_list) { 255 if (READ_ONCE(napi->poll_owner) == smp_processor_id()) 256 return 1; 257 } 258 return 0; 259 } 260 261 /* call with IRQ disabled */ 262 static netdev_tx_t __netpoll_send_skb(struct netpoll *np, struct sk_buff *skb) 263 { 264 netdev_tx_t status = NETDEV_TX_BUSY; 265 netdev_tx_t ret = NET_XMIT_DROP; 266 struct net_device *dev; 267 unsigned long tries; 268 /* It is up to the caller to keep npinfo alive. */ 269 struct netpoll_info *npinfo; 270 271 lockdep_assert_irqs_disabled(); 272 273 dev = np->dev; 274 /* npinfo->txq belongs to np->dev, so retries must stay bound to it. */ 275 skb->dev = dev; 276 rcu_read_lock(); 277 npinfo = rcu_dereference_bh(dev->npinfo); 278 279 if (!npinfo || !netif_running(dev) || !netif_device_present(dev)) { 280 dev_kfree_skb_irq(skb); 281 goto out; 282 } 283 284 /* don't get messages out of order, and no recursion */ 285 if (skb_queue_len(&npinfo->txq) == 0 && !netpoll_owner_active(dev)) { 286 struct netdev_queue *txq; 287 288 txq = netdev_core_pick_tx(dev, skb, NULL); 289 290 /* try until next clock tick */ 291 for (tries = jiffies_to_usecs(1)/USEC_PER_POLL; 292 tries > 0; --tries) { 293 if (HARD_TX_TRYLOCK(dev, txq)) { 294 if (!netif_xmit_stopped(txq)) 295 status = netpoll_start_xmit(skb, dev, txq); 296 297 HARD_TX_UNLOCK(dev, txq); 298 299 if (dev_xmit_complete(status)) 300 break; 301 302 } 303 304 /* tickle device maybe there is some cleanup */ 305 netpoll_poll_dev(np->dev); 306 307 udelay(USEC_PER_POLL); 308 } 309 310 WARN_ONCE(!irqs_disabled(), 311 "netpoll_send_skb_on_dev(): %s enabled interrupts in poll (%pS)\n", 312 dev->name, dev->netdev_ops->ndo_start_xmit); 313 314 } 315 316 if (!dev_xmit_complete(status)) { 317 skb_queue_tail(&npinfo->txq, skb); 318 schedule_delayed_work(&npinfo->tx_work,0); 319 } 320 ret = NETDEV_TX_OK; 321 out: 322 rcu_read_unlock(); 323 return ret; 324 } 325 326 netdev_tx_t netpoll_send_skb(struct netpoll *np, struct sk_buff *skb) 327 { 328 unsigned long flags; 329 netdev_tx_t ret; 330 331 if (unlikely(!np)) { 332 dev_kfree_skb_irq(skb); 333 ret = NET_XMIT_DROP; 334 } else { 335 local_irq_save(flags); 336 ret = __netpoll_send_skb(np, skb); 337 local_irq_restore(flags); 338 } 339 return ret; 340 } 341 EXPORT_SYMBOL(netpoll_send_skb); 342 343 int __netpoll_setup(struct netpoll *np, struct net_device *ndev) 344 { 345 struct netpoll_info *npinfo; 346 const struct net_device_ops *ops; 347 int err; 348 349 if (ndev->priv_flags & IFF_DISABLE_NETPOLL) { 350 np_err(np, "%s doesn't support polling, aborting\n", 351 ndev->name); 352 err = -ENOTSUPP; 353 goto out; 354 } 355 356 npinfo = rtnl_dereference(ndev->npinfo); 357 if (!npinfo) { 358 npinfo = kmalloc_obj(*npinfo); 359 if (!npinfo) { 360 err = -ENOMEM; 361 goto out; 362 } 363 364 sema_init(&npinfo->dev_lock, 1); 365 skb_queue_head_init(&npinfo->txq); 366 INIT_DELAYED_WORK(&npinfo->tx_work, queue_process); 367 368 refcount_set(&npinfo->refcnt, 1); 369 370 ops = ndev->netdev_ops; 371 if (ops->ndo_netpoll_setup) { 372 err = ops->ndo_netpoll_setup(ndev); 373 if (err) 374 goto free_npinfo; 375 } 376 } else { 377 refcount_inc(&npinfo->refcnt); 378 } 379 380 np->dev = ndev; 381 strscpy(np->dev_name, ndev->name, IFNAMSIZ); 382 383 /* last thing to do is link it to the net device structure */ 384 rcu_assign_pointer(ndev->npinfo, npinfo); 385 386 return 0; 387 388 free_npinfo: 389 kfree(npinfo); 390 out: 391 return err; 392 } 393 EXPORT_SYMBOL_GPL(__netpoll_setup); 394 395 static void rcu_cleanup_netpoll_info(struct rcu_head *rcu_head) 396 { 397 struct netpoll_info *npinfo = 398 container_of(rcu_head, struct netpoll_info, rcu); 399 400 skb_queue_purge(&npinfo->txq); 401 kfree(npinfo); 402 } 403 404 static void __netpoll_cleanup(struct netpoll *np) 405 { 406 struct netpoll_info *npinfo; 407 408 npinfo = rtnl_dereference(np->dev->npinfo); 409 if (!npinfo) 410 return; 411 412 /* At this point, there is a single npinfo instance per netdevice, and 413 * its refcnt tracks how many netpoll structures are linked to it. We 414 * only perform npinfo cleanup when the refcnt decrements to zero. 415 */ 416 if (refcount_dec_and_test(&npinfo->refcnt)) { 417 const struct net_device_ops *ops; 418 419 ops = np->dev->netdev_ops; 420 if (ops->ndo_netpoll_cleanup) 421 ops->ndo_netpoll_cleanup(np->dev); 422 423 RCU_INIT_POINTER(np->dev->npinfo, NULL); 424 disable_delayed_work_sync(&npinfo->tx_work); 425 call_rcu(&npinfo->rcu, rcu_cleanup_netpoll_info); 426 } 427 } 428 429 void __netpoll_free(struct netpoll *np) 430 { 431 ASSERT_RTNL(); 432 433 /* Wait for transmitting packets to finish before freeing. */ 434 synchronize_net(); 435 __netpoll_cleanup(np); 436 kfree(np); 437 } 438 EXPORT_SYMBOL_GPL(__netpoll_free); 439 440 void do_netpoll_cleanup(struct netpoll *np) 441 { 442 __netpoll_cleanup(np); 443 netdev_put(np->dev, &np->dev_tracker); 444 np->dev = NULL; 445 } 446 EXPORT_SYMBOL(do_netpoll_cleanup); 447 448 void netpoll_cleanup(struct netpoll *np) 449 { 450 rtnl_lock(); 451 if (!np->dev) 452 goto out; 453 do_netpoll_cleanup(np); 454 out: 455 rtnl_unlock(); 456 } 457 EXPORT_SYMBOL(netpoll_cleanup); 458