1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Common framework for low-level network console, dump, and debugger code 4 * 5 * Sep 8 2003 Matt Mackall <mpm@selenic.com> 6 * 7 * based on the netconsole code from: 8 * 9 * Copyright (C) 2001 Ingo Molnar <mingo@redhat.com> 10 * Copyright (C) 2002 Red Hat, Inc. 11 */ 12 13 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt 14 15 #include <linux/moduleparam.h> 16 #include <linux/kernel.h> 17 #include <linux/netdevice.h> 18 #include <linux/etherdevice.h> 19 #include <linux/string.h> 20 #include <linux/if_arp.h> 21 #include <linux/inetdevice.h> 22 #include <linux/inet.h> 23 #include <linux/interrupt.h> 24 #include <linux/netpoll.h> 25 #include <linux/sched.h> 26 #include <linux/delay.h> 27 #include <linux/rcupdate.h> 28 #include <linux/workqueue.h> 29 #include <linux/slab.h> 30 #include <linux/export.h> 31 #include <linux/if_vlan.h> 32 #include <linux/udp.h> 33 #include <net/tcp.h> 34 #include <net/addrconf.h> 35 #include <net/ndisc.h> 36 #include <trace/events/napi.h> 37 #include <linux/kconfig.h> 38 39 #define USEC_PER_POLL 50 40 41 /* 42 * Cap on skbs parked in npinfo->txq while the device is busy. The queue 43 * exists to ride out a transient stall; a producer that outruns the 44 * device for longer than that must lose packets, not grow it without 45 * bound. Checked without the queue lock, so the queue can overshoot 46 * slightly. 47 */ 48 #define NETPOLL_TXQ_MAX 1024 49 50 /* 51 * carrier_timeout is netconsole-specific and only kept here to preserve the 52 * netpoll.carrier_timeout module-parameter ABI. Its value is exposed to 53 * netconsole through netpoll_get_carrier_timeout(). 54 */ 55 static unsigned int carrier_timeout = 4; 56 module_param(carrier_timeout, uint, 0644); 57 58 unsigned int netpoll_get_carrier_timeout(void) 59 { 60 return carrier_timeout; 61 } 62 EXPORT_SYMBOL_GPL(netpoll_get_carrier_timeout); 63 64 static netdev_tx_t netpoll_start_xmit(struct sk_buff *skb, 65 struct net_device *dev, 66 struct netdev_queue *txq) 67 { 68 netdev_tx_t status = NETDEV_TX_OK; 69 netdev_features_t features; 70 71 features = netif_skb_features(skb); 72 73 if (skb_vlan_tag_present(skb) && 74 !vlan_hw_offload_capable(features, skb->vlan_proto)) { 75 skb = __vlan_hwaccel_push_inside(skb); 76 if (unlikely(!skb)) { 77 /* This is actually a packet drop, but we 78 * don't want the code that calls this 79 * function to try and operate on a NULL skb. 80 */ 81 goto out; 82 } 83 } 84 85 status = netdev_start_xmit(skb, dev, txq, false); 86 87 out: 88 return status; 89 } 90 91 static void queue_process(struct work_struct *work) 92 { 93 struct netpoll_info *npinfo = 94 container_of(work, struct netpoll_info, tx_work.work); 95 struct sk_buff *skb; 96 unsigned long flags; 97 98 while ((skb = skb_dequeue(&npinfo->txq))) { 99 struct net_device *dev = skb->dev; 100 struct netdev_queue *txq; 101 unsigned int q_index; 102 103 if (!netif_device_present(dev) || !netif_running(dev)) { 104 kfree_skb(skb); 105 continue; 106 } 107 108 local_irq_save(flags); 109 /* check if skb->queue_mapping is still valid */ 110 q_index = skb_get_queue_mapping(skb); 111 if (unlikely(q_index >= dev->real_num_tx_queues)) { 112 q_index = q_index % dev->real_num_tx_queues; 113 skb_set_queue_mapping(skb, q_index); 114 } 115 txq = netdev_get_tx_queue(dev, q_index); 116 HARD_TX_LOCK(dev, txq, smp_processor_id()); 117 if (netif_xmit_frozen_or_stopped(txq) || 118 !dev_xmit_complete(netpoll_start_xmit(skb, dev, txq))) { 119 skb_queue_head(&npinfo->txq, skb); 120 HARD_TX_UNLOCK(dev, txq); 121 local_irq_restore(flags); 122 123 schedule_delayed_work(&npinfo->tx_work, HZ/10); 124 return; 125 } 126 HARD_TX_UNLOCK(dev, txq); 127 local_irq_restore(flags); 128 } 129 } 130 131 static int netif_local_xmit_active(struct net_device *dev) 132 { 133 int i; 134 135 for (i = 0; i < dev->num_tx_queues; i++) { 136 struct netdev_queue *txq = netdev_get_tx_queue(dev, i); 137 138 if (netif_tx_owned(txq, smp_processor_id())) 139 return 1; 140 } 141 142 return 0; 143 } 144 145 static void poll_one_napi(struct napi_struct *napi) 146 { 147 int work; 148 149 /* If we set this bit but see that it has already been set, 150 * that indicates that napi has been disabled and we need 151 * to abort this operation 152 */ 153 if (test_and_set_bit(NAPI_STATE_NPSVC, &napi->state)) 154 return; 155 156 /* We explicitly pass the polling call a budget of 0 to 157 * indicate that we are clearing the Tx path only. 158 */ 159 work = napi->poll(napi, 0); 160 WARN_ONCE(work, "%pS exceeded budget in poll\n", napi->poll); 161 trace_napi_poll(napi, work, 0); 162 163 clear_bit(NAPI_STATE_NPSVC, &napi->state); 164 } 165 166 static void poll_napi(struct net_device *dev) 167 { 168 struct napi_struct *napi; 169 int cpu = smp_processor_id(); 170 171 list_for_each_entry_rcu(napi, &dev->napi_list, dev_list) { 172 if (cmpxchg(&napi->poll_owner, -1, cpu) == -1) { 173 poll_one_napi(napi); 174 smp_store_release(&napi->poll_owner, -1); 175 } 176 } 177 } 178 179 void netpoll_poll_dev(struct net_device *dev) 180 { 181 struct netpoll_info *ni = rcu_dereference_bh(dev->npinfo); 182 const struct net_device_ops *ops; 183 184 /* Don't do any rx activity if the dev_lock mutex is held 185 * the dev_open/close paths use this to block netpoll activity 186 * while changing device state 187 */ 188 if (!ni || down_trylock(&ni->dev_lock)) 189 return; 190 191 /* Some drivers will take the same locks in poll and xmit, 192 * we can't poll if local CPU is already in xmit. 193 */ 194 if (!netif_running(dev) || netif_local_xmit_active(dev)) { 195 up(&ni->dev_lock); 196 return; 197 } 198 199 ops = dev->netdev_ops; 200 if (ops->ndo_poll_controller) 201 ops->ndo_poll_controller(dev); 202 203 poll_napi(dev); 204 205 up(&ni->dev_lock); 206 207 netpoll_zap_completion_queue(); 208 } 209 EXPORT_SYMBOL(netpoll_poll_dev); 210 211 void netpoll_poll_disable(struct net_device *dev) 212 { 213 struct netpoll_info *ni; 214 215 might_sleep(); 216 ni = rtnl_dereference(dev->npinfo); 217 if (ni) 218 down(&ni->dev_lock); 219 } 220 221 void netpoll_poll_enable(struct net_device *dev) 222 { 223 struct netpoll_info *ni; 224 225 ni = rtnl_dereference(dev->npinfo); 226 if (ni) 227 up(&ni->dev_lock); 228 } 229 230 void netpoll_zap_completion_queue(void) 231 { 232 unsigned long flags; 233 struct softnet_data *sd = &get_cpu_var(softnet_data); 234 235 if (sd->completion_queue) { 236 struct sk_buff *clist; 237 238 local_irq_save(flags); 239 clist = sd->completion_queue; 240 sd->completion_queue = NULL; 241 local_irq_restore(flags); 242 243 while (clist != NULL) { 244 struct sk_buff *skb = clist; 245 clist = clist->next; 246 if (!skb_irq_freeable(skb)) { 247 refcount_set(&skb->users, 1); 248 dev_kfree_skb_any(skb); /* put this one back */ 249 } else { 250 __kfree_skb(skb); 251 } 252 } 253 } 254 255 put_cpu_var(softnet_data); 256 } 257 EXPORT_SYMBOL_NS_GPL(netpoll_zap_completion_queue, "NETDEV_INTERNAL"); 258 259 static int netpoll_owner_active(struct net_device *dev) 260 { 261 struct napi_struct *napi; 262 263 list_for_each_entry_rcu(napi, &dev->napi_list, dev_list) { 264 if (READ_ONCE(napi->poll_owner) == smp_processor_id()) 265 return 1; 266 } 267 return 0; 268 } 269 270 /* call with IRQ disabled */ 271 static netdev_tx_t __netpoll_send_skb(struct netpoll *np, struct sk_buff *skb) 272 { 273 netdev_tx_t status = NETDEV_TX_BUSY; 274 netdev_tx_t ret = NET_XMIT_DROP; 275 struct net_device *dev; 276 unsigned long tries; 277 /* It is up to the caller to keep npinfo alive. */ 278 struct netpoll_info *npinfo; 279 280 lockdep_assert_irqs_disabled(); 281 282 dev = np->dev; 283 /* npinfo->txq belongs to np->dev, so retries must stay bound to it. */ 284 skb->dev = dev; 285 rcu_read_lock(); 286 npinfo = rcu_dereference_bh(dev->npinfo); 287 288 if (!npinfo || !netif_running(dev) || !netif_device_present(dev)) { 289 dev_kfree_skb_irq(skb); 290 goto out; 291 } 292 293 /* don't get messages out of order, and no recursion */ 294 if (skb_queue_len(&npinfo->txq) == 0 && !netpoll_owner_active(dev)) { 295 struct netdev_queue *txq; 296 297 txq = netdev_core_pick_tx(dev, skb, NULL); 298 299 /* try until next clock tick */ 300 for (tries = jiffies_to_usecs(1)/USEC_PER_POLL; 301 tries > 0; --tries) { 302 if (HARD_TX_TRYLOCK(dev, txq)) { 303 if (!netif_xmit_stopped(txq)) 304 status = netpoll_start_xmit(skb, dev, txq); 305 306 HARD_TX_UNLOCK(dev, txq); 307 308 if (dev_xmit_complete(status)) 309 break; 310 311 } 312 313 /* tickle device maybe there is some cleanup */ 314 netpoll_poll_dev(np->dev); 315 316 udelay(USEC_PER_POLL); 317 } 318 319 WARN_ONCE(!irqs_disabled(), 320 "netpoll_send_skb_on_dev(): %s enabled interrupts in poll (%pS)\n", 321 dev->name, dev->netdev_ops->ndo_start_xmit); 322 323 } 324 325 if (!dev_xmit_complete(status)) { 326 if (skb_queue_len_lockless(&npinfo->txq) >= NETPOLL_TXQ_MAX) { 327 dev_core_stats_tx_dropped_inc(dev); 328 dev_kfree_skb_irq_reason(skb, 329 SKB_DROP_REASON_FULL_RING); 330 goto out; 331 } 332 skb_queue_tail(&npinfo->txq, skb); 333 schedule_delayed_work(&npinfo->tx_work,0); 334 } 335 ret = NETDEV_TX_OK; 336 out: 337 rcu_read_unlock(); 338 return ret; 339 } 340 341 netdev_tx_t netpoll_send_skb(struct netpoll *np, struct sk_buff *skb) 342 { 343 unsigned long flags; 344 netdev_tx_t ret; 345 346 if (unlikely(!np)) { 347 dev_kfree_skb_irq(skb); 348 ret = NET_XMIT_DROP; 349 } else { 350 local_irq_save(flags); 351 ret = __netpoll_send_skb(np, skb); 352 local_irq_restore(flags); 353 } 354 return ret; 355 } 356 EXPORT_SYMBOL(netpoll_send_skb); 357 358 int __netpoll_setup(struct netpoll *np, struct net_device *ndev) 359 { 360 struct netpoll_info *npinfo; 361 const struct net_device_ops *ops; 362 int err; 363 364 if (ndev->priv_flags & IFF_DISABLE_NETPOLL) { 365 np_err(np, "%s doesn't support polling, aborting\n", 366 ndev->name); 367 err = -ENOTSUPP; 368 goto out; 369 } 370 371 npinfo = rtnl_dereference(ndev->npinfo); 372 if (!npinfo) { 373 npinfo = kmalloc_obj(*npinfo); 374 if (!npinfo) { 375 err = -ENOMEM; 376 goto out; 377 } 378 379 sema_init(&npinfo->dev_lock, 1); 380 skb_queue_head_init(&npinfo->txq); 381 INIT_DELAYED_WORK(&npinfo->tx_work, queue_process); 382 383 refcount_set(&npinfo->refcnt, 1); 384 385 ops = ndev->netdev_ops; 386 if (ops->ndo_netpoll_setup) { 387 err = ops->ndo_netpoll_setup(ndev); 388 if (err) 389 goto free_npinfo; 390 } 391 } else { 392 refcount_inc(&npinfo->refcnt); 393 } 394 395 np->dev = ndev; 396 strscpy(np->dev_name, ndev->name, IFNAMSIZ); 397 398 /* last thing to do is link it to the net device structure */ 399 rcu_assign_pointer(ndev->npinfo, npinfo); 400 401 return 0; 402 403 free_npinfo: 404 kfree(npinfo); 405 out: 406 return err; 407 } 408 EXPORT_SYMBOL_GPL(__netpoll_setup); 409 410 static void rcu_cleanup_netpoll_info(struct rcu_head *rcu_head) 411 { 412 struct netpoll_info *npinfo = 413 container_of(rcu_head, struct netpoll_info, rcu); 414 415 skb_queue_purge(&npinfo->txq); 416 kfree(npinfo); 417 } 418 419 static void __netpoll_cleanup(struct netpoll *np) 420 { 421 struct netpoll_info *npinfo; 422 423 npinfo = rtnl_dereference(np->dev->npinfo); 424 if (!npinfo) 425 return; 426 427 /* At this point, there is a single npinfo instance per netdevice, and 428 * its refcnt tracks how many netpoll structures are linked to it. We 429 * only perform npinfo cleanup when the refcnt decrements to zero. 430 */ 431 if (refcount_dec_and_test(&npinfo->refcnt)) { 432 const struct net_device_ops *ops; 433 434 ops = np->dev->netdev_ops; 435 if (ops->ndo_netpoll_cleanup) 436 ops->ndo_netpoll_cleanup(np->dev); 437 438 RCU_INIT_POINTER(np->dev->npinfo, NULL); 439 disable_delayed_work_sync(&npinfo->tx_work); 440 call_rcu(&npinfo->rcu, rcu_cleanup_netpoll_info); 441 } 442 } 443 444 void __netpoll_free(struct netpoll *np) 445 { 446 ASSERT_RTNL(); 447 448 /* Wait for transmitting packets to finish before freeing. */ 449 synchronize_net(); 450 __netpoll_cleanup(np); 451 kfree(np); 452 } 453 EXPORT_SYMBOL_GPL(__netpoll_free); 454 455 void do_netpoll_cleanup(struct netpoll *np) 456 { 457 __netpoll_cleanup(np); 458 netdev_put(np->dev, &np->dev_tracker); 459 np->dev = NULL; 460 } 461 EXPORT_SYMBOL(do_netpoll_cleanup); 462 463 void netpoll_cleanup(struct netpoll *np) 464 { 465 rtnl_lock(); 466 if (!np->dev) 467 goto out; 468 do_netpoll_cleanup(np); 469 out: 470 rtnl_unlock(); 471 } 472 EXPORT_SYMBOL(netpoll_cleanup); 473