1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * NET3 Protocol independent device support routines. 4 * 5 * Derived from the non IP parts of dev.c 1.0.19 6 * Authors: Ross Biro 7 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 8 * Mark Evans, <evansmp@uhura.aston.ac.uk> 9 * 10 * Additional Authors: 11 * Florian la Roche <rzsfl@rz.uni-sb.de> 12 * Alan Cox <gw4pts@gw4pts.ampr.org> 13 * David Hinds <dahinds@users.sourceforge.net> 14 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru> 15 * Adam Sulmicki <adam@cfar.umd.edu> 16 * Pekka Riikonen <priikone@poesidon.pspt.fi> 17 * 18 * Changes: 19 * D.J. Barrow : Fixed bug where dev->refcnt gets set 20 * to 2 if register_netdev gets called 21 * before net_dev_init & also removed a 22 * few lines of code in the process. 23 * Alan Cox : device private ioctl copies fields back. 24 * Alan Cox : Transmit queue code does relevant 25 * stunts to keep the queue safe. 26 * Alan Cox : Fixed double lock. 27 * Alan Cox : Fixed promisc NULL pointer trap 28 * ???????? : Support the full private ioctl range 29 * Alan Cox : Moved ioctl permission check into 30 * drivers 31 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI 32 * Alan Cox : 100 backlog just doesn't cut it when 33 * you start doing multicast video 8) 34 * Alan Cox : Rewrote net_bh and list manager. 35 * Alan Cox : Fix ETH_P_ALL echoback lengths. 36 * Alan Cox : Took out transmit every packet pass 37 * Saved a few bytes in the ioctl handler 38 * Alan Cox : Network driver sets packet type before 39 * calling netif_rx. Saves a function 40 * call a packet. 41 * Alan Cox : Hashed net_bh() 42 * Richard Kooijman: Timestamp fixes. 43 * Alan Cox : Wrong field in SIOCGIFDSTADDR 44 * Alan Cox : Device lock protection. 45 * Alan Cox : Fixed nasty side effect of device close 46 * changes. 47 * Rudi Cilibrasi : Pass the right thing to 48 * set_mac_address() 49 * Dave Miller : 32bit quantity for the device lock to 50 * make it work out on a Sparc. 51 * Bjorn Ekwall : Added KERNELD hack. 52 * Alan Cox : Cleaned up the backlog initialise. 53 * Craig Metz : SIOCGIFCONF fix if space for under 54 * 1 device. 55 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there 56 * is no device open function. 57 * Andi Kleen : Fix error reporting for SIOCGIFCONF 58 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF 59 * Cyrus Durgin : Cleaned for KMOD 60 * Adam Sulmicki : Bug Fix : Network Device Unload 61 * A network device unload needs to purge 62 * the backlog queue. 63 * Paul Rusty Russell : SIOCSIFNAME 64 * Pekka Riikonen : Netdev boot-time settings code 65 * Andrew Morton : Make unregister_netdevice wait 66 * indefinitely on dev->refcnt 67 * J Hadi Salim : - Backlog queue sampling 68 * - netif_rx() feedback 69 */ 70 71 #include <linux/uaccess.h> 72 #include <linux/bitmap.h> 73 #include <linux/capability.h> 74 #include <linux/cpu.h> 75 #include <linux/types.h> 76 #include <linux/kernel.h> 77 #include <linux/hash.h> 78 #include <linux/slab.h> 79 #include <linux/sched.h> 80 #include <linux/sched/isolation.h> 81 #include <linux/sched/mm.h> 82 #include <linux/smpboot.h> 83 #include <linux/mutex.h> 84 #include <linux/rwsem.h> 85 #include <linux/string.h> 86 #include <linux/mm.h> 87 #include <linux/socket.h> 88 #include <linux/sockios.h> 89 #include <linux/errno.h> 90 #include <linux/interrupt.h> 91 #include <linux/if_ether.h> 92 #include <linux/netdevice.h> 93 #include <linux/etherdevice.h> 94 #include <linux/ethtool.h> 95 #include <linux/ethtool_netlink.h> 96 #include <linux/skbuff.h> 97 #include <linux/kthread.h> 98 #include <linux/bpf.h> 99 #include <linux/bpf_trace.h> 100 #include <net/net_namespace.h> 101 #include <net/sock.h> 102 #include <net/busy_poll.h> 103 #include <linux/rtnetlink.h> 104 #include <linux/stat.h> 105 #include <net/dsa.h> 106 #include <net/dst.h> 107 #include <net/dst_metadata.h> 108 #include <net/gro.h> 109 #include <net/netdev_queues.h> 110 #include <net/pkt_sched.h> 111 #include <net/pkt_cls.h> 112 #include <net/checksum.h> 113 #include <net/xfrm.h> 114 #include <net/tcx.h> 115 #include <linux/highmem.h> 116 #include <linux/init.h> 117 #include <linux/module.h> 118 #include <linux/netpoll.h> 119 #include <linux/rcupdate.h> 120 #include <linux/delay.h> 121 #include <net/iw_handler.h> 122 #include <asm/current.h> 123 #include <linux/audit.h> 124 #include <linux/dmaengine.h> 125 #include <linux/err.h> 126 #include <linux/ctype.h> 127 #include <linux/if_arp.h> 128 #include <linux/if_vlan.h> 129 #include <linux/ip.h> 130 #include <net/ip.h> 131 #include <net/mpls.h> 132 #include <net/route.h> 133 #include <linux/ipv6.h> 134 #include <net/ip6_route.h> 135 #include <linux/in.h> 136 #include <linux/jhash.h> 137 #include <linux/random.h> 138 #include <trace/events/napi.h> 139 #include <trace/events/net.h> 140 #include <trace/events/skb.h> 141 #include <trace/events/qdisc.h> 142 #include <trace/events/xdp.h> 143 #include <linux/inetdevice.h> 144 #include <linux/cpu_rmap.h> 145 #include <linux/static_key.h> 146 #include <linux/hashtable.h> 147 #include <linux/vmalloc.h> 148 #include <linux/if_macvlan.h> 149 #include <linux/errqueue.h> 150 #include <linux/hrtimer.h> 151 #include <linux/netfilter_netdev.h> 152 #include <linux/crash_dump.h> 153 #include <linux/sctp.h> 154 #include <net/udp_tunnel.h> 155 #include <linux/net_namespace.h> 156 #include <linux/indirect_call_wrapper.h> 157 #include <net/devlink.h> 158 #include <linux/pm_runtime.h> 159 #include <linux/prandom.h> 160 #include <linux/once_lite.h> 161 #include <net/netdev_lock.h> 162 #include <net/netdev_rx_queue.h> 163 #include <net/page_pool/types.h> 164 #include <net/page_pool/helpers.h> 165 #include <net/page_pool/memory_provider.h> 166 #include <net/rps.h> 167 #include <linux/phy_link_topology.h> 168 169 #include "dev.h" 170 #include "devmem.h" 171 #include "net-sysfs.h" 172 173 static DEFINE_SPINLOCK(ptype_lock); 174 struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly; 175 176 static int netif_rx_internal(struct sk_buff *skb); 177 static int call_netdevice_notifiers_extack(unsigned long val, 178 struct net_device *dev, 179 struct netlink_ext_ack *extack); 180 181 static DEFINE_MUTEX(ifalias_mutex); 182 183 /* protects napi_hash addition/deletion and napi_gen_id */ 184 static DEFINE_SPINLOCK(napi_hash_lock); 185 186 static unsigned int napi_gen_id = NR_CPUS; 187 static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8); 188 189 static inline void dev_base_seq_inc(struct net *net) 190 { 191 unsigned int val = net->dev_base_seq + 1; 192 193 WRITE_ONCE(net->dev_base_seq, val ?: 1); 194 } 195 196 static inline struct hlist_head *dev_name_hash(struct net *net, const char *name) 197 { 198 unsigned int hash = full_name_hash(net, name, strnlen(name, IFNAMSIZ)); 199 200 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)]; 201 } 202 203 static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex) 204 { 205 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)]; 206 } 207 208 #ifndef CONFIG_PREEMPT_RT 209 210 static DEFINE_STATIC_KEY_FALSE(use_backlog_threads_key); 211 212 static int __init setup_backlog_napi_threads(char *arg) 213 { 214 static_branch_enable(&use_backlog_threads_key); 215 return 0; 216 } 217 early_param("thread_backlog_napi", setup_backlog_napi_threads); 218 219 static bool use_backlog_threads(void) 220 { 221 return static_branch_unlikely(&use_backlog_threads_key); 222 } 223 224 #else 225 226 static bool use_backlog_threads(void) 227 { 228 return true; 229 } 230 231 #endif 232 233 static inline void backlog_lock_irq_save(struct softnet_data *sd, 234 unsigned long *flags) 235 { 236 if (IS_ENABLED(CONFIG_PREEMPT_RT)) { 237 spin_lock_irqsave(&sd->input_pkt_queue.lock, *flags); 238 } else { 239 local_irq_save(*flags); 240 if (IS_ENABLED(CONFIG_RPS) || use_backlog_threads()) 241 spin_lock(&sd->input_pkt_queue.lock); 242 } 243 } 244 245 static inline void backlog_lock_irq_disable(struct softnet_data *sd) 246 { 247 if (IS_ENABLED(CONFIG_RPS) || use_backlog_threads()) 248 spin_lock_irq(&sd->input_pkt_queue.lock); 249 else 250 local_irq_disable(); 251 } 252 253 static inline void backlog_unlock_irq_restore(struct softnet_data *sd, 254 unsigned long flags) 255 { 256 if (IS_ENABLED(CONFIG_PREEMPT_RT)) { 257 spin_unlock_irqrestore(&sd->input_pkt_queue.lock, flags); 258 } else { 259 if (IS_ENABLED(CONFIG_RPS) || use_backlog_threads()) 260 spin_unlock(&sd->input_pkt_queue.lock); 261 local_irq_restore(flags); 262 } 263 } 264 265 static inline void backlog_unlock_irq_enable(struct softnet_data *sd) 266 { 267 if (IS_ENABLED(CONFIG_RPS) || use_backlog_threads()) 268 spin_unlock_irq(&sd->input_pkt_queue.lock); 269 else 270 local_irq_enable(); 271 } 272 273 static struct netdev_name_node *netdev_name_node_alloc(struct net_device *dev, 274 const char *name) 275 { 276 struct netdev_name_node *name_node; 277 278 name_node = kmalloc_obj(*name_node); 279 if (!name_node) 280 return NULL; 281 INIT_HLIST_NODE(&name_node->hlist); 282 name_node->dev = dev; 283 name_node->name = name; 284 return name_node; 285 } 286 287 static struct netdev_name_node * 288 netdev_name_node_head_alloc(struct net_device *dev) 289 { 290 struct netdev_name_node *name_node; 291 292 name_node = netdev_name_node_alloc(dev, dev->name); 293 if (!name_node) 294 return NULL; 295 INIT_LIST_HEAD(&name_node->list); 296 return name_node; 297 } 298 299 static void netdev_name_node_free(struct netdev_name_node *name_node) 300 { 301 kfree(name_node); 302 } 303 304 static void netdev_name_node_add(struct net *net, 305 struct netdev_name_node *name_node) 306 { 307 hlist_add_head_rcu(&name_node->hlist, 308 dev_name_hash(net, name_node->name)); 309 } 310 311 static void netdev_name_node_del(struct netdev_name_node *name_node) 312 { 313 hlist_del_rcu(&name_node->hlist); 314 } 315 316 static struct netdev_name_node *netdev_name_node_lookup(struct net *net, 317 const char *name) 318 { 319 struct hlist_head *head = dev_name_hash(net, name); 320 struct netdev_name_node *name_node; 321 322 hlist_for_each_entry(name_node, head, hlist) 323 if (!strcmp(name_node->name, name)) 324 return name_node; 325 return NULL; 326 } 327 328 static struct netdev_name_node *netdev_name_node_lookup_rcu(struct net *net, 329 const char *name) 330 { 331 struct hlist_head *head = dev_name_hash(net, name); 332 struct netdev_name_node *name_node; 333 334 hlist_for_each_entry_rcu(name_node, head, hlist) 335 if (!strcmp(name_node->name, name)) 336 return name_node; 337 return NULL; 338 } 339 340 bool netdev_name_in_use(struct net *net, const char *name) 341 { 342 return netdev_name_node_lookup(net, name); 343 } 344 EXPORT_SYMBOL(netdev_name_in_use); 345 346 int netdev_name_node_alt_create(struct net_device *dev, const char *name) 347 { 348 struct netdev_name_node *name_node; 349 struct net *net = dev_net(dev); 350 351 name_node = netdev_name_node_lookup(net, name); 352 if (name_node) 353 return -EEXIST; 354 name_node = netdev_name_node_alloc(dev, name); 355 if (!name_node) 356 return -ENOMEM; 357 netdev_name_node_add(net, name_node); 358 /* The node that holds dev->name acts as a head of per-device list. */ 359 list_add_tail_rcu(&name_node->list, &dev->name_node->list); 360 361 return 0; 362 } 363 364 static void netdev_name_node_alt_free(struct rcu_head *head) 365 { 366 struct netdev_name_node *name_node = 367 container_of(head, struct netdev_name_node, rcu); 368 369 kfree(name_node->name); 370 netdev_name_node_free(name_node); 371 } 372 373 static void __netdev_name_node_alt_destroy(struct netdev_name_node *name_node) 374 { 375 netdev_name_node_del(name_node); 376 list_del_rcu(&name_node->list); 377 call_rcu(&name_node->rcu, netdev_name_node_alt_free); 378 } 379 380 int netdev_name_node_alt_destroy(struct net_device *dev, const char *name) 381 { 382 struct netdev_name_node *name_node; 383 struct net *net = dev_net(dev); 384 385 name_node = netdev_name_node_lookup(net, name); 386 if (!name_node) 387 return -ENOENT; 388 /* lookup might have found our primary name or a name belonging 389 * to another device. 390 */ 391 if (name_node == dev->name_node || name_node->dev != dev) 392 return -EINVAL; 393 394 __netdev_name_node_alt_destroy(name_node); 395 return 0; 396 } 397 398 static void netdev_name_node_alt_flush(struct net_device *dev) 399 { 400 struct netdev_name_node *name_node, *tmp; 401 402 list_for_each_entry_safe(name_node, tmp, &dev->name_node->list, list) { 403 list_del(&name_node->list); 404 netdev_name_node_alt_free(&name_node->rcu); 405 } 406 } 407 408 /* Device list insertion */ 409 static void list_netdevice(struct net_device *dev) 410 { 411 struct netdev_name_node *name_node; 412 struct net *net = dev_net(dev); 413 414 ASSERT_RTNL(); 415 416 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head); 417 netdev_name_node_add(net, dev->name_node); 418 hlist_add_head_rcu(&dev->index_hlist, 419 dev_index_hash(net, dev->ifindex)); 420 421 netdev_for_each_altname(dev, name_node) 422 netdev_name_node_add(net, name_node); 423 424 /* We reserved the ifindex, this can't fail */ 425 WARN_ON(xa_store(&net->dev_by_index, dev->ifindex, dev, GFP_KERNEL)); 426 427 dev_base_seq_inc(net); 428 } 429 430 /* Device list removal 431 * caller must respect a RCU grace period before freeing/reusing dev 432 */ 433 static void unlist_netdevice(struct net_device *dev) 434 { 435 struct netdev_name_node *name_node; 436 struct net *net = dev_net(dev); 437 438 ASSERT_RTNL(); 439 440 xa_erase(&net->dev_by_index, dev->ifindex); 441 442 netdev_for_each_altname(dev, name_node) 443 netdev_name_node_del(name_node); 444 445 /* Unlink dev from the device chain */ 446 list_del_rcu(&dev->dev_list); 447 netdev_name_node_del(dev->name_node); 448 hlist_del_rcu(&dev->index_hlist); 449 450 dev_base_seq_inc(dev_net(dev)); 451 } 452 453 /* 454 * Our notifier list 455 */ 456 457 static RAW_NOTIFIER_HEAD(netdev_chain); 458 459 /* 460 * Device drivers call our routines to queue packets here. We empty the 461 * queue in the local softnet handler. 462 */ 463 464 DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data) = { 465 .process_queue_bh_lock = INIT_LOCAL_LOCK(process_queue_bh_lock), 466 }; 467 EXPORT_PER_CPU_SYMBOL(softnet_data); 468 469 /* Page_pool has a lockless array/stack to alloc/recycle pages. 470 * PP consumers must pay attention to run APIs in the appropriate context 471 * (e.g. NAPI context). 472 */ 473 DEFINE_PER_CPU(struct page_pool_bh, system_page_pool) = { 474 .bh_lock = INIT_LOCAL_LOCK(bh_lock), 475 }; 476 477 #ifdef CONFIG_LOCKDEP 478 /* 479 * register_netdevice() inits txq->_xmit_lock and sets lockdep class 480 * according to dev->type 481 */ 482 static const unsigned short netdev_lock_type[] = { 483 ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25, 484 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET, 485 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM, 486 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP, 487 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD, 488 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25, 489 ARPHRD_CAN, ARPHRD_MCTP, 490 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP, 491 ARPHRD_RAWHDLC, ARPHRD_RAWIP, 492 ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD, 493 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI, 494 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE, 495 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET, 496 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL, 497 ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM, 498 ARPHRD_IEEE80211_RADIOTAP, 499 ARPHRD_IEEE802154, ARPHRD_IEEE802154_MONITOR, 500 ARPHRD_PHONET, ARPHRD_PHONET_PIPE, 501 ARPHRD_CAIF, ARPHRD_IP6GRE, ARPHRD_NETLINK, ARPHRD_6LOWPAN, 502 ARPHRD_VSOCKMON, 503 ARPHRD_VOID, ARPHRD_NONE}; 504 505 static const char *const netdev_lock_name[] = { 506 "_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25", 507 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET", 508 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM", 509 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP", 510 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD", 511 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25", 512 "_xmit_CAN", "_xmit_MCTP", 513 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP", 514 "_xmit_RAWHDLC", "_xmit_RAWIP", 515 "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD", 516 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI", 517 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE", 518 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET", 519 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL", 520 "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM", 521 "_xmit_IEEE80211_RADIOTAP", 522 "_xmit_IEEE802154", "_xmit_IEEE802154_MONITOR", 523 "_xmit_PHONET", "_xmit_PHONET_PIPE", 524 "_xmit_CAIF", "_xmit_IP6GRE", "_xmit_NETLINK", "_xmit_6LOWPAN", 525 "_xmit_VSOCKMON", 526 "_xmit_VOID", "_xmit_NONE"}; 527 528 static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)]; 529 static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)]; 530 531 static inline unsigned short netdev_lock_pos(unsigned short dev_type) 532 { 533 int i; 534 535 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++) 536 if (netdev_lock_type[i] == dev_type) 537 return i; 538 /* the last key is used by default */ 539 WARN_ONCE(1, "netdev_lock_pos() could not find dev_type=%u\n", dev_type); 540 return ARRAY_SIZE(netdev_lock_type) - 1; 541 } 542 543 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock, 544 unsigned short dev_type) 545 { 546 int i; 547 548 i = netdev_lock_pos(dev_type); 549 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i], 550 netdev_lock_name[i]); 551 } 552 553 static inline void netdev_set_addr_lockdep_class(struct net_device *dev) 554 { 555 int i; 556 557 i = netdev_lock_pos(dev->type); 558 lockdep_set_class_and_name(&dev->addr_list_lock, 559 &netdev_addr_lock_key[i], 560 netdev_lock_name[i]); 561 } 562 #else 563 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock, 564 unsigned short dev_type) 565 { 566 } 567 568 static inline void netdev_set_addr_lockdep_class(struct net_device *dev) 569 { 570 } 571 #endif 572 573 #ifdef CONFIG_PROVE_LOCKING 574 static int netdev_lock_cmp_fn(const struct lockdep_map *a, 575 const struct lockdep_map *b) 576 { 577 if (a == b) 578 return 0; 579 580 /* @a and @b are of same lock class. 581 * cmp_fn won't be called for devices of different classes. 582 * 583 * For the same class only allow nesting under the protection 584 * of rtnl_lock. Note that we can't use lockdep_rtnl_is_held() 585 * here, it always answers UNKNOWN from within lockdep. 586 */ 587 return rtnl_is_locked() ? -1 : 1; 588 } 589 590 /* A virtual device can be locked before the physical device it leases 591 * queues from, see netdev_nl_queue_create_doit(). Keep the two kinds 592 * in separate classes so the dependency graph enforces the order; 593 * netdev_lock_cmp_fn() then only has to rule on same-class nesting. 594 * Other virtual devices stay in the default class. 595 */ 596 void netdev_set_instance_lock_class(struct net_device *dev) 597 { 598 static struct lock_class_key netdev_virt_instance_lock_key; 599 600 if (!netdev_can_create_queue(dev, NULL)) 601 return; 602 603 lockdep_set_class(&dev->lock, &netdev_virt_instance_lock_key); 604 lock_set_cmp_fn(&dev->lock, netdev_lock_cmp_fn, NULL); 605 } 606 EXPORT_SYMBOL_GPL(netdev_set_instance_lock_class); 607 #endif 608 609 /******************************************************************************* 610 * 611 * Protocol management and registration routines 612 * 613 *******************************************************************************/ 614 615 616 /* 617 * Add a protocol ID to the list. Now that the input handler is 618 * smarter we can dispense with all the messy stuff that used to be 619 * here. 620 * 621 * BEWARE!!! Protocol handlers, mangling input packets, 622 * MUST BE last in hash buckets and checking protocol handlers 623 * MUST start from promiscuous ptype_all chain in net_bh. 624 * It is true now, do not change it. 625 * Explanation follows: if protocol handler, mangling packet, will 626 * be the first on list, it is not able to sense, that packet 627 * is cloned and should be copied-on-write, so that it will 628 * change it and subsequent readers will get broken packet. 629 * --ANK (980803) 630 */ 631 632 static inline struct list_head *ptype_head(const struct packet_type *pt) 633 { 634 if (pt->type == htons(ETH_P_ALL)) { 635 if (!pt->af_packet_net && !pt->dev) 636 return NULL; 637 638 return pt->dev ? &pt->dev->ptype_all : 639 &pt->af_packet_net->ptype_all; 640 } 641 642 if (pt->dev) 643 return &pt->dev->ptype_specific; 644 645 return pt->af_packet_net ? &pt->af_packet_net->ptype_specific : 646 &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK]; 647 } 648 649 /** 650 * dev_add_pack - add packet handler 651 * @pt: packet type declaration 652 * 653 * Add a protocol handler to the networking stack. The passed &packet_type 654 * is linked into kernel lists and may not be freed until it has been 655 * removed from the kernel lists. 656 * 657 * This call does not sleep therefore it can not 658 * guarantee all CPU's that are in middle of receiving packets 659 * will see the new packet type (until the next received packet). 660 */ 661 662 void dev_add_pack(struct packet_type *pt) 663 { 664 struct list_head *head = ptype_head(pt); 665 666 if (WARN_ON_ONCE(!head)) 667 return; 668 669 spin_lock(&ptype_lock); 670 list_add_rcu(&pt->list, head); 671 spin_unlock(&ptype_lock); 672 } 673 EXPORT_SYMBOL(dev_add_pack); 674 675 /** 676 * __dev_remove_pack - remove packet handler 677 * @pt: packet type declaration 678 * 679 * Remove a protocol handler that was previously added to the kernel 680 * protocol handlers by dev_add_pack(). The passed &packet_type is removed 681 * from the kernel lists and can be freed or reused once this function 682 * returns. 683 * 684 * The packet type might still be in use by receivers 685 * and must not be freed until after all the CPU's have gone 686 * through a quiescent state. 687 */ 688 void __dev_remove_pack(struct packet_type *pt) 689 { 690 struct list_head *head = ptype_head(pt); 691 struct packet_type *pt1; 692 693 if (!head) 694 return; 695 696 spin_lock(&ptype_lock); 697 698 list_for_each_entry(pt1, head, list) { 699 if (pt == pt1) { 700 list_del_rcu(&pt->list); 701 goto out; 702 } 703 } 704 705 pr_warn("dev_remove_pack: %p not found\n", pt); 706 out: 707 spin_unlock(&ptype_lock); 708 } 709 EXPORT_SYMBOL(__dev_remove_pack); 710 711 /** 712 * dev_remove_pack - remove packet handler 713 * @pt: packet type declaration 714 * 715 * Remove a protocol handler that was previously added to the kernel 716 * protocol handlers by dev_add_pack(). The passed &packet_type is removed 717 * from the kernel lists and can be freed or reused once this function 718 * returns. 719 * 720 * This call sleeps to guarantee that no CPU is looking at the packet 721 * type after return. 722 */ 723 void dev_remove_pack(struct packet_type *pt) 724 { 725 __dev_remove_pack(pt); 726 727 synchronize_net(); 728 } 729 EXPORT_SYMBOL(dev_remove_pack); 730 731 732 /******************************************************************************* 733 * 734 * Device Interface Subroutines 735 * 736 *******************************************************************************/ 737 738 /** 739 * dev_get_iflink - get 'iflink' value of a interface 740 * @dev: targeted interface 741 * 742 * Indicates the ifindex the interface is linked to. 743 * Physical interfaces have the same 'ifindex' and 'iflink' values. 744 */ 745 746 int dev_get_iflink(const struct net_device *dev) 747 { 748 if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink) 749 return dev->netdev_ops->ndo_get_iflink(dev); 750 751 return READ_ONCE(dev->ifindex); 752 } 753 EXPORT_SYMBOL(dev_get_iflink); 754 755 /** 756 * dev_fill_metadata_dst - Retrieve tunnel egress information. 757 * @dev: targeted interface 758 * @skb: The packet. 759 * 760 * For better visibility of tunnel traffic OVS needs to retrieve 761 * egress tunnel information for a packet. Following API allows 762 * user to get this info. 763 */ 764 int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb) 765 { 766 struct ip_tunnel_info *info; 767 768 if (!dev->netdev_ops || !dev->netdev_ops->ndo_fill_metadata_dst) 769 return -EINVAL; 770 771 info = skb_tunnel_info_unclone(skb); 772 if (!info) 773 return -ENOMEM; 774 if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX))) 775 return -EINVAL; 776 777 return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb); 778 } 779 EXPORT_SYMBOL_GPL(dev_fill_metadata_dst); 780 781 static struct net_device_path *dev_fwd_path(struct net_device_path_stack *stack) 782 { 783 if (stack->num_paths + 1 > NET_DEVICE_PATH_STACK_MAX) 784 return NULL; 785 786 return &stack->path[stack->num_paths]; 787 } 788 789 void dev_fill_forward_path_release(struct net_device_path_stack *stack) 790 { 791 struct net_device_path *path; 792 int k; 793 794 if (stack->num_paths == 0) 795 return; 796 797 for (k = stack->num_paths - 1; k >= 0; k--) { 798 path = &stack->path[k]; 799 switch (path->type) { 800 case DEV_PATH_TUN: 801 dst_release(path->tun.dst); 802 break; 803 default: 804 break; 805 } 806 } 807 } 808 EXPORT_SYMBOL_GPL(dev_fill_forward_path_release); 809 810 int dev_fill_forward_path(struct net_device_path_ctx *ctx, 811 struct net_device_path_stack *stack) 812 { 813 const struct net_device *last_dev; 814 struct net_device_path *path; 815 int ret = 0; 816 817 stack->num_paths = 0; 818 while (ctx->dev && ctx->dev->netdev_ops->ndo_fill_forward_path) { 819 last_dev = ctx->dev; 820 path = dev_fwd_path(stack); 821 if (!path) 822 goto err_out; 823 824 memset(path, 0, sizeof(struct net_device_path)); 825 ret = ctx->dev->netdev_ops->ndo_fill_forward_path(ctx, path); 826 if (ret < 0) 827 goto err_out; 828 829 stack->num_paths++; 830 if (last_dev == ctx->dev) 831 goto err_out; 832 } 833 834 if (!ctx->dev) 835 return ret; 836 837 path = dev_fwd_path(stack); 838 if (!path) 839 goto err_out; 840 841 path->type = DEV_PATH_ETHERNET; 842 path->dev = ctx->dev; 843 stack->num_paths++; 844 845 return 0; 846 err_out: 847 dev_fill_forward_path_release(stack); 848 849 return -1; 850 } 851 EXPORT_SYMBOL_GPL(dev_fill_forward_path); 852 853 /* must be called under rcu_read_lock(), as we dont take a reference */ 854 static struct napi_struct *napi_by_id(unsigned int napi_id) 855 { 856 unsigned int hash = napi_id % HASH_SIZE(napi_hash); 857 struct napi_struct *napi; 858 859 hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node) 860 if (napi->napi_id == napi_id) 861 return napi; 862 863 return NULL; 864 } 865 866 /* must be called under rcu_read_lock(), as we dont take a reference */ 867 static struct napi_struct * 868 netdev_napi_by_id(struct net *net, unsigned int napi_id) 869 { 870 struct napi_struct *napi; 871 872 napi = napi_by_id(napi_id); 873 if (!napi) 874 return NULL; 875 876 if (WARN_ON_ONCE(!napi->dev)) 877 return NULL; 878 if (!net_eq(net, dev_net(napi->dev))) 879 return NULL; 880 881 return napi; 882 } 883 884 /** 885 * netdev_napi_by_id_lock() - find a device by NAPI ID and lock it 886 * @net: the applicable net namespace 887 * @napi_id: ID of a NAPI of a target device 888 * 889 * Find a NAPI instance with @napi_id. Lock its device. 890 * The device must be in %NETREG_REGISTERED state for lookup to succeed. 891 * netdev_unlock() must be called to release it. 892 * 893 * Return: pointer to NAPI, its device with lock held, NULL if not found. 894 */ 895 struct napi_struct * 896 netdev_napi_by_id_lock(struct net *net, unsigned int napi_id) 897 { 898 struct napi_struct *napi; 899 struct net_device *dev; 900 901 rcu_read_lock(); 902 napi = netdev_napi_by_id(net, napi_id); 903 if (!napi || READ_ONCE(napi->dev->reg_state) != NETREG_REGISTERED) { 904 rcu_read_unlock(); 905 return NULL; 906 } 907 908 dev = napi->dev; 909 dev_hold(dev); 910 rcu_read_unlock(); 911 912 dev = __netdev_put_lock(dev, net); 913 if (!dev) 914 return NULL; 915 916 rcu_read_lock(); 917 napi = netdev_napi_by_id(net, napi_id); 918 if (napi && napi->dev != dev) 919 napi = NULL; 920 rcu_read_unlock(); 921 922 if (!napi) 923 netdev_unlock(dev); 924 return napi; 925 } 926 927 /** 928 * __dev_get_by_name - find a device by its name 929 * @net: the applicable net namespace 930 * @name: name to find 931 * 932 * Find an interface by name. Must be called under RTNL semaphore. 933 * If the name is found a pointer to the device is returned. 934 * If the name is not found then %NULL is returned. The 935 * reference counters are not incremented so the caller must be 936 * careful with locks. 937 */ 938 939 struct net_device *__dev_get_by_name(struct net *net, const char *name) 940 { 941 struct netdev_name_node *node_name; 942 943 node_name = netdev_name_node_lookup(net, name); 944 return node_name ? node_name->dev : NULL; 945 } 946 EXPORT_SYMBOL(__dev_get_by_name); 947 948 /** 949 * dev_get_by_name_rcu - find a device by its name 950 * @net: the applicable net namespace 951 * @name: name to find 952 * 953 * Find an interface by name. 954 * If the name is found a pointer to the device is returned. 955 * If the name is not found then %NULL is returned. 956 * The reference counters are not incremented so the caller must be 957 * careful with locks. The caller must hold RCU lock. 958 */ 959 960 struct net_device *dev_get_by_name_rcu(struct net *net, const char *name) 961 { 962 struct netdev_name_node *node_name; 963 964 node_name = netdev_name_node_lookup_rcu(net, name); 965 return node_name ? node_name->dev : NULL; 966 } 967 EXPORT_SYMBOL(dev_get_by_name_rcu); 968 969 /* Deprecated for new users, call netdev_get_by_name() instead */ 970 struct net_device *dev_get_by_name(struct net *net, const char *name) 971 { 972 struct net_device *dev; 973 974 rcu_read_lock(); 975 dev = dev_get_by_name_rcu(net, name); 976 dev_hold(dev); 977 rcu_read_unlock(); 978 return dev; 979 } 980 EXPORT_SYMBOL(dev_get_by_name); 981 982 /** 983 * netdev_get_by_name() - find a device by its name 984 * @net: the applicable net namespace 985 * @name: name to find 986 * @tracker: tracking object for the acquired reference 987 * @gfp: allocation flags for the tracker 988 * 989 * Find an interface by name. This can be called from any 990 * context and does its own locking. The returned handle has 991 * the usage count incremented and the caller must use netdev_put() to 992 * release it when it is no longer needed. %NULL is returned if no 993 * matching device is found. 994 */ 995 struct net_device *netdev_get_by_name(struct net *net, const char *name, 996 netdevice_tracker *tracker, gfp_t gfp) 997 { 998 struct net_device *dev; 999 1000 dev = dev_get_by_name(net, name); 1001 if (dev) 1002 netdev_tracker_alloc(dev, tracker, gfp); 1003 return dev; 1004 } 1005 EXPORT_SYMBOL(netdev_get_by_name); 1006 1007 /** 1008 * __dev_get_by_index - find a device by its ifindex 1009 * @net: the applicable net namespace 1010 * @ifindex: index of device 1011 * 1012 * Search for an interface by index. Returns %NULL if the device 1013 * is not found or a pointer to the device. The device has not 1014 * had its reference counter increased so the caller must be careful 1015 * about locking. The caller must hold the RTNL semaphore. 1016 */ 1017 1018 struct net_device *__dev_get_by_index(struct net *net, int ifindex) 1019 { 1020 struct net_device *dev; 1021 struct hlist_head *head = dev_index_hash(net, ifindex); 1022 1023 hlist_for_each_entry(dev, head, index_hlist) 1024 if (dev->ifindex == ifindex) 1025 return dev; 1026 1027 return NULL; 1028 } 1029 EXPORT_SYMBOL(__dev_get_by_index); 1030 1031 /** 1032 * dev_get_by_index_rcu - find a device by its ifindex 1033 * @net: the applicable net namespace 1034 * @ifindex: index of device 1035 * 1036 * Search for an interface by index. Returns %NULL if the device 1037 * is not found or a pointer to the device. The device has not 1038 * had its reference counter increased so the caller must be careful 1039 * about locking. The caller must hold RCU lock. 1040 */ 1041 1042 struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex) 1043 { 1044 struct net_device *dev; 1045 struct hlist_head *head = dev_index_hash(net, ifindex); 1046 1047 hlist_for_each_entry_rcu(dev, head, index_hlist) 1048 if (dev->ifindex == ifindex) 1049 return dev; 1050 1051 return NULL; 1052 } 1053 EXPORT_SYMBOL(dev_get_by_index_rcu); 1054 1055 /* Deprecated for new users, call netdev_get_by_index() instead */ 1056 struct net_device *dev_get_by_index(struct net *net, int ifindex) 1057 { 1058 struct net_device *dev; 1059 1060 rcu_read_lock(); 1061 dev = dev_get_by_index_rcu(net, ifindex); 1062 dev_hold(dev); 1063 rcu_read_unlock(); 1064 return dev; 1065 } 1066 EXPORT_SYMBOL(dev_get_by_index); 1067 1068 /** 1069 * netdev_get_by_index() - find a device by its ifindex 1070 * @net: the applicable net namespace 1071 * @ifindex: index of device 1072 * @tracker: tracking object for the acquired reference 1073 * @gfp: allocation flags for the tracker 1074 * 1075 * Search for an interface by index. Returns NULL if the device 1076 * is not found or a pointer to the device. The device returned has 1077 * had a reference added and the pointer is safe until the user calls 1078 * netdev_put() to indicate they have finished with it. 1079 */ 1080 struct net_device *netdev_get_by_index(struct net *net, int ifindex, 1081 netdevice_tracker *tracker, gfp_t gfp) 1082 { 1083 struct net_device *dev; 1084 1085 dev = dev_get_by_index(net, ifindex); 1086 if (dev) 1087 netdev_tracker_alloc(dev, tracker, gfp); 1088 return dev; 1089 } 1090 EXPORT_SYMBOL(netdev_get_by_index); 1091 1092 /** 1093 * dev_get_by_napi_id - find a device by napi_id 1094 * @napi_id: ID of the NAPI struct 1095 * 1096 * Search for an interface by NAPI ID. Returns %NULL if the device 1097 * is not found or a pointer to the device. The device has not had 1098 * its reference counter increased so the caller must be careful 1099 * about locking. The caller must hold RCU lock. 1100 */ 1101 struct net_device *dev_get_by_napi_id(unsigned int napi_id) 1102 { 1103 struct napi_struct *napi; 1104 1105 WARN_ON_ONCE(!rcu_read_lock_held()); 1106 1107 if (!napi_id_valid(napi_id)) 1108 return NULL; 1109 1110 napi = napi_by_id(napi_id); 1111 1112 return napi ? napi->dev : NULL; 1113 } 1114 1115 /* Release the held reference on the net_device, and if the net_device 1116 * is still registered try to lock the instance lock. If device is being 1117 * unregistered NULL will be returned (but the reference has been released, 1118 * either way!) 1119 * 1120 * This helper is intended for locking net_device after it has been looked up 1121 * using a lockless lookup helper. Lock prevents the instance from going away. 1122 */ 1123 struct net_device * 1124 netdev_put_lock(struct net_device *dev, struct net *net, 1125 netdevice_tracker *tracker) 1126 { 1127 netdev_lock(dev); 1128 if (dev->reg_state > NETREG_REGISTERED || 1129 dev->moving_ns || !net_eq(dev_net(dev), net)) { 1130 netdev_unlock(dev); 1131 netdev_put(dev, tracker); 1132 return NULL; 1133 } 1134 netdev_put(dev, tracker); 1135 return dev; 1136 } 1137 1138 static struct net_device * 1139 __netdev_put_lock_ops_compat(struct net_device *dev, struct net *net) 1140 { 1141 netdev_lock_ops_compat(dev); 1142 if (dev->reg_state > NETREG_REGISTERED || 1143 dev->moving_ns || !net_eq(dev_net(dev), net)) { 1144 netdev_unlock_ops_compat(dev); 1145 dev_put(dev); 1146 return NULL; 1147 } 1148 dev_put(dev); 1149 return dev; 1150 } 1151 1152 /** 1153 * netdev_get_by_index_lock() - find a device by its ifindex 1154 * @net: the applicable net namespace 1155 * @ifindex: index of device 1156 * 1157 * Search for an interface by index. If a valid device 1158 * with @ifindex is found it will be returned with netdev->lock held. 1159 * netdev_unlock() must be called to release it. 1160 * 1161 * Return: pointer to a device with lock held, NULL if not found. 1162 */ 1163 struct net_device *netdev_get_by_index_lock(struct net *net, int ifindex) 1164 { 1165 struct net_device *dev; 1166 1167 dev = dev_get_by_index(net, ifindex); 1168 if (!dev) 1169 return NULL; 1170 1171 return __netdev_put_lock(dev, net); 1172 } 1173 1174 struct net_device * 1175 netdev_get_by_index_lock_ops_compat(struct net *net, int ifindex) 1176 { 1177 struct net_device *dev; 1178 1179 dev = dev_get_by_index(net, ifindex); 1180 if (!dev) 1181 return NULL; 1182 1183 return __netdev_put_lock_ops_compat(dev, net); 1184 } 1185 1186 struct net_device * 1187 netdev_xa_find_lock(struct net *net, struct net_device *dev, 1188 unsigned long *index) 1189 { 1190 if (dev) 1191 netdev_unlock(dev); 1192 1193 do { 1194 rcu_read_lock(); 1195 dev = xa_find(&net->dev_by_index, index, ULONG_MAX, XA_PRESENT); 1196 if (!dev) { 1197 rcu_read_unlock(); 1198 return NULL; 1199 } 1200 dev_hold(dev); 1201 rcu_read_unlock(); 1202 1203 dev = __netdev_put_lock(dev, net); 1204 if (dev) 1205 return dev; 1206 1207 (*index)++; 1208 } while (true); 1209 } 1210 1211 struct net_device * 1212 netdev_xa_find_lock_ops_compat(struct net *net, struct net_device *dev, 1213 unsigned long *index) 1214 { 1215 if (dev) 1216 netdev_unlock_ops_compat(dev); 1217 1218 do { 1219 rcu_read_lock(); 1220 dev = xa_find(&net->dev_by_index, index, ULONG_MAX, XA_PRESENT); 1221 if (!dev) { 1222 rcu_read_unlock(); 1223 return NULL; 1224 } 1225 dev_hold(dev); 1226 rcu_read_unlock(); 1227 1228 dev = __netdev_put_lock_ops_compat(dev, net); 1229 if (dev) 1230 return dev; 1231 1232 (*index)++; 1233 } while (true); 1234 } 1235 1236 static DEFINE_SEQLOCK(netdev_rename_lock); 1237 1238 void netdev_copy_name(struct net_device *dev, char *name) 1239 { 1240 unsigned int seq; 1241 1242 do { 1243 seq = read_seqbegin(&netdev_rename_lock); 1244 strscpy(name, dev->name, IFNAMSIZ); 1245 } while (read_seqretry(&netdev_rename_lock, seq)); 1246 } 1247 1248 /** 1249 * netdev_get_name - get a netdevice name, knowing its ifindex. 1250 * @net: network namespace 1251 * @name: a pointer to the buffer where the name will be stored. 1252 * @ifindex: the ifindex of the interface to get the name from. 1253 */ 1254 int netdev_get_name(struct net *net, char *name, int ifindex) 1255 { 1256 struct net_device *dev; 1257 int ret; 1258 1259 rcu_read_lock(); 1260 1261 dev = dev_get_by_index_rcu(net, ifindex); 1262 if (!dev) { 1263 ret = -ENODEV; 1264 goto out; 1265 } 1266 1267 netdev_copy_name(dev, name); 1268 1269 ret = 0; 1270 out: 1271 rcu_read_unlock(); 1272 return ret; 1273 } 1274 1275 static bool dev_addr_cmp(struct net_device *dev, unsigned short type, 1276 const char *ha) 1277 { 1278 return dev->type == type && !memcmp(dev->dev_addr, ha, dev->addr_len); 1279 } 1280 1281 /** 1282 * dev_getbyhwaddr_rcu - find a device by its hardware address 1283 * @net: the applicable net namespace 1284 * @type: media type of device 1285 * @ha: hardware address 1286 * 1287 * Search for an interface by MAC address. Returns NULL if the device 1288 * is not found or a pointer to the device. 1289 * The caller must hold RCU. 1290 * The returned device has not had its ref count increased 1291 * and the caller must therefore be careful about locking 1292 * 1293 */ 1294 1295 struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type, 1296 const char *ha) 1297 { 1298 struct net_device *dev; 1299 1300 for_each_netdev_rcu(net, dev) 1301 if (dev_addr_cmp(dev, type, ha)) 1302 return dev; 1303 1304 return NULL; 1305 } 1306 EXPORT_SYMBOL(dev_getbyhwaddr_rcu); 1307 1308 /** 1309 * dev_getbyhwaddr() - find a device by its hardware address 1310 * @net: the applicable net namespace 1311 * @type: media type of device 1312 * @ha: hardware address 1313 * 1314 * Similar to dev_getbyhwaddr_rcu(), but the owner needs to hold 1315 * rtnl_lock. 1316 * 1317 * Context: rtnl_lock() must be held. 1318 * Return: pointer to the net_device, or NULL if not found 1319 */ 1320 struct net_device *dev_getbyhwaddr(struct net *net, unsigned short type, 1321 const char *ha) 1322 { 1323 struct net_device *dev; 1324 1325 ASSERT_RTNL(); 1326 for_each_netdev(net, dev) 1327 if (dev_addr_cmp(dev, type, ha)) 1328 return dev; 1329 1330 return NULL; 1331 } 1332 EXPORT_SYMBOL(dev_getbyhwaddr); 1333 1334 struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type) 1335 { 1336 struct net_device *dev, *ret = NULL; 1337 1338 rcu_read_lock(); 1339 for_each_netdev_rcu(net, dev) 1340 if (dev->type == type) { 1341 dev_hold(dev); 1342 ret = dev; 1343 break; 1344 } 1345 rcu_read_unlock(); 1346 return ret; 1347 } 1348 EXPORT_SYMBOL(dev_getfirstbyhwtype); 1349 1350 /** 1351 * netdev_get_by_flags_rcu - find any device with given flags 1352 * @net: the applicable net namespace 1353 * @tracker: tracking object for the acquired reference 1354 * @if_flags: IFF_* values 1355 * @mask: bitmask of bits in if_flags to check 1356 * 1357 * Search for any interface with the given flags. 1358 * 1359 * Context: rcu_read_lock() must be held. 1360 * Returns: NULL if a device is not found or a pointer to the device. 1361 */ 1362 struct net_device *netdev_get_by_flags_rcu(struct net *net, netdevice_tracker *tracker, 1363 unsigned short if_flags, unsigned short mask) 1364 { 1365 struct net_device *dev; 1366 1367 for_each_netdev_rcu(net, dev) { 1368 if (((READ_ONCE(dev->flags) ^ if_flags) & mask) == 0) { 1369 netdev_hold(dev, tracker, GFP_ATOMIC); 1370 return dev; 1371 } 1372 } 1373 1374 return NULL; 1375 } 1376 1377 /** 1378 * dev_valid_name - check if name is okay for network device 1379 * @name: name string 1380 * 1381 * Network device names need to be valid file names to 1382 * allow sysfs to work. We also disallow any kind of 1383 * whitespace. 1384 */ 1385 bool dev_valid_name(const char *name) 1386 { 1387 if (*name == '\0') 1388 return false; 1389 if (strnlen(name, IFNAMSIZ) == IFNAMSIZ) 1390 return false; 1391 if (!strcmp(name, ".") || !strcmp(name, "..")) 1392 return false; 1393 1394 while (*name) { 1395 if (*name == '/' || *name == ':' || isspace(*name)) 1396 return false; 1397 name++; 1398 } 1399 return true; 1400 } 1401 EXPORT_SYMBOL(dev_valid_name); 1402 1403 /** 1404 * __dev_alloc_name - allocate a name for a device 1405 * @net: network namespace to allocate the device name in 1406 * @name: name format string 1407 * @res: result name string 1408 * 1409 * Passed a format string - eg "lt%d" it will try and find a suitable 1410 * id. It scans list of devices to build up a free map, then chooses 1411 * the first empty slot. The caller must hold the dev_base or rtnl lock 1412 * while allocating the name and adding the device in order to avoid 1413 * duplicates. 1414 * Limited to bits_per_byte * page size devices (ie 32K on most platforms). 1415 * Returns the number of the unit assigned or a negative errno code. 1416 */ 1417 1418 static int __dev_alloc_name(struct net *net, const char *name, char *res) 1419 { 1420 int i = 0; 1421 const char *p; 1422 const int max_netdevices = 8*PAGE_SIZE; 1423 unsigned long *inuse; 1424 struct net_device *d; 1425 char buf[IFNAMSIZ]; 1426 1427 /* Verify the string as this thing may have come from the user. 1428 * There must be one "%d" and no other "%" characters. 1429 */ 1430 p = strchr(name, '%'); 1431 if (!p || p[1] != 'd' || strchr(p + 2, '%')) 1432 return -EINVAL; 1433 1434 /* Use one page as a bit array of possible slots */ 1435 inuse = bitmap_zalloc(max_netdevices, GFP_ATOMIC); 1436 if (!inuse) 1437 return -ENOMEM; 1438 1439 for_each_netdev(net, d) { 1440 struct netdev_name_node *name_node; 1441 1442 netdev_for_each_altname(d, name_node) { 1443 if (!sscanf(name_node->name, name, &i)) 1444 continue; 1445 if (i < 0 || i >= max_netdevices) 1446 continue; 1447 1448 /* avoid cases where sscanf is not exact inverse of printf */ 1449 snprintf(buf, IFNAMSIZ, name, i); 1450 if (!strncmp(buf, name_node->name, IFNAMSIZ)) 1451 __set_bit(i, inuse); 1452 } 1453 if (!sscanf(d->name, name, &i)) 1454 continue; 1455 if (i < 0 || i >= max_netdevices) 1456 continue; 1457 1458 /* avoid cases where sscanf is not exact inverse of printf */ 1459 snprintf(buf, IFNAMSIZ, name, i); 1460 if (!strncmp(buf, d->name, IFNAMSIZ)) 1461 __set_bit(i, inuse); 1462 } 1463 1464 i = find_first_zero_bit(inuse, max_netdevices); 1465 bitmap_free(inuse); 1466 if (i == max_netdevices) 1467 return -ENFILE; 1468 1469 /* 'res' and 'name' could overlap, use 'buf' as an intermediate buffer */ 1470 strscpy(buf, name, IFNAMSIZ); 1471 snprintf(res, IFNAMSIZ, buf, i); 1472 return i; 1473 } 1474 1475 /* Returns negative errno or allocated unit id (see __dev_alloc_name()) */ 1476 static int dev_prep_valid_name(struct net *net, struct net_device *dev, 1477 const char *want_name, char *out_name, 1478 int dup_errno) 1479 { 1480 if (!dev_valid_name(want_name)) 1481 return -EINVAL; 1482 1483 if (strchr(want_name, '%')) 1484 return __dev_alloc_name(net, want_name, out_name); 1485 1486 if (netdev_name_in_use(net, want_name)) 1487 return -dup_errno; 1488 if (out_name != want_name) 1489 strscpy(out_name, want_name, IFNAMSIZ); 1490 return 0; 1491 } 1492 1493 /** 1494 * dev_alloc_name - allocate a name for a device 1495 * @dev: device 1496 * @name: name format string 1497 * 1498 * Passed a format string - eg "lt%d" it will try and find a suitable 1499 * id. It scans list of devices to build up a free map, then chooses 1500 * the first empty slot. The caller must hold the dev_base or rtnl lock 1501 * while allocating the name and adding the device in order to avoid 1502 * duplicates. 1503 * Limited to bits_per_byte * page size devices (ie 32K on most platforms). 1504 * Returns the number of the unit assigned or a negative errno code. 1505 */ 1506 1507 int dev_alloc_name(struct net_device *dev, const char *name) 1508 { 1509 return dev_prep_valid_name(dev_net(dev), dev, name, dev->name, ENFILE); 1510 } 1511 EXPORT_SYMBOL(dev_alloc_name); 1512 1513 static int dev_get_valid_name(struct net *net, struct net_device *dev, 1514 const char *name) 1515 { 1516 int ret; 1517 1518 ret = dev_prep_valid_name(net, dev, name, dev->name, EEXIST); 1519 return ret < 0 ? ret : 0; 1520 } 1521 1522 int netif_change_name(struct net_device *dev, const char *newname) 1523 { 1524 struct net *net = dev_net(dev); 1525 unsigned char old_assign_type; 1526 char oldname[IFNAMSIZ]; 1527 int err = 0; 1528 int ret; 1529 1530 ASSERT_RTNL_NET(net); 1531 1532 if (!strncmp(newname, dev->name, IFNAMSIZ)) 1533 return 0; 1534 1535 memcpy(oldname, dev->name, IFNAMSIZ); 1536 1537 write_seqlock_bh(&netdev_rename_lock); 1538 err = dev_get_valid_name(net, dev, newname); 1539 write_sequnlock_bh(&netdev_rename_lock); 1540 1541 if (err < 0) 1542 return err; 1543 1544 if (oldname[0] && !strchr(oldname, '%')) 1545 netdev_info(dev, "renamed from %s%s\n", oldname, 1546 dev->flags & IFF_UP ? " (while UP)" : ""); 1547 1548 old_assign_type = dev->name_assign_type; 1549 WRITE_ONCE(dev->name_assign_type, NET_NAME_RENAMED); 1550 1551 rollback: 1552 ret = device_rename(&dev->dev, dev->name); 1553 if (ret) { 1554 write_seqlock_bh(&netdev_rename_lock); 1555 memcpy(dev->name, oldname, IFNAMSIZ); 1556 write_sequnlock_bh(&netdev_rename_lock); 1557 WRITE_ONCE(dev->name_assign_type, old_assign_type); 1558 return ret; 1559 } 1560 1561 netdev_adjacent_rename_links(dev, oldname); 1562 1563 netdev_name_node_del(dev->name_node); 1564 1565 synchronize_net(); 1566 1567 netdev_name_node_add(net, dev->name_node); 1568 1569 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev); 1570 ret = notifier_to_errno(ret); 1571 1572 if (ret) { 1573 /* err >= 0 after dev_alloc_name() or stores the first errno */ 1574 if (err >= 0) { 1575 err = ret; 1576 write_seqlock_bh(&netdev_rename_lock); 1577 memcpy(dev->name, oldname, IFNAMSIZ); 1578 write_sequnlock_bh(&netdev_rename_lock); 1579 memcpy(oldname, newname, IFNAMSIZ); 1580 WRITE_ONCE(dev->name_assign_type, old_assign_type); 1581 old_assign_type = NET_NAME_RENAMED; 1582 goto rollback; 1583 } else { 1584 netdev_err(dev, "name change rollback failed: %d\n", 1585 ret); 1586 } 1587 } 1588 1589 return err; 1590 } 1591 1592 int netif_set_alias(struct net_device *dev, const char *alias, size_t len) 1593 { 1594 struct dev_ifalias *new_alias = NULL; 1595 1596 if (len >= IFALIASZ) 1597 return -EINVAL; 1598 1599 if (len) { 1600 new_alias = kmalloc(sizeof(*new_alias) + len + 1, GFP_KERNEL); 1601 if (!new_alias) 1602 return -ENOMEM; 1603 1604 memcpy(new_alias->ifalias, alias, len); 1605 new_alias->ifalias[len] = 0; 1606 } 1607 1608 mutex_lock(&ifalias_mutex); 1609 new_alias = rcu_replace_pointer(dev->ifalias, new_alias, 1610 mutex_is_locked(&ifalias_mutex)); 1611 mutex_unlock(&ifalias_mutex); 1612 1613 if (new_alias) 1614 kfree_rcu(new_alias, rcuhead); 1615 1616 return len; 1617 } 1618 1619 /** 1620 * dev_get_alias - get ifalias of a device 1621 * @dev: device 1622 * @name: buffer to store name of ifalias 1623 * @len: size of buffer 1624 * 1625 * get ifalias for a device. Caller must make sure dev cannot go 1626 * away, e.g. rcu read lock or own a reference count to device. 1627 */ 1628 int dev_get_alias(const struct net_device *dev, char *name, size_t len) 1629 { 1630 const struct dev_ifalias *alias; 1631 int ret = 0; 1632 1633 rcu_read_lock(); 1634 alias = rcu_dereference(dev->ifalias); 1635 if (alias) 1636 ret = snprintf(name, len, "%s", alias->ifalias); 1637 rcu_read_unlock(); 1638 1639 return ret; 1640 } 1641 1642 /** 1643 * netdev_features_change - device changes features 1644 * @dev: device to cause notification 1645 * 1646 * Called to indicate a device has changed features. 1647 */ 1648 void netdev_features_change(struct net_device *dev) 1649 { 1650 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev); 1651 } 1652 EXPORT_SYMBOL(netdev_features_change); 1653 1654 void netif_state_change(struct net_device *dev) 1655 { 1656 netdev_assert_locked_ops_compat_or_invisible(dev); 1657 1658 if (dev->flags & IFF_UP) { 1659 struct netdev_notifier_change_info change_info = { 1660 .info.dev = dev, 1661 }; 1662 1663 call_netdevice_notifiers_info(NETDEV_CHANGE, 1664 &change_info.info); 1665 rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL, 0, NULL); 1666 } 1667 } 1668 1669 /** 1670 * __netdev_notify_peers - notify network peers about existence of @dev, 1671 * to be called when rtnl lock is already held. 1672 * @dev: network device 1673 * 1674 * Generate traffic such that interested network peers are aware of 1675 * @dev, such as by generating a gratuitous ARP. This may be used when 1676 * a device wants to inform the rest of the network about some sort of 1677 * reconfiguration such as a failover event or virtual machine 1678 * migration. 1679 */ 1680 void __netdev_notify_peers(struct net_device *dev) 1681 { 1682 ASSERT_RTNL(); 1683 call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev); 1684 call_netdevice_notifiers(NETDEV_RESEND_IGMP, dev); 1685 } 1686 EXPORT_SYMBOL(__netdev_notify_peers); 1687 1688 /** 1689 * netdev_notify_peers - notify network peers about existence of @dev 1690 * @dev: network device 1691 * 1692 * Generate traffic such that interested network peers are aware of 1693 * @dev, such as by generating a gratuitous ARP. This may be used when 1694 * a device wants to inform the rest of the network about some sort of 1695 * reconfiguration such as a failover event or virtual machine 1696 * migration. 1697 */ 1698 void netdev_notify_peers(struct net_device *dev) 1699 { 1700 rtnl_lock(); 1701 __netdev_notify_peers(dev); 1702 rtnl_unlock(); 1703 } 1704 EXPORT_SYMBOL(netdev_notify_peers); 1705 1706 static int napi_threaded_poll(void *data); 1707 1708 static int napi_kthread_create(struct napi_struct *n) 1709 { 1710 int err = 0; 1711 1712 /* Create and wake up the kthread once to put it in 1713 * TASK_INTERRUPTIBLE mode to avoid the blocked task 1714 * warning and work with loadavg. 1715 */ 1716 n->thread = kthread_run(napi_threaded_poll, n, "napi/%s-%d", 1717 n->dev->name, n->napi_id); 1718 if (IS_ERR(n->thread)) { 1719 err = PTR_ERR(n->thread); 1720 pr_err("kthread_run failed with err %d\n", err); 1721 n->thread = NULL; 1722 } 1723 1724 return err; 1725 } 1726 1727 static int __dev_open(struct net_device *dev, struct netlink_ext_ack *extack) 1728 { 1729 const struct net_device_ops *ops = dev->netdev_ops; 1730 int ret; 1731 1732 ASSERT_RTNL(); 1733 dev_addr_check(dev); 1734 1735 if (!netif_device_present(dev)) { 1736 /* may be detached because parent is runtime-suspended */ 1737 if (dev->dev.parent) 1738 pm_runtime_resume(dev->dev.parent); 1739 if (!netif_device_present(dev)) 1740 return -ENODEV; 1741 } 1742 1743 /* Block netpoll from trying to do any rx path servicing. 1744 * If we don't do this there is a chance ndo_poll_controller 1745 * or ndo_poll may be running while we open the device 1746 */ 1747 netpoll_poll_disable(dev); 1748 1749 ret = call_netdevice_notifiers_extack(NETDEV_PRE_UP, dev, extack); 1750 ret = notifier_to_errno(ret); 1751 if (ret) 1752 return ret; 1753 1754 set_bit(__LINK_STATE_START, &dev->state); 1755 1756 netdev_assert_locked_ops_compat(dev); 1757 1758 if (ops->ndo_validate_addr) 1759 ret = ops->ndo_validate_addr(dev); 1760 1761 if (!ret && ops->ndo_open) 1762 ret = ops->ndo_open(dev); 1763 1764 netpoll_poll_enable(dev); 1765 1766 if (ret) 1767 clear_bit(__LINK_STATE_START, &dev->state); 1768 else { 1769 netif_set_up(dev, true); 1770 dev_set_rx_mode(dev); 1771 dev_activate(dev); 1772 add_device_randomness(dev->dev_addr, dev->addr_len); 1773 } 1774 1775 return ret; 1776 } 1777 1778 int netif_open(struct net_device *dev, struct netlink_ext_ack *extack) 1779 { 1780 int ret; 1781 1782 if (dev->flags & IFF_UP) 1783 return 0; 1784 1785 ret = __dev_open(dev, extack); 1786 if (ret < 0) 1787 return ret; 1788 1789 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP | IFF_RUNNING, GFP_KERNEL, 0, NULL); 1790 call_netdevice_notifiers(NETDEV_UP, dev); 1791 1792 return ret; 1793 } 1794 EXPORT_SYMBOL(netif_open); 1795 1796 static void __dev_close_many(struct list_head *head) 1797 { 1798 struct net_device *dev; 1799 1800 ASSERT_RTNL(); 1801 might_sleep(); 1802 1803 list_for_each_entry(dev, head, close_list) { 1804 /* Temporarily disable netpoll until the interface is down */ 1805 netpoll_poll_disable(dev); 1806 1807 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev); 1808 1809 clear_bit(__LINK_STATE_START, &dev->state); 1810 1811 /* Synchronize to scheduled poll. We cannot touch poll list, it 1812 * can be even on different cpu. So just clear netif_running(). 1813 * 1814 * dev->stop() will invoke napi_disable() on all of it's 1815 * napi_struct instances on this device. 1816 */ 1817 smp_mb__after_atomic(); /* Commit netif_running(). */ 1818 } 1819 1820 dev_deactivate_many(head, true); 1821 1822 list_for_each_entry(dev, head, close_list) { 1823 const struct net_device_ops *ops = dev->netdev_ops; 1824 1825 /* 1826 * Call the device specific close. This cannot fail. 1827 * Only if device is UP 1828 * 1829 * We allow it to be called even after a DETACH hot-plug 1830 * event. 1831 */ 1832 1833 netdev_assert_locked_ops_compat(dev); 1834 1835 if (ops->ndo_stop) 1836 ops->ndo_stop(dev); 1837 1838 netif_rx_mode_cancel_retry(dev); 1839 netif_set_up(dev, false); 1840 netpoll_poll_enable(dev); 1841 } 1842 } 1843 1844 static void __dev_close(struct net_device *dev) 1845 { 1846 LIST_HEAD(single); 1847 1848 list_add(&dev->close_list, &single); 1849 __dev_close_many(&single); 1850 list_del(&single); 1851 } 1852 1853 void netif_close_many(struct list_head *head, bool unlink) 1854 { 1855 struct net_device *dev, *tmp; 1856 1857 /* Remove the devices that don't need to be closed */ 1858 list_for_each_entry_safe(dev, tmp, head, close_list) 1859 if (!(dev->flags & IFF_UP)) 1860 list_del_init(&dev->close_list); 1861 1862 __dev_close_many(head); 1863 1864 list_for_each_entry_safe(dev, tmp, head, close_list) { 1865 netdev_assert_locked_ops_compat(dev); 1866 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP | IFF_RUNNING, GFP_KERNEL, 0, NULL); 1867 call_netdevice_notifiers(NETDEV_DOWN, dev); 1868 if (unlink) 1869 list_del_init(&dev->close_list); 1870 } 1871 } 1872 EXPORT_SYMBOL_NS_GPL(netif_close_many, "NETDEV_INTERNAL"); 1873 1874 void netif_close(struct net_device *dev) 1875 { 1876 if (dev->flags & IFF_UP) { 1877 LIST_HEAD(single); 1878 1879 list_add(&dev->close_list, &single); 1880 netif_close_many(&single, true); 1881 list_del(&single); 1882 } 1883 } 1884 EXPORT_SYMBOL(netif_close); 1885 1886 void netif_disable_lro(struct net_device *dev) 1887 { 1888 struct net_device *lower_dev; 1889 struct list_head *iter; 1890 1891 dev->wanted_features &= ~NETIF_F_LRO; 1892 netdev_update_features(dev); 1893 1894 if (unlikely(dev->features & NETIF_F_LRO)) 1895 netdev_WARN(dev, "failed to disable LRO!\n"); 1896 1897 netdev_for_each_lower_dev(dev, lower_dev, iter) { 1898 netdev_lock_ops(lower_dev); 1899 netif_disable_lro(lower_dev); 1900 netdev_unlock_ops(lower_dev); 1901 } 1902 } 1903 1904 /** 1905 * dev_disable_gro_hw - disable HW Generic Receive Offload on a device 1906 * @dev: device 1907 * 1908 * Disable HW Generic Receive Offload (GRO_HW) on a net device. Must be 1909 * called under RTNL. This is needed if Generic XDP is installed on 1910 * the device. 1911 */ 1912 static void dev_disable_gro_hw(struct net_device *dev) 1913 { 1914 dev->wanted_features &= ~NETIF_F_GRO_HW; 1915 netdev_update_features(dev); 1916 1917 if (unlikely(dev->features & NETIF_F_GRO_HW)) 1918 netdev_WARN(dev, "failed to disable GRO_HW!\n"); 1919 } 1920 1921 const char *netdev_cmd_to_name(enum netdev_cmd cmd) 1922 { 1923 #define N(val) \ 1924 case NETDEV_##val: \ 1925 return "NETDEV_" __stringify(val); 1926 switch (cmd) { 1927 N(UP) N(DOWN) N(REBOOT) N(CHANGE) N(REGISTER) N(UNREGISTER) 1928 N(CHANGEMTU) N(CHANGEADDR) N(GOING_DOWN) N(CHANGENAME) N(FEAT_CHANGE) 1929 N(BONDING_FAILOVER) N(PRE_UP) N(PRE_TYPE_CHANGE) N(POST_TYPE_CHANGE) 1930 N(POST_INIT) N(PRE_UNINIT) N(RELEASE) N(NOTIFY_PEERS) N(JOIN) 1931 N(CHANGEUPPER) N(RESEND_IGMP) N(PRECHANGEMTU) N(CHANGEINFODATA) 1932 N(BONDING_INFO) N(PRECHANGEUPPER) N(CHANGELOWERSTATE) 1933 N(UDP_TUNNEL_PUSH_INFO) N(UDP_TUNNEL_DROP_INFO) N(CHANGE_TX_QUEUE_LEN) 1934 N(CVLAN_FILTER_PUSH_INFO) N(CVLAN_FILTER_DROP_INFO) 1935 N(SVLAN_FILTER_PUSH_INFO) N(SVLAN_FILTER_DROP_INFO) 1936 N(PRE_CHANGEADDR) N(OFFLOAD_XSTATS_ENABLE) N(OFFLOAD_XSTATS_DISABLE) 1937 N(OFFLOAD_XSTATS_REPORT_USED) N(OFFLOAD_XSTATS_REPORT_DELTA) 1938 N(XDP_FEAT_CHANGE) 1939 } 1940 #undef N 1941 return "UNKNOWN_NETDEV_EVENT"; 1942 } 1943 EXPORT_SYMBOL_GPL(netdev_cmd_to_name); 1944 1945 static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val, 1946 struct net_device *dev) 1947 { 1948 struct netdev_notifier_info info = { 1949 .dev = dev, 1950 }; 1951 1952 return nb->notifier_call(nb, val, &info); 1953 } 1954 1955 static int call_netdevice_register_notifiers(struct notifier_block *nb, 1956 struct net_device *dev) 1957 { 1958 int err; 1959 1960 err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev); 1961 err = notifier_to_errno(err); 1962 if (err) 1963 return err; 1964 1965 if (!(dev->flags & IFF_UP)) 1966 return 0; 1967 1968 call_netdevice_notifier(nb, NETDEV_UP, dev); 1969 return 0; 1970 } 1971 1972 static void call_netdevice_unregister_notifiers(struct notifier_block *nb, 1973 struct net_device *dev) 1974 { 1975 if (dev->flags & IFF_UP) { 1976 netdev_lock_ops(dev); 1977 call_netdevice_notifier(nb, NETDEV_GOING_DOWN, 1978 dev); 1979 call_netdevice_notifier(nb, NETDEV_DOWN, dev); 1980 netdev_unlock_ops(dev); 1981 } 1982 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev); 1983 } 1984 1985 static int call_netdevice_register_net_notifiers(struct notifier_block *nb, 1986 struct net *net) 1987 { 1988 struct net_device *dev; 1989 int err; 1990 1991 for_each_netdev(net, dev) { 1992 netdev_lock_ops(dev); 1993 err = call_netdevice_register_notifiers(nb, dev); 1994 netdev_unlock_ops(dev); 1995 if (err) 1996 goto rollback; 1997 } 1998 return 0; 1999 2000 rollback: 2001 for_each_netdev_continue_reverse(net, dev) 2002 call_netdevice_unregister_notifiers(nb, dev); 2003 return err; 2004 } 2005 2006 static void call_netdevice_unregister_net_notifiers(struct notifier_block *nb, 2007 struct net *net) 2008 { 2009 struct net_device *dev; 2010 2011 for_each_netdev(net, dev) 2012 call_netdevice_unregister_notifiers(nb, dev); 2013 } 2014 2015 static int dev_boot_phase = 1; 2016 2017 /** 2018 * register_netdevice_notifier - register a network notifier block 2019 * @nb: notifier 2020 * 2021 * Register a notifier to be called when network device events occur. 2022 * The notifier passed is linked into the kernel structures and must 2023 * not be reused until it has been unregistered. A negative errno code 2024 * is returned on a failure. 2025 * 2026 * When registered all registration and up events are replayed 2027 * to the new notifier to allow device to have a race free 2028 * view of the network device list. 2029 */ 2030 2031 int register_netdevice_notifier(struct notifier_block *nb) 2032 { 2033 struct net *net; 2034 int err; 2035 2036 /* Close race with setup_net() and cleanup_net() */ 2037 down_write(&pernet_ops_rwsem); 2038 2039 /* When RTNL is removed, we need protection for netdev_chain. */ 2040 rtnl_lock(); 2041 2042 err = raw_notifier_chain_register(&netdev_chain, nb); 2043 if (err) 2044 goto unlock; 2045 if (dev_boot_phase) 2046 goto unlock; 2047 for_each_net(net) { 2048 __rtnl_net_lock(net); 2049 err = call_netdevice_register_net_notifiers(nb, net); 2050 __rtnl_net_unlock(net); 2051 if (err) 2052 goto rollback; 2053 } 2054 2055 unlock: 2056 rtnl_unlock(); 2057 up_write(&pernet_ops_rwsem); 2058 return err; 2059 2060 rollback: 2061 for_each_net_continue_reverse(net) { 2062 __rtnl_net_lock(net); 2063 call_netdevice_unregister_net_notifiers(nb, net); 2064 __rtnl_net_unlock(net); 2065 } 2066 2067 raw_notifier_chain_unregister(&netdev_chain, nb); 2068 goto unlock; 2069 } 2070 EXPORT_SYMBOL(register_netdevice_notifier); 2071 2072 /** 2073 * unregister_netdevice_notifier - unregister a network notifier block 2074 * @nb: notifier 2075 * 2076 * Unregister a notifier previously registered by 2077 * register_netdevice_notifier(). The notifier is unlinked into the 2078 * kernel structures and may then be reused. A negative errno code 2079 * is returned on a failure. 2080 * 2081 * After unregistering unregister and down device events are synthesized 2082 * for all devices on the device list to the removed notifier to remove 2083 * the need for special case cleanup code. 2084 */ 2085 2086 int unregister_netdevice_notifier(struct notifier_block *nb) 2087 { 2088 struct net *net; 2089 int err; 2090 2091 /* Close race with setup_net() and cleanup_net() */ 2092 down_write(&pernet_ops_rwsem); 2093 rtnl_lock(); 2094 err = raw_notifier_chain_unregister(&netdev_chain, nb); 2095 if (err) 2096 goto unlock; 2097 2098 for_each_net(net) { 2099 __rtnl_net_lock(net); 2100 call_netdevice_unregister_net_notifiers(nb, net); 2101 __rtnl_net_unlock(net); 2102 } 2103 2104 unlock: 2105 rtnl_unlock(); 2106 up_write(&pernet_ops_rwsem); 2107 return err; 2108 } 2109 EXPORT_SYMBOL(unregister_netdevice_notifier); 2110 2111 static int __register_netdevice_notifier_net(struct net *net, 2112 struct notifier_block *nb, 2113 bool ignore_call_fail) 2114 { 2115 int err; 2116 2117 err = raw_notifier_chain_register(&net->netdev_chain, nb); 2118 if (err) 2119 return err; 2120 if (dev_boot_phase) 2121 return 0; 2122 2123 err = call_netdevice_register_net_notifiers(nb, net); 2124 if (err && !ignore_call_fail) 2125 goto chain_unregister; 2126 2127 return 0; 2128 2129 chain_unregister: 2130 raw_notifier_chain_unregister(&net->netdev_chain, nb); 2131 return err; 2132 } 2133 2134 static int __unregister_netdevice_notifier_net(struct net *net, 2135 struct notifier_block *nb) 2136 { 2137 int err; 2138 2139 err = raw_notifier_chain_unregister(&net->netdev_chain, nb); 2140 if (err) 2141 return err; 2142 2143 call_netdevice_unregister_net_notifiers(nb, net); 2144 return 0; 2145 } 2146 2147 /** 2148 * register_netdevice_notifier_net - register a per-netns network notifier block 2149 * @net: network namespace 2150 * @nb: notifier 2151 * 2152 * Register a notifier to be called when network device events occur. 2153 * The notifier passed is linked into the kernel structures and must 2154 * not be reused until it has been unregistered. A negative errno code 2155 * is returned on a failure. 2156 * 2157 * When registered all registration and up events are replayed 2158 * to the new notifier to allow device to have a race free 2159 * view of the network device list. 2160 */ 2161 2162 int register_netdevice_notifier_net(struct net *net, struct notifier_block *nb) 2163 { 2164 int err; 2165 2166 rtnl_net_lock(net); 2167 err = __register_netdevice_notifier_net(net, nb, false); 2168 rtnl_net_unlock(net); 2169 2170 return err; 2171 } 2172 EXPORT_SYMBOL(register_netdevice_notifier_net); 2173 2174 /** 2175 * unregister_netdevice_notifier_net - unregister a per-netns 2176 * network notifier block 2177 * @net: network namespace 2178 * @nb: notifier 2179 * 2180 * Unregister a notifier previously registered by 2181 * register_netdevice_notifier_net(). The notifier is unlinked from the 2182 * kernel structures and may then be reused. A negative errno code 2183 * is returned on a failure. 2184 * 2185 * After unregistering unregister and down device events are synthesized 2186 * for all devices on the device list to the removed notifier to remove 2187 * the need for special case cleanup code. 2188 */ 2189 2190 int unregister_netdevice_notifier_net(struct net *net, 2191 struct notifier_block *nb) 2192 { 2193 int err; 2194 2195 rtnl_net_lock(net); 2196 err = __unregister_netdevice_notifier_net(net, nb); 2197 rtnl_net_unlock(net); 2198 2199 return err; 2200 } 2201 EXPORT_SYMBOL(unregister_netdevice_notifier_net); 2202 2203 static void __move_netdevice_notifier_net(struct net *src_net, 2204 struct net *dst_net, 2205 struct notifier_block *nb) 2206 { 2207 __unregister_netdevice_notifier_net(src_net, nb); 2208 __register_netdevice_notifier_net(dst_net, nb, true); 2209 } 2210 2211 static void rtnl_net_dev_lock(struct net_device *dev) 2212 { 2213 bool again; 2214 2215 do { 2216 struct net *net; 2217 2218 again = false; 2219 2220 /* netns might be being dismantled. */ 2221 rcu_read_lock(); 2222 net = dev_net_rcu(dev); 2223 net_passive_inc(net); 2224 rcu_read_unlock(); 2225 2226 rtnl_net_lock(net); 2227 2228 #ifdef CONFIG_NET_NS 2229 /* dev might have been moved to another netns. */ 2230 if (!net_eq(net, rcu_access_pointer(dev->nd_net.net))) { 2231 rtnl_net_unlock(net); 2232 net_passive_dec(net); 2233 again = true; 2234 } 2235 #endif 2236 } while (again); 2237 } 2238 2239 static void rtnl_net_dev_unlock(struct net_device *dev) 2240 { 2241 struct net *net = dev_net(dev); 2242 2243 rtnl_net_unlock(net); 2244 net_passive_dec(net); 2245 } 2246 2247 int register_netdevice_notifier_dev_net(struct net_device *dev, 2248 struct notifier_block *nb, 2249 struct netdev_net_notifier *nn) 2250 { 2251 int err; 2252 2253 rtnl_net_dev_lock(dev); 2254 err = __register_netdevice_notifier_net(dev_net(dev), nb, false); 2255 if (!err) { 2256 nn->nb = nb; 2257 list_add(&nn->list, &dev->net_notifier_list); 2258 } 2259 rtnl_net_dev_unlock(dev); 2260 2261 return err; 2262 } 2263 EXPORT_SYMBOL(register_netdevice_notifier_dev_net); 2264 2265 int unregister_netdevice_notifier_dev_net(struct net_device *dev, 2266 struct notifier_block *nb, 2267 struct netdev_net_notifier *nn) 2268 { 2269 int err; 2270 2271 rtnl_net_dev_lock(dev); 2272 list_del(&nn->list); 2273 err = __unregister_netdevice_notifier_net(dev_net(dev), nb); 2274 rtnl_net_dev_unlock(dev); 2275 2276 return err; 2277 } 2278 EXPORT_SYMBOL(unregister_netdevice_notifier_dev_net); 2279 2280 static void move_netdevice_notifiers_dev_net(struct net_device *dev, 2281 struct net *net) 2282 { 2283 struct netdev_net_notifier *nn; 2284 2285 list_for_each_entry(nn, &dev->net_notifier_list, list) 2286 __move_netdevice_notifier_net(dev_net(dev), net, nn->nb); 2287 } 2288 2289 /** 2290 * call_netdevice_notifiers_info - call all network notifier blocks 2291 * @val: value passed unmodified to notifier function 2292 * @info: notifier information data 2293 * 2294 * Call all network notifier blocks. Parameters and return value 2295 * are as for raw_notifier_call_chain(). 2296 */ 2297 2298 int call_netdevice_notifiers_info(unsigned long val, 2299 struct netdev_notifier_info *info) 2300 { 2301 struct net *net = dev_net(info->dev); 2302 int ret; 2303 2304 ASSERT_RTNL(); 2305 2306 /* Run per-netns notifier block chain first, then run the global one. 2307 * Hopefully, one day, the global one is going to be removed after 2308 * all notifier block registrators get converted to be per-netns. 2309 */ 2310 ret = raw_notifier_call_chain(&net->netdev_chain, val, info); 2311 if (ret & NOTIFY_STOP_MASK) 2312 return ret; 2313 return raw_notifier_call_chain(&netdev_chain, val, info); 2314 } 2315 2316 /** 2317 * call_netdevice_notifiers_info_robust - call per-netns notifier blocks 2318 * for and rollback on error 2319 * @val_up: value passed unmodified to notifier function 2320 * @val_down: value passed unmodified to the notifier function when 2321 * recovering from an error on @val_up 2322 * @info: notifier information data 2323 * 2324 * Call all per-netns network notifier blocks, but not notifier blocks on 2325 * the global notifier chain. Parameters and return value are as for 2326 * raw_notifier_call_chain_robust(). 2327 */ 2328 2329 static int 2330 call_netdevice_notifiers_info_robust(unsigned long val_up, 2331 unsigned long val_down, 2332 struct netdev_notifier_info *info) 2333 { 2334 struct net *net = dev_net(info->dev); 2335 2336 ASSERT_RTNL(); 2337 2338 return raw_notifier_call_chain_robust(&net->netdev_chain, 2339 val_up, val_down, info); 2340 } 2341 2342 static int call_netdevice_notifiers_extack(unsigned long val, 2343 struct net_device *dev, 2344 struct netlink_ext_ack *extack) 2345 { 2346 struct netdev_notifier_info info = { 2347 .dev = dev, 2348 .extack = extack, 2349 }; 2350 2351 return call_netdevice_notifiers_info(val, &info); 2352 } 2353 2354 /** 2355 * call_netdevice_notifiers - call all network notifier blocks 2356 * @val: value passed unmodified to notifier function 2357 * @dev: net_device pointer passed unmodified to notifier function 2358 * 2359 * Call all network notifier blocks. Parameters and return value 2360 * are as for raw_notifier_call_chain(). 2361 */ 2362 2363 int call_netdevice_notifiers(unsigned long val, struct net_device *dev) 2364 { 2365 return call_netdevice_notifiers_extack(val, dev, NULL); 2366 } 2367 EXPORT_SYMBOL(call_netdevice_notifiers); 2368 2369 /** 2370 * call_netdevice_notifiers_mtu - call all network notifier blocks 2371 * @val: value passed unmodified to notifier function 2372 * @dev: net_device pointer passed unmodified to notifier function 2373 * @arg: additional u32 argument passed to the notifier function 2374 * 2375 * Call all network notifier blocks. Parameters and return value 2376 * are as for raw_notifier_call_chain(). 2377 */ 2378 static int call_netdevice_notifiers_mtu(unsigned long val, 2379 struct net_device *dev, u32 arg) 2380 { 2381 struct netdev_notifier_info_ext info = { 2382 .info.dev = dev, 2383 .ext.mtu = arg, 2384 }; 2385 2386 BUILD_BUG_ON(offsetof(struct netdev_notifier_info_ext, info) != 0); 2387 2388 return call_netdevice_notifiers_info(val, &info.info); 2389 } 2390 2391 #ifdef CONFIG_NET_INGRESS 2392 static DEFINE_STATIC_KEY_FALSE(ingress_needed_key); 2393 2394 void net_inc_ingress_queue(void) 2395 { 2396 static_branch_inc(&ingress_needed_key); 2397 } 2398 EXPORT_SYMBOL_GPL(net_inc_ingress_queue); 2399 2400 void net_dec_ingress_queue(void) 2401 { 2402 static_branch_dec(&ingress_needed_key); 2403 } 2404 EXPORT_SYMBOL_GPL(net_dec_ingress_queue); 2405 #endif 2406 2407 #ifdef CONFIG_NET_EGRESS 2408 static DEFINE_STATIC_KEY_FALSE(egress_needed_key); 2409 2410 void net_inc_egress_queue(void) 2411 { 2412 static_branch_inc(&egress_needed_key); 2413 } 2414 EXPORT_SYMBOL_GPL(net_inc_egress_queue); 2415 2416 void net_dec_egress_queue(void) 2417 { 2418 static_branch_dec(&egress_needed_key); 2419 } 2420 EXPORT_SYMBOL_GPL(net_dec_egress_queue); 2421 #endif 2422 2423 #ifdef CONFIG_NET_CLS_ACT 2424 DEFINE_STATIC_KEY_FALSE(tcf_sw_enabled_key); 2425 EXPORT_SYMBOL(tcf_sw_enabled_key); 2426 #endif 2427 2428 DEFINE_STATIC_KEY_FALSE(netstamp_needed_key); 2429 EXPORT_SYMBOL(netstamp_needed_key); 2430 #ifdef CONFIG_JUMP_LABEL 2431 static atomic_t netstamp_needed_deferred; 2432 static atomic_t netstamp_wanted; 2433 static void netstamp_clear(struct work_struct *work) 2434 { 2435 int deferred = atomic_xchg(&netstamp_needed_deferred, 0); 2436 int wanted; 2437 2438 wanted = atomic_add_return(deferred, &netstamp_wanted); 2439 if (wanted > 0) 2440 static_branch_enable(&netstamp_needed_key); 2441 else 2442 static_branch_disable(&netstamp_needed_key); 2443 } 2444 static DECLARE_WORK(netstamp_work, netstamp_clear); 2445 #endif 2446 2447 void net_enable_timestamp(void) 2448 { 2449 #ifdef CONFIG_JUMP_LABEL 2450 int wanted = atomic_read(&netstamp_wanted); 2451 2452 while (wanted > 0) { 2453 if (atomic_try_cmpxchg(&netstamp_wanted, &wanted, wanted + 1)) 2454 return; 2455 } 2456 atomic_inc(&netstamp_needed_deferred); 2457 schedule_work(&netstamp_work); 2458 #else 2459 static_branch_inc(&netstamp_needed_key); 2460 #endif 2461 } 2462 EXPORT_SYMBOL(net_enable_timestamp); 2463 2464 void net_disable_timestamp(void) 2465 { 2466 #ifdef CONFIG_JUMP_LABEL 2467 int wanted = atomic_read(&netstamp_wanted); 2468 2469 while (wanted > 1) { 2470 if (atomic_try_cmpxchg(&netstamp_wanted, &wanted, wanted - 1)) 2471 return; 2472 } 2473 atomic_dec(&netstamp_needed_deferred); 2474 schedule_work(&netstamp_work); 2475 #else 2476 static_branch_dec(&netstamp_needed_key); 2477 #endif 2478 } 2479 EXPORT_SYMBOL(net_disable_timestamp); 2480 2481 static inline void net_timestamp_set(struct sk_buff *skb) 2482 { 2483 skb->tstamp = 0; 2484 skb->tstamp_type = SKB_CLOCK_REALTIME; 2485 if (static_branch_unlikely(&netstamp_needed_key)) 2486 skb->tstamp = ktime_get_real(); 2487 } 2488 2489 #define net_timestamp_check(COND, SKB) \ 2490 if (static_branch_unlikely(&netstamp_needed_key)) { \ 2491 if ((COND) && !(SKB)->tstamp) \ 2492 (SKB)->tstamp = ktime_get_real(); \ 2493 } \ 2494 2495 bool is_skb_forwardable(const struct net_device *dev, const struct sk_buff *skb) 2496 { 2497 return __is_skb_forwardable(dev, skb, true); 2498 } 2499 EXPORT_SYMBOL_GPL(is_skb_forwardable); 2500 2501 static int __dev_forward_skb2(struct net_device *dev, struct sk_buff *skb, 2502 bool check_mtu) 2503 { 2504 enum skb_drop_reason reason; 2505 int ret; 2506 2507 ret = ____dev_forward_skb(dev, skb, check_mtu); 2508 if (unlikely(ret)) 2509 return ret; 2510 2511 reason = eth_type_trans_careful(skb, dev); 2512 if (unlikely(reason)) { 2513 dev_core_stats_rx_dropped_inc(dev); 2514 kfree_skb_reason(skb, reason); 2515 return NET_RX_DROP; 2516 } 2517 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN); 2518 2519 return 0; 2520 } 2521 2522 int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb) 2523 { 2524 return __dev_forward_skb2(dev, skb, true); 2525 } 2526 EXPORT_SYMBOL_GPL(__dev_forward_skb); 2527 2528 /** 2529 * dev_forward_skb - loopback an skb to another netif 2530 * 2531 * @dev: destination network device 2532 * @skb: buffer to forward 2533 * 2534 * return values: 2535 * NET_RX_SUCCESS (no congestion) 2536 * NET_RX_DROP (packet was dropped, but freed) 2537 * 2538 * dev_forward_skb can be used for injecting an skb from the 2539 * start_xmit function of one device into the receive queue 2540 * of another device. 2541 * 2542 * The receiving device may be in another namespace, so 2543 * we have to clear all information in the skb that could 2544 * impact namespace isolation. 2545 */ 2546 int dev_forward_skb(struct net_device *dev, struct sk_buff *skb) 2547 { 2548 return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb); 2549 } 2550 EXPORT_SYMBOL_GPL(dev_forward_skb); 2551 2552 int dev_forward_skb_nomtu(struct net_device *dev, struct sk_buff *skb) 2553 { 2554 return __dev_forward_skb2(dev, skb, false) ?: netif_rx_internal(skb); 2555 } 2556 2557 static int deliver_skb(struct sk_buff *skb, 2558 struct packet_type *pt_prev, 2559 struct net_device *orig_dev) 2560 { 2561 if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC))) 2562 return -ENOMEM; 2563 refcount_inc(&skb->users); 2564 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev); 2565 } 2566 2567 static inline void deliver_ptype_list_skb(struct sk_buff *skb, 2568 struct packet_type **pt, 2569 struct net_device *orig_dev, 2570 __be16 type, 2571 struct list_head *ptype_list) 2572 { 2573 struct packet_type *ptype, *pt_prev = *pt; 2574 2575 list_for_each_entry_rcu(ptype, ptype_list, list) { 2576 if (ptype->type != type) 2577 continue; 2578 if (unlikely(pt_prev)) 2579 deliver_skb(skb, pt_prev, orig_dev); 2580 pt_prev = ptype; 2581 } 2582 *pt = pt_prev; 2583 } 2584 2585 static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb) 2586 { 2587 if (!ptype->af_packet_priv || !skb->sk) 2588 return false; 2589 2590 if (ptype->id_match) 2591 return ptype->id_match(ptype, skb->sk); 2592 else if ((struct sock *)ptype->af_packet_priv == skb->sk) 2593 return true; 2594 2595 return false; 2596 } 2597 2598 /** 2599 * dev_nit_active_rcu - return true if any network interface taps are in use 2600 * 2601 * The caller must hold the RCU lock 2602 * 2603 * @dev: network device to check for the presence of taps 2604 */ 2605 bool dev_nit_active_rcu(const struct net_device *dev) 2606 { 2607 /* Callers may hold either RCU or RCU BH lock */ 2608 WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held()); 2609 2610 return !list_empty(&dev_net(dev)->ptype_all) || 2611 !list_empty(&dev->ptype_all); 2612 } 2613 EXPORT_SYMBOL_GPL(dev_nit_active_rcu); 2614 2615 /* 2616 * Support routine. Sends outgoing frames to any network 2617 * taps currently in use. 2618 */ 2619 2620 void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev) 2621 { 2622 struct packet_type *ptype, *pt_prev = NULL; 2623 struct list_head *ptype_list; 2624 struct sk_buff *skb2 = NULL; 2625 2626 rcu_read_lock(); 2627 ptype_list = &dev_net_rcu(dev)->ptype_all; 2628 again: 2629 list_for_each_entry_rcu(ptype, ptype_list, list) { 2630 if (READ_ONCE(ptype->ignore_outgoing)) 2631 continue; 2632 2633 /* Never send packets back to the socket 2634 * they originated from - MvS (miquels@drinkel.ow.org) 2635 */ 2636 if (skb_loop_sk(ptype, skb)) 2637 continue; 2638 2639 if (unlikely(pt_prev)) { 2640 deliver_skb(skb2, pt_prev, skb->dev); 2641 pt_prev = ptype; 2642 continue; 2643 } 2644 2645 /* need to clone skb, done only once */ 2646 skb2 = skb_clone(skb, GFP_ATOMIC); 2647 if (!skb2) 2648 goto out_unlock; 2649 2650 net_timestamp_set(skb2); 2651 2652 /* skb->nh should be correctly 2653 * set by sender, so that the second statement is 2654 * just protection against buggy protocols. 2655 */ 2656 skb_reset_mac_header(skb2); 2657 2658 if (skb_network_header(skb2) < skb2->data || 2659 skb_network_header(skb2) > skb_tail_pointer(skb2)) { 2660 net_crit_ratelimited("protocol %04x is buggy, dev %s\n", 2661 ntohs(skb2->protocol), 2662 dev->name); 2663 skb_reset_network_header(skb2); 2664 } 2665 2666 skb2->transport_header = skb2->network_header; 2667 skb2->pkt_type = PACKET_OUTGOING; 2668 pt_prev = ptype; 2669 } 2670 2671 if (ptype_list != &dev->ptype_all) { 2672 ptype_list = &dev->ptype_all; 2673 goto again; 2674 } 2675 out_unlock: 2676 if (pt_prev) { 2677 if (!skb_orphan_frags_rx(skb2, GFP_ATOMIC)) 2678 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev); 2679 else 2680 kfree_skb(skb2); 2681 } 2682 rcu_read_unlock(); 2683 } 2684 EXPORT_SYMBOL_GPL(dev_queue_xmit_nit); 2685 2686 /** 2687 * netif_setup_tc - Handle tc mappings on real_num_tx_queues change 2688 * @dev: Network device 2689 * @txq: number of queues available 2690 * 2691 * If real_num_tx_queues is changed the tc mappings may no longer be 2692 * valid. To resolve this verify the tc mapping remains valid and if 2693 * not NULL the mapping. With no priorities mapping to this 2694 * offset/count pair it will no longer be used. In the worst case TC0 2695 * is invalid nothing can be done so disable priority mappings. If is 2696 * expected that drivers will fix this mapping if they can before 2697 * calling netif_set_real_num_tx_queues. 2698 */ 2699 static void netif_setup_tc(struct net_device *dev, unsigned int txq) 2700 { 2701 struct netdev_tc_txq res; 2702 int i; 2703 2704 res.combined = READ_ONCE(dev->tc_to_txq[0].combined); 2705 2706 /* If TC0 is invalidated disable TC mapping */ 2707 if (res.offset + res.count > txq) { 2708 netdev_warn(dev, "Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n"); 2709 WRITE_ONCE(dev->num_tc, 0); 2710 return; 2711 } 2712 2713 /* Invalidated prio to tc mappings set to TC0 */ 2714 for (i = 1; i < TC_BITMASK + 1; i++) { 2715 int q = netdev_get_prio_tc_map(dev, i); 2716 2717 res.combined = READ_ONCE(dev->tc_to_txq[q].combined); 2718 if (res.offset + res.count > txq) { 2719 netdev_warn(dev, "Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n", 2720 i, q); 2721 netdev_set_prio_tc_map(dev, i, 0); 2722 } 2723 } 2724 } 2725 2726 int netdev_txq_to_tc(struct net_device *dev, unsigned int txq) 2727 { 2728 if (READ_ONCE(dev->num_tc)) { 2729 struct netdev_tc_txq *tc = &dev->tc_to_txq[0]; 2730 int i; 2731 2732 /* walk through the TCs and see if it falls into any of them */ 2733 for (i = 0; i < TC_MAX_QUEUE; i++, tc++) { 2734 struct netdev_tc_txq res; 2735 2736 res.combined = READ_ONCE(tc->combined); 2737 if ((txq - res.offset) < res.count) 2738 return i; 2739 } 2740 2741 /* didn't find it, just return -1 to indicate no match */ 2742 return -1; 2743 } 2744 2745 return 0; 2746 } 2747 EXPORT_SYMBOL(netdev_txq_to_tc); 2748 2749 #ifdef CONFIG_XPS 2750 static struct static_key xps_needed __read_mostly; 2751 static struct static_key xps_rxqs_needed __read_mostly; 2752 static DEFINE_MUTEX(xps_map_mutex); 2753 #define xmap_dereference(P) \ 2754 rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex)) 2755 2756 static bool remove_xps_queue(struct xps_dev_maps *dev_maps, 2757 struct xps_dev_maps *old_maps, int tci, u16 index) 2758 { 2759 struct xps_map *map = NULL; 2760 int pos; 2761 2762 map = xmap_dereference(dev_maps->attr_map[tci]); 2763 if (!map) 2764 return false; 2765 2766 for (pos = map->len; pos--;) { 2767 if (map->queues[pos] != index) 2768 continue; 2769 2770 if (map->len > 1) { 2771 map->queues[pos] = map->queues[--map->len]; 2772 break; 2773 } 2774 2775 if (old_maps) 2776 RCU_INIT_POINTER(old_maps->attr_map[tci], NULL); 2777 RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL); 2778 kfree_rcu(map, rcu); 2779 return false; 2780 } 2781 2782 return true; 2783 } 2784 2785 static bool remove_xps_queue_cpu(struct net_device *dev, 2786 struct xps_dev_maps *dev_maps, 2787 int cpu, u16 offset, u16 count) 2788 { 2789 int num_tc = dev_maps->num_tc; 2790 bool active = false; 2791 int tci; 2792 2793 for (tci = cpu * num_tc; num_tc--; tci++) { 2794 int i, j; 2795 2796 for (i = count, j = offset; i--; j++) { 2797 if (!remove_xps_queue(dev_maps, NULL, tci, j)) 2798 break; 2799 } 2800 2801 active |= i < 0; 2802 } 2803 2804 return active; 2805 } 2806 2807 static void reset_xps_maps(struct net_device *dev, 2808 struct xps_dev_maps *dev_maps, 2809 enum xps_map_type type) 2810 { 2811 static_key_slow_dec_cpuslocked(&xps_needed); 2812 if (type == XPS_RXQS) 2813 static_key_slow_dec_cpuslocked(&xps_rxqs_needed); 2814 2815 RCU_INIT_POINTER(dev->xps_maps[type], NULL); 2816 2817 kfree_rcu(dev_maps, rcu); 2818 } 2819 2820 static void clean_xps_maps(struct net_device *dev, enum xps_map_type type, 2821 u16 offset, u16 count) 2822 { 2823 struct xps_dev_maps *dev_maps; 2824 bool active = false; 2825 int i, j; 2826 2827 dev_maps = xmap_dereference(dev->xps_maps[type]); 2828 if (!dev_maps) 2829 return; 2830 2831 for (j = 0; j < dev_maps->nr_ids; j++) 2832 active |= remove_xps_queue_cpu(dev, dev_maps, j, offset, count); 2833 if (!active) 2834 reset_xps_maps(dev, dev_maps, type); 2835 2836 if (type == XPS_CPUS) { 2837 for (i = offset + (count - 1); count--; i--) 2838 netdev_queue_numa_node_write( 2839 netdev_get_tx_queue(dev, i), NUMA_NO_NODE); 2840 } 2841 } 2842 2843 static void netif_reset_xps_queues(struct net_device *dev, u16 offset, 2844 u16 count) 2845 { 2846 if (!static_key_false(&xps_needed)) 2847 return; 2848 2849 cpus_read_lock(); 2850 mutex_lock(&xps_map_mutex); 2851 2852 if (static_key_false(&xps_rxqs_needed)) 2853 clean_xps_maps(dev, XPS_RXQS, offset, count); 2854 2855 clean_xps_maps(dev, XPS_CPUS, offset, count); 2856 2857 mutex_unlock(&xps_map_mutex); 2858 cpus_read_unlock(); 2859 } 2860 2861 static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index) 2862 { 2863 netif_reset_xps_queues(dev, index, dev->num_tx_queues - index); 2864 } 2865 2866 static struct xps_map *expand_xps_map(struct xps_map *map, int attr_index, 2867 u16 index, bool is_rxqs_map) 2868 { 2869 struct xps_map *new_map; 2870 int alloc_len = XPS_MIN_MAP_ALLOC; 2871 int i, pos; 2872 2873 for (pos = 0; map && pos < map->len; pos++) { 2874 if (map->queues[pos] != index) 2875 continue; 2876 return map; 2877 } 2878 2879 /* Need to add tx-queue to this CPU's/rx-queue's existing map */ 2880 if (map) { 2881 if (pos < map->alloc_len) 2882 return map; 2883 2884 alloc_len = map->alloc_len * 2; 2885 } 2886 2887 /* Need to allocate new map to store tx-queue on this CPU's/rx-queue's 2888 * map 2889 */ 2890 if (is_rxqs_map) 2891 new_map = kzalloc(XPS_MAP_SIZE(alloc_len), GFP_KERNEL); 2892 else 2893 new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL, 2894 cpu_to_node(attr_index)); 2895 if (!new_map) 2896 return NULL; 2897 2898 for (i = 0; i < pos; i++) 2899 new_map->queues[i] = map->queues[i]; 2900 new_map->alloc_len = alloc_len; 2901 new_map->len = pos; 2902 2903 return new_map; 2904 } 2905 2906 /* Copy xps maps at a given index */ 2907 static void xps_copy_dev_maps(struct xps_dev_maps *dev_maps, 2908 struct xps_dev_maps *new_dev_maps, int index, 2909 int tc, bool skip_tc) 2910 { 2911 int i, tci = index * dev_maps->num_tc; 2912 struct xps_map *map; 2913 2914 /* copy maps belonging to foreign traffic classes */ 2915 for (i = 0; i < dev_maps->num_tc; i++, tci++) { 2916 if (i == tc && skip_tc) 2917 continue; 2918 2919 /* fill in the new device map from the old device map */ 2920 map = xmap_dereference(dev_maps->attr_map[tci]); 2921 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map); 2922 } 2923 } 2924 2925 /* Must be called under cpus_read_lock */ 2926 int __netif_set_xps_queue(struct net_device *dev, const unsigned long *mask, 2927 u16 index, enum xps_map_type type) 2928 { 2929 struct xps_dev_maps *dev_maps, *new_dev_maps = NULL, *old_dev_maps = NULL; 2930 int maps_sz, num_tc = 1, tc = 0, dev_num_tc; 2931 const unsigned long *online_mask = NULL; 2932 bool active = false, copy = false; 2933 int i, j, tci, numa_node_id = -2; 2934 struct xps_map *map, *new_map; 2935 unsigned int nr_ids; 2936 2937 WARN_ON_ONCE(index >= dev->num_tx_queues); 2938 2939 dev_num_tc = READ_ONCE(dev->num_tc); 2940 if (dev_num_tc) { 2941 /* Do not allow XPS on subordinate device directly */ 2942 num_tc = dev_num_tc; 2943 if (num_tc < 0) 2944 return -EINVAL; 2945 2946 /* If queue belongs to subordinate dev use its map */ 2947 dev = netdev_get_tx_queue(dev, index)->sb_dev ? : dev; 2948 2949 tc = netdev_txq_to_tc(dev, index); 2950 if (tc < 0 || tc >= num_tc) 2951 return -EINVAL; 2952 } 2953 2954 mutex_lock(&xps_map_mutex); 2955 2956 dev_maps = xmap_dereference(dev->xps_maps[type]); 2957 if (type == XPS_RXQS) { 2958 maps_sz = XPS_RXQ_DEV_MAPS_SIZE(num_tc, dev->num_rx_queues); 2959 nr_ids = dev->num_rx_queues; 2960 } else { 2961 maps_sz = XPS_CPU_DEV_MAPS_SIZE(num_tc); 2962 if (num_possible_cpus() > 1) 2963 online_mask = cpumask_bits(cpu_online_mask); 2964 nr_ids = nr_cpu_ids; 2965 } 2966 2967 if (maps_sz < L1_CACHE_BYTES) 2968 maps_sz = L1_CACHE_BYTES; 2969 2970 /* The old dev_maps could be larger or smaller than the one we're 2971 * setting up now, as dev->num_tc or nr_ids could have been updated in 2972 * between. We could try to be smart, but let's be safe instead and only 2973 * copy foreign traffic classes if the two map sizes match. 2974 */ 2975 if (dev_maps && 2976 dev_maps->num_tc == num_tc && dev_maps->nr_ids == nr_ids) 2977 copy = true; 2978 2979 /* allocate memory for queue storage */ 2980 for (j = -1; j = netif_attrmask_next_and(j, online_mask, mask, nr_ids), 2981 j < nr_ids;) { 2982 if (!new_dev_maps) { 2983 new_dev_maps = kzalloc(maps_sz, GFP_KERNEL); 2984 if (!new_dev_maps) { 2985 mutex_unlock(&xps_map_mutex); 2986 return -ENOMEM; 2987 } 2988 2989 new_dev_maps->nr_ids = nr_ids; 2990 new_dev_maps->num_tc = num_tc; 2991 } 2992 2993 tci = j * num_tc + tc; 2994 map = copy ? xmap_dereference(dev_maps->attr_map[tci]) : NULL; 2995 2996 map = expand_xps_map(map, j, index, type == XPS_RXQS); 2997 if (!map) 2998 goto error; 2999 3000 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map); 3001 } 3002 3003 if (!new_dev_maps) 3004 goto out_no_new_maps; 3005 3006 if (!dev_maps) { 3007 /* Increment static keys at most once per type */ 3008 static_key_slow_inc_cpuslocked(&xps_needed); 3009 if (type == XPS_RXQS) 3010 static_key_slow_inc_cpuslocked(&xps_rxqs_needed); 3011 } 3012 3013 for (j = 0; j < nr_ids; j++) { 3014 bool skip_tc = false; 3015 3016 tci = j * num_tc + tc; 3017 if (netif_attr_test_mask(j, mask, nr_ids) && 3018 netif_attr_test_online(j, online_mask, nr_ids)) { 3019 /* add tx-queue to CPU/rx-queue maps */ 3020 int pos = 0; 3021 3022 skip_tc = true; 3023 3024 map = xmap_dereference(new_dev_maps->attr_map[tci]); 3025 while ((pos < map->len) && (map->queues[pos] != index)) 3026 pos++; 3027 3028 if (pos == map->len) 3029 map->queues[map->len++] = index; 3030 #ifdef CONFIG_NUMA 3031 if (type == XPS_CPUS) { 3032 if (numa_node_id == -2) 3033 numa_node_id = cpu_to_node(j); 3034 else if (numa_node_id != cpu_to_node(j)) 3035 numa_node_id = -1; 3036 } 3037 #endif 3038 } 3039 3040 if (copy) 3041 xps_copy_dev_maps(dev_maps, new_dev_maps, j, tc, 3042 skip_tc); 3043 } 3044 3045 rcu_assign_pointer(dev->xps_maps[type], new_dev_maps); 3046 3047 /* Cleanup old maps */ 3048 if (!dev_maps) 3049 goto out_no_old_maps; 3050 3051 for (j = 0; j < dev_maps->nr_ids; j++) { 3052 for (i = dev_maps->num_tc, tci = j * dev_maps->num_tc; i--; tci++) { 3053 map = xmap_dereference(dev_maps->attr_map[tci]); 3054 if (!map) 3055 continue; 3056 3057 if (copy) { 3058 new_map = xmap_dereference(new_dev_maps->attr_map[tci]); 3059 if (map == new_map) 3060 continue; 3061 } 3062 3063 RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL); 3064 kfree_rcu(map, rcu); 3065 } 3066 } 3067 3068 old_dev_maps = dev_maps; 3069 3070 out_no_old_maps: 3071 dev_maps = new_dev_maps; 3072 active = true; 3073 3074 out_no_new_maps: 3075 if (type == XPS_CPUS) 3076 /* update Tx queue numa node */ 3077 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index), 3078 (numa_node_id >= 0) ? 3079 numa_node_id : NUMA_NO_NODE); 3080 3081 if (!dev_maps) 3082 goto out_no_maps; 3083 3084 /* removes tx-queue from unused CPUs/rx-queues */ 3085 for (j = 0; j < dev_maps->nr_ids; j++) { 3086 tci = j * dev_maps->num_tc; 3087 3088 for (i = 0; i < dev_maps->num_tc; i++, tci++) { 3089 if (i == tc && 3090 netif_attr_test_mask(j, mask, dev_maps->nr_ids) && 3091 netif_attr_test_online(j, online_mask, dev_maps->nr_ids)) 3092 continue; 3093 3094 active |= remove_xps_queue(dev_maps, 3095 copy ? old_dev_maps : NULL, 3096 tci, index); 3097 } 3098 } 3099 3100 if (old_dev_maps) 3101 kfree_rcu(old_dev_maps, rcu); 3102 3103 /* free map if not active */ 3104 if (!active) 3105 reset_xps_maps(dev, dev_maps, type); 3106 3107 out_no_maps: 3108 mutex_unlock(&xps_map_mutex); 3109 3110 return 0; 3111 error: 3112 /* remove any maps that we added */ 3113 for (j = 0; j < nr_ids; j++) { 3114 for (i = num_tc, tci = j * num_tc; i--; tci++) { 3115 new_map = xmap_dereference(new_dev_maps->attr_map[tci]); 3116 map = copy ? 3117 xmap_dereference(dev_maps->attr_map[tci]) : 3118 NULL; 3119 if (new_map && new_map != map) 3120 kfree(new_map); 3121 } 3122 } 3123 3124 mutex_unlock(&xps_map_mutex); 3125 3126 kfree(new_dev_maps); 3127 return -ENOMEM; 3128 } 3129 EXPORT_SYMBOL_GPL(__netif_set_xps_queue); 3130 3131 int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask, 3132 u16 index) 3133 { 3134 int ret; 3135 3136 cpus_read_lock(); 3137 ret = __netif_set_xps_queue(dev, cpumask_bits(mask), index, XPS_CPUS); 3138 cpus_read_unlock(); 3139 3140 return ret; 3141 } 3142 EXPORT_SYMBOL(netif_set_xps_queue); 3143 3144 #endif 3145 static void netdev_unbind_all_sb_channels(struct net_device *dev) 3146 { 3147 struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues]; 3148 3149 /* Unbind any subordinate channels */ 3150 while (txq-- != &dev->_tx[0]) { 3151 if (txq->sb_dev) 3152 netdev_unbind_sb_channel(dev, txq->sb_dev); 3153 } 3154 } 3155 3156 void netdev_reset_tc(struct net_device *dev) 3157 { 3158 int i; 3159 3160 #ifdef CONFIG_XPS 3161 netif_reset_xps_queues_gt(dev, 0); 3162 #endif 3163 netdev_unbind_all_sb_channels(dev); 3164 3165 /* Reset TC configuration of device */ 3166 WRITE_ONCE(dev->num_tc, 0); 3167 for (i = 0; i < TC_MAX_QUEUE; i++) 3168 WRITE_ONCE(dev->tc_to_txq[i].combined, 0); 3169 for (i = 0; i <= TC_BITMASK; i++) 3170 WRITE_ONCE(dev->prio_tc_map[i], 0); 3171 } 3172 EXPORT_SYMBOL(netdev_reset_tc); 3173 3174 int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset) 3175 { 3176 struct netdev_tc_txq res = { 3177 .count = count, 3178 .offset = offset, 3179 }; 3180 3181 if (tc >= READ_ONCE(dev->num_tc)) 3182 return -EINVAL; 3183 3184 #ifdef CONFIG_XPS 3185 netif_reset_xps_queues(dev, offset, count); 3186 #endif 3187 WRITE_ONCE(dev->tc_to_txq[tc].combined, res.combined); 3188 return 0; 3189 } 3190 EXPORT_SYMBOL(netdev_set_tc_queue); 3191 3192 int netdev_set_num_tc(struct net_device *dev, u8 num_tc) 3193 { 3194 if (num_tc > TC_MAX_QUEUE) 3195 return -EINVAL; 3196 3197 #ifdef CONFIG_XPS 3198 netif_reset_xps_queues_gt(dev, 0); 3199 #endif 3200 netdev_unbind_all_sb_channels(dev); 3201 3202 WRITE_ONCE(dev->num_tc, num_tc); 3203 return 0; 3204 } 3205 EXPORT_SYMBOL(netdev_set_num_tc); 3206 3207 void netdev_unbind_sb_channel(struct net_device *dev, 3208 struct net_device *sb_dev) 3209 { 3210 struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues]; 3211 int i; 3212 3213 #ifdef CONFIG_XPS 3214 netif_reset_xps_queues_gt(sb_dev, 0); 3215 #endif 3216 for (i = 0; i < TC_MAX_QUEUE; i++) 3217 WRITE_ONCE(sb_dev->tc_to_txq[i].combined, 0); 3218 for (i = 0; i <= TC_BITMASK; i++) 3219 WRITE_ONCE(sb_dev->prio_tc_map[i], 0); 3220 3221 while (txq-- != &dev->_tx[0]) { 3222 if (txq->sb_dev == sb_dev) 3223 txq->sb_dev = NULL; 3224 } 3225 } 3226 EXPORT_SYMBOL(netdev_unbind_sb_channel); 3227 3228 int netdev_bind_sb_channel_queue(struct net_device *dev, 3229 struct net_device *sb_dev, 3230 u8 tc, u16 count, u16 offset) 3231 { 3232 /* Make certain the sb_dev and dev are already configured */ 3233 if (READ_ONCE(sb_dev->num_tc) >= 0 || tc >= READ_ONCE(dev->num_tc)) 3234 return -EINVAL; 3235 3236 /* We cannot hand out queues we don't have */ 3237 if ((offset + count) > dev->real_num_tx_queues) 3238 return -EINVAL; 3239 3240 /* Record the mapping */ 3241 struct netdev_tc_txq res = { 3242 .count = count, 3243 .offset = offset, 3244 }; 3245 3246 WRITE_ONCE(sb_dev->tc_to_txq[tc].combined, res.combined); 3247 3248 /* Provide a way for Tx queue to find the tc_to_txq map or 3249 * XPS map for itself. 3250 */ 3251 while (count--) 3252 netdev_get_tx_queue(dev, count + offset)->sb_dev = sb_dev; 3253 3254 return 0; 3255 } 3256 EXPORT_SYMBOL(netdev_bind_sb_channel_queue); 3257 3258 int netdev_set_sb_channel(struct net_device *dev, u16 channel) 3259 { 3260 /* Do not use a multiqueue device to represent a subordinate channel */ 3261 if (netif_is_multiqueue(dev)) 3262 return -ENODEV; 3263 3264 /* We allow channels 1 - 32767 to be used for subordinate channels. 3265 * Channel 0 is meant to be "native" mode and used only to represent 3266 * the main root device. We allow writing 0 to reset the device back 3267 * to normal mode after being used as a subordinate channel. 3268 */ 3269 if (channel > S16_MAX) 3270 return -EINVAL; 3271 3272 WRITE_ONCE(dev->num_tc, -channel); 3273 3274 return 0; 3275 } 3276 EXPORT_SYMBOL(netdev_set_sb_channel); 3277 3278 /* 3279 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues 3280 * greater than real_num_tx_queues stale skbs on the qdisc must be flushed. 3281 */ 3282 int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq) 3283 { 3284 bool disabling; 3285 int rc; 3286 3287 disabling = txq < dev->real_num_tx_queues; 3288 3289 if (txq < 1 || txq > dev->num_tx_queues) 3290 return -EINVAL; 3291 3292 if (dev->reg_state == NETREG_REGISTERED || 3293 dev->reg_state == NETREG_UNREGISTERING) { 3294 netdev_assert_locked_ops_compat(dev); 3295 3296 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues, 3297 txq); 3298 if (rc) 3299 return rc; 3300 3301 if (READ_ONCE(dev->num_tc)) 3302 netif_setup_tc(dev, txq); 3303 3304 net_shaper_set_real_num_tx_queues(dev, txq); 3305 3306 dev_qdisc_change_real_num_tx(dev, txq); 3307 3308 dev->real_num_tx_queues = txq; 3309 3310 if (disabling) { 3311 synchronize_net(); 3312 qdisc_reset_all_tx_gt(dev, txq); 3313 #ifdef CONFIG_XPS 3314 netif_reset_xps_queues_gt(dev, txq); 3315 #endif 3316 } 3317 } else { 3318 dev->real_num_tx_queues = txq; 3319 } 3320 3321 return 0; 3322 } 3323 EXPORT_SYMBOL(netif_set_real_num_tx_queues); 3324 3325 /** 3326 * netif_set_real_num_rx_queues - set actual number of RX queues used 3327 * @dev: Network device 3328 * @rxq: Actual number of RX queues 3329 * 3330 * This must be called either with the rtnl_lock held or before 3331 * registration of the net device. Returns 0 on success, or a 3332 * negative error code. If called before registration, it always 3333 * succeeds. 3334 */ 3335 int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq) 3336 { 3337 int rc; 3338 3339 if (rxq < 1 || rxq > dev->num_rx_queues) 3340 return -EINVAL; 3341 3342 if (dev->reg_state == NETREG_REGISTERED) { 3343 netdev_assert_locked_ops_compat(dev); 3344 3345 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues, 3346 rxq); 3347 if (rc) 3348 return rc; 3349 } 3350 3351 dev->real_num_rx_queues = rxq; 3352 return 0; 3353 } 3354 EXPORT_SYMBOL(netif_set_real_num_rx_queues); 3355 3356 /** 3357 * netif_set_real_num_queues - set actual number of RX and TX queues used 3358 * @dev: Network device 3359 * @txq: Actual number of TX queues 3360 * @rxq: Actual number of RX queues 3361 * 3362 * Set the real number of both TX and RX queues. 3363 * Does nothing if the number of queues is already correct. 3364 */ 3365 int netif_set_real_num_queues(struct net_device *dev, 3366 unsigned int txq, unsigned int rxq) 3367 { 3368 unsigned int old_rxq = dev->real_num_rx_queues; 3369 int err; 3370 3371 if (txq < 1 || txq > dev->num_tx_queues || 3372 rxq < 1 || rxq > dev->num_rx_queues) 3373 return -EINVAL; 3374 3375 /* Start from increases, so the error path only does decreases - 3376 * decreases can't fail. 3377 */ 3378 if (rxq > dev->real_num_rx_queues) { 3379 err = netif_set_real_num_rx_queues(dev, rxq); 3380 if (err) 3381 return err; 3382 } 3383 if (txq > dev->real_num_tx_queues) { 3384 err = netif_set_real_num_tx_queues(dev, txq); 3385 if (err) 3386 goto undo_rx; 3387 } 3388 if (rxq < dev->real_num_rx_queues) 3389 WARN_ON(netif_set_real_num_rx_queues(dev, rxq)); 3390 if (txq < dev->real_num_tx_queues) 3391 WARN_ON(netif_set_real_num_tx_queues(dev, txq)); 3392 3393 return 0; 3394 undo_rx: 3395 WARN_ON(netif_set_real_num_rx_queues(dev, old_rxq)); 3396 return err; 3397 } 3398 EXPORT_SYMBOL(netif_set_real_num_queues); 3399 3400 /** 3401 * netif_set_tso_max_size() - set the max size of TSO frames supported 3402 * @dev: netdev to update 3403 * @size: max skb->len of a TSO frame 3404 * 3405 * Set the limit on the size of TSO super-frames the device can handle. 3406 * Unless explicitly set the stack will assume the value of 3407 * %GSO_LEGACY_MAX_SIZE. 3408 */ 3409 void netif_set_tso_max_size(struct net_device *dev, unsigned int size) 3410 { 3411 dev->tso_max_size = min(GSO_MAX_SIZE, size); 3412 if (size < READ_ONCE(dev->gso_max_size)) 3413 netif_set_gso_max_size(dev, size); 3414 if (size < READ_ONCE(dev->gso_ipv4_max_size)) 3415 netif_set_gso_ipv4_max_size(dev, size); 3416 } 3417 EXPORT_SYMBOL(netif_set_tso_max_size); 3418 3419 /** 3420 * netif_set_tso_max_segs() - set the max number of segs supported for TSO 3421 * @dev: netdev to update 3422 * @segs: max number of TCP segments 3423 * 3424 * Set the limit on the number of TCP segments the device can generate from 3425 * a single TSO super-frame. 3426 * Unless explicitly set the stack will assume the value of %GSO_MAX_SEGS. 3427 */ 3428 void netif_set_tso_max_segs(struct net_device *dev, unsigned int segs) 3429 { 3430 dev->tso_max_segs = segs; 3431 if (segs < READ_ONCE(dev->gso_max_segs)) 3432 netif_set_gso_max_segs(dev, segs); 3433 } 3434 EXPORT_SYMBOL(netif_set_tso_max_segs); 3435 3436 /** 3437 * netif_inherit_tso_max() - copy all TSO limits from a lower device to an upper 3438 * @to: netdev to update 3439 * @from: netdev from which to copy the limits 3440 */ 3441 void netif_inherit_tso_max(struct net_device *to, const struct net_device *from) 3442 { 3443 netif_set_tso_max_size(to, from->tso_max_size); 3444 netif_set_tso_max_segs(to, from->tso_max_segs); 3445 } 3446 EXPORT_SYMBOL(netif_inherit_tso_max); 3447 3448 /** 3449 * netif_get_num_default_rss_queues - default number of RSS queues 3450 * 3451 * Default value is the number of physical cores if there are only 1 or 2, or 3452 * divided by 2 if there are more. 3453 */ 3454 int netif_get_num_default_rss_queues(void) 3455 { 3456 cpumask_var_t cpus; 3457 int cpu, count = 0; 3458 3459 if (unlikely(is_kdump_kernel() || !zalloc_cpumask_var(&cpus, GFP_KERNEL))) 3460 return 1; 3461 3462 cpumask_copy(cpus, cpu_online_mask); 3463 for_each_cpu(cpu, cpus) { 3464 ++count; 3465 cpumask_andnot(cpus, cpus, topology_sibling_cpumask(cpu)); 3466 } 3467 free_cpumask_var(cpus); 3468 3469 return count > 2 ? DIV_ROUND_UP(count, 2) : count; 3470 } 3471 EXPORT_SYMBOL(netif_get_num_default_rss_queues); 3472 3473 static void __netif_reschedule(struct Qdisc *q) 3474 { 3475 struct softnet_data *sd; 3476 unsigned long flags; 3477 3478 local_irq_save(flags); 3479 sd = this_cpu_ptr(&softnet_data); 3480 q->next_sched = NULL; 3481 *sd->output_queue_tailp = q; 3482 sd->output_queue_tailp = &q->next_sched; 3483 raise_softirq_irqoff(NET_TX_SOFTIRQ); 3484 local_irq_restore(flags); 3485 } 3486 3487 void __netif_schedule(struct Qdisc *q) 3488 { 3489 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) 3490 return; 3491 3492 /* If q->defer_list is not empty, at least one thread is 3493 * in __dev_xmit_skb() before llist_del_all(&q->defer_list). 3494 * This thread will attempt to run the queue. 3495 */ 3496 if (!llist_empty(&q->defer_list)) 3497 return; 3498 3499 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state)) 3500 __netif_reschedule(q); 3501 } 3502 EXPORT_SYMBOL(__netif_schedule); 3503 3504 struct dev_kfree_skb_cb { 3505 enum skb_drop_reason reason; 3506 }; 3507 3508 static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb) 3509 { 3510 return (struct dev_kfree_skb_cb *)skb->cb; 3511 } 3512 3513 void netif_schedule_queue(struct netdev_queue *txq) 3514 { 3515 rcu_read_lock(); 3516 if (!netif_xmit_stopped(txq)) { 3517 struct Qdisc *q = rcu_dereference(txq->qdisc); 3518 3519 __netif_schedule(q); 3520 } 3521 rcu_read_unlock(); 3522 } 3523 EXPORT_SYMBOL(netif_schedule_queue); 3524 3525 void netif_tx_wake_queue(struct netdev_queue *dev_queue) 3526 { 3527 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) { 3528 struct Qdisc *q; 3529 3530 rcu_read_lock(); 3531 q = rcu_dereference(dev_queue->qdisc); 3532 __netif_schedule(q); 3533 rcu_read_unlock(); 3534 } 3535 } 3536 EXPORT_SYMBOL(netif_tx_wake_queue); 3537 3538 void dev_kfree_skb_irq_reason(struct sk_buff *skb, enum skb_drop_reason reason) 3539 { 3540 unsigned long flags; 3541 3542 if (unlikely(!skb)) 3543 return; 3544 3545 if (likely(refcount_read(&skb->users) == 1)) { 3546 smp_rmb(); 3547 refcount_set(&skb->users, 0); 3548 } else if (likely(!refcount_dec_and_test(&skb->users))) { 3549 return; 3550 } 3551 get_kfree_skb_cb(skb)->reason = reason; 3552 local_irq_save(flags); 3553 skb->next = __this_cpu_read(softnet_data.completion_queue); 3554 __this_cpu_write(softnet_data.completion_queue, skb); 3555 raise_softirq_irqoff(NET_TX_SOFTIRQ); 3556 local_irq_restore(flags); 3557 } 3558 EXPORT_SYMBOL(dev_kfree_skb_irq_reason); 3559 3560 void dev_kfree_skb_any_reason(struct sk_buff *skb, enum skb_drop_reason reason) 3561 { 3562 if (in_hardirq() || irqs_disabled()) 3563 dev_kfree_skb_irq_reason(skb, reason); 3564 else 3565 kfree_skb_reason(skb, reason); 3566 } 3567 EXPORT_SYMBOL(dev_kfree_skb_any_reason); 3568 3569 3570 /** 3571 * netif_device_detach - mark device as removed 3572 * @dev: network device 3573 * 3574 * Mark device as removed from system and therefore no longer available. 3575 */ 3576 void netif_device_detach(struct net_device *dev) 3577 { 3578 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) && 3579 netif_running(dev)) { 3580 netif_tx_stop_all_queues(dev); 3581 } 3582 } 3583 EXPORT_SYMBOL(netif_device_detach); 3584 3585 /** 3586 * netif_device_attach - mark device as attached 3587 * @dev: network device 3588 * 3589 * Mark device as attached from system and restart if needed. 3590 */ 3591 void netif_device_attach(struct net_device *dev) 3592 { 3593 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) && 3594 netif_running(dev)) { 3595 netif_tx_wake_all_queues(dev); 3596 netdev_watchdog_up(dev); 3597 } 3598 } 3599 EXPORT_SYMBOL(netif_device_attach); 3600 3601 /* 3602 * Returns a Tx hash based on the given packet descriptor a Tx queues' number 3603 * to be used as a distribution range. 3604 */ 3605 static u16 skb_tx_hash(const struct net_device *dev, 3606 const struct net_device *sb_dev, 3607 struct sk_buff *skb) 3608 { 3609 u32 hash; 3610 u16 qoffset = 0; 3611 u16 qcount = dev->real_num_tx_queues; 3612 3613 if (READ_ONCE(dev->num_tc)) { 3614 u8 tc = netdev_get_prio_tc_map(dev, skb->priority); 3615 struct netdev_tc_txq res; 3616 3617 res.combined = READ_ONCE(sb_dev->tc_to_txq[tc].combined); 3618 qoffset = res.offset; 3619 qcount = res.count; 3620 if (unlikely(!qcount)) { 3621 net_warn_ratelimited("%s: invalid qcount, qoffset %u for tc %u\n", 3622 sb_dev->name, qoffset, tc); 3623 qoffset = 0; 3624 qcount = dev->real_num_tx_queues; 3625 } 3626 } 3627 3628 if (skb_rx_queue_recorded(skb)) { 3629 DEBUG_NET_WARN_ON_ONCE(qcount == 0); 3630 hash = skb_get_rx_queue(skb); 3631 if (hash >= qoffset) 3632 hash -= qoffset; 3633 while (unlikely(hash >= qcount)) 3634 hash -= qcount; 3635 return hash + qoffset; 3636 } 3637 3638 return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset; 3639 } 3640 3641 void skb_warn_bad_offload(const struct sk_buff *skb) 3642 { 3643 static const netdev_features_t null_features; 3644 struct net_device *dev = skb->dev; 3645 const char *name = ""; 3646 3647 if (!net_ratelimit()) 3648 return; 3649 3650 if (dev) { 3651 if (dev->dev.parent) 3652 name = dev_driver_string(dev->dev.parent); 3653 else 3654 name = netdev_name(dev); 3655 } 3656 skb_dump(KERN_WARNING, skb, false); 3657 WARN(1, "%s: caps=(%pNF, %pNF)\n", 3658 name, dev ? &dev->features : &null_features, 3659 skb->sk ? &skb->sk->sk_route_caps : &null_features); 3660 } 3661 3662 /* 3663 * Invalidate hardware checksum when packet is to be mangled, and 3664 * complete checksum manually on outgoing path. 3665 */ 3666 int skb_checksum_help(struct sk_buff *skb) 3667 { 3668 __wsum csum; 3669 int ret = 0, offset; 3670 3671 if (skb->ip_summed == CHECKSUM_COMPLETE) 3672 goto out_set_summed; 3673 3674 if (unlikely(skb_is_gso(skb))) { 3675 skb_warn_bad_offload(skb); 3676 return -EINVAL; 3677 } 3678 3679 if (!skb_frags_readable(skb)) { 3680 return -EFAULT; 3681 } 3682 3683 /* Before computing a checksum, we should make sure no frag could 3684 * be modified by an external entity : checksum could be wrong. 3685 */ 3686 if (skb_has_shared_frag(skb)) { 3687 ret = __skb_linearize(skb); 3688 if (ret) 3689 goto out; 3690 } 3691 3692 offset = skb_checksum_start_offset(skb); 3693 ret = -EINVAL; 3694 if (unlikely(offset >= skb_headlen(skb))) { 3695 DO_ONCE_LITE(skb_dump, KERN_ERR, skb, false); 3696 WARN_ONCE(true, "offset (%d) >= skb_headlen() (%u)\n", 3697 offset, skb_headlen(skb)); 3698 goto out; 3699 } 3700 csum = skb_checksum(skb, offset, skb->len - offset, 0); 3701 3702 offset += skb->csum_offset; 3703 if (unlikely(offset + sizeof(__sum16) > skb_headlen(skb))) { 3704 DO_ONCE_LITE(skb_dump, KERN_ERR, skb, false); 3705 WARN_ONCE(true, "offset+2 (%zu) > skb_headlen() (%u)\n", 3706 offset + sizeof(__sum16), skb_headlen(skb)); 3707 goto out; 3708 } 3709 ret = skb_ensure_writable(skb, offset + sizeof(__sum16)); 3710 if (ret) 3711 goto out; 3712 3713 *(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0; 3714 out_set_summed: 3715 skb->ip_summed = CHECKSUM_NONE; 3716 out: 3717 return ret; 3718 } 3719 EXPORT_SYMBOL(skb_checksum_help); 3720 3721 #ifdef CONFIG_NET_CRC32C 3722 int skb_crc32c_csum_help(struct sk_buff *skb) 3723 { 3724 u32 crc; 3725 int ret = 0, offset, start; 3726 3727 if (skb->ip_summed != CHECKSUM_PARTIAL) 3728 goto out; 3729 3730 if (unlikely(skb_is_gso(skb))) 3731 goto out; 3732 3733 /* Before computing a checksum, we should make sure no frag could 3734 * be modified by an external entity : checksum could be wrong. 3735 */ 3736 if (unlikely(skb_has_shared_frag(skb))) { 3737 ret = __skb_linearize(skb); 3738 if (ret) 3739 goto out; 3740 } 3741 start = skb_checksum_start_offset(skb); 3742 offset = start + offsetof(struct sctphdr, checksum); 3743 if (WARN_ON_ONCE(offset >= skb_headlen(skb))) { 3744 ret = -EINVAL; 3745 goto out; 3746 } 3747 3748 ret = skb_ensure_writable(skb, offset + sizeof(__le32)); 3749 if (ret) 3750 goto out; 3751 3752 crc = ~skb_crc32c(skb, start, skb->len - start, ~0); 3753 *(__le32 *)(skb->data + offset) = cpu_to_le32(crc); 3754 skb_reset_csum_not_inet(skb); 3755 out: 3756 return ret; 3757 } 3758 EXPORT_SYMBOL(skb_crc32c_csum_help); 3759 #endif /* CONFIG_NET_CRC32C */ 3760 3761 __be16 skb_network_protocol(struct sk_buff *skb, int *depth) 3762 { 3763 __be16 type = skb->protocol; 3764 3765 /* Tunnel gso handlers can set protocol to ethernet. */ 3766 if (type == htons(ETH_P_TEB)) { 3767 struct ethhdr *eth; 3768 3769 if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr)))) 3770 return 0; 3771 3772 eth = (struct ethhdr *)skb->data; 3773 type = eth->h_proto; 3774 } 3775 3776 return vlan_get_protocol_and_depth(skb, type, depth); 3777 } 3778 3779 3780 /* Take action when hardware reception checksum errors are detected. */ 3781 #ifdef CONFIG_BUG 3782 static void do_netdev_rx_csum_fault(struct net_device *dev, struct sk_buff *skb) 3783 { 3784 netdev_err(dev, "hw csum failure\n"); 3785 skb_dump(KERN_ERR, skb, true); 3786 dump_stack(); 3787 } 3788 3789 void netdev_rx_csum_fault(struct net_device *dev, struct sk_buff *skb) 3790 { 3791 DO_ONCE_LITE(do_netdev_rx_csum_fault, dev, skb); 3792 } 3793 EXPORT_SYMBOL(netdev_rx_csum_fault); 3794 #endif 3795 3796 /* XXX: check that highmem exists at all on the given machine. */ 3797 static int illegal_highdma(struct net_device *dev, struct sk_buff *skb) 3798 { 3799 #ifdef CONFIG_HIGHMEM 3800 int i; 3801 3802 if (!(dev->features & NETIF_F_HIGHDMA)) { 3803 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 3804 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 3805 struct page *page = skb_frag_page(frag); 3806 3807 if (page && PageHighMem(page)) 3808 return 1; 3809 } 3810 } 3811 #endif 3812 return 0; 3813 } 3814 3815 /* If MPLS offload request, verify we are testing hardware MPLS features 3816 * instead of standard features for the netdev. 3817 */ 3818 #if IS_ENABLED(CONFIG_NET_MPLS_GSO) 3819 static netdev_features_t net_mpls_features(struct sk_buff *skb, 3820 netdev_features_t features, 3821 __be16 type) 3822 { 3823 if (eth_p_mpls(type)) 3824 features &= skb->dev->mpls_features; 3825 3826 return features; 3827 } 3828 #else 3829 static netdev_features_t net_mpls_features(struct sk_buff *skb, 3830 netdev_features_t features, 3831 __be16 type) 3832 { 3833 return features; 3834 } 3835 #endif 3836 3837 static netdev_features_t harmonize_features(struct sk_buff *skb, 3838 netdev_features_t features) 3839 { 3840 __be16 type; 3841 3842 type = skb_network_protocol(skb, NULL); 3843 features = net_mpls_features(skb, features, type); 3844 3845 if (skb->ip_summed != CHECKSUM_NONE && 3846 !can_checksum_protocol(features, type)) { 3847 features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 3848 } 3849 if (illegal_highdma(skb->dev, skb)) 3850 features &= ~NETIF_F_SG; 3851 3852 return features; 3853 } 3854 3855 netdev_features_t passthru_features_check(struct sk_buff *skb, 3856 struct net_device *dev, 3857 netdev_features_t features) 3858 { 3859 return features; 3860 } 3861 EXPORT_SYMBOL(passthru_features_check); 3862 3863 static netdev_features_t dflt_features_check(struct sk_buff *skb, 3864 struct net_device *dev, 3865 netdev_features_t features) 3866 { 3867 return vlan_features_check(skb, features); 3868 } 3869 3870 static bool __skb_has_ipv6_ext_hdr(const struct sk_buff *skb, int nhoff) 3871 { 3872 const struct ipv6hdr *ip6h; 3873 struct ipv6hdr _ip6h; 3874 3875 ip6h = skb_header_pointer(skb, nhoff, sizeof(_ip6h), &_ip6h); 3876 return ip6h && ip6h->version == 6 && ipv6_ext_hdr(ip6h->nexthdr); 3877 } 3878 3879 static bool skb_has_ipv6_extension_hdr(const struct sk_buff *skb) 3880 { 3881 if (vlan_get_protocol(skb) == htons(ETH_P_IPV6)) { 3882 if (__skb_has_ipv6_ext_hdr(skb, skb_network_offset(skb))) 3883 return true; 3884 3885 /* Catch tunnels without skb->encapsulation (e.g., virtio). */ 3886 if (!skb->encapsulation && 3887 skb_transport_header_was_set(skb) && 3888 skb_network_header_len(skb) != sizeof(struct ipv6hdr)) 3889 return true; 3890 } 3891 3892 /* Tunnels without an inner network header, such as SCTP-in-UDP or 3893 * PSP, have no inner IP header and thus no inner extension header. 3894 */ 3895 if (skb->encapsulation && skb_inner_network_header_was_set(skb) && 3896 __skb_has_ipv6_ext_hdr(skb, skb_inner_network_offset(skb))) 3897 return true; 3898 3899 return false; 3900 } 3901 3902 static unsigned int 3903 netif_get_gso_max_size(const struct net_device *dev, const struct sk_buff *skb) 3904 { 3905 /* pairs with WRITE_ONCE() in netif_set_gso(_ipv4)_max_size() */ 3906 return vlan_get_protocol(skb) == htons(ETH_P_IPV6) ? 3907 READ_ONCE(dev->gso_max_size) : 3908 READ_ONCE(dev->gso_ipv4_max_size); 3909 } 3910 3911 static bool 3912 gso_within_dev_limits(const struct sk_buff *skb, const struct net_device *dev) 3913 { 3914 if (skb_shinfo(skb)->gso_segs > READ_ONCE(dev->gso_max_segs)) 3915 return false; 3916 3917 if (unlikely(skb->len >= netif_get_gso_max_size(dev, skb))) 3918 return false; 3919 3920 return true; 3921 } 3922 3923 static netdev_features_t gso_features_check(const struct sk_buff *skb, 3924 struct net_device *dev, 3925 netdev_features_t features, 3926 bool check_limits) 3927 { 3928 if (check_limits && !gso_within_dev_limits(skb, dev)) 3929 return features & ~NETIF_F_GSO_MASK; 3930 3931 if (!skb_shinfo(skb)->gso_type) { 3932 skb_warn_bad_offload(skb); 3933 return features & ~NETIF_F_GSO_MASK; 3934 } 3935 3936 /* Support for GSO partial features requires software 3937 * intervention before we can actually process the packets 3938 * so we need to strip support for any partial features now 3939 * and we can pull them back in after we have partially 3940 * segmented the frame. 3941 */ 3942 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL)) 3943 features &= ~dev->gso_partial_features; 3944 3945 /* Make sure to clear the IPv4 ID mangling feature if the IPv4 header 3946 * has the potential to be fragmented so that TSO does not generate 3947 * segments with the same ID. For encapsulated packets, the ID mangling 3948 * feature is guaranteed not to use the same ID for the outer IPv4 3949 * headers of the generated segments if the headers have the potential 3950 * to be fragmented, so there is no need to clear the IPv4 ID mangling 3951 * feature (see the section about NETIF_F_TSO_MANGLEID in 3952 * segmentation-offloads.rst). 3953 */ 3954 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) { 3955 const struct iphdr *iph; 3956 struct iphdr _iph; 3957 int nhoff = skb->encapsulation ? 3958 skb_inner_network_offset(skb) : 3959 skb_network_offset(skb); 3960 3961 iph = skb_header_pointer(skb, nhoff, sizeof(_iph), &_iph); 3962 3963 if (!iph || !(iph->frag_off & htons(IP_DF))) 3964 features &= ~dev->mangleid_features; 3965 } 3966 3967 /* NETIF_F_IPV6_CSUM does not support IPv6 extension headers, 3968 * so neither does TSO that depends on it. 3969 */ 3970 if (features & NETIF_F_IPV6_CSUM && 3971 skb_has_ipv6_extension_hdr(skb)) 3972 features &= ~(NETIF_F_IPV6_CSUM | NETIF_F_TSO6 | NETIF_F_GSO_UDP_L4); 3973 3974 return features; 3975 } 3976 3977 netdev_features_t 3978 __netif_skb_features(struct sk_buff *skb, bool check_gso_limits) 3979 { 3980 struct net_device *dev = skb->dev; 3981 netdev_features_t features = dev->features; 3982 3983 if (skb_is_gso(skb)) 3984 features = gso_features_check(skb, dev, features, 3985 check_gso_limits); 3986 3987 /* If encapsulation offload request, verify we are testing 3988 * hardware encapsulation features instead of standard 3989 * features for the netdev 3990 */ 3991 if (skb->encapsulation) 3992 features &= dev->hw_enc_features; 3993 3994 if (skb_vlan_tagged(skb)) 3995 features = netdev_intersect_features(features, 3996 dev->vlan_features | 3997 NETIF_F_HW_VLAN_CTAG_TX | 3998 NETIF_F_HW_VLAN_STAG_TX); 3999 4000 if (dev->netdev_ops->ndo_features_check) 4001 features &= dev->netdev_ops->ndo_features_check(skb, dev, 4002 features); 4003 else 4004 features &= dflt_features_check(skb, dev, features); 4005 4006 return harmonize_features(skb, features); 4007 } 4008 EXPORT_SYMBOL(__netif_skb_features); 4009 4010 static unsigned int 4011 skb_gso_output_max_segs(struct sk_buff *skb, struct net_device *dev) 4012 { 4013 unsigned int mss = skb_shinfo(skb)->gso_size; 4014 unsigned int gso_max_size, hdr_len, max_segs; 4015 netdev_features_t features; 4016 struct tcphdr _tcph, *th; 4017 4018 if (!skb_is_gso(skb) || !skb_is_gso_tcp(skb) || 4019 skb->encapsulation || skb_has_frag_list(skb)) 4020 return 0; 4021 4022 /* 4023 * The transport offset can be unset or stale, so hdr_len is only taken 4024 * from a header at the checksum start. 4025 */ 4026 if (!skb_transport_header_was_set(skb) || 4027 skb_checksum_start(skb) != skb_transport_header(skb)) 4028 return 0; 4029 4030 th = skb_header_pointer(skb, skb_transport_offset(skb), sizeof(_tcph), 4031 &_tcph); 4032 if (!th || __tcp_hdrlen(th) < sizeof(*th)) 4033 return 0; 4034 4035 hdr_len = skb_transport_header(skb) - skb_mac_header(skb) + 4036 __tcp_hdrlen(th); 4037 4038 /* The output is still a GSO skb: recheck without the limit checks */ 4039 features = __netif_skb_features(skb, false) | NETIF_F_GSO_ROBUST; 4040 if (!net_gso_ok(features, skb_shinfo(skb)->gso_type)) 4041 return 0; 4042 4043 gso_max_size = netif_get_gso_max_size(dev, skb); 4044 gso_max_size = min(gso_max_size, GSO_LEGACY_MAX_SIZE); 4045 4046 /* 4047 * gso_within_dev_limits() accepts gso_segs == gso_max_segs but 4048 * rejects skb->len >= gso_max_size, so only the size limit needs the 4049 * - 1; the inner min() keeps that subtraction from wrapping. 4050 */ 4051 max_segs = (gso_max_size - min(gso_max_size, hdr_len + 1)) / mss; 4052 max_segs = min(max_segs, READ_ONCE(dev->gso_max_segs)); 4053 4054 return max_segs; 4055 } 4056 4057 static int xmit_one(struct sk_buff *skb, struct net_device *dev, 4058 struct netdev_queue *txq, bool more) 4059 { 4060 unsigned int len; 4061 int rc; 4062 4063 if (dev_nit_active_rcu(dev)) 4064 dev_queue_xmit_nit(skb, dev); 4065 4066 len = skb->len; 4067 trace_net_dev_start_xmit(skb, dev); 4068 rc = netdev_start_xmit(skb, dev, txq, more); 4069 trace_net_dev_xmit(skb, rc, dev, len); 4070 4071 return rc; 4072 } 4073 4074 struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev, 4075 struct netdev_queue *txq, int *ret) 4076 { 4077 struct sk_buff *skb = first; 4078 int rc = NETDEV_TX_OK; 4079 4080 while (skb) { 4081 struct sk_buff *next = skb->next; 4082 4083 skb_mark_not_on_list(skb); 4084 rc = xmit_one(skb, dev, txq, next != NULL); 4085 if (unlikely(!dev_xmit_complete(rc))) { 4086 skb->next = next; 4087 goto out; 4088 } 4089 4090 skb = next; 4091 if (netif_tx_queue_stopped(txq) && skb) { 4092 rc = NETDEV_TX_BUSY; 4093 break; 4094 } 4095 } 4096 4097 out: 4098 *ret = rc; 4099 return skb; 4100 } 4101 4102 static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb, 4103 netdev_features_t features) 4104 { 4105 if (skb_vlan_tag_present(skb) && 4106 !vlan_hw_offload_capable(features, skb->vlan_proto)) 4107 skb = __vlan_hwaccel_push_inside(skb); 4108 return skb; 4109 } 4110 4111 int skb_csum_hwoffload_help(struct sk_buff *skb, 4112 const netdev_features_t features) 4113 { 4114 if (unlikely(skb_csum_is_sctp(skb))) 4115 return !!(features & NETIF_F_SCTP_CRC) ? 0 : 4116 skb_crc32c_csum_help(skb); 4117 4118 if (features & NETIF_F_HW_CSUM) 4119 return 0; 4120 4121 if (features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) { 4122 if (skb_has_ipv6_extension_hdr(skb)) 4123 goto sw_checksum; 4124 4125 switch (skb->csum_offset) { 4126 case offsetof(struct tcphdr, check): 4127 case offsetof(struct udphdr, check): 4128 return 0; 4129 } 4130 } 4131 4132 sw_checksum: 4133 return skb_checksum_help(skb); 4134 } 4135 EXPORT_SYMBOL(skb_csum_hwoffload_help); 4136 4137 /* Checks if this SKB belongs to an HW offloaded socket 4138 * and whether any SW fallbacks are required based on dev. 4139 * Check decrypted mark in case skb_orphan() cleared socket. 4140 */ 4141 static struct sk_buff *sk_validate_xmit_skb(struct sk_buff *skb, 4142 struct net_device *dev) 4143 { 4144 #ifdef CONFIG_SOCK_VALIDATE_XMIT 4145 struct sk_buff *(*sk_validate)(struct sock *sk, struct net_device *dev, 4146 struct sk_buff *skb); 4147 struct sock *sk = skb->sk; 4148 4149 sk_validate = NULL; 4150 if (sk) { 4151 if (sk_fullsock(sk)) 4152 sk_validate = sk->sk_validate_xmit_skb; 4153 else if (sk_is_inet(sk) && sk->sk_state == TCP_TIME_WAIT) 4154 sk_validate = inet_twsk(sk)->tw_validate_xmit_skb; 4155 } 4156 4157 if (sk_validate) { 4158 skb = sk_validate(sk, dev, skb); 4159 } else if (unlikely(skb_is_decrypted(skb))) { 4160 pr_warn_ratelimited("unencrypted skb with no associated socket - dropping\n"); 4161 kfree_skb(skb); 4162 skb = NULL; 4163 } 4164 #endif 4165 4166 return skb; 4167 } 4168 4169 static struct sk_buff *validate_xmit_unreadable_skb(struct sk_buff *skb, 4170 struct net_device *dev) 4171 { 4172 struct skb_shared_info *shinfo; 4173 struct net_iov *niov; 4174 4175 if (likely(skb_frags_readable(skb) || 4176 dev->netmem_tx == NETMEM_TX_NO_DMA)) 4177 goto out; 4178 4179 if (dev->netmem_tx == NETMEM_TX_NONE) 4180 goto out_free; 4181 4182 shinfo = skb_shinfo(skb); 4183 4184 if (shinfo->nr_frags > 0) { 4185 niov = netmem_to_net_iov(skb_frag_netmem(&shinfo->frags[0])); 4186 if (net_is_devmem_iov(niov) && 4187 READ_ONCE(net_devmem_iov_binding(niov)->dev) != dev) 4188 goto out_free; 4189 } 4190 4191 out: 4192 return skb; 4193 4194 out_free: 4195 kfree_skb(skb); 4196 return NULL; 4197 } 4198 4199 /* Returns the skb on success, NULL if dropped, or ERR_PTR(-EINPROGRESS) 4200 * if stolen by async xfrm crypto (delivered via xfrm_dev_resume()). 4201 */ 4202 static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev, bool *again) 4203 { 4204 netdev_features_t features; 4205 4206 skb = validate_xmit_unreadable_skb(skb, dev); 4207 if (unlikely(!skb)) 4208 goto out_null; 4209 4210 /* Only fraglist GSO, whose segments already carry valid checksums, 4211 * can be handed to a device as CHECKSUM_UNNECESSARY. Any other GSO 4212 * skb has no csum_start for the device to segment from. NONE means 4213 * the same on transmit, but makes netif_needs_gso() send it to the 4214 * GSO code, which turns TCP into CHECKSUM_PARTIAL for TSO and 4215 * segments anything the device cannot offload in software. 4216 */ 4217 if (unlikely(skb->ip_summed == CHECKSUM_UNNECESSARY && 4218 skb_is_gso(skb) && 4219 !(skb_shinfo(skb)->gso_type & SKB_GSO_FRAGLIST))) 4220 skb->ip_summed = CHECKSUM_NONE; 4221 4222 features = netif_skb_features(skb); 4223 skb = validate_xmit_vlan(skb, features); 4224 if (unlikely(!skb)) 4225 goto out_null; 4226 4227 skb = sk_validate_xmit_skb(skb, dev); 4228 if (unlikely(!skb)) 4229 goto out_null; 4230 4231 if (netif_needs_gso(skb, features)) { 4232 unsigned int max_segs = 0; 4233 struct sk_buff *segs; 4234 4235 if (unlikely(!gso_within_dev_limits(skb, dev))) 4236 max_segs = skb_gso_output_max_segs(skb, dev); 4237 4238 segs = __skb_gso_segment(skb, features, true, max_segs); 4239 if (IS_ERR(segs)) { 4240 goto out_kfree_skb; 4241 } else if (segs) { 4242 consume_skb(skb); 4243 skb = segs; 4244 } 4245 } else { 4246 if (skb_needs_linearize(skb, features) && 4247 __skb_linearize(skb)) 4248 goto out_kfree_skb; 4249 4250 /* If packet is not checksummed and device does not 4251 * support checksumming for this protocol, complete 4252 * checksumming here. 4253 */ 4254 if (skb->ip_summed == CHECKSUM_PARTIAL) { 4255 if (skb->encapsulation) 4256 skb_set_inner_transport_header(skb, 4257 skb_checksum_start_offset(skb)); 4258 else 4259 skb_set_transport_header(skb, 4260 skb_checksum_start_offset(skb)); 4261 if (skb_csum_hwoffload_help(skb, features)) 4262 goto out_kfree_skb; 4263 } 4264 } 4265 4266 skb = validate_xmit_xfrm(skb, features, again); 4267 4268 return skb; 4269 4270 out_kfree_skb: 4271 kfree_skb(skb); 4272 out_null: 4273 dev_core_stats_tx_dropped_inc(dev); 4274 return NULL; 4275 } 4276 4277 struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev, bool *again) 4278 { 4279 struct sk_buff *next, *head = NULL, *tail; 4280 4281 for (; skb != NULL; skb = next) { 4282 next = skb->next; 4283 skb_mark_not_on_list(skb); 4284 4285 /* in case skb won't be segmented, point to itself */ 4286 skb->prev = skb; 4287 4288 skb = validate_xmit_skb(skb, dev, again); 4289 if (IS_ERR_OR_NULL(skb)) 4290 continue; 4291 4292 if (!head) 4293 head = skb; 4294 else 4295 tail->next = skb; 4296 /* If skb was segmented, skb->prev points to 4297 * the last segment. If not, it still contains skb. 4298 */ 4299 tail = skb->prev; 4300 } 4301 return head; 4302 } 4303 EXPORT_SYMBOL_GPL(validate_xmit_skb_list); 4304 4305 static enum skb_drop_reason qdisc_pkt_len_segs_init(struct sk_buff *skb) 4306 { 4307 struct skb_shared_info *shinfo = skb_shinfo(skb); 4308 unsigned int hdr_len, tlen; 4309 u32 gso_segs; 4310 4311 qdisc_skb_cb(skb)->pkt_len = skb->len; 4312 if (!shinfo->gso_size) { 4313 qdisc_skb_cb(skb)->pkt_segs = 1; 4314 return SKB_NOT_DROPPED_YET; 4315 } 4316 4317 qdisc_skb_cb(skb)->pkt_segs = gso_segs = shinfo->gso_segs; 4318 4319 /* To get more precise estimation of bytes sent on wire, 4320 * we add to pkt_len the headers size of all segments 4321 */ 4322 4323 /* mac layer + network layer */ 4324 if (!skb->encapsulation) { 4325 if (unlikely(!skb_transport_header_was_set(skb))) 4326 return SKB_NOT_DROPPED_YET; 4327 hdr_len = skb_transport_offset(skb); 4328 } else { 4329 hdr_len = skb_inner_transport_offset(skb); 4330 } 4331 /* + transport layer */ 4332 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) { 4333 const struct tcphdr *th; 4334 4335 /* A header in front of skb->data gives a negative offset. */ 4336 if ((int)hdr_len < 0 || 4337 !pskb_may_pull(skb, hdr_len + sizeof(struct tcphdr))) 4338 return SKB_DROP_REASON_SKB_BAD_GSO; 4339 4340 th = (const struct tcphdr *)(skb->data + hdr_len); 4341 tlen = __tcp_hdrlen(th); 4342 if (tlen < sizeof(*th)) 4343 return SKB_DROP_REASON_SKB_BAD_GSO; 4344 hdr_len += tlen; 4345 if (!pskb_may_pull(skb, hdr_len)) 4346 return SKB_DROP_REASON_SKB_BAD_GSO; 4347 } else if (shinfo->gso_type & SKB_GSO_UDP_L4) { 4348 if (!pskb_may_pull(skb, hdr_len + sizeof(struct udphdr))) 4349 return SKB_DROP_REASON_SKB_BAD_GSO; 4350 hdr_len += sizeof(struct udphdr); 4351 } 4352 4353 /* prior pskb_may_pull() might have changed skb->head. */ 4354 shinfo = skb_shinfo(skb); 4355 if (unlikely(shinfo->gso_type & SKB_GSO_DODGY)) { 4356 int payload = skb->len - hdr_len; 4357 4358 /* Malicious packet. */ 4359 if (payload <= 0) 4360 return SKB_DROP_REASON_SKB_BAD_GSO; 4361 gso_segs = DIV_ROUND_UP(payload, shinfo->gso_size); 4362 /* shinfo->gso_segs is a 16bit field. 4363 * Also make sure untrusted gso_size/hdr_len can not inflate 4364 * qdisc_pkt_len() beyond QDISC_PKT_LEN_MAX, or deficit 4365 * schedulers could spin in their refill loops. 4366 */ 4367 if (unlikely(gso_segs > GSO_MAX_SEGS || 4368 skb->len + (u64)(gso_segs - 1) * hdr_len > 4369 QDISC_PKT_LEN_MAX)) 4370 return SKB_DROP_REASON_SKB_BAD_GSO; 4371 shinfo->gso_segs = gso_segs; 4372 qdisc_skb_cb(skb)->pkt_segs = gso_segs; 4373 } 4374 qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len; 4375 return SKB_NOT_DROPPED_YET; 4376 } 4377 4378 static int dev_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *q, 4379 struct sk_buff **to_free, 4380 struct netdev_queue *txq) 4381 { 4382 int rc; 4383 4384 rc = q->enqueue(skb, q, to_free) & NET_XMIT_MASK; 4385 if (rc == NET_XMIT_SUCCESS) 4386 trace_qdisc_enqueue(q, txq, skb); 4387 return rc; 4388 } 4389 4390 static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q, 4391 struct net_device *dev, 4392 struct netdev_queue *txq) 4393 { 4394 struct sk_buff *next, *to_free = NULL, *to_free2 = NULL; 4395 spinlock_t *root_lock = qdisc_lock(q); 4396 struct llist_node *ll_list, *first_n; 4397 unsigned long defer_count = 0; 4398 int rc; 4399 4400 qdisc_calculate_pkt_len(skb, q); 4401 4402 tcf_set_qdisc_drop_reason(skb, QDISC_DROP_GENERIC); 4403 4404 if (q->flags & TCQ_F_NOLOCK) { 4405 if (q->flags & TCQ_F_CAN_BYPASS && nolock_qdisc_is_empty(q) && 4406 qdisc_run_begin(q)) { 4407 /* Retest nolock_qdisc_is_empty() within the protection 4408 * of q->seqlock to protect from racing with requeuing. 4409 */ 4410 if (unlikely(!nolock_qdisc_is_empty(q))) { 4411 rc = dev_qdisc_enqueue(skb, q, &to_free, txq); 4412 __qdisc_run(q); 4413 to_free2 = qdisc_run_end(q); 4414 4415 goto free_skbs; 4416 } 4417 4418 qdisc_bstats_cpu_update(q, skb); 4419 if (sch_direct_xmit(skb, q, dev, txq, NULL, true) && 4420 !nolock_qdisc_is_empty(q)) 4421 __qdisc_run(q); 4422 4423 to_free2 = qdisc_run_end(q); 4424 rc = NET_XMIT_SUCCESS; 4425 goto free_skbs; 4426 } 4427 4428 rc = dev_qdisc_enqueue(skb, q, &to_free, txq); 4429 to_free2 = qdisc_run(q); 4430 goto free_skbs; 4431 } 4432 4433 /* Open code llist_add(&skb->ll_node, &q->defer_list) + queue limit. 4434 * In the try_cmpxchg() loop, we want to increment q->defer_count 4435 * at most once to limit the number of skbs in defer_list. 4436 * We perform the defer_count increment only if the list is not empty, 4437 * because some arches have slow atomic_long_inc_return(). 4438 */ 4439 first_n = READ_ONCE(q->defer_list.first); 4440 do { 4441 if (first_n && !defer_count) { 4442 defer_count = atomic_long_inc_return(&q->defer_count); 4443 if (unlikely(defer_count > READ_ONCE(net_hotdata.qdisc_max_burst))) { 4444 kfree_skb_reason(skb, SKB_DROP_REASON_QDISC_BURST_DROP); 4445 return NET_XMIT_DROP; 4446 } 4447 } 4448 skb->ll_node.next = first_n; 4449 } while (!try_cmpxchg(&q->defer_list.first, &first_n, &skb->ll_node)); 4450 4451 /* If defer_list was not empty, we know the cpu which queued 4452 * the first skb will process the whole list for us. 4453 */ 4454 if (first_n) 4455 return NET_XMIT_SUCCESS; 4456 4457 spin_lock(root_lock); 4458 4459 ll_list = llist_del_all(&q->defer_list); 4460 /* There is a small race because we clear defer_count not atomically 4461 * with the prior llist_del_all(). This means defer_list could grow 4462 * over qdisc_max_burst. 4463 */ 4464 atomic_long_set(&q->defer_count, 0); 4465 4466 ll_list = llist_reverse_order(ll_list); 4467 4468 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) { 4469 llist_for_each_entry_safe(skb, next, ll_list, ll_node) 4470 __qdisc_drop(skb, &to_free); 4471 rc = NET_XMIT_DROP; 4472 goto unlock; 4473 } 4474 if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) && 4475 !llist_next(ll_list) && qdisc_run_begin(q)) { 4476 /* 4477 * This is a work-conserving queue; there are no old skbs 4478 * waiting to be sent out; and the qdisc is not running - 4479 * xmit the skb directly. 4480 */ 4481 4482 DEBUG_NET_WARN_ON_ONCE(skb != llist_entry(ll_list, 4483 struct sk_buff, 4484 ll_node)); 4485 qdisc_bstats_update(q, skb); 4486 if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) 4487 __qdisc_run(q); 4488 to_free2 = qdisc_run_end(q); 4489 rc = NET_XMIT_SUCCESS; 4490 } else { 4491 int count = 0; 4492 4493 llist_for_each_entry_safe(skb, next, ll_list, ll_node) { 4494 if (next) { 4495 prefetch(next); 4496 prefetch(&next->priority); 4497 skb_mark_not_on_list(skb); 4498 } 4499 rc = dev_qdisc_enqueue(skb, q, &to_free, txq); 4500 count++; 4501 } 4502 to_free2 = qdisc_run(q); 4503 if (count != 1) 4504 rc = NET_XMIT_SUCCESS; 4505 } 4506 unlock: 4507 spin_unlock(root_lock); 4508 4509 free_skbs: 4510 tcf_kfree_skb_list(to_free, q, txq, dev); 4511 tcf_kfree_skb_list(to_free2, q, txq, dev); 4512 return rc; 4513 } 4514 4515 #if IS_ENABLED(CONFIG_CGROUP_NET_PRIO) 4516 static void skb_update_prio(struct sk_buff *skb) 4517 { 4518 const struct netprio_map *map; 4519 const struct sock *sk; 4520 unsigned int prioidx; 4521 4522 if (skb->priority) 4523 return; 4524 map = rcu_dereference_bh(skb->dev->priomap); 4525 if (!map) 4526 return; 4527 sk = skb_to_full_sk(skb); 4528 if (!sk) 4529 return; 4530 4531 prioidx = sock_cgroup_prioidx(&sk->sk_cgrp_data); 4532 4533 if (prioidx < map->priomap_len) 4534 skb->priority = map->priomap[prioidx]; 4535 } 4536 #else 4537 #define skb_update_prio(skb) 4538 #endif 4539 4540 /** 4541 * dev_loopback_xmit - loop back @skb 4542 * @net: network namespace this loopback is happening in 4543 * @sk: sk needed to be a netfilter okfn 4544 * @skb: buffer to transmit 4545 */ 4546 int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *skb) 4547 { 4548 skb_reset_mac_header(skb); 4549 __skb_pull(skb, skb_network_offset(skb)); 4550 skb->pkt_type = PACKET_LOOPBACK; 4551 if (skb->ip_summed == CHECKSUM_NONE) 4552 skb->ip_summed = CHECKSUM_UNNECESSARY; 4553 DEBUG_NET_WARN_ON_ONCE(!skb_dst(skb)); 4554 skb_dst_force(skb); 4555 netif_rx(skb); 4556 return 0; 4557 } 4558 EXPORT_SYMBOL(dev_loopback_xmit); 4559 4560 #ifdef CONFIG_NET_EGRESS 4561 static struct netdev_queue * 4562 netdev_tx_queue_mapping(struct net_device *dev, struct sk_buff *skb) 4563 { 4564 int queue = skb_get_queue_mapping(skb); 4565 int capped; 4566 4567 capped = netdev_cap_txqueue(dev, queue); 4568 if (unlikely(capped != queue)) 4569 skb_set_queue_mapping(skb, capped); 4570 4571 return netdev_get_tx_queue(dev, capped); 4572 } 4573 4574 #ifndef CONFIG_PREEMPT_RT 4575 static bool netdev_xmit_txqueue_skipped(void) 4576 { 4577 return __this_cpu_read(softnet_data.xmit.skip_txqueue); 4578 } 4579 4580 bool netdev_xmit_skip_txqueue(bool skip) 4581 { 4582 bool prev = netdev_xmit_txqueue_skipped(); 4583 4584 __this_cpu_write(softnet_data.xmit.skip_txqueue, skip); 4585 4586 return prev; 4587 } 4588 EXPORT_SYMBOL_GPL(netdev_xmit_skip_txqueue); 4589 4590 #else 4591 static bool netdev_xmit_txqueue_skipped(void) 4592 { 4593 return current->net_xmit.skip_txqueue; 4594 } 4595 4596 bool netdev_xmit_skip_txqueue(bool skip) 4597 { 4598 bool prev = netdev_xmit_txqueue_skipped(); 4599 4600 current->net_xmit.skip_txqueue = skip; 4601 4602 return prev; 4603 } 4604 EXPORT_SYMBOL_GPL(netdev_xmit_skip_txqueue); 4605 #endif 4606 #endif /* CONFIG_NET_EGRESS */ 4607 4608 #ifdef CONFIG_NET_XGRESS 4609 static int tc_run(struct tcx_entry *entry, struct sk_buff *skb, 4610 enum skb_drop_reason *drop_reason) 4611 { 4612 int ret = TC_ACT_UNSPEC; 4613 #ifdef CONFIG_NET_CLS_ACT 4614 struct mini_Qdisc *miniq = rcu_dereference_bh(entry->miniq); 4615 struct tcf_result res; 4616 4617 if (!miniq) 4618 return ret; 4619 4620 /* Global bypass */ 4621 if (!static_branch_likely(&tcf_sw_enabled_key)) 4622 return ret; 4623 4624 /* Block-wise bypass */ 4625 if (tcf_block_bypass_sw(miniq->block)) 4626 return ret; 4627 4628 tc_skb_cb(skb)->mru = 0; 4629 qdisc_skb_cb(skb)->post_ct = false; 4630 tcf_set_drop_reason(skb, *drop_reason); 4631 4632 mini_qdisc_bstats_cpu_update(miniq, skb); 4633 ret = tcf_classify(skb, miniq->block, miniq->filter_list, &res, false); 4634 /* Only tcf related quirks below. */ 4635 switch (ret) { 4636 case TC_ACT_SHOT: 4637 *drop_reason = tcf_get_drop_reason(skb); 4638 mini_qdisc_qstats_cpu_drop(miniq); 4639 break; 4640 case TC_ACT_OK: 4641 case TC_ACT_RECLASSIFY: 4642 skb->tc_index = TC_H_MIN(res.classid); 4643 break; 4644 } 4645 #endif /* CONFIG_NET_CLS_ACT */ 4646 return ret; 4647 } 4648 4649 static DEFINE_STATIC_KEY_FALSE(tcx_needed_key); 4650 4651 void tcx_inc(void) 4652 { 4653 static_branch_inc(&tcx_needed_key); 4654 } 4655 4656 void tcx_dec(void) 4657 { 4658 static_branch_dec(&tcx_needed_key); 4659 } 4660 4661 static __always_inline enum tcx_action_base 4662 tcx_run(const struct bpf_mprog_entry *entry, struct sk_buff *skb, 4663 const bool needs_mac) 4664 { 4665 const struct bpf_mprog_fp *fp; 4666 const struct bpf_prog *prog; 4667 int ret = TCX_NEXT; 4668 4669 if (needs_mac) 4670 __skb_push(skb, skb->mac_len); 4671 bpf_mprog_foreach_prog(entry, fp, prog) { 4672 bpf_compute_data_pointers(skb); 4673 ret = bpf_prog_run(prog, skb); 4674 if (ret != TCX_NEXT) 4675 break; 4676 } 4677 if (needs_mac) 4678 __skb_pull(skb, skb->mac_len); 4679 return tcx_action_code(skb, ret); 4680 } 4681 4682 static __always_inline struct sk_buff * 4683 sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret, 4684 struct net_device *orig_dev, bool *another) 4685 { 4686 struct bpf_mprog_entry *entry = rcu_dereference_bh(skb->dev->tcx_ingress); 4687 enum skb_drop_reason drop_reason = SKB_DROP_REASON_TC_INGRESS; 4688 struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx; 4689 int sch_ret; 4690 4691 if (!entry) 4692 return skb; 4693 4694 bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx); 4695 if (unlikely(*pt_prev)) { 4696 *ret = deliver_skb(skb, *pt_prev, orig_dev); 4697 *pt_prev = NULL; 4698 } 4699 4700 qdisc_pkt_len_segs_init(skb); 4701 tcx_set_ingress(skb, true); 4702 4703 if (static_branch_unlikely(&tcx_needed_key)) { 4704 sch_ret = tcx_run(entry, skb, true); 4705 if (sch_ret != TC_ACT_UNSPEC) 4706 goto ingress_verdict; 4707 } 4708 sch_ret = tc_run(tcx_entry(entry), skb, &drop_reason); 4709 ingress_verdict: 4710 switch (sch_ret) { 4711 case TC_ACT_REDIRECT: 4712 /* skb_mac_header check was done by BPF, so we can safely 4713 * push the L2 header back before redirecting to another 4714 * netdev. 4715 */ 4716 __skb_push(skb, skb->mac_len); 4717 if (skb_do_redirect(skb) == -EAGAIN) { 4718 __skb_pull(skb, skb->mac_len); 4719 *another = true; 4720 break; 4721 } 4722 *ret = NET_RX_SUCCESS; 4723 bpf_net_ctx_clear(bpf_net_ctx); 4724 return NULL; 4725 case TC_ACT_SHOT: 4726 kfree_skb_reason(skb, drop_reason); 4727 *ret = NET_RX_DROP; 4728 bpf_net_ctx_clear(bpf_net_ctx); 4729 return NULL; 4730 /* used by tc_run */ 4731 case TC_ACT_STOLEN: 4732 case TC_ACT_QUEUED: 4733 case TC_ACT_TRAP: 4734 consume_skb(skb); 4735 fallthrough; 4736 case TC_ACT_CONSUMED: 4737 *ret = NET_RX_SUCCESS; 4738 bpf_net_ctx_clear(bpf_net_ctx); 4739 return NULL; 4740 } 4741 bpf_net_ctx_clear(bpf_net_ctx); 4742 4743 return skb; 4744 } 4745 4746 static __always_inline struct sk_buff * 4747 sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev) 4748 { 4749 struct bpf_mprog_entry *entry = rcu_dereference_bh(dev->tcx_egress); 4750 enum skb_drop_reason drop_reason = SKB_DROP_REASON_TC_EGRESS; 4751 struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx; 4752 int sch_ret; 4753 4754 if (!entry) 4755 return skb; 4756 4757 bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx); 4758 4759 /* qdisc_skb_cb(skb)->pkt_len & tcx_set_ingress() was 4760 * already set by the caller. 4761 */ 4762 if (static_branch_unlikely(&tcx_needed_key)) { 4763 sch_ret = tcx_run(entry, skb, false); 4764 if (sch_ret != TC_ACT_UNSPEC) 4765 goto egress_verdict; 4766 } 4767 sch_ret = tc_run(tcx_entry(entry), skb, &drop_reason); 4768 egress_verdict: 4769 switch (sch_ret) { 4770 case TC_ACT_REDIRECT: 4771 /* No need to push/pop skb's mac_header here on egress! */ 4772 skb_do_redirect(skb); 4773 *ret = NET_XMIT_SUCCESS; 4774 bpf_net_ctx_clear(bpf_net_ctx); 4775 return NULL; 4776 case TC_ACT_SHOT: 4777 kfree_skb_reason(skb, drop_reason); 4778 *ret = NET_XMIT_DROP; 4779 bpf_net_ctx_clear(bpf_net_ctx); 4780 return NULL; 4781 /* used by tc_run */ 4782 case TC_ACT_STOLEN: 4783 case TC_ACT_QUEUED: 4784 case TC_ACT_TRAP: 4785 consume_skb(skb); 4786 fallthrough; 4787 case TC_ACT_CONSUMED: 4788 *ret = NET_XMIT_SUCCESS; 4789 bpf_net_ctx_clear(bpf_net_ctx); 4790 return NULL; 4791 } 4792 bpf_net_ctx_clear(bpf_net_ctx); 4793 4794 return skb; 4795 } 4796 #else 4797 static __always_inline struct sk_buff * 4798 sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret, 4799 struct net_device *orig_dev, bool *another) 4800 { 4801 return skb; 4802 } 4803 4804 static __always_inline struct sk_buff * 4805 sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev) 4806 { 4807 return skb; 4808 } 4809 #endif /* CONFIG_NET_XGRESS */ 4810 4811 #ifdef CONFIG_XPS 4812 static int __get_xps_queue_idx(struct net_device *dev, struct sk_buff *skb, 4813 struct xps_dev_maps *dev_maps, unsigned int tci) 4814 { 4815 int tc = netdev_get_prio_tc_map(dev, skb->priority); 4816 struct xps_map *map; 4817 int queue_index = -1; 4818 4819 if (tc >= dev_maps->num_tc || tci >= dev_maps->nr_ids) 4820 return queue_index; 4821 4822 tci *= dev_maps->num_tc; 4823 tci += tc; 4824 4825 map = rcu_dereference(dev_maps->attr_map[tci]); 4826 if (map) { 4827 if (map->len == 1) 4828 queue_index = map->queues[0]; 4829 else 4830 queue_index = map->queues[reciprocal_scale( 4831 skb_get_hash(skb), map->len)]; 4832 if (unlikely(queue_index >= dev->real_num_tx_queues)) 4833 queue_index = -1; 4834 } 4835 return queue_index; 4836 } 4837 #endif 4838 4839 static int get_xps_queue(struct net_device *dev, struct net_device *sb_dev, 4840 struct sk_buff *skb) 4841 { 4842 #ifdef CONFIG_XPS 4843 struct xps_dev_maps *dev_maps; 4844 struct sock *sk = skb->sk; 4845 int queue_index = -1; 4846 4847 if (!static_key_false(&xps_needed)) 4848 return -1; 4849 4850 rcu_read_lock(); 4851 if (!static_key_false(&xps_rxqs_needed)) 4852 goto get_cpus_map; 4853 4854 dev_maps = rcu_dereference(sb_dev->xps_maps[XPS_RXQS]); 4855 if (dev_maps) { 4856 int tci = sk_rx_queue_get(sk); 4857 4858 if (tci >= 0) 4859 queue_index = __get_xps_queue_idx(dev, skb, dev_maps, 4860 tci); 4861 } 4862 4863 get_cpus_map: 4864 if (queue_index < 0) { 4865 dev_maps = rcu_dereference(sb_dev->xps_maps[XPS_CPUS]); 4866 if (dev_maps) { 4867 unsigned int tci = skb->sender_cpu - 1; 4868 4869 queue_index = __get_xps_queue_idx(dev, skb, dev_maps, 4870 tci); 4871 } 4872 } 4873 rcu_read_unlock(); 4874 4875 return queue_index; 4876 #else 4877 return -1; 4878 #endif 4879 } 4880 4881 u16 dev_pick_tx_zero(struct net_device *dev, struct sk_buff *skb, 4882 struct net_device *sb_dev) 4883 { 4884 return 0; 4885 } 4886 EXPORT_SYMBOL(dev_pick_tx_zero); 4887 4888 int sk_tx_queue_get(const struct sock *sk) 4889 { 4890 int resel, val; 4891 4892 if (!sk) 4893 return -1; 4894 /* Paired with WRITE_ONCE() in sk_tx_queue_clear() 4895 * and sk_tx_queue_set(). 4896 */ 4897 val = READ_ONCE(sk->sk_tx_queue_mapping); 4898 4899 if (val == NO_QUEUE_MAPPING) 4900 return -1; 4901 4902 if (!sk_fullsock(sk)) 4903 return val; 4904 4905 resel = READ_ONCE(sock_net(sk)->core.sysctl_txq_reselection); 4906 if (resel && time_is_before_jiffies( 4907 READ_ONCE(sk->sk_tx_queue_mapping_jiffies) + resel)) 4908 return -1; 4909 4910 return val; 4911 } 4912 EXPORT_SYMBOL(sk_tx_queue_get); 4913 4914 u16 netdev_pick_tx(struct net_device *dev, struct sk_buff *skb, 4915 struct net_device *sb_dev) 4916 { 4917 struct sock *sk = skb->sk; 4918 int queue_index = sk_tx_queue_get(sk); 4919 4920 sb_dev = sb_dev ? : dev; 4921 4922 if (queue_index < 0 || skb->ooo_okay || 4923 queue_index >= dev->real_num_tx_queues) { 4924 int new_index = get_xps_queue(dev, sb_dev, skb); 4925 4926 if (new_index < 0) 4927 new_index = skb_tx_hash(dev, sb_dev, skb); 4928 4929 if (sk && sk_fullsock(sk) && 4930 rcu_access_pointer(sk->sk_dst_cache)) 4931 sk_tx_queue_set(sk, new_index); 4932 4933 queue_index = new_index; 4934 } 4935 4936 return queue_index; 4937 } 4938 EXPORT_SYMBOL(netdev_pick_tx); 4939 4940 struct netdev_queue *netdev_core_pick_tx(struct net_device *dev, 4941 struct sk_buff *skb, 4942 struct net_device *sb_dev) 4943 { 4944 int queue_index = 0; 4945 4946 #ifdef CONFIG_XPS 4947 u32 sender_cpu = skb->sender_cpu - 1; 4948 4949 if (sender_cpu >= (u32)NR_CPUS) 4950 skb->sender_cpu = raw_smp_processor_id() + 1; 4951 #endif 4952 4953 if (dev->real_num_tx_queues != 1) { 4954 const struct net_device_ops *ops = dev->netdev_ops; 4955 4956 if (ops->ndo_select_queue) 4957 queue_index = ops->ndo_select_queue(dev, skb, sb_dev); 4958 else 4959 queue_index = netdev_pick_tx(dev, skb, sb_dev); 4960 4961 queue_index = netdev_cap_txqueue(dev, queue_index); 4962 } 4963 4964 skb_set_queue_mapping(skb, queue_index); 4965 return netdev_get_tx_queue(dev, queue_index); 4966 } 4967 4968 /** 4969 * __dev_queue_xmit() - transmit a buffer 4970 * @skb: buffer to transmit 4971 * @sb_dev: suboordinate device used for L2 forwarding offload 4972 * 4973 * Queue a buffer for transmission to a network device. The caller must 4974 * have set the device and priority and built the buffer before calling 4975 * this function. The function can be called from an interrupt. 4976 * 4977 * When calling this method, interrupts MUST be enabled. This is because 4978 * the BH enable code must have IRQs enabled so that it will not deadlock. 4979 * 4980 * Regardless of the return value, the skb is consumed, so it is currently 4981 * difficult to retry a send to this method. (You can bump the ref count 4982 * before sending to hold a reference for retry if you are careful.) 4983 * 4984 * Return: 4985 * * 0 - buffer successfully transmitted 4986 * * positive qdisc return code - NET_XMIT_DROP etc. 4987 * * negative errno - other errors 4988 */ 4989 int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev) 4990 { 4991 struct net_device *dev = skb->dev; 4992 struct netdev_queue *txq = NULL; 4993 enum skb_drop_reason reason; 4994 int cpu, rc = -ENOMEM; 4995 bool again = false; 4996 struct Qdisc *q; 4997 4998 skb_reset_mac_header(skb); 4999 skb_assert_len(skb); 5000 5001 if (unlikely(skb_shinfo(skb)->tx_flags & 5002 (SKBTX_SCHED_TSTAMP | SKBTX_BPF))) 5003 __skb_tstamp_tx(skb, NULL, NULL, skb->sk, SCM_TSTAMP_SCHED); 5004 5005 reason = qdisc_pkt_len_segs_init(skb); 5006 if (unlikely(reason)) { 5007 dev_core_stats_tx_dropped_inc(dev); 5008 kfree_skb_reason(skb, reason); 5009 return -EINVAL; 5010 } 5011 /* Disable soft irqs for various locks below. Also 5012 * stops preemption for RCU. 5013 */ 5014 rcu_read_lock_bh(); 5015 5016 skb_update_prio(skb); 5017 5018 tcx_set_ingress(skb, false); 5019 #ifdef CONFIG_NET_EGRESS 5020 if (static_branch_unlikely(&egress_needed_key)) { 5021 bool skip_txq; 5022 5023 if (nf_hook_egress_active()) { 5024 skb = nf_hook_egress(skb, &rc, dev); 5025 if (!skb) 5026 goto out; 5027 } 5028 5029 skip_txq = netdev_xmit_skip_txqueue(false); 5030 5031 nf_skip_egress(skb, true); 5032 skb = sch_handle_egress(skb, &rc, dev); 5033 if (!skb) { 5034 netdev_xmit_skip_txqueue(skip_txq); 5035 goto out; 5036 } 5037 nf_skip_egress(skb, false); 5038 5039 if (netdev_xmit_skip_txqueue(skip_txq)) 5040 txq = netdev_tx_queue_mapping(dev, skb); 5041 } 5042 #endif 5043 /* If device/qdisc don't need skb->dst, release it right now while 5044 * its hot in this cpu cache. 5045 */ 5046 if (dev->priv_flags & IFF_XMIT_DST_RELEASE) 5047 skb_dst_drop(skb); 5048 else 5049 skb_dst_force(skb); 5050 5051 if (!txq) 5052 txq = netdev_core_pick_tx(dev, skb, sb_dev); 5053 5054 q = rcu_dereference_bh(txq->qdisc); 5055 5056 trace_net_dev_queue(skb); 5057 if (q->enqueue) { 5058 rc = __dev_xmit_skb(skb, q, dev, txq); 5059 goto out; 5060 } 5061 5062 /* The device has no queue. Common case for software devices: 5063 * loopback, all the sorts of tunnels... 5064 5065 * Really, it is unlikely that netif_tx_lock protection is necessary 5066 * here. (f.e. loopback and IP tunnels are clean ignoring statistics 5067 * counters.) 5068 * However, it is possible, that they rely on protection 5069 * made by us here. 5070 5071 * Check this and shot the lock. It is not prone from deadlocks. 5072 *Either shot noqueue qdisc, it is even simpler 8) 5073 */ 5074 if (unlikely(!(dev->flags & IFF_UP))) { 5075 reason = SKB_DROP_REASON_DEV_READY; 5076 goto drop; 5077 } 5078 5079 cpu = smp_processor_id(); /* ok because BHs are off */ 5080 5081 if (likely(!netif_tx_owned(txq, cpu))) { 5082 bool is_list = false; 5083 5084 if (dev_xmit_recursion()) 5085 goto recursion_alert; 5086 5087 skb = validate_xmit_skb(skb, dev, &again); 5088 if (IS_ERR_OR_NULL(skb)) { 5089 if (PTR_ERR(skb) == -EINPROGRESS) 5090 rc = NET_XMIT_SUCCESS; 5091 goto out; 5092 } 5093 5094 HARD_TX_LOCK(dev, txq, cpu); 5095 5096 if (!netif_xmit_stopped(txq)) { 5097 is_list = !!skb->next; 5098 5099 dev_xmit_recursion_inc(); 5100 skb = dev_hard_start_xmit(skb, dev, txq, &rc); 5101 dev_xmit_recursion_dec(); 5102 5103 /* GSO segments a single SKB into a list of frames. 5104 * TCP expects error to mean none of the data was sent. 5105 */ 5106 if (is_list) 5107 rc = NETDEV_TX_OK; 5108 } 5109 HARD_TX_UNLOCK(dev, txq); 5110 if (!skb) /* xmit completed */ 5111 goto out; 5112 5113 net_crit_ratelimited("Virtual device %s asks to queue packet!\n", 5114 dev->name); 5115 /* NETDEV_TX_BUSY or queue was stopped */ 5116 if (!is_list) 5117 rc = -ENETDOWN; 5118 } else { 5119 /* Recursion is detected! It is possible unfortunately. */ 5120 recursion_alert: 5121 net_crit_ratelimited("Dead loop on virtual device %s (net %llu), fix it urgently!\n", 5122 dev->name, dev_net(dev)->net_cookie); 5123 5124 rc = -ENETDOWN; 5125 } 5126 5127 reason = SKB_DROP_REASON_RECURSION_LIMIT; 5128 drop: 5129 rcu_read_unlock_bh(); 5130 5131 dev_core_stats_tx_dropped_inc(dev); 5132 kfree_skb_list_reason(skb, reason); 5133 return rc; 5134 out: 5135 rcu_read_unlock_bh(); 5136 return rc; 5137 } 5138 EXPORT_SYMBOL(__dev_queue_xmit); 5139 5140 int __dev_direct_xmit(struct sk_buff *skb, u16 queue_id) 5141 { 5142 struct net_device *dev = skb->dev; 5143 struct sk_buff *orig_skb = skb; 5144 struct netdev_queue *txq; 5145 int ret = NETDEV_TX_BUSY; 5146 bool again = false; 5147 5148 if (unlikely(!netif_running(dev) || 5149 !netif_carrier_ok(dev))) 5150 goto drop; 5151 5152 skb = validate_xmit_skb_list(skb, dev, &again); 5153 if (skb != orig_skb) 5154 goto drop; 5155 5156 skb_set_queue_mapping(skb, queue_id); 5157 txq = skb_get_tx_queue(dev, skb); 5158 5159 local_bh_disable(); 5160 5161 dev_xmit_recursion_inc(); 5162 HARD_TX_LOCK(dev, txq, smp_processor_id()); 5163 if (!netif_xmit_frozen_or_drv_stopped(txq)) 5164 ret = netdev_start_xmit(skb, dev, txq, false); 5165 HARD_TX_UNLOCK(dev, txq); 5166 dev_xmit_recursion_dec(); 5167 5168 local_bh_enable(); 5169 return ret; 5170 drop: 5171 dev_core_stats_tx_dropped_inc(dev); 5172 kfree_skb_list(skb); 5173 return NET_XMIT_DROP; 5174 } 5175 EXPORT_SYMBOL(__dev_direct_xmit); 5176 5177 /************************************************************************* 5178 * Receiver routines 5179 *************************************************************************/ 5180 static DEFINE_PER_CPU(struct task_struct *, backlog_napi); 5181 5182 int weight_p __read_mostly = 64; /* old backlog weight */ 5183 int dev_weight_rx_bias __read_mostly = 1; /* bias for backlog weight */ 5184 int dev_weight_tx_bias __read_mostly = 1; /* bias for output_queue quota */ 5185 5186 /* Called with irq disabled */ 5187 static inline void ____napi_schedule(struct softnet_data *sd, 5188 struct napi_struct *napi) 5189 { 5190 struct task_struct *thread; 5191 5192 lockdep_assert_irqs_disabled(); 5193 5194 if (test_bit(NAPI_STATE_THREADED, &napi->state)) { 5195 /* Paired with smp_mb__before_atomic() in 5196 * napi_enable()/netif_set_threaded(). 5197 * Use READ_ONCE() to guarantee a complete 5198 * read on napi->thread. Only call 5199 * wake_up_process() when it's not NULL. 5200 */ 5201 thread = READ_ONCE(napi->thread); 5202 if (thread) { 5203 if (use_backlog_threads() && thread == raw_cpu_read(backlog_napi)) 5204 goto use_local_napi; 5205 5206 set_bit(NAPI_STATE_SCHED_THREADED, &napi->state); 5207 wake_up_process(thread); 5208 return; 5209 } 5210 } 5211 5212 use_local_napi: 5213 DEBUG_NET_WARN_ON_ONCE(!list_empty(&napi->poll_list)); 5214 list_add_tail(&napi->poll_list, &sd->poll_list); 5215 WRITE_ONCE(napi->list_owner, smp_processor_id()); 5216 /* If not called from net_rx_action() 5217 * we have to raise NET_RX_SOFTIRQ. 5218 */ 5219 if (!sd->in_net_rx_action) 5220 raise_softirq_irqoff(NET_RX_SOFTIRQ); 5221 } 5222 5223 #ifdef CONFIG_RPS 5224 5225 struct static_key_false rps_needed __read_mostly; 5226 EXPORT_SYMBOL(rps_needed); 5227 struct static_key_false rfs_needed __read_mostly; 5228 EXPORT_SYMBOL(rfs_needed); 5229 5230 static u32 rfs_slot(u32 hash, rps_tag_ptr tag_ptr) 5231 { 5232 return hash_32(hash, rps_tag_to_log(tag_ptr)); 5233 } 5234 5235 #ifdef CONFIG_RFS_ACCEL 5236 /** 5237 * rps_flow_is_active - check whether the flow is recently active. 5238 * @rflow: Specific flow to check activity. 5239 * @log: ilog2(hashsize). 5240 * @cpu: CPU saved in @rflow. 5241 * 5242 * If the CPU has processed many packets since the flow's last activity 5243 * (beyond 10 times the table size), the flow is considered stale. 5244 * 5245 * Return: true if flow was recently active. 5246 */ 5247 static bool rps_flow_is_active(struct rps_dev_flow *rflow, 5248 u8 log, 5249 unsigned int cpu) 5250 { 5251 unsigned int flow_last_active; 5252 unsigned int sd_input_head; 5253 5254 if (cpu >= nr_cpu_ids) 5255 return false; 5256 5257 sd_input_head = READ_ONCE(per_cpu(softnet_data, cpu).input_queue_head); 5258 flow_last_active = READ_ONCE(rflow->last_qtail); 5259 5260 return (int)(sd_input_head - flow_last_active) < 5261 (int)(10 << log); 5262 } 5263 #endif 5264 5265 static struct rps_dev_flow * 5266 set_rps_cpu(struct net_device *dev, struct sk_buff *skb, 5267 struct rps_dev_flow *rflow, u16 next_cpu, u32 hash) 5268 { 5269 if (next_cpu < nr_cpu_ids) { 5270 u32 head; 5271 #ifdef CONFIG_RFS_ACCEL 5272 struct netdev_rx_queue *rxqueue; 5273 struct rps_dev_flow *flow_table; 5274 struct rps_dev_flow *old_rflow; 5275 struct rps_dev_flow *tmp_rflow; 5276 rps_tag_ptr q_tag_ptr; 5277 unsigned int tmp_cpu; 5278 u16 rxq_index; 5279 u32 flow_id; 5280 int rc; 5281 5282 /* Should we steer this flow to a different hardware queue? */ 5283 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap || 5284 !(dev->features & NETIF_F_NTUPLE)) 5285 goto out; 5286 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu); 5287 if (rxq_index == skb_get_rx_queue(skb)) 5288 goto out; 5289 5290 rxqueue = dev->_rx + rxq_index; 5291 q_tag_ptr = READ_ONCE(rxqueue->rps_flow_table); 5292 if (!q_tag_ptr) 5293 goto out; 5294 5295 flow_id = rfs_slot(hash, q_tag_ptr); 5296 flow_table = rps_tag_to_table(q_tag_ptr); 5297 tmp_rflow = flow_table + flow_id; 5298 tmp_cpu = READ_ONCE(tmp_rflow->cpu); 5299 5300 if (READ_ONCE(tmp_rflow->filter) != RPS_NO_FILTER) { 5301 if (rps_flow_is_active(tmp_rflow, 5302 rps_tag_to_log(q_tag_ptr), 5303 tmp_cpu)) { 5304 if (hash != READ_ONCE(tmp_rflow->hash) || 5305 next_cpu == tmp_cpu) 5306 goto out; 5307 } 5308 } 5309 5310 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb, 5311 rxq_index, flow_id); 5312 if (rc < 0) 5313 goto out; 5314 5315 old_rflow = rflow; 5316 rflow = tmp_rflow; 5317 WRITE_ONCE(rflow->filter, rc); 5318 WRITE_ONCE(rflow->hash, hash); 5319 5320 if (old_rflow->filter == rc) 5321 WRITE_ONCE(old_rflow->filter, RPS_NO_FILTER); 5322 out: 5323 #endif 5324 head = READ_ONCE(per_cpu(softnet_data, next_cpu).input_queue_head); 5325 rps_input_queue_tail_save(&rflow->last_qtail, head); 5326 } 5327 5328 WRITE_ONCE(rflow->cpu, next_cpu); 5329 return rflow; 5330 } 5331 5332 /* 5333 * get_rps_cpu is called from netif_receive_skb and returns the target 5334 * CPU from the RPS map of the receiving queue for a given skb. 5335 * rcu_read_lock must be held on entry. 5336 */ 5337 static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb, 5338 struct rps_dev_flow **rflowp) 5339 { 5340 struct netdev_rx_queue *rxqueue = dev->_rx; 5341 rps_tag_ptr global_tag_ptr, q_tag_ptr; 5342 struct rps_map *map; 5343 int cpu = -1; 5344 u32 tcpu; 5345 u32 hash; 5346 5347 if (skb_rx_queue_recorded(skb)) { 5348 u16 index = skb_get_rx_queue(skb); 5349 5350 if (unlikely(index >= dev->real_num_rx_queues)) { 5351 WARN_ONCE(dev->real_num_rx_queues > 1, 5352 "%s received packet on queue %u, but number " 5353 "of RX queues is %u\n", 5354 dev->name, index, dev->real_num_rx_queues); 5355 goto done; 5356 } 5357 rxqueue += index; 5358 } 5359 5360 /* Avoid computing hash if RFS/RPS is not active for this rxqueue */ 5361 5362 q_tag_ptr = READ_ONCE(rxqueue->rps_flow_table); 5363 map = rcu_dereference(rxqueue->rps_map); 5364 if (!q_tag_ptr && !map) 5365 goto done; 5366 5367 skb_reset_network_header(skb); 5368 hash = skb_get_hash(skb); 5369 if (!hash) 5370 goto done; 5371 5372 global_tag_ptr = READ_ONCE(net_hotdata.rps_sock_flow_table); 5373 if (q_tag_ptr && global_tag_ptr) { 5374 struct rps_sock_flow_table *sock_flow_table; 5375 struct rps_dev_flow *flow_table; 5376 struct rps_dev_flow *rflow; 5377 u32 next_cpu; 5378 u32 flow_id; 5379 u32 ident; 5380 5381 /* First check into global flow table if there is a match. 5382 * This READ_ONCE() pairs with WRITE_ONCE() from rps_record_sock_flow(). 5383 */ 5384 flow_id = hash & rps_tag_to_mask(global_tag_ptr); 5385 sock_flow_table = rps_tag_to_table(global_tag_ptr); 5386 ident = READ_ONCE(sock_flow_table[flow_id].ent); 5387 if ((ident ^ hash) & ~net_hotdata.rps_cpu_mask) 5388 goto try_rps; 5389 5390 next_cpu = ident & net_hotdata.rps_cpu_mask; 5391 5392 /* OK, now we know there is a match, 5393 * we can look at the local (per receive queue) flow table 5394 */ 5395 flow_id = rfs_slot(hash, q_tag_ptr); 5396 flow_table = rps_tag_to_table(q_tag_ptr); 5397 rflow = flow_table + flow_id; 5398 tcpu = rflow->cpu; 5399 5400 /* 5401 * If the desired CPU (where last recvmsg was done) is 5402 * different from current CPU (one in the rx-queue flow 5403 * table entry), switch if one of the following holds: 5404 * - Current CPU is unset (>= nr_cpu_ids). 5405 * - Current CPU is offline. 5406 * - The current CPU's queue tail has advanced beyond the 5407 * last packet that was enqueued using this table entry. 5408 * This guarantees that all previous packets for the flow 5409 * have been dequeued, thus preserving in order delivery. 5410 */ 5411 if (unlikely(tcpu != next_cpu) && 5412 (tcpu >= nr_cpu_ids || !cpu_online(tcpu) || 5413 ((int)(READ_ONCE(per_cpu(softnet_data, tcpu).input_queue_head) - 5414 rflow->last_qtail)) >= 0)) { 5415 tcpu = next_cpu; 5416 rflow = set_rps_cpu(dev, skb, rflow, next_cpu, hash); 5417 } 5418 5419 if (tcpu < nr_cpu_ids && cpu_online(tcpu)) { 5420 *rflowp = rflow; 5421 cpu = tcpu; 5422 goto done; 5423 } 5424 } 5425 5426 try_rps: 5427 5428 if (map) { 5429 tcpu = map->cpus[reciprocal_scale(hash, map->len)]; 5430 if (cpu_online(tcpu)) { 5431 cpu = tcpu; 5432 goto done; 5433 } 5434 } 5435 5436 done: 5437 return cpu; 5438 } 5439 5440 #ifdef CONFIG_RFS_ACCEL 5441 5442 /** 5443 * rps_may_expire_flow - check whether an RFS hardware filter may be removed 5444 * @dev: Device on which the filter was set 5445 * @rxq_index: RX queue index 5446 * @flow_id: Flow ID passed to ndo_rx_flow_steer() 5447 * @filter_id: Filter ID returned by ndo_rx_flow_steer() 5448 * 5449 * Drivers that implement ndo_rx_flow_steer() should periodically call 5450 * this function for each installed filter and remove the filters for 5451 * which it returns %true. 5452 */ 5453 bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index, 5454 u32 flow_id, u16 filter_id) 5455 { 5456 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index; 5457 struct rps_dev_flow *flow_table; 5458 struct rps_dev_flow *rflow; 5459 rps_tag_ptr q_tag_ptr; 5460 bool expire = true; 5461 u8 log; 5462 5463 rcu_read_lock(); 5464 q_tag_ptr = READ_ONCE(rxqueue->rps_flow_table); 5465 log = rps_tag_to_log(q_tag_ptr); 5466 if (q_tag_ptr && flow_id < (1UL << log)) { 5467 unsigned int cpu; 5468 5469 flow_table = rps_tag_to_table(q_tag_ptr); 5470 rflow = flow_table + flow_id; 5471 cpu = READ_ONCE(rflow->cpu); 5472 if (READ_ONCE(rflow->filter) == filter_id && 5473 rps_flow_is_active(rflow, log, cpu)) 5474 expire = false; 5475 } 5476 rcu_read_unlock(); 5477 return expire; 5478 } 5479 EXPORT_SYMBOL(rps_may_expire_flow); 5480 5481 #endif /* CONFIG_RFS_ACCEL */ 5482 5483 /* Called from hardirq (IPI) context */ 5484 static void rps_trigger_softirq(void *data) 5485 { 5486 struct softnet_data *sd = data; 5487 5488 ____napi_schedule(sd, &sd->backlog); 5489 /* Pairs with READ_ONCE() in softnet_seq_show() */ 5490 WRITE_ONCE(sd->received_rps, sd->received_rps + 1); 5491 } 5492 5493 #endif /* CONFIG_RPS */ 5494 5495 /* Called from hardirq (IPI) context */ 5496 static void trigger_rx_softirq(void *data) 5497 { 5498 struct softnet_data *sd = data; 5499 5500 __raise_softirq_irqoff(NET_RX_SOFTIRQ); 5501 smp_store_release(&sd->defer_ipi_scheduled, 0); 5502 } 5503 5504 /* 5505 * After we queued a packet into sd->input_pkt_queue, 5506 * we need to make sure this queue is serviced soon. 5507 * 5508 * - If this is another cpu queue, link it to our rps_ipi_list, 5509 * and make sure we will process rps_ipi_list from net_rx_action(). 5510 * 5511 * - If this is our own queue, NAPI schedule our backlog. 5512 * Note that this also raises NET_RX_SOFTIRQ. 5513 */ 5514 static void napi_schedule_rps(struct softnet_data *sd) 5515 { 5516 struct softnet_data *mysd = this_cpu_ptr(&softnet_data); 5517 5518 #ifdef CONFIG_RPS 5519 if (sd != mysd) { 5520 if (use_backlog_threads()) { 5521 __napi_schedule_irqoff(&sd->backlog); 5522 return; 5523 } 5524 5525 sd->rps_ipi_next = mysd->rps_ipi_list; 5526 mysd->rps_ipi_list = sd; 5527 5528 /* If not called from net_rx_action() or napi_threaded_poll() 5529 * we have to raise NET_RX_SOFTIRQ. 5530 */ 5531 if (!mysd->in_net_rx_action && !mysd->in_napi_threaded_poll) 5532 __raise_softirq_irqoff(NET_RX_SOFTIRQ); 5533 return; 5534 } 5535 #endif /* CONFIG_RPS */ 5536 __napi_schedule_irqoff(&mysd->backlog); 5537 } 5538 5539 void kick_defer_list_purge(unsigned int cpu) 5540 { 5541 struct softnet_data *sd = &per_cpu(softnet_data, cpu); 5542 unsigned long flags; 5543 5544 if (use_backlog_threads()) { 5545 backlog_lock_irq_save(sd, &flags); 5546 5547 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) 5548 __napi_schedule_irqoff(&sd->backlog); 5549 5550 backlog_unlock_irq_restore(sd, flags); 5551 5552 } else if (!cmpxchg(&sd->defer_ipi_scheduled, 0, 1)) { 5553 if (smp_call_function_single_async(cpu, &sd->defer_csd)) 5554 WRITE_ONCE(sd->defer_ipi_scheduled, 0); 5555 } 5556 } 5557 5558 #ifdef CONFIG_NET_FLOW_LIMIT 5559 int netdev_flow_limit_table_len __read_mostly = (1 << 12); 5560 #endif 5561 5562 static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen, 5563 int max_backlog) 5564 { 5565 #ifdef CONFIG_NET_FLOW_LIMIT 5566 unsigned int old_flow, new_flow; 5567 const struct softnet_data *sd; 5568 struct sd_flow_limit *fl; 5569 5570 if (likely(qlen < (max_backlog >> 1))) 5571 return false; 5572 5573 sd = this_cpu_ptr(&softnet_data); 5574 5575 rcu_read_lock(); 5576 fl = rcu_dereference(sd->flow_limit); 5577 if (fl) { 5578 new_flow = hash_32(skb_get_hash(skb), fl->log_buckets); 5579 old_flow = fl->history[fl->history_head]; 5580 fl->history[fl->history_head] = new_flow; 5581 5582 fl->history_head++; 5583 fl->history_head &= FLOW_LIMIT_HISTORY - 1; 5584 5585 if (likely(fl->buckets[old_flow])) 5586 fl->buckets[old_flow]--; 5587 5588 if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) { 5589 /* Pairs with READ_ONCE() in softnet_seq_show() */ 5590 WRITE_ONCE(fl->count, fl->count + 1); 5591 rcu_read_unlock(); 5592 return true; 5593 } 5594 } 5595 rcu_read_unlock(); 5596 #endif 5597 return false; 5598 } 5599 5600 /* 5601 * enqueue_to_backlog is called to queue an skb to a per CPU backlog 5602 * queue (may be a remote CPU queue). 5603 */ 5604 static int enqueue_to_backlog(struct sk_buff *skb, int cpu, 5605 unsigned int *qtail) 5606 { 5607 enum skb_drop_reason reason; 5608 struct softnet_data *sd; 5609 unsigned long flags; 5610 unsigned int qlen; 5611 int max_backlog; 5612 u32 tail; 5613 5614 reason = SKB_DROP_REASON_DEV_READY; 5615 5616 sd = &per_cpu(softnet_data, cpu); 5617 5618 qlen = skb_queue_len_lockless(&sd->input_pkt_queue); 5619 max_backlog = READ_ONCE(net_hotdata.max_backlog); 5620 if (unlikely(qlen > max_backlog) || 5621 skb_flow_limit(skb, qlen, max_backlog)) 5622 goto cpu_backlog_drop; 5623 backlog_lock_irq_save(sd, &flags); 5624 qlen = skb_queue_len(&sd->input_pkt_queue); 5625 if (likely(qlen <= max_backlog)) { 5626 if (unlikely(!netif_running(skb->dev))) { 5627 backlog_unlock_irq_restore(sd, flags); 5628 goto bad_dev; 5629 } 5630 if (!qlen) { 5631 /* Schedule NAPI for backlog device. We can use 5632 * non atomic operation as we own the queue lock. 5633 */ 5634 if (!__test_and_set_bit(NAPI_STATE_SCHED, 5635 &sd->backlog.state)) 5636 napi_schedule_rps(sd); 5637 } 5638 __skb_queue_tail(&sd->input_pkt_queue, skb); 5639 tail = rps_input_queue_tail_incr(sd); 5640 backlog_unlock_irq_restore(sd, flags); 5641 5642 /* save the tail outside of the critical section */ 5643 rps_input_queue_tail_save(qtail, tail); 5644 return NET_RX_SUCCESS; 5645 } 5646 5647 backlog_unlock_irq_restore(sd, flags); 5648 5649 cpu_backlog_drop: 5650 reason = SKB_DROP_REASON_CPU_BACKLOG; 5651 numa_drop_add(&sd->drop_counters, 1); 5652 bad_dev: 5653 dev_core_stats_rx_dropped_inc(skb->dev); 5654 kfree_skb_reason(skb, reason); 5655 return NET_RX_DROP; 5656 } 5657 5658 static struct netdev_rx_queue *netif_get_rxqueue(struct sk_buff *skb) 5659 { 5660 struct net_device *dev = skb->dev; 5661 struct netdev_rx_queue *rxqueue; 5662 5663 rxqueue = dev->_rx; 5664 5665 if (skb_rx_queue_recorded(skb)) { 5666 u16 index = skb_get_rx_queue(skb); 5667 5668 if (unlikely(index >= dev->real_num_rx_queues)) { 5669 WARN_ONCE(dev->real_num_rx_queues > 1, 5670 "%s received packet on queue %u, but number " 5671 "of RX queues is %u\n", 5672 dev->name, index, dev->real_num_rx_queues); 5673 5674 return rxqueue; /* Return first rxqueue */ 5675 } 5676 rxqueue += index; 5677 } 5678 return rxqueue; 5679 } 5680 5681 u32 bpf_prog_run_generic_xdp(struct sk_buff *skb, struct xdp_buff *xdp, 5682 const struct bpf_prog *xdp_prog) 5683 { 5684 void *orig_data, *orig_data_end, *hard_start; 5685 struct netdev_rx_queue *rxqueue; 5686 bool orig_bcast, orig_host; 5687 u32 mac_len, frame_sz; 5688 __be16 orig_eth_type; 5689 struct ethhdr *eth; 5690 u32 metalen, act; 5691 int off; 5692 5693 /* The XDP program wants to see the packet starting at the MAC 5694 * header. 5695 */ 5696 mac_len = skb->data - skb_mac_header(skb); 5697 hard_start = skb->data - skb_headroom(skb); 5698 5699 /* SKB "head" area always have tailroom for skb_shared_info */ 5700 frame_sz = (void *)skb_end_pointer(skb) - hard_start; 5701 frame_sz += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 5702 5703 rxqueue = netif_get_rxqueue(skb); 5704 xdp_init_buff(xdp, frame_sz, &rxqueue->xdp_rxq); 5705 xdp_prepare_buff(xdp, hard_start, skb_headroom(skb) - mac_len, 5706 skb_headlen(skb) + mac_len, true); 5707 if (skb_is_nonlinear(skb)) { 5708 skb_shinfo(skb)->xdp_frags_size = skb->data_len; 5709 xdp_buff_set_frags_flag(xdp); 5710 } else { 5711 xdp_buff_clear_frags_flag(xdp); 5712 } 5713 5714 orig_data_end = xdp->data_end; 5715 orig_data = xdp->data; 5716 eth = (struct ethhdr *)xdp->data; 5717 orig_host = ether_addr_equal_64bits(eth->h_dest, skb->dev->dev_addr); 5718 orig_bcast = is_multicast_ether_addr_64bits(eth->h_dest); 5719 orig_eth_type = eth->h_proto; 5720 5721 act = bpf_prog_run_xdp(xdp_prog, xdp); 5722 5723 /* check if bpf_xdp_adjust_head was used */ 5724 off = xdp->data - orig_data; 5725 if (off) { 5726 if (off > 0) 5727 __skb_pull(skb, off); 5728 else if (off < 0) 5729 __skb_push(skb, -off); 5730 5731 skb->mac_header += off; 5732 skb_reset_network_header(skb); 5733 } 5734 5735 /* check if bpf_xdp_adjust_tail was used */ 5736 off = xdp->data_end - orig_data_end; 5737 if (off != 0) { 5738 skb_set_tail_pointer(skb, xdp->data_end - xdp->data); 5739 skb->len += off; /* positive on grow, negative on shrink */ 5740 } 5741 5742 /* XDP frag metadata (e.g. nr_frags) are updated in eBPF helpers 5743 * (e.g. bpf_xdp_adjust_tail). Remove the old fragment contribution 5744 * from skb->len before updating data_len, then add the new one back. 5745 */ 5746 skb->len -= skb->data_len; 5747 if (xdp_buff_has_frags(xdp)) { 5748 skb->data_len = skb_shinfo(skb)->xdp_frags_size; 5749 skb->len += skb->data_len; 5750 } else { 5751 skb->data_len = 0; 5752 } 5753 5754 /* check if XDP changed eth hdr such SKB needs update */ 5755 eth = (struct ethhdr *)xdp->data; 5756 if ((orig_eth_type != eth->h_proto) || 5757 (orig_host != ether_addr_equal_64bits(eth->h_dest, 5758 skb->dev->dev_addr)) || 5759 (orig_bcast != is_multicast_ether_addr_64bits(eth->h_dest))) { 5760 __skb_push(skb, ETH_HLEN); 5761 skb->pkt_type = PACKET_HOST; 5762 skb->protocol = eth_type_trans(skb, skb->dev); 5763 } 5764 5765 /* Redirect/Tx gives L2 packet, code that will reuse skb must __skb_pull 5766 * before calling us again on redirect path. We do not call do_redirect 5767 * as we leave that up to the caller. 5768 * 5769 * Caller is responsible for managing lifetime of skb (i.e. calling 5770 * kfree_skb in response to actions it cannot handle/XDP_DROP). 5771 */ 5772 switch (act) { 5773 case XDP_REDIRECT: 5774 case XDP_TX: 5775 __skb_push(skb, mac_len); 5776 break; 5777 case XDP_PASS: 5778 metalen = xdp->data - xdp->data_meta; 5779 if (metalen) 5780 skb_metadata_set(skb, metalen); 5781 break; 5782 } 5783 5784 return act; 5785 } 5786 5787 static int 5788 netif_skb_check_for_xdp(struct sk_buff **pskb, const struct bpf_prog *prog) 5789 { 5790 struct sk_buff *skb = *pskb; 5791 int err, hroom, troom; 5792 5793 local_lock_nested_bh(&system_page_pool.bh_lock); 5794 err = skb_cow_data_for_xdp(this_cpu_read(system_page_pool.pool), pskb, prog); 5795 local_unlock_nested_bh(&system_page_pool.bh_lock); 5796 if (!err) 5797 return 0; 5798 5799 /* In case we have to go down the path and also linearize, 5800 * then lets do the pskb_expand_head() work just once here. 5801 */ 5802 hroom = XDP_PACKET_HEADROOM - skb_headroom(skb); 5803 troom = skb->tail + skb->data_len - skb->end; 5804 err = pskb_expand_head(skb, 5805 hroom > 0 ? ALIGN(hroom, NET_SKB_PAD) : 0, 5806 troom > 0 ? troom + 128 : 0, GFP_ATOMIC); 5807 if (err) 5808 return err; 5809 5810 return skb_linearize(skb); 5811 } 5812 5813 static u32 netif_receive_generic_xdp(struct sk_buff **pskb, 5814 struct xdp_buff *xdp, 5815 const struct bpf_prog *xdp_prog) 5816 { 5817 struct sk_buff *skb = *pskb; 5818 u32 mac_len, act = XDP_DROP; 5819 5820 /* Reinjected packets coming from act_mirred or similar should 5821 * not get XDP generic processing. 5822 */ 5823 if (skb_is_redirected(skb)) 5824 return XDP_PASS; 5825 5826 /* XDP packets must have sufficient headroom of XDP_PACKET_HEADROOM 5827 * bytes. This is the guarantee that also native XDP provides, 5828 * thus we need to do it here as well. 5829 */ 5830 mac_len = skb->data - skb_mac_header(skb); 5831 __skb_push(skb, mac_len); 5832 5833 if (skb_cloned(skb) || skb_is_nonlinear(skb) || 5834 skb_headroom(skb) < XDP_PACKET_HEADROOM) { 5835 if (netif_skb_check_for_xdp(pskb, xdp_prog)) 5836 goto do_drop; 5837 } 5838 5839 __skb_pull(*pskb, mac_len); 5840 5841 act = bpf_prog_run_generic_xdp(*pskb, xdp, xdp_prog); 5842 switch (act) { 5843 case XDP_REDIRECT: 5844 case XDP_TX: 5845 case XDP_PASS: 5846 break; 5847 default: 5848 bpf_warn_invalid_xdp_action((*pskb)->dev, xdp_prog, act); 5849 fallthrough; 5850 case XDP_ABORTED: 5851 trace_xdp_exception((*pskb)->dev, xdp_prog, act); 5852 fallthrough; 5853 case XDP_DROP: 5854 do_drop: 5855 kfree_skb(*pskb); 5856 break; 5857 } 5858 5859 return act; 5860 } 5861 5862 /* When doing generic XDP we have to bypass the qdisc layer and the 5863 * network taps in order to match in-driver-XDP behavior. This also means 5864 * that XDP packets are able to starve other packets going through a qdisc, 5865 * and DDOS attacks will be more effective. In-driver-XDP use dedicated TX 5866 * queues, so they do not have this starvation issue. 5867 */ 5868 void generic_xdp_tx(struct sk_buff *skb, const struct bpf_prog *xdp_prog) 5869 { 5870 struct net_device *dev = skb->dev; 5871 struct netdev_queue *txq; 5872 bool free_skb = true; 5873 int cpu, rc; 5874 5875 txq = netdev_core_pick_tx(dev, skb, NULL); 5876 cpu = smp_processor_id(); 5877 HARD_TX_LOCK(dev, txq, cpu); 5878 if (!netif_xmit_frozen_or_drv_stopped(txq)) { 5879 rc = netdev_start_xmit(skb, dev, txq, 0); 5880 if (dev_xmit_complete(rc)) 5881 free_skb = false; 5882 } 5883 HARD_TX_UNLOCK(dev, txq); 5884 if (free_skb) { 5885 trace_xdp_exception(dev, xdp_prog, XDP_TX); 5886 dev_core_stats_tx_dropped_inc(dev); 5887 kfree_skb(skb); 5888 } 5889 } 5890 5891 static DEFINE_STATIC_KEY_FALSE(generic_xdp_needed_key); 5892 5893 int do_xdp_generic(const struct bpf_prog *xdp_prog, struct sk_buff **pskb) 5894 { 5895 struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx; 5896 5897 if (xdp_prog) { 5898 struct xdp_buff xdp; 5899 u32 act; 5900 int err; 5901 5902 bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx); 5903 act = netif_receive_generic_xdp(pskb, &xdp, xdp_prog); 5904 if (act != XDP_PASS) { 5905 switch (act) { 5906 case XDP_REDIRECT: 5907 err = xdp_do_generic_redirect((*pskb)->dev, *pskb, 5908 &xdp, xdp_prog); 5909 if (err) 5910 goto out_redir; 5911 break; 5912 case XDP_TX: 5913 generic_xdp_tx(*pskb, xdp_prog); 5914 break; 5915 } 5916 bpf_net_ctx_clear(bpf_net_ctx); 5917 return XDP_DROP; 5918 } 5919 bpf_net_ctx_clear(bpf_net_ctx); 5920 } 5921 return XDP_PASS; 5922 out_redir: 5923 bpf_net_ctx_clear(bpf_net_ctx); 5924 kfree_skb_reason(*pskb, SKB_DROP_REASON_XDP); 5925 return XDP_DROP; 5926 } 5927 EXPORT_SYMBOL_GPL(do_xdp_generic); 5928 5929 static int netif_rx_internal(struct sk_buff *skb) 5930 { 5931 int ret; 5932 5933 net_timestamp_check(READ_ONCE(net_hotdata.tstamp_prequeue), skb); 5934 5935 trace_netif_rx(skb); 5936 5937 #ifdef CONFIG_RPS 5938 if (static_branch_unlikely(&rps_needed)) { 5939 struct rps_dev_flow voidflow, *rflow = &voidflow; 5940 int cpu; 5941 5942 rcu_read_lock(); 5943 5944 cpu = get_rps_cpu(skb->dev, skb, &rflow); 5945 if (cpu < 0) 5946 cpu = smp_processor_id(); 5947 5948 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail); 5949 5950 rcu_read_unlock(); 5951 } else 5952 #endif 5953 { 5954 unsigned int qtail; 5955 5956 ret = enqueue_to_backlog(skb, smp_processor_id(), &qtail); 5957 } 5958 return ret; 5959 } 5960 5961 /** 5962 * __netif_rx - Slightly optimized version of netif_rx 5963 * @skb: buffer to post 5964 * 5965 * This behaves as netif_rx except that it does not disable bottom halves. 5966 * As a result this function may only be invoked from the interrupt context 5967 * (either hard or soft interrupt). 5968 */ 5969 int __netif_rx(struct sk_buff *skb) 5970 { 5971 int ret; 5972 5973 lockdep_assert_once(hardirq_count() | softirq_count()); 5974 5975 trace_netif_rx_entry(skb); 5976 ret = netif_rx_internal(skb); 5977 trace_netif_rx_exit(ret); 5978 return ret; 5979 } 5980 EXPORT_SYMBOL(__netif_rx); 5981 5982 /** 5983 * netif_rx - post buffer to the network code 5984 * @skb: buffer to post 5985 * 5986 * This function receives a packet from a device driver and queues it for 5987 * the upper (protocol) levels to process via the backlog NAPI device. It 5988 * always succeeds. The buffer may be dropped during processing for 5989 * congestion control or by the protocol layers. 5990 * The network buffer is passed via the backlog NAPI device. Modern NIC 5991 * driver should use NAPI and GRO. 5992 * This function can used from interrupt and from process context. The 5993 * caller from process context must not disable interrupts before invoking 5994 * this function. 5995 * 5996 * return values: 5997 * NET_RX_SUCCESS (no congestion) 5998 * NET_RX_DROP (packet was dropped) 5999 * 6000 */ 6001 int netif_rx(struct sk_buff *skb) 6002 { 6003 bool need_bh_off = !(hardirq_count() | softirq_count()); 6004 int ret; 6005 6006 if (need_bh_off) 6007 local_bh_disable(); 6008 trace_netif_rx_entry(skb); 6009 ret = netif_rx_internal(skb); 6010 trace_netif_rx_exit(ret); 6011 if (need_bh_off) 6012 local_bh_enable(); 6013 return ret; 6014 } 6015 EXPORT_SYMBOL(netif_rx); 6016 6017 static __latent_entropy void net_tx_action(void) 6018 { 6019 struct softnet_data *sd = this_cpu_ptr(&softnet_data); 6020 6021 if (sd->completion_queue) { 6022 struct sk_buff *clist; 6023 6024 local_irq_disable(); 6025 clist = sd->completion_queue; 6026 sd->completion_queue = NULL; 6027 local_irq_enable(); 6028 6029 while (clist) { 6030 struct sk_buff *skb = clist; 6031 6032 clist = clist->next; 6033 6034 WARN_ON(refcount_read(&skb->users)); 6035 if (likely(get_kfree_skb_cb(skb)->reason == SKB_CONSUMED)) 6036 trace_consume_skb(skb, net_tx_action); 6037 else 6038 trace_kfree_skb(skb, net_tx_action, 6039 get_kfree_skb_cb(skb)->reason, NULL); 6040 6041 if (skb->fclone != SKB_FCLONE_UNAVAILABLE) 6042 __kfree_skb(skb); 6043 else 6044 __napi_kfree_skb(skb, 6045 get_kfree_skb_cb(skb)->reason); 6046 } 6047 } 6048 6049 if (sd->output_queue) { 6050 struct Qdisc *head; 6051 6052 local_irq_disable(); 6053 head = sd->output_queue; 6054 sd->output_queue = NULL; 6055 sd->output_queue_tailp = &sd->output_queue; 6056 local_irq_enable(); 6057 6058 rcu_read_lock(); 6059 6060 while (head) { 6061 spinlock_t *root_lock = NULL; 6062 struct sk_buff *to_free; 6063 struct Qdisc *q = head; 6064 6065 head = head->next_sched; 6066 6067 /* We need to make sure head->next_sched is read 6068 * before clearing __QDISC_STATE_SCHED 6069 */ 6070 smp_mb__before_atomic(); 6071 6072 if (!(q->flags & TCQ_F_NOLOCK)) { 6073 root_lock = qdisc_lock(q); 6074 spin_lock(root_lock); 6075 } else if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, 6076 &q->state))) { 6077 /* There is a synchronize_net() between 6078 * STATE_DEACTIVATED flag being set and 6079 * qdisc_reset()/some_qdisc_is_busy() in 6080 * dev_deactivate(), so we can safely bail out 6081 * early here to avoid data race between 6082 * qdisc_deactivate() and some_qdisc_is_busy() 6083 * for lockless qdisc. 6084 */ 6085 clear_bit(__QDISC_STATE_SCHED, &q->state); 6086 continue; 6087 } 6088 6089 clear_bit(__QDISC_STATE_SCHED, &q->state); 6090 to_free = qdisc_run(q); 6091 if (root_lock) 6092 spin_unlock(root_lock); 6093 tcf_kfree_skb_list(to_free, q, NULL, qdisc_dev(q)); 6094 } 6095 6096 rcu_read_unlock(); 6097 } 6098 6099 xfrm_dev_backlog(sd); 6100 } 6101 6102 /** 6103 * netdev_is_rx_handler_busy - check if receive handler is registered 6104 * @dev: device to check 6105 * 6106 * Check if a receive handler is already registered for a given device. 6107 * Return true if there one. 6108 * 6109 * The caller must hold the rtnl_mutex. 6110 */ 6111 bool netdev_is_rx_handler_busy(struct net_device *dev) 6112 { 6113 ASSERT_RTNL(); 6114 return dev && rtnl_dereference(dev->rx_handler); 6115 } 6116 EXPORT_SYMBOL_GPL(netdev_is_rx_handler_busy); 6117 6118 /** 6119 * netdev_rx_handler_register - register receive handler 6120 * @dev: device to register a handler for 6121 * @rx_handler: receive handler to register 6122 * @rx_handler_data: data pointer that is used by rx handler 6123 * 6124 * Register a receive handler for a device. This handler will then be 6125 * called from __netif_receive_skb. A negative errno code is returned 6126 * on a failure. 6127 * 6128 * The caller must hold the rtnl_mutex. 6129 * 6130 * For a general description of rx_handler, see enum rx_handler_result. 6131 */ 6132 int netdev_rx_handler_register(struct net_device *dev, 6133 rx_handler_func_t *rx_handler, 6134 void *rx_handler_data) 6135 { 6136 if (netdev_is_rx_handler_busy(dev)) 6137 return -EBUSY; 6138 6139 if (dev->priv_flags & IFF_NO_RX_HANDLER) 6140 return -EINVAL; 6141 6142 /* Note: rx_handler_data must be set before rx_handler */ 6143 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data); 6144 rcu_assign_pointer(dev->rx_handler, rx_handler); 6145 6146 return 0; 6147 } 6148 EXPORT_SYMBOL_GPL(netdev_rx_handler_register); 6149 6150 /** 6151 * netdev_rx_handler_unregister - unregister receive handler 6152 * @dev: device to unregister a handler from 6153 * 6154 * Unregister a receive handler from a device. 6155 * 6156 * The caller must hold the rtnl_mutex. 6157 */ 6158 void netdev_rx_handler_unregister(struct net_device *dev) 6159 { 6160 6161 ASSERT_RTNL(); 6162 RCU_INIT_POINTER(dev->rx_handler, NULL); 6163 /* a reader seeing a non NULL rx_handler in a rcu_read_lock() 6164 * section has a guarantee to see a non NULL rx_handler_data 6165 * as well. 6166 */ 6167 synchronize_net(); 6168 RCU_INIT_POINTER(dev->rx_handler_data, NULL); 6169 } 6170 EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister); 6171 6172 /* 6173 * Limit the use of PFMEMALLOC reserves to those protocols that implement 6174 * the special handling of PFMEMALLOC skbs. 6175 */ 6176 static bool skb_pfmemalloc_protocol(struct sk_buff *skb) 6177 { 6178 switch (skb->protocol) { 6179 case htons(ETH_P_ARP): 6180 case htons(ETH_P_IP): 6181 case htons(ETH_P_IPV6): 6182 case htons(ETH_P_8021Q): 6183 case htons(ETH_P_8021AD): 6184 return true; 6185 default: 6186 return false; 6187 } 6188 } 6189 6190 static inline int nf_ingress(struct sk_buff *skb, struct packet_type **pt_prev, 6191 int *ret, struct net_device *orig_dev) 6192 { 6193 if (nf_hook_ingress_active(skb)) { 6194 int ingress_retval; 6195 6196 if (unlikely(*pt_prev)) { 6197 *ret = deliver_skb(skb, *pt_prev, orig_dev); 6198 *pt_prev = NULL; 6199 } 6200 6201 rcu_read_lock(); 6202 ingress_retval = nf_hook_ingress(skb); 6203 rcu_read_unlock(); 6204 return ingress_retval; 6205 } 6206 return 0; 6207 } 6208 6209 static int __netif_receive_skb_core(struct sk_buff **pskb, bool pfmemalloc, 6210 struct packet_type **ppt_prev) 6211 { 6212 enum skb_drop_reason drop_reason = SKB_DROP_REASON_UNHANDLED_PROTO; 6213 struct packet_type *ptype, *pt_prev; 6214 rx_handler_func_t *rx_handler; 6215 struct sk_buff *skb = *pskb; 6216 struct net_device *orig_dev; 6217 bool deliver_exact = false; 6218 int ret = NET_RX_DROP; 6219 __be16 type; 6220 6221 net_timestamp_check(!READ_ONCE(net_hotdata.tstamp_prequeue), skb); 6222 6223 trace_netif_receive_skb(skb); 6224 6225 orig_dev = skb->dev; 6226 6227 skb_reset_network_header(skb); 6228 #if !defined(CONFIG_DEBUG_NET) 6229 /* We plan to no longer reset the transport header here. 6230 * Give some time to fuzzers and dev build to catch bugs 6231 * in network stacks. 6232 */ 6233 if (!skb_transport_header_was_set(skb)) 6234 skb_reset_transport_header(skb); 6235 #endif 6236 skb_reset_mac_len(skb); 6237 6238 pt_prev = NULL; 6239 6240 another_round: 6241 skb->skb_iif = skb->dev->ifindex; 6242 6243 __this_cpu_inc(softnet_data.processed); 6244 6245 if (static_branch_unlikely(&generic_xdp_needed_key)) { 6246 int ret2; 6247 6248 migrate_disable(); 6249 ret2 = do_xdp_generic(rcu_dereference(skb->dev->xdp_prog), 6250 &skb); 6251 migrate_enable(); 6252 6253 if (ret2 != XDP_PASS) { 6254 ret = NET_RX_DROP; 6255 goto out; 6256 } 6257 } 6258 6259 if (eth_type_vlan(skb->protocol)) { 6260 skb = skb_vlan_untag(skb); 6261 if (unlikely(!skb)) 6262 goto out; 6263 } 6264 6265 if (skb_skip_tc_classify(skb)) 6266 goto skip_classify; 6267 6268 if (pfmemalloc) 6269 goto skip_taps; 6270 6271 list_for_each_entry_rcu(ptype, &dev_net_rcu(skb->dev)->ptype_all, 6272 list) { 6273 if (unlikely(pt_prev)) 6274 ret = deliver_skb(skb, pt_prev, orig_dev); 6275 pt_prev = ptype; 6276 } 6277 6278 list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) { 6279 if (unlikely(pt_prev)) 6280 ret = deliver_skb(skb, pt_prev, orig_dev); 6281 pt_prev = ptype; 6282 } 6283 6284 skip_taps: 6285 #ifdef CONFIG_NET_INGRESS 6286 if (static_branch_unlikely(&ingress_needed_key)) { 6287 bool another = false; 6288 6289 nf_skip_egress(skb, true); 6290 skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev, 6291 &another); 6292 if (another) 6293 goto another_round; 6294 if (!skb) 6295 goto out; 6296 6297 nf_skip_egress(skb, false); 6298 if (nf_ingress(skb, &pt_prev, &ret, orig_dev) < 0) 6299 goto out; 6300 } 6301 #endif 6302 skb_reset_redirect(skb); 6303 skip_classify: 6304 if (pfmemalloc && !skb_pfmemalloc_protocol(skb)) { 6305 drop_reason = SKB_DROP_REASON_PFMEMALLOC; 6306 goto drop; 6307 } 6308 6309 if (skb_vlan_tag_present(skb)) { 6310 if (unlikely(pt_prev)) { 6311 ret = deliver_skb(skb, pt_prev, orig_dev); 6312 pt_prev = NULL; 6313 } 6314 if (vlan_do_receive(&skb)) 6315 goto another_round; 6316 else if (unlikely(!skb)) 6317 goto out; 6318 } 6319 6320 rx_handler = rcu_dereference(skb->dev->rx_handler); 6321 if (rx_handler) { 6322 if (unlikely(pt_prev)) { 6323 ret = deliver_skb(skb, pt_prev, orig_dev); 6324 pt_prev = NULL; 6325 } 6326 switch (rx_handler(&skb)) { 6327 case RX_HANDLER_CONSUMED: 6328 ret = NET_RX_SUCCESS; 6329 goto out; 6330 case RX_HANDLER_ANOTHER: 6331 goto another_round; 6332 case RX_HANDLER_EXACT: 6333 deliver_exact = true; 6334 break; 6335 case RX_HANDLER_PASS: 6336 break; 6337 default: 6338 BUG(); 6339 } 6340 } 6341 6342 if (unlikely(skb_vlan_tag_present(skb)) && !netdev_uses_dsa(skb->dev)) { 6343 unsigned int priority_tag_depth = 1; 6344 6345 check_vlan_id: 6346 if (skb_vlan_tag_get_id(skb)) { 6347 /* Vlan id is non 0 and vlan_do_receive() above couldn't 6348 * find vlan device. 6349 */ 6350 skb->pkt_type = PACKET_OTHERHOST; 6351 } else if (eth_type_vlan(skb->protocol)) { 6352 /* Outer header is 802.1P with vlan 0, inner header is 6353 * 802.1Q or 802.1AD and vlan_do_receive() above could 6354 * not find vlan dev for vlan id 0. 6355 */ 6356 if (unlikely(priority_tag_depth >= VLAN_MAX_DEPTH)) { 6357 drop_reason = SKB_DROP_REASON_RECURSION_LIMIT; 6358 goto drop; 6359 } 6360 6361 priority_tag_depth++; 6362 __vlan_hwaccel_clear_tag(skb); 6363 skb = skb_vlan_untag(skb); 6364 if (unlikely(!skb)) 6365 goto out; 6366 if (vlan_do_receive(&skb)) 6367 /* After stripping off 802.1P header with vlan 0 6368 * vlan dev is found for inner header. 6369 */ 6370 goto another_round; 6371 else if (unlikely(!skb)) 6372 goto out; 6373 else 6374 /* We have stripped outer 802.1P vlan 0 header. 6375 * But could not find vlan dev. 6376 * check again for vlan id to set OTHERHOST. 6377 */ 6378 goto check_vlan_id; 6379 } 6380 /* Note: we might in the future use prio bits 6381 * and set skb->priority like in vlan_do_receive() 6382 * For the time being, just ignore Priority Code Point 6383 */ 6384 __vlan_hwaccel_clear_tag(skb); 6385 } 6386 6387 type = skb->protocol; 6388 6389 /* deliver only exact match when indicated */ 6390 if (likely(!deliver_exact)) { 6391 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type, 6392 &ptype_base[ntohs(type) & 6393 PTYPE_HASH_MASK]); 6394 6395 /* orig_dev and skb->dev could belong to different netns; 6396 * Even in such case we need to traverse only the list 6397 * coming from skb->dev, as the ptype owner (packet socket) 6398 * will use dev_net(skb->dev) to do namespace filtering. 6399 */ 6400 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type, 6401 &dev_net_rcu(skb->dev)->ptype_specific); 6402 } 6403 6404 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type, 6405 &orig_dev->ptype_specific); 6406 6407 if (unlikely(skb->dev != orig_dev)) { 6408 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type, 6409 &skb->dev->ptype_specific); 6410 } 6411 6412 if (pt_prev) { 6413 *ppt_prev = pt_prev; 6414 } else { 6415 drop: 6416 if (!deliver_exact) 6417 dev_core_stats_rx_dropped_inc(skb->dev); 6418 else 6419 dev_core_stats_rx_nohandler_inc(skb->dev); 6420 6421 kfree_skb_reason(skb, drop_reason); 6422 /* Jamal, now you will not able to escape explaining 6423 * me how you were going to use this. :-) 6424 */ 6425 ret = NET_RX_DROP; 6426 } 6427 6428 out: 6429 /* The invariant here is that if *ppt_prev is not NULL 6430 * then skb should also be non-NULL. 6431 * 6432 * Apparently *ppt_prev assignment above holds this invariant due to 6433 * skb dereferencing near it. 6434 */ 6435 *pskb = skb; 6436 return ret; 6437 } 6438 6439 static int __netif_receive_skb_one_core(struct sk_buff *skb, bool pfmemalloc) 6440 { 6441 struct net_device *orig_dev = skb->dev; 6442 struct packet_type *pt_prev = NULL; 6443 int ret; 6444 6445 ret = __netif_receive_skb_core(&skb, pfmemalloc, &pt_prev); 6446 if (pt_prev) 6447 ret = INDIRECT_CALL_INET(pt_prev->func, ipv6_rcv, ip_rcv, skb, 6448 skb->dev, pt_prev, orig_dev); 6449 return ret; 6450 } 6451 6452 /** 6453 * netif_receive_skb_core - special purpose version of netif_receive_skb 6454 * @skb: buffer to process 6455 * 6456 * More direct receive version of netif_receive_skb(). It should 6457 * only be used by callers that have a need to skip RPS and Generic XDP. 6458 * Caller must also take care of handling if ``(page_is_)pfmemalloc``. 6459 * 6460 * This function may only be called from softirq context and interrupts 6461 * should be enabled. 6462 * 6463 * Return values (usually ignored): 6464 * NET_RX_SUCCESS: no congestion 6465 * NET_RX_DROP: packet was dropped 6466 */ 6467 int netif_receive_skb_core(struct sk_buff *skb) 6468 { 6469 int ret; 6470 6471 rcu_read_lock(); 6472 ret = __netif_receive_skb_one_core(skb, false); 6473 rcu_read_unlock(); 6474 6475 return ret; 6476 } 6477 EXPORT_SYMBOL(netif_receive_skb_core); 6478 6479 static inline void __netif_receive_skb_list_ptype(struct list_head *head, 6480 struct packet_type *pt_prev, 6481 struct net_device *orig_dev) 6482 { 6483 struct sk_buff *skb, *next; 6484 6485 if (!pt_prev) 6486 return; 6487 if (list_empty(head)) 6488 return; 6489 if (pt_prev->list_func != NULL) 6490 INDIRECT_CALL_INET(pt_prev->list_func, ipv6_list_rcv, 6491 ip_list_rcv, head, pt_prev, orig_dev); 6492 else 6493 list_for_each_entry_safe(skb, next, head, list) { 6494 skb_list_del_init(skb); 6495 pt_prev->func(skb, skb->dev, pt_prev, orig_dev); 6496 } 6497 } 6498 6499 static void __netif_receive_skb_list_core(struct list_head *head, bool pfmemalloc) 6500 { 6501 /* Fast-path assumptions: 6502 * - There is no RX handler. 6503 * - Only one packet_type matches. 6504 * If either of these fails, we will end up doing some per-packet 6505 * processing in-line, then handling the 'last ptype' for the whole 6506 * sublist. This can't cause out-of-order delivery to any single ptype, 6507 * because the 'last ptype' must be constant across the sublist, and all 6508 * other ptypes are handled per-packet. 6509 */ 6510 /* Current (common) ptype of sublist */ 6511 struct packet_type *pt_curr = NULL; 6512 /* Current (common) orig_dev of sublist */ 6513 struct net_device *od_curr = NULL; 6514 struct sk_buff *skb, *next; 6515 LIST_HEAD(sublist); 6516 6517 list_for_each_entry_safe(skb, next, head, list) { 6518 struct net_device *orig_dev = skb->dev; 6519 struct packet_type *pt_prev = NULL; 6520 6521 skb_list_del_init(skb); 6522 __netif_receive_skb_core(&skb, pfmemalloc, &pt_prev); 6523 if (!pt_prev) 6524 continue; 6525 if (pt_curr != pt_prev || od_curr != orig_dev) { 6526 /* dispatch old sublist */ 6527 __netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr); 6528 /* start new sublist */ 6529 INIT_LIST_HEAD(&sublist); 6530 pt_curr = pt_prev; 6531 od_curr = orig_dev; 6532 } 6533 list_add_tail(&skb->list, &sublist); 6534 } 6535 6536 /* dispatch final sublist */ 6537 __netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr); 6538 } 6539 6540 static int __netif_receive_skb(struct sk_buff *skb) 6541 { 6542 int ret; 6543 6544 if (sk_memalloc_socks() && skb_pfmemalloc(skb)) { 6545 unsigned int noreclaim_flag; 6546 6547 /* 6548 * PFMEMALLOC skbs are special, they should 6549 * - be delivered to SOCK_MEMALLOC sockets only 6550 * - stay away from userspace 6551 * - have bounded memory usage 6552 * 6553 * Use PF_MEMALLOC as this saves us from propagating the allocation 6554 * context down to all allocation sites. 6555 */ 6556 noreclaim_flag = memalloc_noreclaim_save(); 6557 ret = __netif_receive_skb_one_core(skb, true); 6558 memalloc_noreclaim_restore(noreclaim_flag); 6559 } else 6560 ret = __netif_receive_skb_one_core(skb, false); 6561 6562 return ret; 6563 } 6564 6565 static void __netif_receive_skb_list(struct list_head *head) 6566 { 6567 unsigned long noreclaim_flag = 0; 6568 struct sk_buff *skb, *next; 6569 bool pfmemalloc = false; /* Is current sublist PF_MEMALLOC? */ 6570 6571 list_for_each_entry_safe(skb, next, head, list) { 6572 if ((sk_memalloc_socks() && skb_pfmemalloc(skb)) != pfmemalloc) { 6573 struct list_head sublist; 6574 6575 /* Handle the previous sublist */ 6576 list_cut_before(&sublist, head, &skb->list); 6577 if (!list_empty(&sublist)) 6578 __netif_receive_skb_list_core(&sublist, pfmemalloc); 6579 pfmemalloc = !pfmemalloc; 6580 /* See comments in __netif_receive_skb */ 6581 if (pfmemalloc) 6582 noreclaim_flag = memalloc_noreclaim_save(); 6583 else 6584 memalloc_noreclaim_restore(noreclaim_flag); 6585 } 6586 } 6587 /* Handle the remaining sublist */ 6588 if (!list_empty(head)) 6589 __netif_receive_skb_list_core(head, pfmemalloc); 6590 /* Restore pflags */ 6591 if (pfmemalloc) 6592 memalloc_noreclaim_restore(noreclaim_flag); 6593 } 6594 6595 static int generic_xdp_install(struct net_device *dev, struct netdev_bpf *xdp) 6596 { 6597 struct bpf_prog *old = rtnl_dereference(dev->xdp_prog); 6598 struct bpf_prog *new = xdp->prog; 6599 int ret = 0; 6600 6601 switch (xdp->command) { 6602 case XDP_SETUP_PROG: 6603 rcu_assign_pointer(dev->xdp_prog, new); 6604 if (old) 6605 bpf_prog_put(old); 6606 6607 if (old && !new) { 6608 static_branch_dec(&generic_xdp_needed_key); 6609 } else if (new && !old) { 6610 static_branch_inc(&generic_xdp_needed_key); 6611 netif_disable_lro(dev); 6612 dev_disable_gro_hw(dev); 6613 } 6614 break; 6615 6616 default: 6617 ret = -EINVAL; 6618 break; 6619 } 6620 6621 return ret; 6622 } 6623 6624 static int netif_receive_skb_internal(struct sk_buff *skb) 6625 { 6626 int ret; 6627 6628 net_timestamp_check(READ_ONCE(net_hotdata.tstamp_prequeue), skb); 6629 6630 if (skb_defer_rx_timestamp(skb)) 6631 return NET_RX_SUCCESS; 6632 6633 rcu_read_lock(); 6634 #ifdef CONFIG_RPS 6635 if (static_branch_unlikely(&rps_needed)) { 6636 struct rps_dev_flow voidflow, *rflow = &voidflow; 6637 int cpu = get_rps_cpu(skb->dev, skb, &rflow); 6638 6639 if (cpu >= 0) { 6640 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail); 6641 rcu_read_unlock(); 6642 return ret; 6643 } 6644 } 6645 #endif 6646 ret = __netif_receive_skb(skb); 6647 rcu_read_unlock(); 6648 return ret; 6649 } 6650 6651 void netif_receive_skb_list_internal(struct list_head *head) 6652 { 6653 struct sk_buff *skb, *next; 6654 LIST_HEAD(sublist); 6655 6656 list_for_each_entry_safe(skb, next, head, list) { 6657 net_timestamp_check(READ_ONCE(net_hotdata.tstamp_prequeue), 6658 skb); 6659 skb_list_del_init(skb); 6660 if (!skb_defer_rx_timestamp(skb)) 6661 list_add_tail(&skb->list, &sublist); 6662 } 6663 list_splice_init(&sublist, head); 6664 6665 rcu_read_lock(); 6666 #ifdef CONFIG_RPS 6667 if (static_branch_unlikely(&rps_needed)) { 6668 list_for_each_entry_safe(skb, next, head, list) { 6669 struct rps_dev_flow voidflow, *rflow = &voidflow; 6670 int cpu = get_rps_cpu(skb->dev, skb, &rflow); 6671 6672 if (cpu >= 0) { 6673 /* Will be handled, remove from list */ 6674 skb_list_del_init(skb); 6675 enqueue_to_backlog(skb, cpu, &rflow->last_qtail); 6676 } 6677 } 6678 } 6679 #endif 6680 __netif_receive_skb_list(head); 6681 rcu_read_unlock(); 6682 } 6683 6684 /** 6685 * netif_receive_skb - process receive buffer from network 6686 * @skb: buffer to process 6687 * 6688 * netif_receive_skb() is the main receive data processing function. 6689 * It always succeeds. The buffer may be dropped during processing 6690 * for congestion control or by the protocol layers. 6691 * 6692 * This function may only be called from softirq context and interrupts 6693 * should be enabled. 6694 * 6695 * Return values (usually ignored): 6696 * NET_RX_SUCCESS: no congestion 6697 * NET_RX_DROP: packet was dropped 6698 */ 6699 int netif_receive_skb(struct sk_buff *skb) 6700 { 6701 int ret; 6702 6703 trace_netif_receive_skb_entry(skb); 6704 6705 ret = netif_receive_skb_internal(skb); 6706 trace_netif_receive_skb_exit(ret); 6707 6708 return ret; 6709 } 6710 EXPORT_SYMBOL(netif_receive_skb); 6711 6712 /** 6713 * netif_receive_skb_list - process many receive buffers from network 6714 * @head: list of skbs to process. 6715 * 6716 * Since return value of netif_receive_skb() is normally ignored, and 6717 * wouldn't be meaningful for a list, this function returns void. 6718 * 6719 * This function may only be called from softirq context and interrupts 6720 * should be enabled. 6721 */ 6722 void netif_receive_skb_list(struct list_head *head) 6723 { 6724 struct sk_buff *skb; 6725 6726 if (list_empty(head)) 6727 return; 6728 if (trace_netif_receive_skb_list_entry_enabled()) { 6729 list_for_each_entry(skb, head, list) 6730 trace_netif_receive_skb_list_entry(skb); 6731 } 6732 netif_receive_skb_list_internal(head); 6733 trace_netif_receive_skb_list_exit(0); 6734 } 6735 EXPORT_SYMBOL(netif_receive_skb_list); 6736 6737 /* Network device is going away, flush any packets still pending */ 6738 static void flush_backlog(struct work_struct *work) 6739 { 6740 struct sk_buff *skb, *tmp; 6741 struct sk_buff_head list; 6742 struct softnet_data *sd; 6743 6744 __skb_queue_head_init(&list); 6745 local_bh_disable(); 6746 sd = this_cpu_ptr(&softnet_data); 6747 6748 backlog_lock_irq_disable(sd); 6749 skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) { 6750 if (READ_ONCE(skb->dev->reg_state) == NETREG_UNREGISTERING) { 6751 __skb_unlink(skb, &sd->input_pkt_queue); 6752 __skb_queue_tail(&list, skb); 6753 rps_input_queue_head_incr(sd); 6754 } 6755 } 6756 backlog_unlock_irq_enable(sd); 6757 6758 local_lock_nested_bh(&softnet_data.process_queue_bh_lock); 6759 skb_queue_walk_safe(&sd->process_queue, skb, tmp) { 6760 if (READ_ONCE(skb->dev->reg_state) == NETREG_UNREGISTERING) { 6761 __skb_unlink(skb, &sd->process_queue); 6762 __skb_queue_tail(&list, skb); 6763 rps_input_queue_head_incr(sd); 6764 } 6765 } 6766 local_unlock_nested_bh(&softnet_data.process_queue_bh_lock); 6767 local_bh_enable(); 6768 6769 __skb_queue_purge_reason(&list, SKB_DROP_REASON_DEV_READY); 6770 } 6771 6772 static bool flush_required(int cpu) 6773 { 6774 #if IS_ENABLED(CONFIG_RPS) 6775 struct softnet_data *sd = &per_cpu(softnet_data, cpu); 6776 bool do_flush; 6777 6778 backlog_lock_irq_disable(sd); 6779 6780 /* as insertion into process_queue happens with the rps lock held, 6781 * process_queue access may race only with dequeue 6782 */ 6783 do_flush = !skb_queue_empty(&sd->input_pkt_queue) || 6784 !skb_queue_empty_lockless(&sd->process_queue); 6785 backlog_unlock_irq_enable(sd); 6786 6787 return do_flush; 6788 #endif 6789 /* without RPS we can't safely check input_pkt_queue: during a 6790 * concurrent remote skb_queue_splice() we can detect as empty both 6791 * input_pkt_queue and process_queue even if the latter could end-up 6792 * containing a lot of packets. 6793 */ 6794 return true; 6795 } 6796 6797 struct flush_backlogs { 6798 cpumask_t flush_cpus; 6799 struct work_struct w[]; 6800 }; 6801 6802 static struct flush_backlogs *flush_backlogs_alloc(void) 6803 { 6804 return kmalloc_flex(struct flush_backlogs, w, nr_cpu_ids); 6805 } 6806 6807 static struct flush_backlogs *flush_backlogs_fallback; 6808 static DEFINE_MUTEX(flush_backlogs_mutex); 6809 6810 static void flush_all_backlogs(void) 6811 { 6812 struct flush_backlogs *ptr = flush_backlogs_alloc(); 6813 unsigned int cpu; 6814 6815 if (!ptr) { 6816 mutex_lock(&flush_backlogs_mutex); 6817 ptr = flush_backlogs_fallback; 6818 } 6819 cpumask_clear(&ptr->flush_cpus); 6820 6821 cpus_read_lock(); 6822 6823 for_each_online_cpu(cpu) { 6824 if (flush_required(cpu)) { 6825 INIT_WORK(&ptr->w[cpu], flush_backlog); 6826 queue_work_on(cpu, system_highpri_wq, &ptr->w[cpu]); 6827 __cpumask_set_cpu(cpu, &ptr->flush_cpus); 6828 } 6829 } 6830 6831 /* we can have in flight packet[s] on the cpus we are not flushing, 6832 * synchronize_net() in unregister_netdevice_many() will take care of 6833 * them. 6834 */ 6835 for_each_cpu(cpu, &ptr->flush_cpus) 6836 flush_work(&ptr->w[cpu]); 6837 6838 cpus_read_unlock(); 6839 6840 if (ptr != flush_backlogs_fallback) 6841 kfree(ptr); 6842 else 6843 mutex_unlock(&flush_backlogs_mutex); 6844 } 6845 6846 static void net_rps_send_ipi(struct softnet_data *remsd) 6847 { 6848 #ifdef CONFIG_RPS 6849 while (remsd) { 6850 struct softnet_data *next = remsd->rps_ipi_next; 6851 6852 if (cpu_online(remsd->cpu)) 6853 smp_call_function_single_async(remsd->cpu, &remsd->csd); 6854 remsd = next; 6855 } 6856 #endif 6857 } 6858 6859 /* 6860 * net_rps_action_and_irq_enable sends any pending IPI's for rps. 6861 * Note: called with local irq disabled, but exits with local irq enabled. 6862 */ 6863 static void net_rps_action_and_irq_enable(struct softnet_data *sd) 6864 { 6865 #ifdef CONFIG_RPS 6866 struct softnet_data *remsd = sd->rps_ipi_list; 6867 6868 if (!use_backlog_threads() && remsd) { 6869 sd->rps_ipi_list = NULL; 6870 6871 local_irq_enable(); 6872 6873 /* Send pending IPI's to kick RPS processing on remote cpus. */ 6874 net_rps_send_ipi(remsd); 6875 } else 6876 #endif 6877 local_irq_enable(); 6878 } 6879 6880 static bool sd_has_rps_ipi_waiting(struct softnet_data *sd) 6881 { 6882 #ifdef CONFIG_RPS 6883 return !use_backlog_threads() && sd->rps_ipi_list; 6884 #else 6885 return false; 6886 #endif 6887 } 6888 6889 static int process_backlog(struct napi_struct *napi, int quota) 6890 { 6891 struct softnet_data *sd = container_of(napi, struct softnet_data, backlog); 6892 bool again = true; 6893 int work = 0; 6894 6895 /* Check if we have pending ipi, its better to send them now, 6896 * not waiting net_rx_action() end. 6897 */ 6898 if (sd_has_rps_ipi_waiting(sd)) { 6899 local_irq_disable(); 6900 net_rps_action_and_irq_enable(sd); 6901 } 6902 6903 napi->weight = READ_ONCE(net_hotdata.dev_rx_weight); 6904 while (again) { 6905 struct sk_buff *skb; 6906 6907 local_lock_nested_bh(&softnet_data.process_queue_bh_lock); 6908 while ((skb = __skb_dequeue(&sd->process_queue))) { 6909 local_unlock_nested_bh(&softnet_data.process_queue_bh_lock); 6910 rcu_read_lock(); 6911 __netif_receive_skb(skb); 6912 rcu_read_unlock(); 6913 if (++work >= quota) { 6914 rps_input_queue_head_add(sd, work); 6915 return work; 6916 } 6917 6918 local_lock_nested_bh(&softnet_data.process_queue_bh_lock); 6919 } 6920 local_unlock_nested_bh(&softnet_data.process_queue_bh_lock); 6921 6922 backlog_lock_irq_disable(sd); 6923 if (skb_queue_empty(&sd->input_pkt_queue)) { 6924 /* 6925 * Inline a custom version of __napi_complete(). 6926 * only current cpu owns and manipulates this napi, 6927 * and NAPI_STATE_SCHED is the only possible flag set 6928 * on backlog. 6929 * We can use a plain write instead of clear_bit(), 6930 * and we dont need an smp_mb() memory barrier. 6931 */ 6932 napi->state &= NAPIF_STATE_THREADED; 6933 again = false; 6934 } else { 6935 local_lock_nested_bh(&softnet_data.process_queue_bh_lock); 6936 skb_queue_splice_tail_init(&sd->input_pkt_queue, 6937 &sd->process_queue); 6938 local_unlock_nested_bh(&softnet_data.process_queue_bh_lock); 6939 } 6940 backlog_unlock_irq_enable(sd); 6941 } 6942 6943 if (work) 6944 rps_input_queue_head_add(sd, work); 6945 return work; 6946 } 6947 6948 /** 6949 * __napi_schedule - schedule for receive 6950 * @n: entry to schedule 6951 * 6952 * The entry's receive function will be scheduled to run. 6953 * Consider using __napi_schedule_irqoff() if hard irqs are masked. 6954 */ 6955 void __napi_schedule(struct napi_struct *n) 6956 { 6957 unsigned long flags; 6958 6959 local_irq_save(flags); 6960 ____napi_schedule(this_cpu_ptr(&softnet_data), n); 6961 local_irq_restore(flags); 6962 } 6963 EXPORT_SYMBOL(__napi_schedule); 6964 6965 /** 6966 * napi_schedule_prep - check if napi can be scheduled 6967 * @n: napi context 6968 * 6969 * Test if NAPI routine is already running, and if not mark 6970 * it as running. This is used as a condition variable to 6971 * insure only one NAPI poll instance runs. We also make 6972 * sure there is no pending NAPI disable. 6973 */ 6974 bool napi_schedule_prep(struct napi_struct *n) 6975 { 6976 unsigned long new, val = READ_ONCE(n->state); 6977 6978 do { 6979 if (unlikely(val & NAPIF_STATE_DISABLE)) 6980 return false; 6981 new = val | NAPIF_STATE_SCHED; 6982 6983 /* Sets STATE_MISSED bit if STATE_SCHED was already set 6984 * This was suggested by Alexander Duyck, as compiler 6985 * emits better code than : 6986 * if (val & NAPIF_STATE_SCHED) 6987 * new |= NAPIF_STATE_MISSED; 6988 */ 6989 new |= (val & NAPIF_STATE_SCHED) / NAPIF_STATE_SCHED * 6990 NAPIF_STATE_MISSED; 6991 } while (!try_cmpxchg(&n->state, &val, new)); 6992 6993 return !(val & NAPIF_STATE_SCHED); 6994 } 6995 EXPORT_SYMBOL(napi_schedule_prep); 6996 6997 /** 6998 * __napi_schedule_irqoff - schedule for receive 6999 * @n: entry to schedule 7000 * 7001 * Variant of __napi_schedule() assuming hard irqs are masked. 7002 * 7003 * On PREEMPT_RT enabled kernels this maps to __napi_schedule() 7004 * because the interrupt disabled assumption might not be true 7005 * due to force-threaded interrupts and spinlock substitution. 7006 */ 7007 void __napi_schedule_irqoff(struct napi_struct *n) 7008 { 7009 if (!IS_ENABLED(CONFIG_PREEMPT_RT)) 7010 ____napi_schedule(this_cpu_ptr(&softnet_data), n); 7011 else 7012 __napi_schedule(n); 7013 } 7014 EXPORT_SYMBOL(__napi_schedule_irqoff); 7015 7016 bool napi_complete_done(struct napi_struct *n, int work_done) 7017 { 7018 unsigned long flags, val, new, timeout = 0; 7019 bool ret = true; 7020 7021 /* 7022 * 1) Don't let napi dequeue from the cpu poll list 7023 * just in case its running on a different cpu. 7024 * 2) If we are busy polling, do nothing here, we have 7025 * the guarantee we will be called later. 7026 */ 7027 if (unlikely(n->state & (NAPIF_STATE_NPSVC | 7028 NAPIF_STATE_IN_BUSY_POLL))) 7029 return false; 7030 7031 if (work_done) { 7032 if (n->gro.bitmask) 7033 timeout = napi_get_gro_flush_timeout(n); 7034 n->defer_hard_irqs_count = napi_get_defer_hard_irqs(n); 7035 } 7036 if (n->defer_hard_irqs_count > 0) { 7037 n->defer_hard_irqs_count--; 7038 timeout = napi_get_gro_flush_timeout(n); 7039 if (timeout) 7040 ret = false; 7041 } 7042 7043 /* 7044 * When the NAPI instance uses a timeout and keeps postponing 7045 * it, we need to bound somehow the time packets are kept in 7046 * the GRO layer. 7047 */ 7048 gro_flush_normal(&n->gro, !!timeout); 7049 7050 if (unlikely(!list_empty(&n->poll_list))) { 7051 /* If n->poll_list is not empty, we need to mask irqs */ 7052 local_irq_save(flags); 7053 list_del_init(&n->poll_list); 7054 local_irq_restore(flags); 7055 } 7056 WRITE_ONCE(n->list_owner, -1); 7057 7058 val = READ_ONCE(n->state); 7059 do { 7060 WARN_ON_ONCE(!(val & NAPIF_STATE_SCHED)); 7061 7062 new = val & ~(NAPIF_STATE_MISSED | NAPIF_STATE_SCHED | 7063 NAPIF_STATE_SCHED_THREADED | 7064 NAPIF_STATE_PREFER_BUSY_POLL); 7065 7066 /* If STATE_MISSED was set, leave STATE_SCHED set, 7067 * because we will call napi->poll() one more time. 7068 * This C code was suggested by Alexander Duyck to help gcc. 7069 */ 7070 new |= (val & NAPIF_STATE_MISSED) / NAPIF_STATE_MISSED * 7071 NAPIF_STATE_SCHED; 7072 } while (!try_cmpxchg(&n->state, &val, new)); 7073 7074 if (unlikely(val & NAPIF_STATE_MISSED)) { 7075 __napi_schedule(n); 7076 return false; 7077 } 7078 7079 if (timeout) 7080 hrtimer_start(&n->timer, ns_to_ktime(timeout), 7081 HRTIMER_MODE_REL_PINNED); 7082 return ret; 7083 } 7084 EXPORT_SYMBOL(napi_complete_done); 7085 7086 static void __skb_defer_free_flush(struct skb_defer_node *sdn, int budget) 7087 { 7088 struct llist_node *free_list; 7089 struct sk_buff *skb, *next; 7090 long n = 0; 7091 7092 if (llist_empty(&sdn->defer_list)) 7093 return; 7094 free_list = llist_del_all(&sdn->defer_list); 7095 7096 llist_for_each_entry_safe(skb, next, free_list, ll_node) { 7097 prefetch(next); 7098 napi_consume_skb(skb, budget); 7099 n++; 7100 } 7101 /* Subtract exactly what was drained. Never store an absolute value: 7102 * mixing one with the relative updates in skb_attempt_defer_free() 7103 * lets the counter go negative, and it is read as unsigned there. 7104 */ 7105 atomic_long_sub(n, &sdn->defer_count); 7106 } 7107 7108 void skb_defer_node_flush(struct skb_defer_node *sdn) 7109 { 7110 __skb_defer_free_flush(sdn, 0); 7111 } 7112 7113 static void skb_defer_free_flush(void) 7114 { 7115 struct skb_defer_node *sdn; 7116 int node; 7117 7118 for_each_online_node(node) { 7119 sdn = this_cpu_ptr(net_hotdata.skb_defer_nodes) + node; 7120 __skb_defer_free_flush(sdn, 1); 7121 } 7122 } 7123 7124 #if defined(CONFIG_NET_RX_BUSY_POLL) 7125 7126 enum { 7127 NAPI_F_PREFER_BUSY_POLL = 1, 7128 NAPI_F_END_ON_RESCHED = 2, 7129 }; 7130 7131 static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock, 7132 unsigned flags, u16 budget) 7133 { 7134 struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx; 7135 unsigned long timeout = 0; 7136 int rc; 7137 7138 /* Busy polling means there is a high chance device driver hard irq 7139 * could not grab NAPI_STATE_SCHED, and that NAPI_STATE_MISSED was 7140 * set in napi_schedule_prep(). 7141 * Since we either call napi->poll() once more or start the timer, 7142 * we can safely clear NAPI_STATE_MISSED. 7143 * 7144 * Note: x86 could use a single "lock and ..." instruction 7145 * to perform these two clear_bit() 7146 */ 7147 clear_bit(NAPI_STATE_MISSED, &napi->state); 7148 clear_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state); 7149 7150 local_bh_disable(); 7151 bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx); 7152 7153 if (flags & NAPI_F_PREFER_BUSY_POLL) { 7154 napi->defer_hard_irqs_count = napi_get_defer_hard_irqs(napi); 7155 if (napi->defer_hard_irqs_count) 7156 timeout = napi_get_gro_flush_timeout(napi); 7157 } 7158 if (timeout) { 7159 netpoll_poll_unlock(have_poll_lock); 7160 7161 /* Drain aged GRO packets before clearing SCHED since the NAPI 7162 * won't run again until after the timer fires. When HZ < 1000, 7163 * GRO age comparison is too coarse, so flush everything. 7164 */ 7165 gro_flush_normal(&napi->gro, HZ >= 1000); 7166 7167 clear_bit(NAPI_STATE_SCHED, &napi->state); 7168 hrtimer_start(&napi->timer, ns_to_ktime(timeout), 7169 HRTIMER_MODE_REL_PINNED); 7170 } else { 7171 /* Use driver poll to re-enable device interrupts. */ 7172 rc = napi->poll(napi, budget); 7173 /* Unless rc == budget we no longer own the NAPI instance, 7174 * IRQ may fire on another CPU, poll this NAPI, and enter GRO. 7175 */ 7176 trace_napi_poll(napi, rc, budget); 7177 netpoll_poll_unlock(have_poll_lock); 7178 if (rc == budget) { 7179 gro_normal_list(&napi->gro); 7180 __napi_schedule(napi); 7181 } 7182 } 7183 7184 bpf_net_ctx_clear(bpf_net_ctx); 7185 local_bh_enable(); 7186 } 7187 7188 static void __napi_busy_loop(unsigned int napi_id, 7189 bool (*loop_end)(void *, unsigned long), 7190 void *loop_end_arg, unsigned flags, u16 budget) 7191 { 7192 unsigned long start_time = loop_end ? busy_loop_current_time() : 0; 7193 int (*napi_poll)(struct napi_struct *napi, int budget); 7194 struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx; 7195 void *have_poll_lock = NULL; 7196 struct napi_struct *napi; 7197 7198 WARN_ON_ONCE(!rcu_read_lock_held()); 7199 7200 restart: 7201 napi_poll = NULL; 7202 7203 napi = napi_by_id(napi_id); 7204 if (!napi) 7205 return; 7206 7207 if (!IS_ENABLED(CONFIG_PREEMPT_RT)) 7208 preempt_disable(); 7209 for (;;) { 7210 int work = 0; 7211 7212 local_bh_disable(); 7213 bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx); 7214 if (!napi_poll) { 7215 unsigned long val = READ_ONCE(napi->state); 7216 7217 /* If multiple threads are competing for this napi, 7218 * we avoid dirtying napi->state as much as we can. 7219 */ 7220 if (val & (NAPIF_STATE_DISABLE | NAPIF_STATE_SCHED | 7221 NAPIF_STATE_IN_BUSY_POLL)) { 7222 if (flags & NAPI_F_PREFER_BUSY_POLL) 7223 set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state); 7224 goto count; 7225 } 7226 if (cmpxchg(&napi->state, val, 7227 val | NAPIF_STATE_IN_BUSY_POLL | 7228 NAPIF_STATE_SCHED) != val) { 7229 if (flags & NAPI_F_PREFER_BUSY_POLL) 7230 set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state); 7231 goto count; 7232 } 7233 have_poll_lock = netpoll_poll_lock(napi); 7234 napi_poll = napi->poll; 7235 } 7236 work = napi_poll(napi, budget); 7237 trace_napi_poll(napi, work, budget); 7238 gro_normal_list(&napi->gro); 7239 count: 7240 if (work > 0) 7241 __NET_ADD_STATS(dev_net(napi->dev), 7242 LINUX_MIB_BUSYPOLLRXPACKETS, work); 7243 skb_defer_free_flush(); 7244 bpf_net_ctx_clear(bpf_net_ctx); 7245 local_bh_enable(); 7246 7247 if (!loop_end || loop_end(loop_end_arg, start_time)) 7248 break; 7249 7250 if (unlikely(need_resched())) { 7251 if (flags & NAPI_F_END_ON_RESCHED) 7252 break; 7253 if (napi_poll) 7254 busy_poll_stop(napi, have_poll_lock, flags, budget); 7255 if (!IS_ENABLED(CONFIG_PREEMPT_RT)) 7256 preempt_enable(); 7257 rcu_read_unlock(); 7258 cond_resched(); 7259 rcu_read_lock(); 7260 if (loop_end(loop_end_arg, start_time)) 7261 return; 7262 goto restart; 7263 } 7264 cpu_relax(); 7265 } 7266 if (napi_poll) 7267 busy_poll_stop(napi, have_poll_lock, flags, budget); 7268 if (!IS_ENABLED(CONFIG_PREEMPT_RT)) 7269 preempt_enable(); 7270 } 7271 7272 void napi_busy_loop_rcu(unsigned int napi_id, 7273 bool (*loop_end)(void *, unsigned long), 7274 void *loop_end_arg, bool prefer_busy_poll, u16 budget) 7275 { 7276 unsigned flags = NAPI_F_END_ON_RESCHED; 7277 7278 if (prefer_busy_poll) 7279 flags |= NAPI_F_PREFER_BUSY_POLL; 7280 7281 __napi_busy_loop(napi_id, loop_end, loop_end_arg, flags, budget); 7282 } 7283 7284 void napi_busy_loop(unsigned int napi_id, 7285 bool (*loop_end)(void *, unsigned long), 7286 void *loop_end_arg, bool prefer_busy_poll, u16 budget) 7287 { 7288 unsigned flags = prefer_busy_poll ? NAPI_F_PREFER_BUSY_POLL : 0; 7289 7290 rcu_read_lock(); 7291 __napi_busy_loop(napi_id, loop_end, loop_end_arg, flags, budget); 7292 rcu_read_unlock(); 7293 } 7294 EXPORT_SYMBOL(napi_busy_loop); 7295 7296 void napi_suspend_irqs(unsigned int napi_id) 7297 { 7298 struct napi_struct *napi; 7299 7300 rcu_read_lock(); 7301 napi = napi_by_id(napi_id); 7302 if (napi) { 7303 unsigned long timeout = napi_get_irq_suspend_timeout(napi); 7304 7305 if (timeout) 7306 hrtimer_start(&napi->timer, ns_to_ktime(timeout), 7307 HRTIMER_MODE_REL_PINNED); 7308 } 7309 rcu_read_unlock(); 7310 } 7311 7312 void napi_resume_irqs(unsigned int napi_id) 7313 { 7314 struct napi_struct *napi; 7315 7316 rcu_read_lock(); 7317 napi = napi_by_id(napi_id); 7318 if (napi) { 7319 /* If irq_suspend_timeout is set to 0 between the call to 7320 * napi_suspend_irqs and now, the original value still 7321 * determines the safety timeout as intended and napi_watchdog 7322 * will resume irq processing. 7323 */ 7324 if (napi_get_irq_suspend_timeout(napi)) { 7325 local_bh_disable(); 7326 napi_schedule(napi); 7327 local_bh_enable(); 7328 } 7329 } 7330 rcu_read_unlock(); 7331 } 7332 7333 #endif /* CONFIG_NET_RX_BUSY_POLL */ 7334 7335 static void __napi_hash_add_with_id(struct napi_struct *napi, 7336 unsigned int napi_id) 7337 { 7338 napi->gro.cached_napi_id = napi_id; 7339 7340 WRITE_ONCE(napi->napi_id, napi_id); 7341 hlist_add_head_rcu(&napi->napi_hash_node, 7342 &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]); 7343 } 7344 7345 static void napi_hash_add_with_id(struct napi_struct *napi, 7346 unsigned int napi_id) 7347 { 7348 unsigned long flags; 7349 7350 spin_lock_irqsave(&napi_hash_lock, flags); 7351 WARN_ON_ONCE(napi_by_id(napi_id)); 7352 __napi_hash_add_with_id(napi, napi_id); 7353 spin_unlock_irqrestore(&napi_hash_lock, flags); 7354 } 7355 7356 static void napi_hash_add(struct napi_struct *napi) 7357 { 7358 unsigned long flags; 7359 7360 if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state)) 7361 return; 7362 7363 spin_lock_irqsave(&napi_hash_lock, flags); 7364 7365 /* 0..NR_CPUS range is reserved for sender_cpu use */ 7366 do { 7367 if (unlikely(!napi_id_valid(++napi_gen_id))) 7368 napi_gen_id = MIN_NAPI_ID; 7369 } while (napi_by_id(napi_gen_id)); 7370 7371 __napi_hash_add_with_id(napi, napi_gen_id); 7372 7373 spin_unlock_irqrestore(&napi_hash_lock, flags); 7374 } 7375 7376 /* Warning : caller is responsible to make sure rcu grace period 7377 * is respected before freeing memory containing @napi 7378 */ 7379 static void napi_hash_del(struct napi_struct *napi) 7380 { 7381 unsigned long flags; 7382 7383 spin_lock_irqsave(&napi_hash_lock, flags); 7384 7385 hlist_del_init_rcu(&napi->napi_hash_node); 7386 7387 spin_unlock_irqrestore(&napi_hash_lock, flags); 7388 } 7389 7390 static enum hrtimer_restart napi_watchdog(struct hrtimer *timer) 7391 { 7392 struct napi_struct *napi; 7393 7394 napi = container_of(timer, struct napi_struct, timer); 7395 7396 /* Note : we use a relaxed variant of napi_schedule_prep() not setting 7397 * NAPI_STATE_MISSED, since we do not react to a device IRQ. 7398 */ 7399 if (!napi_disable_pending(napi) && 7400 !test_and_set_bit(NAPI_STATE_SCHED, &napi->state)) { 7401 clear_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state); 7402 __napi_schedule_irqoff(napi); 7403 } 7404 7405 return HRTIMER_NORESTART; 7406 } 7407 7408 static void napi_stop_kthread(struct napi_struct *napi) 7409 { 7410 unsigned long val, new; 7411 7412 /* Instance is disabled, we can immediately stop the kthread */ 7413 if (test_bit(NAPI_STATE_DISABLED, &napi->state)) { 7414 clear_bit(NAPI_STATE_THREADED, &napi->state); 7415 clear_bit(NAPI_STATE_THREADED_BUSY_POLL, &napi->state); 7416 goto stop; 7417 } 7418 7419 /* Wait until the napi STATE_THREADED is unset. */ 7420 while (true) { 7421 val = READ_ONCE(napi->state); 7422 7423 /* If napi kthread own this napi or the napi is idle, 7424 * STATE_THREADED can be unset here. 7425 */ 7426 if ((val & NAPIF_STATE_SCHED_THREADED) || 7427 !(val & NAPIF_STATE_SCHED)) { 7428 new = val & (~(NAPIF_STATE_THREADED | 7429 NAPIF_STATE_THREADED_BUSY_POLL)); 7430 } else { 7431 msleep(20); 7432 continue; 7433 } 7434 7435 if (try_cmpxchg(&napi->state, &val, new)) 7436 break; 7437 } 7438 7439 /* Once STATE_THREADED is unset, wait for SCHED_THREADED to be unset by 7440 * the kthread. 7441 */ 7442 while (true) { 7443 if (!test_bit(NAPI_STATE_SCHED_THREADED, &napi->state)) 7444 break; 7445 7446 msleep(20); 7447 } 7448 7449 stop: 7450 kthread_stop(napi->thread); 7451 napi->thread = NULL; 7452 } 7453 7454 static void napi_set_threaded_state(struct napi_struct *napi, 7455 enum netdev_napi_threaded threaded_mode) 7456 { 7457 bool threaded = threaded_mode != NETDEV_NAPI_THREADED_DISABLED; 7458 bool busy_poll = threaded_mode == NETDEV_NAPI_THREADED_BUSY_POLL; 7459 7460 assign_bit(NAPI_STATE_THREADED, &napi->state, threaded); 7461 assign_bit(NAPI_STATE_THREADED_BUSY_POLL, &napi->state, busy_poll); 7462 } 7463 7464 int napi_set_threaded(struct napi_struct *napi, 7465 enum netdev_napi_threaded threaded) 7466 { 7467 if (threaded) { 7468 if (!napi->thread) { 7469 int err = napi_kthread_create(napi); 7470 7471 if (err) 7472 return err; 7473 } 7474 } 7475 7476 if (napi->config) 7477 napi->config->threaded = threaded; 7478 7479 /* Setting/unsetting threaded mode on a napi might not immediately 7480 * take effect, if the current napi instance is actively being 7481 * polled. In this case, the switch between threaded mode and 7482 * softirq mode will happen in the next round of napi_schedule(). 7483 * This should not cause hiccups/stalls to the live traffic. 7484 */ 7485 if (!threaded && napi->thread) { 7486 napi_stop_kthread(napi); 7487 } else { 7488 /* Make sure kthread is created before THREADED bit is set. */ 7489 smp_mb__before_atomic(); 7490 napi_set_threaded_state(napi, threaded); 7491 } 7492 7493 return 0; 7494 } 7495 7496 int netif_set_threaded(struct net_device *dev, 7497 enum netdev_napi_threaded threaded) 7498 { 7499 struct napi_struct *napi; 7500 int i, err = 0; 7501 7502 netdev_assert_locked_or_invisible(dev); 7503 7504 if (threaded) { 7505 list_for_each_entry(napi, &dev->napi_list, dev_list) { 7506 if (!napi->thread) { 7507 err = napi_kthread_create(napi); 7508 if (err) { 7509 threaded = NETDEV_NAPI_THREADED_DISABLED; 7510 break; 7511 } 7512 } 7513 } 7514 } 7515 7516 WRITE_ONCE(dev->threaded, threaded); 7517 7518 /* The error should not occur as the kthreads are already created. */ 7519 list_for_each_entry(napi, &dev->napi_list, dev_list) 7520 WARN_ON_ONCE(napi_set_threaded(napi, threaded)); 7521 7522 /* Override the config for all NAPIs even if currently not listed */ 7523 for (i = 0; i < dev->num_napi_configs; i++) 7524 dev->napi_config[i].threaded = threaded; 7525 7526 return err; 7527 } 7528 7529 /** 7530 * netif_threaded_enable() - enable threaded NAPIs 7531 * @dev: net_device instance 7532 * 7533 * Enable threaded mode for the NAPI instances of the device. This may be useful 7534 * for devices where multiple NAPI instances get scheduled by a single 7535 * interrupt. Threaded NAPI allows moving the NAPI processing to cores other 7536 * than the core where IRQ is mapped. 7537 * 7538 * This function should be called before @dev is registered. 7539 */ 7540 void netif_threaded_enable(struct net_device *dev) 7541 { 7542 WARN_ON_ONCE(netif_set_threaded(dev, NETDEV_NAPI_THREADED_ENABLED)); 7543 } 7544 EXPORT_SYMBOL(netif_threaded_enable); 7545 7546 /** 7547 * netif_queue_set_napi - Associate queue with the napi 7548 * @dev: device to which NAPI and queue belong 7549 * @queue_index: Index of queue 7550 * @type: queue type as RX or TX 7551 * @napi: NAPI context, pass NULL to clear previously set NAPI 7552 * 7553 * Set queue with its corresponding napi context. This should be done after 7554 * registering the NAPI handler for the queue-vector and the queues have been 7555 * mapped to the corresponding interrupt vector. 7556 */ 7557 void netif_queue_set_napi(struct net_device *dev, unsigned int queue_index, 7558 enum netdev_queue_type type, struct napi_struct *napi) 7559 { 7560 struct netdev_rx_queue *rxq; 7561 struct netdev_queue *txq; 7562 7563 if (WARN_ON_ONCE(napi && !napi->dev)) 7564 return; 7565 netdev_assert_locked_ops_compat_or_invisible(dev); 7566 7567 switch (type) { 7568 case NETDEV_QUEUE_TYPE_RX: 7569 rxq = __netif_get_rx_queue(dev, queue_index); 7570 rxq->napi = napi; 7571 return; 7572 case NETDEV_QUEUE_TYPE_TX: 7573 txq = netdev_get_tx_queue(dev, queue_index); 7574 txq->napi = napi; 7575 return; 7576 default: 7577 return; 7578 } 7579 } 7580 EXPORT_SYMBOL(netif_queue_set_napi); 7581 7582 static void 7583 netif_napi_irq_notify(struct irq_affinity_notify *notify, 7584 const cpumask_t *mask) 7585 { 7586 struct napi_struct *napi = 7587 container_of(notify, struct napi_struct, notify); 7588 #ifdef CONFIG_RFS_ACCEL 7589 struct cpu_rmap *rmap = napi->dev->rx_cpu_rmap; 7590 int err; 7591 #endif 7592 7593 if (napi->config && napi->dev->irq_affinity_auto) 7594 cpumask_copy(&napi->config->affinity_mask, mask); 7595 7596 #ifdef CONFIG_RFS_ACCEL 7597 if (napi->dev->rx_cpu_rmap_auto) { 7598 err = cpu_rmap_update(rmap, napi->napi_rmap_idx, mask); 7599 if (err) 7600 netdev_warn(napi->dev, "RMAP update failed (%d)\n", 7601 err); 7602 } 7603 #endif 7604 } 7605 7606 #ifdef CONFIG_RFS_ACCEL 7607 static void netif_napi_affinity_release(struct kref *ref) 7608 { 7609 struct napi_struct *napi = 7610 container_of(ref, struct napi_struct, notify.kref); 7611 struct cpu_rmap *rmap = napi->dev->rx_cpu_rmap; 7612 7613 netdev_assert_locked(napi->dev); 7614 WARN_ON(test_and_clear_bit(NAPI_STATE_HAS_NOTIFIER, 7615 &napi->state)); 7616 7617 if (!napi->dev->rx_cpu_rmap_auto) 7618 return; 7619 rmap->obj[napi->napi_rmap_idx] = NULL; 7620 napi->napi_rmap_idx = -1; 7621 cpu_rmap_put(rmap); 7622 } 7623 7624 int netif_enable_cpu_rmap(struct net_device *dev, unsigned int num_irqs) 7625 { 7626 if (dev->rx_cpu_rmap_auto) 7627 return 0; 7628 7629 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(num_irqs); 7630 if (!dev->rx_cpu_rmap) 7631 return -ENOMEM; 7632 7633 dev->rx_cpu_rmap_auto = true; 7634 return 0; 7635 } 7636 EXPORT_SYMBOL(netif_enable_cpu_rmap); 7637 7638 static void netif_del_cpu_rmap(struct net_device *dev) 7639 { 7640 struct cpu_rmap *rmap = dev->rx_cpu_rmap; 7641 7642 if (!dev->rx_cpu_rmap_auto) 7643 return; 7644 7645 /* Free the rmap */ 7646 cpu_rmap_put(rmap); 7647 dev->rx_cpu_rmap = NULL; 7648 dev->rx_cpu_rmap_auto = false; 7649 } 7650 7651 #else 7652 static void netif_napi_affinity_release(struct kref *ref) 7653 { 7654 } 7655 7656 int netif_enable_cpu_rmap(struct net_device *dev, unsigned int num_irqs) 7657 { 7658 return 0; 7659 } 7660 EXPORT_SYMBOL(netif_enable_cpu_rmap); 7661 7662 static void netif_del_cpu_rmap(struct net_device *dev) 7663 { 7664 } 7665 #endif 7666 7667 void netif_set_affinity_auto(struct net_device *dev) 7668 { 7669 unsigned int i, maxqs, numa; 7670 7671 maxqs = max(dev->num_tx_queues, dev->num_rx_queues); 7672 numa = dev_to_node(&dev->dev); 7673 7674 for (i = 0; i < maxqs; i++) 7675 cpumask_set_cpu(cpumask_local_spread(i, numa), 7676 &dev->napi_config[i].affinity_mask); 7677 7678 dev->irq_affinity_auto = true; 7679 } 7680 EXPORT_SYMBOL(netif_set_affinity_auto); 7681 7682 void netif_napi_set_irq_locked(struct napi_struct *napi, int irq) 7683 { 7684 int rc; 7685 7686 netdev_assert_locked_or_invisible(napi->dev); 7687 7688 if (napi->irq == irq) 7689 return; 7690 7691 /* Remove existing resources */ 7692 if (test_and_clear_bit(NAPI_STATE_HAS_NOTIFIER, &napi->state)) 7693 irq_set_affinity_notifier(napi->irq, NULL); 7694 7695 napi->irq = irq; 7696 if (irq < 0 || 7697 (!napi->dev->rx_cpu_rmap_auto && !napi->dev->irq_affinity_auto)) 7698 return; 7699 7700 /* Abort for buggy drivers */ 7701 if (napi->dev->irq_affinity_auto && WARN_ON_ONCE(!napi->config)) 7702 return; 7703 7704 #ifdef CONFIG_RFS_ACCEL 7705 if (napi->dev->rx_cpu_rmap_auto) { 7706 rc = cpu_rmap_add(napi->dev->rx_cpu_rmap, napi); 7707 if (rc < 0) 7708 return; 7709 7710 cpu_rmap_get(napi->dev->rx_cpu_rmap); 7711 napi->napi_rmap_idx = rc; 7712 } 7713 #endif 7714 7715 /* Use core IRQ notifier */ 7716 napi->notify.notify = netif_napi_irq_notify; 7717 napi->notify.release = netif_napi_affinity_release; 7718 rc = irq_set_affinity_notifier(irq, &napi->notify); 7719 if (rc) { 7720 netdev_warn(napi->dev, "Unable to set IRQ notifier (%d)\n", 7721 rc); 7722 goto put_rmap; 7723 } 7724 7725 set_bit(NAPI_STATE_HAS_NOTIFIER, &napi->state); 7726 return; 7727 7728 put_rmap: 7729 #ifdef CONFIG_RFS_ACCEL 7730 if (napi->dev->rx_cpu_rmap_auto) { 7731 napi->dev->rx_cpu_rmap->obj[napi->napi_rmap_idx] = NULL; 7732 cpu_rmap_put(napi->dev->rx_cpu_rmap); 7733 napi->napi_rmap_idx = -1; 7734 } 7735 #endif 7736 napi->notify.notify = NULL; 7737 napi->notify.release = NULL; 7738 } 7739 EXPORT_SYMBOL(netif_napi_set_irq_locked); 7740 7741 static void napi_restore_config(struct napi_struct *n) 7742 { 7743 n->defer_hard_irqs = n->config->defer_hard_irqs; 7744 n->gro_flush_timeout = n->config->gro_flush_timeout; 7745 n->irq_suspend_timeout = n->config->irq_suspend_timeout; 7746 7747 if (n->dev->irq_affinity_auto && 7748 test_bit(NAPI_STATE_HAS_NOTIFIER, &n->state)) 7749 irq_set_affinity(n->irq, &n->config->affinity_mask); 7750 7751 /* a NAPI ID might be stored in the config, if so use it. if not, use 7752 * napi_hash_add to generate one for us. 7753 */ 7754 if (n->config->napi_id) { 7755 napi_hash_add_with_id(n, n->config->napi_id); 7756 } else { 7757 napi_hash_add(n); 7758 n->config->napi_id = n->napi_id; 7759 } 7760 7761 WARN_ON_ONCE(napi_set_threaded(n, n->config->threaded)); 7762 } 7763 7764 static void napi_save_config(struct napi_struct *n) 7765 { 7766 n->config->defer_hard_irqs = n->defer_hard_irqs; 7767 n->config->gro_flush_timeout = n->gro_flush_timeout; 7768 n->config->irq_suspend_timeout = n->irq_suspend_timeout; 7769 napi_hash_del(n); 7770 } 7771 7772 /* Netlink wants the NAPI list to be sorted by ID, if adding a NAPI which will 7773 * inherit an existing ID try to insert it at the right position. 7774 */ 7775 static void 7776 netif_napi_dev_list_add(struct net_device *dev, struct napi_struct *napi) 7777 { 7778 unsigned int new_id, pos_id; 7779 struct list_head *higher; 7780 struct napi_struct *pos; 7781 7782 new_id = UINT_MAX; 7783 if (napi->config && napi->config->napi_id) 7784 new_id = napi->config->napi_id; 7785 7786 higher = &dev->napi_list; 7787 list_for_each_entry(pos, &dev->napi_list, dev_list) { 7788 if (napi_id_valid(pos->napi_id)) 7789 pos_id = pos->napi_id; 7790 else if (pos->config) 7791 pos_id = pos->config->napi_id; 7792 else 7793 pos_id = UINT_MAX; 7794 7795 if (pos_id <= new_id) 7796 break; 7797 higher = &pos->dev_list; 7798 } 7799 list_add_rcu(&napi->dev_list, higher); /* adds after higher */ 7800 } 7801 7802 /* Double check that napi_get_frags() allocates skbs with 7803 * skb->head being backed by slab, not a page fragment. 7804 * This is to make sure bug fixed in 3226b158e67c 7805 * ("net: avoid 32 x truesize under-estimation for tiny skbs") 7806 * does not accidentally come back. 7807 */ 7808 static void napi_get_frags_check(struct napi_struct *napi) 7809 { 7810 struct sk_buff *skb; 7811 7812 local_bh_disable(); 7813 skb = napi_get_frags(napi); 7814 WARN_ON_ONCE(skb && skb->head_frag); 7815 napi_free_frags(napi); 7816 local_bh_enable(); 7817 } 7818 7819 void netif_napi_add_weight_locked(struct net_device *dev, 7820 struct napi_struct *napi, 7821 int (*poll)(struct napi_struct *, int), 7822 int weight) 7823 { 7824 netdev_assert_locked(dev); 7825 if (WARN_ON(test_and_set_bit(NAPI_STATE_LISTED, &napi->state))) 7826 return; 7827 7828 INIT_LIST_HEAD(&napi->poll_list); 7829 INIT_HLIST_NODE(&napi->napi_hash_node); 7830 hrtimer_setup(&napi->timer, napi_watchdog, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); 7831 gro_init(&napi->gro); 7832 napi->skb = NULL; 7833 napi->poll = poll; 7834 if (weight > NAPI_POLL_WEIGHT) 7835 netdev_err_once(dev, "%s() called with weight %d\n", __func__, 7836 weight); 7837 napi->weight = weight; 7838 napi->dev = dev; 7839 #ifdef CONFIG_NETPOLL 7840 napi->poll_owner = -1; 7841 #endif 7842 napi->list_owner = -1; 7843 set_bit(NAPI_STATE_SCHED, &napi->state); 7844 set_bit(NAPI_STATE_NPSVC, &napi->state); 7845 set_bit(NAPI_STATE_DISABLED, &napi->state); 7846 netif_napi_dev_list_add(dev, napi); 7847 7848 /* default settings from sysfs are applied to all NAPIs. any per-NAPI 7849 * configuration will be loaded in napi_enable 7850 */ 7851 napi_set_defer_hard_irqs(napi, READ_ONCE(dev->napi_defer_hard_irqs)); 7852 napi_set_gro_flush_timeout(napi, READ_ONCE(dev->gro_flush_timeout)); 7853 7854 napi_get_frags_check(napi); 7855 /* Create kthread for this napi if dev->threaded is set. 7856 * Clear dev->threaded if kthread creation failed so that 7857 * threaded mode will not be enabled in napi_enable(). 7858 */ 7859 if (napi_get_threaded_config(dev, napi)) 7860 if (napi_kthread_create(napi)) 7861 dev->threaded = NETDEV_NAPI_THREADED_DISABLED; 7862 netif_napi_set_irq_locked(napi, -1); 7863 } 7864 EXPORT_SYMBOL(netif_napi_add_weight_locked); 7865 7866 void napi_disable_locked(struct napi_struct *n) 7867 { 7868 unsigned long val, new; 7869 7870 might_sleep(); 7871 netdev_assert_locked(n->dev); 7872 7873 set_bit(NAPI_STATE_DISABLE, &n->state); 7874 7875 val = READ_ONCE(n->state); 7876 do { 7877 while (val & (NAPIF_STATE_SCHED | NAPIF_STATE_NPSVC)) { 7878 usleep_range(20, 200); 7879 val = READ_ONCE(n->state); 7880 } 7881 7882 new = val | NAPIF_STATE_SCHED | NAPIF_STATE_NPSVC | 7883 NAPIF_STATE_DISABLED; 7884 new &= ~(NAPIF_STATE_THREADED | 7885 NAPIF_STATE_THREADED_BUSY_POLL | 7886 NAPIF_STATE_PREFER_BUSY_POLL); 7887 } while (!try_cmpxchg(&n->state, &val, new)); 7888 7889 hrtimer_cancel(&n->timer); 7890 7891 if (n->config) 7892 napi_save_config(n); 7893 else 7894 napi_hash_del(n); 7895 7896 clear_bit(NAPI_STATE_DISABLE, &n->state); 7897 } 7898 EXPORT_SYMBOL(napi_disable_locked); 7899 7900 /** 7901 * napi_disable() - prevent NAPI from scheduling 7902 * @n: NAPI context 7903 * 7904 * Stop NAPI from being scheduled on this context. 7905 * Waits till any outstanding processing completes. 7906 * Takes netdev_lock() for associated net_device. 7907 */ 7908 void napi_disable(struct napi_struct *n) 7909 { 7910 netdev_lock(n->dev); 7911 napi_disable_locked(n); 7912 netdev_unlock(n->dev); 7913 } 7914 EXPORT_SYMBOL(napi_disable); 7915 7916 void napi_enable_locked(struct napi_struct *n) 7917 { 7918 unsigned long new, val = READ_ONCE(n->state); 7919 7920 if (n->config) 7921 napi_restore_config(n); 7922 else 7923 napi_hash_add(n); 7924 7925 do { 7926 BUG_ON(!test_bit(NAPI_STATE_SCHED, &val)); 7927 7928 new = val & ~(NAPIF_STATE_SCHED | NAPIF_STATE_NPSVC | 7929 NAPIF_STATE_DISABLED); 7930 if (n->dev->threaded && n->thread) 7931 new |= NAPIF_STATE_THREADED; 7932 } while (!try_cmpxchg(&n->state, &val, new)); 7933 } 7934 EXPORT_SYMBOL(napi_enable_locked); 7935 7936 /** 7937 * napi_enable() - enable NAPI scheduling 7938 * @n: NAPI context 7939 * 7940 * Enable scheduling of a NAPI instance. 7941 * Must be paired with napi_disable(). 7942 * Takes netdev_lock() for associated net_device. 7943 */ 7944 void napi_enable(struct napi_struct *n) 7945 { 7946 netdev_lock(n->dev); 7947 napi_enable_locked(n); 7948 netdev_unlock(n->dev); 7949 } 7950 EXPORT_SYMBOL(napi_enable); 7951 7952 /* Must be called in process context */ 7953 void __netif_napi_del_locked(struct napi_struct *napi) 7954 { 7955 netdev_assert_locked(napi->dev); 7956 7957 if (!test_and_clear_bit(NAPI_STATE_LISTED, &napi->state)) 7958 return; 7959 7960 /* Make sure NAPI is disabled (or was never enabled). */ 7961 WARN_ON(!test_bit(NAPI_STATE_SCHED, &napi->state)); 7962 7963 if (test_and_clear_bit(NAPI_STATE_HAS_NOTIFIER, &napi->state)) 7964 irq_set_affinity_notifier(napi->irq, NULL); 7965 7966 if (napi->config) { 7967 napi->index = -1; 7968 napi->config = NULL; 7969 } 7970 7971 list_del_rcu(&napi->dev_list); 7972 napi_free_frags(napi); 7973 7974 gro_cleanup(&napi->gro); 7975 7976 if (napi->thread) { 7977 kthread_stop(napi->thread); 7978 napi->thread = NULL; 7979 } 7980 } 7981 EXPORT_SYMBOL(__netif_napi_del_locked); 7982 7983 static int __napi_poll(struct napi_struct *n, bool *repoll) 7984 { 7985 int work, weight; 7986 7987 weight = n->weight; 7988 7989 /* This NAPI_STATE_SCHED test is for avoiding a race 7990 * with netpoll's poll_napi(). Only the entity which 7991 * obtains the lock and sees NAPI_STATE_SCHED set will 7992 * actually make the ->poll() call. Therefore we avoid 7993 * accidentally calling ->poll() when NAPI is not scheduled. 7994 */ 7995 work = 0; 7996 if (napi_is_scheduled(n)) { 7997 work = n->poll(n, weight); 7998 trace_napi_poll(n, work, weight); 7999 8000 xdp_do_check_flushed(n); 8001 } 8002 8003 if (unlikely(work > weight)) 8004 netdev_err_once(n->dev, "NAPI poll function %pS returned %d, exceeding its budget of %d.\n", 8005 n->poll, work, weight); 8006 8007 if (likely(work < weight)) 8008 return work; 8009 8010 /* Drivers must not modify the NAPI state if they 8011 * consume the entire weight. In such cases this code 8012 * still "owns" the NAPI instance and therefore can 8013 * move the instance around on the list at-will. 8014 */ 8015 if (unlikely(napi_disable_pending(n))) { 8016 napi_complete(n); 8017 return work; 8018 } 8019 8020 /* The NAPI context has more processing work, but busy-polling 8021 * is preferred. Exit early. 8022 */ 8023 if (napi_prefer_busy_poll(n)) { 8024 if (napi_complete_done(n, work)) { 8025 /* If timeout is not set, we need to make sure 8026 * that the NAPI is re-scheduled. 8027 */ 8028 napi_schedule(n); 8029 } 8030 return work; 8031 } 8032 8033 /* Flush too old packets. If HZ < 1000, flush all packets */ 8034 gro_flush_normal(&n->gro, HZ >= 1000); 8035 8036 /* Some drivers may have called napi_schedule 8037 * prior to exhausting their budget. 8038 */ 8039 if (unlikely(!list_empty(&n->poll_list))) { 8040 pr_warn_once("%s: Budget exhausted after napi rescheduled\n", 8041 n->dev ? n->dev->name : "backlog"); 8042 return work; 8043 } 8044 8045 *repoll = true; 8046 8047 return work; 8048 } 8049 8050 static int napi_poll(struct napi_struct *n, struct list_head *repoll) 8051 { 8052 bool do_repoll = false; 8053 void *have; 8054 int work; 8055 8056 list_del_init(&n->poll_list); 8057 8058 have = netpoll_poll_lock(n); 8059 8060 work = __napi_poll(n, &do_repoll); 8061 8062 if (do_repoll) { 8063 #if defined(CONFIG_DEBUG_NET) 8064 if (unlikely(!napi_is_scheduled(n))) 8065 pr_crit("repoll requested for device %s %ps but napi is not scheduled.\n", 8066 n->dev->name, n->poll); 8067 #endif 8068 list_add_tail(&n->poll_list, repoll); 8069 } 8070 netpoll_poll_unlock(have); 8071 8072 return work; 8073 } 8074 8075 static int napi_thread_wait(struct napi_struct *napi) 8076 { 8077 set_current_state(TASK_INTERRUPTIBLE); 8078 8079 while (!kthread_should_stop()) { 8080 /* Testing SCHED_THREADED bit here to make sure the current 8081 * kthread owns this napi and could poll on this napi. 8082 * Testing SCHED bit is not enough because SCHED bit might be 8083 * set by some other busy poll thread or by napi_disable(). 8084 */ 8085 if (test_bit(NAPI_STATE_SCHED_THREADED, &napi->state)) { 8086 WARN_ON(!list_empty(&napi->poll_list)); 8087 __set_current_state(TASK_RUNNING); 8088 return 0; 8089 } 8090 8091 schedule(); 8092 set_current_state(TASK_INTERRUPTIBLE); 8093 } 8094 __set_current_state(TASK_RUNNING); 8095 8096 return -1; 8097 } 8098 8099 static void napi_threaded_poll_loop(struct napi_struct *napi, 8100 unsigned long *busy_poll_last_qs) 8101 { 8102 unsigned long last_qs = busy_poll_last_qs ? *busy_poll_last_qs : jiffies; 8103 struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx; 8104 struct softnet_data *sd; 8105 8106 for (;;) { 8107 bool repoll = false; 8108 void *have; 8109 8110 local_bh_disable(); 8111 bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx); 8112 8113 sd = this_cpu_ptr(&softnet_data); 8114 sd->in_napi_threaded_poll = true; 8115 8116 have = netpoll_poll_lock(napi); 8117 __napi_poll(napi, &repoll); 8118 netpoll_poll_unlock(have); 8119 8120 sd->in_napi_threaded_poll = false; 8121 barrier(); 8122 8123 if (sd_has_rps_ipi_waiting(sd)) { 8124 local_irq_disable(); 8125 net_rps_action_and_irq_enable(sd); 8126 } 8127 skb_defer_free_flush(); 8128 bpf_net_ctx_clear(bpf_net_ctx); 8129 8130 /* When busy poll is enabled, the old packets are not flushed in 8131 * napi_complete_done. So flush them here. 8132 */ 8133 if (busy_poll_last_qs) 8134 gro_flush_normal(&napi->gro, HZ >= 1000); 8135 local_bh_enable(); 8136 8137 /* Call cond_resched here to avoid watchdog warnings. */ 8138 if (repoll || busy_poll_last_qs) { 8139 rcu_softirq_qs_periodic(last_qs); 8140 cond_resched(); 8141 } 8142 8143 if (!repoll) 8144 break; 8145 } 8146 8147 if (busy_poll_last_qs) 8148 *busy_poll_last_qs = last_qs; 8149 } 8150 8151 static int napi_threaded_poll(void *data) 8152 { 8153 struct napi_struct *napi = data; 8154 unsigned long last_qs = jiffies; 8155 bool want_busy_poll; 8156 bool in_busy_poll; 8157 unsigned long val; 8158 8159 while (!napi_thread_wait(napi)) { 8160 val = READ_ONCE(napi->state); 8161 8162 want_busy_poll = val & NAPIF_STATE_THREADED_BUSY_POLL; 8163 in_busy_poll = val & NAPIF_STATE_IN_BUSY_POLL; 8164 8165 if (unlikely(val & NAPIF_STATE_DISABLE)) 8166 want_busy_poll = false; 8167 8168 if (want_busy_poll != in_busy_poll) 8169 assign_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state, 8170 want_busy_poll); 8171 8172 napi_threaded_poll_loop(napi, want_busy_poll ? &last_qs : NULL); 8173 } 8174 8175 return 0; 8176 } 8177 8178 static __latent_entropy void net_rx_action(void) 8179 { 8180 struct softnet_data *sd = this_cpu_ptr(&softnet_data); 8181 unsigned long time_limit = jiffies + 8182 usecs_to_jiffies(READ_ONCE(net_hotdata.netdev_budget_usecs)); 8183 struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx; 8184 int budget = READ_ONCE(net_hotdata.netdev_budget); 8185 LIST_HEAD(list); 8186 LIST_HEAD(repoll); 8187 8188 bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx); 8189 start: 8190 sd->in_net_rx_action = true; 8191 local_irq_disable(); 8192 list_splice_init(&sd->poll_list, &list); 8193 local_irq_enable(); 8194 8195 for (;;) { 8196 struct napi_struct *n; 8197 8198 skb_defer_free_flush(); 8199 8200 if (list_empty(&list)) { 8201 if (list_empty(&repoll)) { 8202 sd->in_net_rx_action = false; 8203 barrier(); 8204 /* We need to check if ____napi_schedule() 8205 * had refilled poll_list while 8206 * sd->in_net_rx_action was true. 8207 */ 8208 if (!list_empty(&sd->poll_list)) 8209 goto start; 8210 if (!sd_has_rps_ipi_waiting(sd)) 8211 goto end; 8212 } 8213 break; 8214 } 8215 8216 n = list_first_entry(&list, struct napi_struct, poll_list); 8217 budget -= napi_poll(n, &repoll); 8218 8219 /* If softirq window is exhausted then punt. 8220 * Allow this to run for 2 jiffies since which will allow 8221 * an average latency of 1.5/HZ. 8222 */ 8223 if (unlikely(budget <= 0 || 8224 time_after_eq(jiffies, time_limit))) { 8225 /* Pairs with READ_ONCE() in softnet_seq_show() */ 8226 WRITE_ONCE(sd->time_squeeze, sd->time_squeeze + 1); 8227 break; 8228 } 8229 } 8230 8231 local_irq_disable(); 8232 8233 list_splice_tail_init(&sd->poll_list, &list); 8234 list_splice_tail(&repoll, &list); 8235 list_splice(&list, &sd->poll_list); 8236 if (!list_empty(&sd->poll_list)) 8237 __raise_softirq_irqoff(NET_RX_SOFTIRQ); 8238 else 8239 sd->in_net_rx_action = false; 8240 8241 net_rps_action_and_irq_enable(sd); 8242 end: 8243 bpf_net_ctx_clear(bpf_net_ctx); 8244 } 8245 8246 struct netdev_adjacent { 8247 struct net_device *dev; 8248 netdevice_tracker dev_tracker; 8249 8250 /* upper master flag, there can only be one master device per list */ 8251 bool master; 8252 8253 /* lookup ignore flag */ 8254 bool ignore; 8255 8256 /* counter for the number of times this device was added to us */ 8257 u16 ref_nr; 8258 8259 /* private field for the users */ 8260 void *private; 8261 8262 struct list_head list; 8263 struct rcu_head rcu; 8264 }; 8265 8266 static struct netdev_adjacent *__netdev_find_adj(struct net_device *adj_dev, 8267 struct list_head *adj_list) 8268 { 8269 struct netdev_adjacent *adj; 8270 8271 list_for_each_entry(adj, adj_list, list) { 8272 if (adj->dev == adj_dev) 8273 return adj; 8274 } 8275 return NULL; 8276 } 8277 8278 static int ____netdev_has_upper_dev(struct net_device *upper_dev, 8279 struct netdev_nested_priv *priv) 8280 { 8281 struct net_device *dev = (struct net_device *)priv->data; 8282 8283 return upper_dev == dev; 8284 } 8285 8286 /** 8287 * netdev_has_upper_dev - Check if device is linked to an upper device 8288 * @dev: device 8289 * @upper_dev: upper device to check 8290 * 8291 * Find out if a device is linked to specified upper device and return true 8292 * in case it is. Note that this checks only immediate upper device, 8293 * not through a complete stack of devices. The caller must hold the RTNL lock. 8294 */ 8295 bool netdev_has_upper_dev(struct net_device *dev, 8296 struct net_device *upper_dev) 8297 { 8298 struct netdev_nested_priv priv = { 8299 .data = (void *)upper_dev, 8300 }; 8301 8302 ASSERT_RTNL(); 8303 8304 return netdev_walk_all_upper_dev_rcu(dev, ____netdev_has_upper_dev, 8305 &priv); 8306 } 8307 EXPORT_SYMBOL(netdev_has_upper_dev); 8308 8309 /** 8310 * netdev_has_upper_dev_all_rcu - Check if device is linked to an upper device 8311 * @dev: device 8312 * @upper_dev: upper device to check 8313 * 8314 * Find out if a device is linked to specified upper device and return true 8315 * in case it is. Note that this checks the entire upper device chain. 8316 * The caller must hold rcu lock. 8317 */ 8318 8319 bool netdev_has_upper_dev_all_rcu(struct net_device *dev, 8320 struct net_device *upper_dev) 8321 { 8322 struct netdev_nested_priv priv = { 8323 .data = (void *)upper_dev, 8324 }; 8325 8326 return !!netdev_walk_all_upper_dev_rcu(dev, ____netdev_has_upper_dev, 8327 &priv); 8328 } 8329 EXPORT_SYMBOL(netdev_has_upper_dev_all_rcu); 8330 8331 /** 8332 * netdev_has_any_upper_dev - Check if device is linked to some device 8333 * @dev: device 8334 * 8335 * Find out if a device is linked to an upper device and return true in case 8336 * it is. The caller must hold the RTNL lock. 8337 */ 8338 bool netdev_has_any_upper_dev(struct net_device *dev) 8339 { 8340 ASSERT_RTNL(); 8341 8342 return !list_empty(&dev->adj_list.upper); 8343 } 8344 EXPORT_SYMBOL(netdev_has_any_upper_dev); 8345 8346 /** 8347 * netdev_master_upper_dev_get - Get master upper device 8348 * @dev: device 8349 * 8350 * Find a master upper device and return pointer to it or NULL in case 8351 * it's not there. The caller must hold the RTNL lock. 8352 */ 8353 struct net_device *netdev_master_upper_dev_get(struct net_device *dev) 8354 { 8355 struct netdev_adjacent *upper; 8356 8357 ASSERT_RTNL(); 8358 8359 if (list_empty(&dev->adj_list.upper)) 8360 return NULL; 8361 8362 upper = list_first_entry(&dev->adj_list.upper, 8363 struct netdev_adjacent, list); 8364 if (likely(upper->master)) 8365 return upper->dev; 8366 return NULL; 8367 } 8368 EXPORT_SYMBOL(netdev_master_upper_dev_get); 8369 8370 static struct net_device *__netdev_master_upper_dev_get(struct net_device *dev) 8371 { 8372 struct netdev_adjacent *upper; 8373 8374 ASSERT_RTNL(); 8375 8376 if (list_empty(&dev->adj_list.upper)) 8377 return NULL; 8378 8379 upper = list_first_entry(&dev->adj_list.upper, 8380 struct netdev_adjacent, list); 8381 if (likely(upper->master) && !upper->ignore) 8382 return upper->dev; 8383 return NULL; 8384 } 8385 8386 /** 8387 * netdev_has_any_lower_dev - Check if device is linked to some device 8388 * @dev: device 8389 * 8390 * Find out if a device is linked to a lower device and return true in case 8391 * it is. The caller must hold the RTNL lock. 8392 */ 8393 static bool netdev_has_any_lower_dev(struct net_device *dev) 8394 { 8395 ASSERT_RTNL(); 8396 8397 return !list_empty(&dev->adj_list.lower); 8398 } 8399 8400 void *netdev_adjacent_get_private(struct list_head *adj_list) 8401 { 8402 struct netdev_adjacent *adj; 8403 8404 adj = list_entry(adj_list, struct netdev_adjacent, list); 8405 8406 return adj->private; 8407 } 8408 EXPORT_SYMBOL(netdev_adjacent_get_private); 8409 8410 /** 8411 * netdev_upper_get_next_dev_rcu - Get the next dev from upper list 8412 * @dev: device 8413 * @iter: list_head ** of the current position 8414 * 8415 * Gets the next device from the dev's upper list, starting from iter 8416 * position. The caller must hold RCU read lock. 8417 */ 8418 struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev, 8419 struct list_head **iter) 8420 { 8421 struct netdev_adjacent *upper; 8422 8423 WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held() && 8424 !lockdep_rtnl_is_held()); 8425 8426 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); 8427 8428 if (&upper->list == &dev->adj_list.upper) 8429 return NULL; 8430 8431 *iter = &upper->list; 8432 8433 return upper->dev; 8434 } 8435 EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu); 8436 8437 static struct net_device *__netdev_next_upper_dev(struct net_device *dev, 8438 struct list_head **iter, 8439 bool *ignore) 8440 { 8441 struct netdev_adjacent *upper; 8442 8443 upper = list_entry((*iter)->next, struct netdev_adjacent, list); 8444 8445 if (&upper->list == &dev->adj_list.upper) 8446 return NULL; 8447 8448 *iter = &upper->list; 8449 *ignore = upper->ignore; 8450 8451 return upper->dev; 8452 } 8453 8454 static struct net_device *netdev_next_upper_dev_rcu(struct net_device *dev, 8455 struct list_head **iter) 8456 { 8457 struct netdev_adjacent *upper; 8458 8459 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held()); 8460 8461 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); 8462 8463 if (&upper->list == &dev->adj_list.upper) 8464 return NULL; 8465 8466 *iter = &upper->list; 8467 8468 return upper->dev; 8469 } 8470 8471 static int __netdev_walk_all_upper_dev(struct net_device *dev, 8472 int (*fn)(struct net_device *dev, 8473 struct netdev_nested_priv *priv), 8474 struct netdev_nested_priv *priv) 8475 { 8476 struct net_device *udev, *next, *now, *dev_stack[MAX_NEST_DEV + 1]; 8477 struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1]; 8478 int ret, cur = 0; 8479 bool ignore; 8480 8481 now = dev; 8482 iter = &dev->adj_list.upper; 8483 8484 while (1) { 8485 if (now != dev) { 8486 ret = fn(now, priv); 8487 if (ret) 8488 return ret; 8489 } 8490 8491 next = NULL; 8492 while (1) { 8493 udev = __netdev_next_upper_dev(now, &iter, &ignore); 8494 if (!udev) 8495 break; 8496 if (ignore) 8497 continue; 8498 8499 next = udev; 8500 niter = &udev->adj_list.upper; 8501 dev_stack[cur] = now; 8502 iter_stack[cur++] = iter; 8503 break; 8504 } 8505 8506 if (!next) { 8507 if (!cur) 8508 return 0; 8509 next = dev_stack[--cur]; 8510 niter = iter_stack[cur]; 8511 } 8512 8513 now = next; 8514 iter = niter; 8515 } 8516 8517 return 0; 8518 } 8519 8520 int netdev_walk_all_upper_dev_rcu(struct net_device *dev, 8521 int (*fn)(struct net_device *dev, 8522 struct netdev_nested_priv *priv), 8523 struct netdev_nested_priv *priv) 8524 { 8525 struct net_device *udev, *next, *now, *dev_stack[MAX_NEST_DEV + 1]; 8526 struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1]; 8527 int ret, cur = 0; 8528 8529 now = dev; 8530 iter = &dev->adj_list.upper; 8531 8532 while (1) { 8533 if (now != dev) { 8534 ret = fn(now, priv); 8535 if (ret) 8536 return ret; 8537 } 8538 8539 next = NULL; 8540 while (1) { 8541 udev = netdev_next_upper_dev_rcu(now, &iter); 8542 if (!udev) 8543 break; 8544 8545 next = udev; 8546 niter = &udev->adj_list.upper; 8547 dev_stack[cur] = now; 8548 iter_stack[cur++] = iter; 8549 break; 8550 } 8551 8552 if (!next) { 8553 if (!cur) 8554 return 0; 8555 next = dev_stack[--cur]; 8556 niter = iter_stack[cur]; 8557 } 8558 8559 now = next; 8560 iter = niter; 8561 } 8562 8563 return 0; 8564 } 8565 EXPORT_SYMBOL_GPL(netdev_walk_all_upper_dev_rcu); 8566 8567 static bool __netdev_has_upper_dev(struct net_device *dev, 8568 struct net_device *upper_dev) 8569 { 8570 struct netdev_nested_priv priv = { 8571 .flags = 0, 8572 .data = (void *)upper_dev, 8573 }; 8574 8575 ASSERT_RTNL(); 8576 8577 return __netdev_walk_all_upper_dev(dev, ____netdev_has_upper_dev, 8578 &priv); 8579 } 8580 8581 /** 8582 * netdev_lower_get_next_private - Get the next ->private from the 8583 * lower neighbour list 8584 * @dev: device 8585 * @iter: list_head ** of the current position 8586 * 8587 * Gets the next netdev_adjacent->private from the dev's lower neighbour 8588 * list, starting from iter position. The caller must hold either hold the 8589 * RTNL lock or its own locking that guarantees that the neighbour lower 8590 * list will remain unchanged. 8591 */ 8592 void *netdev_lower_get_next_private(struct net_device *dev, 8593 struct list_head **iter) 8594 { 8595 struct netdev_adjacent *lower; 8596 8597 lower = list_entry(*iter, struct netdev_adjacent, list); 8598 8599 if (&lower->list == &dev->adj_list.lower) 8600 return NULL; 8601 8602 *iter = lower->list.next; 8603 8604 return lower->private; 8605 } 8606 EXPORT_SYMBOL(netdev_lower_get_next_private); 8607 8608 /** 8609 * netdev_lower_get_next_private_rcu - Get the next ->private from the 8610 * lower neighbour list, RCU 8611 * variant 8612 * @dev: device 8613 * @iter: list_head ** of the current position 8614 * 8615 * Gets the next netdev_adjacent->private from the dev's lower neighbour 8616 * list, starting from iter position. The caller must hold RCU read lock. 8617 */ 8618 void *netdev_lower_get_next_private_rcu(struct net_device *dev, 8619 struct list_head **iter) 8620 { 8621 struct netdev_adjacent *lower; 8622 8623 WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held()); 8624 8625 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); 8626 8627 if (&lower->list == &dev->adj_list.lower) 8628 return NULL; 8629 8630 *iter = &lower->list; 8631 8632 return lower->private; 8633 } 8634 EXPORT_SYMBOL(netdev_lower_get_next_private_rcu); 8635 8636 /** 8637 * netdev_lower_get_next - Get the next device from the lower neighbour 8638 * list 8639 * @dev: device 8640 * @iter: list_head ** of the current position 8641 * 8642 * Gets the next netdev_adjacent from the dev's lower neighbour 8643 * list, starting from iter position. The caller must hold RTNL lock or 8644 * its own locking that guarantees that the neighbour lower 8645 * list will remain unchanged. 8646 */ 8647 void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter) 8648 { 8649 struct netdev_adjacent *lower; 8650 8651 lower = list_entry(*iter, struct netdev_adjacent, list); 8652 8653 if (&lower->list == &dev->adj_list.lower) 8654 return NULL; 8655 8656 *iter = lower->list.next; 8657 8658 return lower->dev; 8659 } 8660 EXPORT_SYMBOL(netdev_lower_get_next); 8661 8662 static struct net_device *netdev_next_lower_dev(struct net_device *dev, 8663 struct list_head **iter) 8664 { 8665 struct netdev_adjacent *lower; 8666 8667 lower = list_entry((*iter)->next, struct netdev_adjacent, list); 8668 8669 if (&lower->list == &dev->adj_list.lower) 8670 return NULL; 8671 8672 *iter = &lower->list; 8673 8674 return lower->dev; 8675 } 8676 8677 static struct net_device *__netdev_next_lower_dev(struct net_device *dev, 8678 struct list_head **iter, 8679 bool *ignore) 8680 { 8681 struct netdev_adjacent *lower; 8682 8683 lower = list_entry((*iter)->next, struct netdev_adjacent, list); 8684 8685 if (&lower->list == &dev->adj_list.lower) 8686 return NULL; 8687 8688 *iter = &lower->list; 8689 *ignore = lower->ignore; 8690 8691 return lower->dev; 8692 } 8693 8694 int netdev_walk_all_lower_dev(struct net_device *dev, 8695 int (*fn)(struct net_device *dev, 8696 struct netdev_nested_priv *priv), 8697 struct netdev_nested_priv *priv) 8698 { 8699 struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1]; 8700 struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1]; 8701 int ret, cur = 0; 8702 8703 now = dev; 8704 iter = &dev->adj_list.lower; 8705 8706 while (1) { 8707 if (now != dev) { 8708 ret = fn(now, priv); 8709 if (ret) 8710 return ret; 8711 } 8712 8713 next = NULL; 8714 while (1) { 8715 ldev = netdev_next_lower_dev(now, &iter); 8716 if (!ldev) 8717 break; 8718 8719 next = ldev; 8720 niter = &ldev->adj_list.lower; 8721 dev_stack[cur] = now; 8722 iter_stack[cur++] = iter; 8723 break; 8724 } 8725 8726 if (!next) { 8727 if (!cur) 8728 return 0; 8729 next = dev_stack[--cur]; 8730 niter = iter_stack[cur]; 8731 } 8732 8733 now = next; 8734 iter = niter; 8735 } 8736 8737 return 0; 8738 } 8739 EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev); 8740 8741 static int __netdev_walk_all_lower_dev(struct net_device *dev, 8742 int (*fn)(struct net_device *dev, 8743 struct netdev_nested_priv *priv), 8744 struct netdev_nested_priv *priv) 8745 { 8746 struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1]; 8747 struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1]; 8748 int ret, cur = 0; 8749 bool ignore; 8750 8751 now = dev; 8752 iter = &dev->adj_list.lower; 8753 8754 while (1) { 8755 if (now != dev) { 8756 ret = fn(now, priv); 8757 if (ret) 8758 return ret; 8759 } 8760 8761 next = NULL; 8762 while (1) { 8763 ldev = __netdev_next_lower_dev(now, &iter, &ignore); 8764 if (!ldev) 8765 break; 8766 if (ignore) 8767 continue; 8768 8769 next = ldev; 8770 niter = &ldev->adj_list.lower; 8771 dev_stack[cur] = now; 8772 iter_stack[cur++] = iter; 8773 break; 8774 } 8775 8776 if (!next) { 8777 if (!cur) 8778 return 0; 8779 next = dev_stack[--cur]; 8780 niter = iter_stack[cur]; 8781 } 8782 8783 now = next; 8784 iter = niter; 8785 } 8786 8787 return 0; 8788 } 8789 8790 struct net_device *netdev_next_lower_dev_rcu(struct net_device *dev, 8791 struct list_head **iter) 8792 { 8793 struct netdev_adjacent *lower; 8794 8795 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); 8796 if (&lower->list == &dev->adj_list.lower) 8797 return NULL; 8798 8799 *iter = &lower->list; 8800 8801 return lower->dev; 8802 } 8803 EXPORT_SYMBOL(netdev_next_lower_dev_rcu); 8804 8805 static u8 __netdev_upper_depth(struct net_device *dev) 8806 { 8807 struct net_device *udev; 8808 struct list_head *iter; 8809 u8 max_depth = 0; 8810 bool ignore; 8811 8812 for (iter = &dev->adj_list.upper, 8813 udev = __netdev_next_upper_dev(dev, &iter, &ignore); 8814 udev; 8815 udev = __netdev_next_upper_dev(dev, &iter, &ignore)) { 8816 if (ignore) 8817 continue; 8818 if (max_depth < udev->upper_level) 8819 max_depth = udev->upper_level; 8820 } 8821 8822 return max_depth; 8823 } 8824 8825 static u8 __netdev_lower_depth(struct net_device *dev) 8826 { 8827 struct net_device *ldev; 8828 struct list_head *iter; 8829 u8 max_depth = 0; 8830 bool ignore; 8831 8832 for (iter = &dev->adj_list.lower, 8833 ldev = __netdev_next_lower_dev(dev, &iter, &ignore); 8834 ldev; 8835 ldev = __netdev_next_lower_dev(dev, &iter, &ignore)) { 8836 if (ignore) 8837 continue; 8838 if (max_depth < ldev->lower_level) 8839 max_depth = ldev->lower_level; 8840 } 8841 8842 return max_depth; 8843 } 8844 8845 static int __netdev_update_upper_level(struct net_device *dev, 8846 struct netdev_nested_priv *__unused) 8847 { 8848 dev->upper_level = __netdev_upper_depth(dev) + 1; 8849 return 0; 8850 } 8851 8852 #ifdef CONFIG_LOCKDEP 8853 static LIST_HEAD(net_unlink_list); 8854 8855 static void net_unlink_todo(struct net_device *dev) 8856 { 8857 if (list_empty(&dev->unlink_list)) 8858 list_add_tail(&dev->unlink_list, &net_unlink_list); 8859 } 8860 #endif 8861 8862 static int __netdev_update_lower_level(struct net_device *dev, 8863 struct netdev_nested_priv *priv) 8864 { 8865 dev->lower_level = __netdev_lower_depth(dev) + 1; 8866 8867 #ifdef CONFIG_LOCKDEP 8868 if (!priv) 8869 return 0; 8870 8871 if (priv->flags & NESTED_SYNC_IMM) 8872 dev->nested_level = dev->lower_level - 1; 8873 if (priv->flags & NESTED_SYNC_TODO) 8874 net_unlink_todo(dev); 8875 #endif 8876 return 0; 8877 } 8878 8879 int netdev_walk_all_lower_dev_rcu(struct net_device *dev, 8880 int (*fn)(struct net_device *dev, 8881 struct netdev_nested_priv *priv), 8882 struct netdev_nested_priv *priv) 8883 { 8884 struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1]; 8885 struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1]; 8886 int ret, cur = 0; 8887 8888 now = dev; 8889 iter = &dev->adj_list.lower; 8890 8891 while (1) { 8892 if (now != dev) { 8893 ret = fn(now, priv); 8894 if (ret) 8895 return ret; 8896 } 8897 8898 next = NULL; 8899 while (1) { 8900 ldev = netdev_next_lower_dev_rcu(now, &iter); 8901 if (!ldev) 8902 break; 8903 8904 next = ldev; 8905 niter = &ldev->adj_list.lower; 8906 dev_stack[cur] = now; 8907 iter_stack[cur++] = iter; 8908 break; 8909 } 8910 8911 if (!next) { 8912 if (!cur) 8913 return 0; 8914 next = dev_stack[--cur]; 8915 niter = iter_stack[cur]; 8916 } 8917 8918 now = next; 8919 iter = niter; 8920 } 8921 8922 return 0; 8923 } 8924 EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev_rcu); 8925 8926 /** 8927 * netdev_lower_get_first_private_rcu - Get the first ->private from the 8928 * lower neighbour list, RCU 8929 * variant 8930 * @dev: device 8931 * 8932 * Gets the first netdev_adjacent->private from the dev's lower neighbour 8933 * list. The caller must hold RCU read lock. 8934 */ 8935 void *netdev_lower_get_first_private_rcu(struct net_device *dev) 8936 { 8937 struct netdev_adjacent *lower; 8938 8939 lower = list_first_or_null_rcu(&dev->adj_list.lower, 8940 struct netdev_adjacent, list); 8941 if (lower) 8942 return lower->private; 8943 return NULL; 8944 } 8945 EXPORT_SYMBOL(netdev_lower_get_first_private_rcu); 8946 8947 /** 8948 * netdev_master_upper_dev_get_rcu - Get master upper device 8949 * @dev: device 8950 * 8951 * Find a master upper device and return pointer to it or NULL in case 8952 * it's not there. The caller must hold the RCU read lock. 8953 */ 8954 struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev) 8955 { 8956 struct netdev_adjacent *upper; 8957 8958 upper = list_first_or_null_rcu(&dev->adj_list.upper, 8959 struct netdev_adjacent, list); 8960 if (upper && likely(upper->master)) 8961 return upper->dev; 8962 return NULL; 8963 } 8964 EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu); 8965 8966 static int netdev_adjacent_sysfs_add(struct net_device *dev, 8967 struct net_device *adj_dev, 8968 struct list_head *dev_list) 8969 { 8970 char linkname[IFNAMSIZ+7]; 8971 8972 sprintf(linkname, dev_list == &dev->adj_list.upper ? 8973 "upper_%s" : "lower_%s", adj_dev->name); 8974 return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj), 8975 linkname); 8976 } 8977 static void netdev_adjacent_sysfs_del(struct net_device *dev, 8978 char *name, 8979 struct list_head *dev_list) 8980 { 8981 char linkname[IFNAMSIZ+7]; 8982 8983 sprintf(linkname, dev_list == &dev->adj_list.upper ? 8984 "upper_%s" : "lower_%s", name); 8985 sysfs_remove_link(&(dev->dev.kobj), linkname); 8986 } 8987 8988 static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev, 8989 struct net_device *adj_dev, 8990 struct list_head *dev_list) 8991 { 8992 return (dev_list == &dev->adj_list.upper || 8993 dev_list == &dev->adj_list.lower) && 8994 net_eq(dev_net(dev), dev_net(adj_dev)); 8995 } 8996 8997 static int __netdev_adjacent_dev_insert(struct net_device *dev, 8998 struct net_device *adj_dev, 8999 struct list_head *dev_list, 9000 void *private, bool master) 9001 { 9002 struct netdev_adjacent *adj; 9003 int ret; 9004 9005 adj = __netdev_find_adj(adj_dev, dev_list); 9006 9007 if (adj) { 9008 adj->ref_nr += 1; 9009 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d\n", 9010 dev->name, adj_dev->name, adj->ref_nr); 9011 9012 return 0; 9013 } 9014 9015 adj = kmalloc_obj(*adj); 9016 if (!adj) 9017 return -ENOMEM; 9018 9019 adj->dev = adj_dev; 9020 adj->master = master; 9021 adj->ref_nr = 1; 9022 adj->private = private; 9023 adj->ignore = false; 9024 netdev_hold(adj_dev, &adj->dev_tracker, GFP_KERNEL); 9025 9026 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d; dev_hold on %s\n", 9027 dev->name, adj_dev->name, adj->ref_nr, adj_dev->name); 9028 9029 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) { 9030 ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list); 9031 if (ret) 9032 goto free_adj; 9033 } 9034 9035 /* Ensure that master link is always the first item in list. */ 9036 if (master) { 9037 ret = sysfs_create_link(&(dev->dev.kobj), 9038 &(adj_dev->dev.kobj), "master"); 9039 if (ret) 9040 goto remove_symlinks; 9041 9042 list_add_rcu(&adj->list, dev_list); 9043 } else { 9044 list_add_tail_rcu(&adj->list, dev_list); 9045 } 9046 9047 return 0; 9048 9049 remove_symlinks: 9050 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) 9051 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list); 9052 free_adj: 9053 netdev_put(adj_dev, &adj->dev_tracker); 9054 kfree(adj); 9055 9056 return ret; 9057 } 9058 9059 static void __netdev_adjacent_dev_remove(struct net_device *dev, 9060 struct net_device *adj_dev, 9061 u16 ref_nr, 9062 struct list_head *dev_list) 9063 { 9064 struct netdev_adjacent *adj; 9065 9066 pr_debug("Remove adjacency: dev %s adj_dev %s ref_nr %d\n", 9067 dev->name, adj_dev->name, ref_nr); 9068 9069 adj = __netdev_find_adj(adj_dev, dev_list); 9070 9071 if (!adj) { 9072 pr_err("Adjacency does not exist for device %s from %s\n", 9073 dev->name, adj_dev->name); 9074 WARN_ON(1); 9075 return; 9076 } 9077 9078 if (adj->ref_nr > ref_nr) { 9079 pr_debug("adjacency: %s to %s ref_nr - %d = %d\n", 9080 dev->name, adj_dev->name, ref_nr, 9081 adj->ref_nr - ref_nr); 9082 adj->ref_nr -= ref_nr; 9083 return; 9084 } 9085 9086 if (adj->master) 9087 sysfs_remove_link(&(dev->dev.kobj), "master"); 9088 9089 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) 9090 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list); 9091 9092 list_del_rcu(&adj->list); 9093 pr_debug("adjacency: dev_put for %s, because link removed from %s to %s\n", 9094 adj_dev->name, dev->name, adj_dev->name); 9095 netdev_put(adj_dev, &adj->dev_tracker); 9096 kfree_rcu(adj, rcu); 9097 } 9098 9099 static int __netdev_adjacent_dev_link_lists(struct net_device *dev, 9100 struct net_device *upper_dev, 9101 struct list_head *up_list, 9102 struct list_head *down_list, 9103 void *private, bool master) 9104 { 9105 int ret; 9106 9107 ret = __netdev_adjacent_dev_insert(dev, upper_dev, up_list, 9108 private, master); 9109 if (ret) 9110 return ret; 9111 9112 ret = __netdev_adjacent_dev_insert(upper_dev, dev, down_list, 9113 private, false); 9114 if (ret) { 9115 __netdev_adjacent_dev_remove(dev, upper_dev, 1, up_list); 9116 return ret; 9117 } 9118 9119 return 0; 9120 } 9121 9122 static void __netdev_adjacent_dev_unlink_lists(struct net_device *dev, 9123 struct net_device *upper_dev, 9124 u16 ref_nr, 9125 struct list_head *up_list, 9126 struct list_head *down_list) 9127 { 9128 __netdev_adjacent_dev_remove(dev, upper_dev, ref_nr, up_list); 9129 __netdev_adjacent_dev_remove(upper_dev, dev, ref_nr, down_list); 9130 } 9131 9132 static int __netdev_adjacent_dev_link_neighbour(struct net_device *dev, 9133 struct net_device *upper_dev, 9134 void *private, bool master) 9135 { 9136 return __netdev_adjacent_dev_link_lists(dev, upper_dev, 9137 &dev->adj_list.upper, 9138 &upper_dev->adj_list.lower, 9139 private, master); 9140 } 9141 9142 static void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev, 9143 struct net_device *upper_dev) 9144 { 9145 __netdev_adjacent_dev_unlink_lists(dev, upper_dev, 1, 9146 &dev->adj_list.upper, 9147 &upper_dev->adj_list.lower); 9148 } 9149 9150 static int __netdev_upper_dev_link(struct net_device *dev, 9151 struct net_device *upper_dev, bool master, 9152 void *upper_priv, void *upper_info, 9153 struct netdev_nested_priv *priv, 9154 struct netlink_ext_ack *extack) 9155 { 9156 struct netdev_notifier_changeupper_info changeupper_info = { 9157 .info = { 9158 .dev = dev, 9159 .extack = extack, 9160 }, 9161 .upper_dev = upper_dev, 9162 .master = master, 9163 .linking = true, 9164 .upper_info = upper_info, 9165 }; 9166 struct net_device *master_dev; 9167 int ret = 0; 9168 9169 ASSERT_RTNL(); 9170 9171 if (dev == upper_dev) 9172 return -EBUSY; 9173 9174 /* To prevent loops, check if dev is not upper device to upper_dev. */ 9175 if (__netdev_has_upper_dev(upper_dev, dev)) 9176 return -EBUSY; 9177 9178 if ((dev->lower_level + upper_dev->upper_level) > MAX_NEST_DEV) 9179 return -EMLINK; 9180 9181 if (!master) { 9182 if (__netdev_has_upper_dev(dev, upper_dev)) 9183 return -EEXIST; 9184 } else { 9185 master_dev = __netdev_master_upper_dev_get(dev); 9186 if (master_dev) 9187 return master_dev == upper_dev ? -EEXIST : -EBUSY; 9188 } 9189 9190 ret = call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER, 9191 &changeupper_info.info); 9192 ret = notifier_to_errno(ret); 9193 if (ret) 9194 return ret; 9195 9196 ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, upper_priv, 9197 master); 9198 if (ret) 9199 return ret; 9200 9201 ret = call_netdevice_notifiers_info(NETDEV_CHANGEUPPER, 9202 &changeupper_info.info); 9203 ret = notifier_to_errno(ret); 9204 if (ret) 9205 goto rollback; 9206 9207 __netdev_update_upper_level(dev, NULL); 9208 __netdev_walk_all_lower_dev(dev, __netdev_update_upper_level, NULL); 9209 9210 __netdev_update_lower_level(upper_dev, priv); 9211 __netdev_walk_all_upper_dev(upper_dev, __netdev_update_lower_level, 9212 priv); 9213 9214 return 0; 9215 9216 rollback: 9217 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev); 9218 9219 return ret; 9220 } 9221 9222 /** 9223 * netdev_upper_dev_link - Add a link to the upper device 9224 * @dev: device 9225 * @upper_dev: new upper device 9226 * @extack: netlink extended ack 9227 * 9228 * Adds a link to device which is upper to this one. The caller must hold 9229 * the RTNL lock. On a failure a negative errno code is returned. 9230 * On success the reference counts are adjusted and the function 9231 * returns zero. 9232 */ 9233 int netdev_upper_dev_link(struct net_device *dev, 9234 struct net_device *upper_dev, 9235 struct netlink_ext_ack *extack) 9236 { 9237 struct netdev_nested_priv priv = { 9238 .flags = NESTED_SYNC_IMM | NESTED_SYNC_TODO, 9239 .data = NULL, 9240 }; 9241 9242 return __netdev_upper_dev_link(dev, upper_dev, false, 9243 NULL, NULL, &priv, extack); 9244 } 9245 EXPORT_SYMBOL(netdev_upper_dev_link); 9246 9247 /** 9248 * netdev_master_upper_dev_link - Add a master link to the upper device 9249 * @dev: device 9250 * @upper_dev: new upper device 9251 * @upper_priv: upper device private 9252 * @upper_info: upper info to be passed down via notifier 9253 * @extack: netlink extended ack 9254 * 9255 * Adds a link to device which is upper to this one. In this case, only 9256 * one master upper device can be linked, although other non-master devices 9257 * might be linked as well. The caller must hold the RTNL lock. 9258 * On a failure a negative errno code is returned. On success the reference 9259 * counts are adjusted and the function returns zero. 9260 */ 9261 int netdev_master_upper_dev_link(struct net_device *dev, 9262 struct net_device *upper_dev, 9263 void *upper_priv, void *upper_info, 9264 struct netlink_ext_ack *extack) 9265 { 9266 struct netdev_nested_priv priv = { 9267 .flags = NESTED_SYNC_IMM | NESTED_SYNC_TODO, 9268 .data = NULL, 9269 }; 9270 9271 return __netdev_upper_dev_link(dev, upper_dev, true, 9272 upper_priv, upper_info, &priv, extack); 9273 } 9274 EXPORT_SYMBOL(netdev_master_upper_dev_link); 9275 9276 static void __netdev_upper_dev_unlink(struct net_device *dev, 9277 struct net_device *upper_dev, 9278 struct netdev_nested_priv *priv) 9279 { 9280 struct netdev_notifier_changeupper_info changeupper_info = { 9281 .info = { 9282 .dev = dev, 9283 }, 9284 .upper_dev = upper_dev, 9285 .linking = false, 9286 }; 9287 9288 ASSERT_RTNL(); 9289 9290 changeupper_info.master = netdev_master_upper_dev_get(dev) == upper_dev; 9291 9292 call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER, 9293 &changeupper_info.info); 9294 9295 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev); 9296 9297 call_netdevice_notifiers_info(NETDEV_CHANGEUPPER, 9298 &changeupper_info.info); 9299 9300 __netdev_update_upper_level(dev, NULL); 9301 __netdev_walk_all_lower_dev(dev, __netdev_update_upper_level, NULL); 9302 9303 __netdev_update_lower_level(upper_dev, priv); 9304 __netdev_walk_all_upper_dev(upper_dev, __netdev_update_lower_level, 9305 priv); 9306 } 9307 9308 /** 9309 * netdev_upper_dev_unlink - Removes a link to upper device 9310 * @dev: device 9311 * @upper_dev: new upper device 9312 * 9313 * Removes a link to device which is upper to this one. The caller must hold 9314 * the RTNL lock. 9315 */ 9316 void netdev_upper_dev_unlink(struct net_device *dev, 9317 struct net_device *upper_dev) 9318 { 9319 struct netdev_nested_priv priv = { 9320 .flags = NESTED_SYNC_TODO, 9321 .data = NULL, 9322 }; 9323 9324 __netdev_upper_dev_unlink(dev, upper_dev, &priv); 9325 } 9326 EXPORT_SYMBOL(netdev_upper_dev_unlink); 9327 9328 static void __netdev_adjacent_dev_set(struct net_device *upper_dev, 9329 struct net_device *lower_dev, 9330 bool val) 9331 { 9332 struct netdev_adjacent *adj; 9333 9334 adj = __netdev_find_adj(lower_dev, &upper_dev->adj_list.lower); 9335 if (adj) 9336 adj->ignore = val; 9337 9338 adj = __netdev_find_adj(upper_dev, &lower_dev->adj_list.upper); 9339 if (adj) 9340 adj->ignore = val; 9341 } 9342 9343 static void netdev_adjacent_dev_disable(struct net_device *upper_dev, 9344 struct net_device *lower_dev) 9345 { 9346 __netdev_adjacent_dev_set(upper_dev, lower_dev, true); 9347 } 9348 9349 static void netdev_adjacent_dev_enable(struct net_device *upper_dev, 9350 struct net_device *lower_dev) 9351 { 9352 __netdev_adjacent_dev_set(upper_dev, lower_dev, false); 9353 } 9354 9355 int netdev_adjacent_change_prepare(struct net_device *old_dev, 9356 struct net_device *new_dev, 9357 struct net_device *dev, 9358 struct netlink_ext_ack *extack) 9359 { 9360 struct netdev_nested_priv priv = { 9361 .flags = 0, 9362 .data = NULL, 9363 }; 9364 int err; 9365 9366 if (!new_dev) 9367 return 0; 9368 9369 if (old_dev && new_dev != old_dev) 9370 netdev_adjacent_dev_disable(dev, old_dev); 9371 err = __netdev_upper_dev_link(new_dev, dev, false, NULL, NULL, &priv, 9372 extack); 9373 if (err) { 9374 if (old_dev && new_dev != old_dev) 9375 netdev_adjacent_dev_enable(dev, old_dev); 9376 return err; 9377 } 9378 9379 return 0; 9380 } 9381 EXPORT_SYMBOL(netdev_adjacent_change_prepare); 9382 9383 void netdev_adjacent_change_commit(struct net_device *old_dev, 9384 struct net_device *new_dev, 9385 struct net_device *dev) 9386 { 9387 struct netdev_nested_priv priv = { 9388 .flags = NESTED_SYNC_IMM | NESTED_SYNC_TODO, 9389 .data = NULL, 9390 }; 9391 9392 if (!new_dev || !old_dev) 9393 return; 9394 9395 if (new_dev == old_dev) 9396 return; 9397 9398 netdev_adjacent_dev_enable(dev, old_dev); 9399 __netdev_upper_dev_unlink(old_dev, dev, &priv); 9400 } 9401 EXPORT_SYMBOL(netdev_adjacent_change_commit); 9402 9403 void netdev_adjacent_change_abort(struct net_device *old_dev, 9404 struct net_device *new_dev, 9405 struct net_device *dev) 9406 { 9407 struct netdev_nested_priv priv = { 9408 .flags = 0, 9409 .data = NULL, 9410 }; 9411 9412 if (!new_dev) 9413 return; 9414 9415 if (old_dev && new_dev != old_dev) 9416 netdev_adjacent_dev_enable(dev, old_dev); 9417 9418 __netdev_upper_dev_unlink(new_dev, dev, &priv); 9419 } 9420 EXPORT_SYMBOL(netdev_adjacent_change_abort); 9421 9422 /** 9423 * netdev_bonding_info_change - Dispatch event about slave change 9424 * @dev: device 9425 * @bonding_info: info to dispatch 9426 * 9427 * Send NETDEV_BONDING_INFO to netdev notifiers with info. 9428 * The caller must hold the RTNL lock. 9429 */ 9430 void netdev_bonding_info_change(struct net_device *dev, 9431 struct netdev_bonding_info *bonding_info) 9432 { 9433 struct netdev_notifier_bonding_info info = { 9434 .info.dev = dev, 9435 }; 9436 9437 memcpy(&info.bonding_info, bonding_info, 9438 sizeof(struct netdev_bonding_info)); 9439 call_netdevice_notifiers_info(NETDEV_BONDING_INFO, 9440 &info.info); 9441 } 9442 EXPORT_SYMBOL(netdev_bonding_info_change); 9443 9444 static int netdev_offload_xstats_enable_l3(struct net_device *dev, 9445 struct netlink_ext_ack *extack) 9446 { 9447 struct netdev_notifier_offload_xstats_info info = { 9448 .info.dev = dev, 9449 .info.extack = extack, 9450 .type = NETDEV_OFFLOAD_XSTATS_TYPE_L3, 9451 }; 9452 int err; 9453 int rc; 9454 9455 dev->offload_xstats_l3 = kzalloc_obj(*dev->offload_xstats_l3); 9456 if (!dev->offload_xstats_l3) 9457 return -ENOMEM; 9458 9459 rc = call_netdevice_notifiers_info_robust(NETDEV_OFFLOAD_XSTATS_ENABLE, 9460 NETDEV_OFFLOAD_XSTATS_DISABLE, 9461 &info.info); 9462 err = notifier_to_errno(rc); 9463 if (err) 9464 goto free_stats; 9465 9466 return 0; 9467 9468 free_stats: 9469 kfree(dev->offload_xstats_l3); 9470 dev->offload_xstats_l3 = NULL; 9471 return err; 9472 } 9473 9474 int netdev_offload_xstats_enable(struct net_device *dev, 9475 enum netdev_offload_xstats_type type, 9476 struct netlink_ext_ack *extack) 9477 { 9478 ASSERT_RTNL(); 9479 9480 if (netdev_offload_xstats_enabled(dev, type)) 9481 return -EALREADY; 9482 9483 switch (type) { 9484 case NETDEV_OFFLOAD_XSTATS_TYPE_L3: 9485 return netdev_offload_xstats_enable_l3(dev, extack); 9486 } 9487 9488 WARN_ON(1); 9489 return -EINVAL; 9490 } 9491 EXPORT_SYMBOL(netdev_offload_xstats_enable); 9492 9493 static void netdev_offload_xstats_disable_l3(struct net_device *dev) 9494 { 9495 struct netdev_notifier_offload_xstats_info info = { 9496 .info.dev = dev, 9497 .type = NETDEV_OFFLOAD_XSTATS_TYPE_L3, 9498 }; 9499 9500 call_netdevice_notifiers_info(NETDEV_OFFLOAD_XSTATS_DISABLE, 9501 &info.info); 9502 kfree(dev->offload_xstats_l3); 9503 dev->offload_xstats_l3 = NULL; 9504 } 9505 9506 int netdev_offload_xstats_disable(struct net_device *dev, 9507 enum netdev_offload_xstats_type type) 9508 { 9509 ASSERT_RTNL(); 9510 9511 if (!netdev_offload_xstats_enabled(dev, type)) 9512 return -EALREADY; 9513 9514 switch (type) { 9515 case NETDEV_OFFLOAD_XSTATS_TYPE_L3: 9516 netdev_offload_xstats_disable_l3(dev); 9517 return 0; 9518 } 9519 9520 WARN_ON(1); 9521 return -EINVAL; 9522 } 9523 EXPORT_SYMBOL(netdev_offload_xstats_disable); 9524 9525 static void netdev_offload_xstats_disable_all(struct net_device *dev) 9526 { 9527 netdev_offload_xstats_disable(dev, NETDEV_OFFLOAD_XSTATS_TYPE_L3); 9528 } 9529 9530 static struct rtnl_hw_stats64 * 9531 netdev_offload_xstats_get_ptr(const struct net_device *dev, 9532 enum netdev_offload_xstats_type type) 9533 { 9534 switch (type) { 9535 case NETDEV_OFFLOAD_XSTATS_TYPE_L3: 9536 return dev->offload_xstats_l3; 9537 } 9538 9539 WARN_ON(1); 9540 return NULL; 9541 } 9542 9543 bool netdev_offload_xstats_enabled(const struct net_device *dev, 9544 enum netdev_offload_xstats_type type) 9545 { 9546 ASSERT_RTNL(); 9547 9548 return netdev_offload_xstats_get_ptr(dev, type); 9549 } 9550 EXPORT_SYMBOL(netdev_offload_xstats_enabled); 9551 9552 struct netdev_notifier_offload_xstats_ru { 9553 bool used; 9554 }; 9555 9556 struct netdev_notifier_offload_xstats_rd { 9557 struct rtnl_hw_stats64 stats; 9558 bool used; 9559 }; 9560 9561 static void netdev_hw_stats64_add(struct rtnl_hw_stats64 *dest, 9562 const struct rtnl_hw_stats64 *src) 9563 { 9564 dest->rx_packets += src->rx_packets; 9565 dest->tx_packets += src->tx_packets; 9566 dest->rx_bytes += src->rx_bytes; 9567 dest->tx_bytes += src->tx_bytes; 9568 dest->rx_errors += src->rx_errors; 9569 dest->tx_errors += src->tx_errors; 9570 dest->rx_dropped += src->rx_dropped; 9571 dest->tx_dropped += src->tx_dropped; 9572 dest->multicast += src->multicast; 9573 } 9574 9575 static int netdev_offload_xstats_get_used(struct net_device *dev, 9576 enum netdev_offload_xstats_type type, 9577 bool *p_used, 9578 struct netlink_ext_ack *extack) 9579 { 9580 struct netdev_notifier_offload_xstats_ru report_used = {}; 9581 struct netdev_notifier_offload_xstats_info info = { 9582 .info.dev = dev, 9583 .info.extack = extack, 9584 .type = type, 9585 .report_used = &report_used, 9586 }; 9587 int rc; 9588 9589 WARN_ON(!netdev_offload_xstats_enabled(dev, type)); 9590 rc = call_netdevice_notifiers_info(NETDEV_OFFLOAD_XSTATS_REPORT_USED, 9591 &info.info); 9592 *p_used = report_used.used; 9593 return notifier_to_errno(rc); 9594 } 9595 9596 static int netdev_offload_xstats_get_stats(struct net_device *dev, 9597 enum netdev_offload_xstats_type type, 9598 struct rtnl_hw_stats64 *p_stats, 9599 bool *p_used, 9600 struct netlink_ext_ack *extack) 9601 { 9602 struct netdev_notifier_offload_xstats_rd report_delta = {}; 9603 struct netdev_notifier_offload_xstats_info info = { 9604 .info.dev = dev, 9605 .info.extack = extack, 9606 .type = type, 9607 .report_delta = &report_delta, 9608 }; 9609 struct rtnl_hw_stats64 *stats; 9610 int rc; 9611 9612 stats = netdev_offload_xstats_get_ptr(dev, type); 9613 if (WARN_ON(!stats)) 9614 return -EINVAL; 9615 9616 rc = call_netdevice_notifiers_info(NETDEV_OFFLOAD_XSTATS_REPORT_DELTA, 9617 &info.info); 9618 9619 /* Cache whatever we got, even if there was an error, otherwise the 9620 * successful stats retrievals would get lost. 9621 */ 9622 netdev_hw_stats64_add(stats, &report_delta.stats); 9623 9624 if (p_stats) 9625 *p_stats = *stats; 9626 *p_used = report_delta.used; 9627 9628 return notifier_to_errno(rc); 9629 } 9630 9631 int netdev_offload_xstats_get(struct net_device *dev, 9632 enum netdev_offload_xstats_type type, 9633 struct rtnl_hw_stats64 *p_stats, bool *p_used, 9634 struct netlink_ext_ack *extack) 9635 { 9636 ASSERT_RTNL(); 9637 9638 if (p_stats) 9639 return netdev_offload_xstats_get_stats(dev, type, p_stats, 9640 p_used, extack); 9641 else 9642 return netdev_offload_xstats_get_used(dev, type, p_used, 9643 extack); 9644 } 9645 EXPORT_SYMBOL(netdev_offload_xstats_get); 9646 9647 void 9648 netdev_offload_xstats_report_delta(struct netdev_notifier_offload_xstats_rd *report_delta, 9649 const struct rtnl_hw_stats64 *stats) 9650 { 9651 report_delta->used = true; 9652 netdev_hw_stats64_add(&report_delta->stats, stats); 9653 } 9654 EXPORT_SYMBOL(netdev_offload_xstats_report_delta); 9655 9656 void 9657 netdev_offload_xstats_report_used(struct netdev_notifier_offload_xstats_ru *report_used) 9658 { 9659 report_used->used = true; 9660 } 9661 EXPORT_SYMBOL(netdev_offload_xstats_report_used); 9662 9663 void netdev_offload_xstats_push_delta(struct net_device *dev, 9664 enum netdev_offload_xstats_type type, 9665 const struct rtnl_hw_stats64 *p_stats) 9666 { 9667 struct rtnl_hw_stats64 *stats; 9668 9669 ASSERT_RTNL(); 9670 9671 stats = netdev_offload_xstats_get_ptr(dev, type); 9672 if (WARN_ON(!stats)) 9673 return; 9674 9675 netdev_hw_stats64_add(stats, p_stats); 9676 } 9677 EXPORT_SYMBOL(netdev_offload_xstats_push_delta); 9678 9679 /** 9680 * netdev_get_xmit_slave - Get the xmit slave of master device 9681 * @dev: device 9682 * @skb: The packet 9683 * @all_slaves: assume all the slaves are active 9684 * 9685 * The reference counters are not incremented so the caller must be 9686 * careful with locks. The caller must hold RCU lock. 9687 * %NULL is returned if no slave is found. 9688 */ 9689 9690 struct net_device *netdev_get_xmit_slave(struct net_device *dev, 9691 struct sk_buff *skb, 9692 bool all_slaves) 9693 { 9694 const struct net_device_ops *ops = dev->netdev_ops; 9695 9696 if (!ops->ndo_get_xmit_slave) 9697 return NULL; 9698 return ops->ndo_get_xmit_slave(dev, skb, all_slaves); 9699 } 9700 EXPORT_SYMBOL(netdev_get_xmit_slave); 9701 9702 static struct net_device *netdev_sk_get_lower_dev(struct net_device *dev, 9703 struct sock *sk) 9704 { 9705 const struct net_device_ops *ops = dev->netdev_ops; 9706 9707 if (!ops->ndo_sk_get_lower_dev) 9708 return NULL; 9709 return ops->ndo_sk_get_lower_dev(dev, sk); 9710 } 9711 9712 /** 9713 * netdev_sk_get_lowest_dev - Get the lowest device in chain given device and socket 9714 * @dev: device 9715 * @sk: the socket 9716 * 9717 * %NULL is returned if no lower device is found. 9718 */ 9719 9720 struct net_device *netdev_sk_get_lowest_dev(struct net_device *dev, 9721 struct sock *sk) 9722 { 9723 struct net_device *lower; 9724 9725 lower = netdev_sk_get_lower_dev(dev, sk); 9726 while (lower) { 9727 dev = lower; 9728 lower = netdev_sk_get_lower_dev(dev, sk); 9729 } 9730 9731 return dev; 9732 } 9733 EXPORT_SYMBOL(netdev_sk_get_lowest_dev); 9734 9735 static void netdev_adjacent_add_links(struct net_device *dev) 9736 { 9737 struct netdev_adjacent *iter; 9738 9739 struct net *net = dev_net(dev); 9740 9741 list_for_each_entry(iter, &dev->adj_list.upper, list) { 9742 if (!net_eq(net, dev_net(iter->dev))) 9743 continue; 9744 netdev_adjacent_sysfs_add(iter->dev, dev, 9745 &iter->dev->adj_list.lower); 9746 netdev_adjacent_sysfs_add(dev, iter->dev, 9747 &dev->adj_list.upper); 9748 } 9749 9750 list_for_each_entry(iter, &dev->adj_list.lower, list) { 9751 if (!net_eq(net, dev_net(iter->dev))) 9752 continue; 9753 netdev_adjacent_sysfs_add(iter->dev, dev, 9754 &iter->dev->adj_list.upper); 9755 netdev_adjacent_sysfs_add(dev, iter->dev, 9756 &dev->adj_list.lower); 9757 } 9758 } 9759 9760 static void netdev_adjacent_del_links(struct net_device *dev) 9761 { 9762 struct netdev_adjacent *iter; 9763 9764 struct net *net = dev_net(dev); 9765 9766 list_for_each_entry(iter, &dev->adj_list.upper, list) { 9767 if (!net_eq(net, dev_net(iter->dev))) 9768 continue; 9769 netdev_adjacent_sysfs_del(iter->dev, dev->name, 9770 &iter->dev->adj_list.lower); 9771 netdev_adjacent_sysfs_del(dev, iter->dev->name, 9772 &dev->adj_list.upper); 9773 } 9774 9775 list_for_each_entry(iter, &dev->adj_list.lower, list) { 9776 if (!net_eq(net, dev_net(iter->dev))) 9777 continue; 9778 netdev_adjacent_sysfs_del(iter->dev, dev->name, 9779 &iter->dev->adj_list.upper); 9780 netdev_adjacent_sysfs_del(dev, iter->dev->name, 9781 &dev->adj_list.lower); 9782 } 9783 } 9784 9785 void netdev_adjacent_rename_links(struct net_device *dev, char *oldname) 9786 { 9787 struct netdev_adjacent *iter; 9788 9789 struct net *net = dev_net(dev); 9790 9791 list_for_each_entry(iter, &dev->adj_list.upper, list) { 9792 if (!net_eq(net, dev_net(iter->dev))) 9793 continue; 9794 netdev_adjacent_sysfs_del(iter->dev, oldname, 9795 &iter->dev->adj_list.lower); 9796 netdev_adjacent_sysfs_add(iter->dev, dev, 9797 &iter->dev->adj_list.lower); 9798 } 9799 9800 list_for_each_entry(iter, &dev->adj_list.lower, list) { 9801 if (!net_eq(net, dev_net(iter->dev))) 9802 continue; 9803 netdev_adjacent_sysfs_del(iter->dev, oldname, 9804 &iter->dev->adj_list.upper); 9805 netdev_adjacent_sysfs_add(iter->dev, dev, 9806 &iter->dev->adj_list.upper); 9807 } 9808 } 9809 9810 void *netdev_lower_dev_get_private(struct net_device *dev, 9811 struct net_device *lower_dev) 9812 { 9813 struct netdev_adjacent *lower; 9814 9815 if (!lower_dev) 9816 return NULL; 9817 lower = __netdev_find_adj(lower_dev, &dev->adj_list.lower); 9818 if (!lower) 9819 return NULL; 9820 9821 return lower->private; 9822 } 9823 EXPORT_SYMBOL(netdev_lower_dev_get_private); 9824 9825 9826 /** 9827 * netdev_lower_state_changed - Dispatch event about lower device state change 9828 * @lower_dev: device 9829 * @lower_state_info: state to dispatch 9830 * 9831 * Send NETDEV_CHANGELOWERSTATE to netdev notifiers with info. 9832 * The caller must hold the RTNL lock. 9833 */ 9834 void netdev_lower_state_changed(struct net_device *lower_dev, 9835 void *lower_state_info) 9836 { 9837 struct netdev_notifier_changelowerstate_info changelowerstate_info = { 9838 .info.dev = lower_dev, 9839 }; 9840 9841 ASSERT_RTNL(); 9842 changelowerstate_info.lower_state_info = lower_state_info; 9843 call_netdevice_notifiers_info(NETDEV_CHANGELOWERSTATE, 9844 &changelowerstate_info.info); 9845 } 9846 EXPORT_SYMBOL(netdev_lower_state_changed); 9847 9848 static void dev_change_rx_flags(struct net_device *dev, int flags) 9849 { 9850 const struct net_device_ops *ops = dev->netdev_ops; 9851 9852 if (ops->ndo_change_rx_flags) 9853 ops->ndo_change_rx_flags(dev, flags); 9854 } 9855 9856 int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify) 9857 { 9858 unsigned int old_flags = dev->flags; 9859 unsigned int promiscuity, flags; 9860 kuid_t uid; 9861 kgid_t gid; 9862 9863 netdev_assert_locked_ops_compat(dev); 9864 9865 promiscuity = dev->promiscuity + inc; 9866 if (promiscuity == 0) { 9867 /* 9868 * Avoid overflow. 9869 * If inc causes overflow, untouch promisc and return error. 9870 */ 9871 if (unlikely(inc > 0)) { 9872 netdev_warn(dev, "promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n"); 9873 return -EOVERFLOW; 9874 } 9875 flags = old_flags & ~IFF_PROMISC; 9876 } else { 9877 flags = old_flags | IFF_PROMISC; 9878 } 9879 WRITE_ONCE(dev->promiscuity, promiscuity); 9880 if (flags != old_flags) { 9881 WRITE_ONCE(dev->flags, flags); 9882 netdev_info(dev, "%s promiscuous mode\n", 9883 dev->flags & IFF_PROMISC ? "entered" : "left"); 9884 if (audit_enabled) { 9885 current_uid_gid(&uid, &gid); 9886 audit_log(audit_context(), GFP_ATOMIC, 9887 AUDIT_ANOM_PROMISCUOUS, 9888 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u", 9889 dev->name, (dev->flags & IFF_PROMISC), 9890 (old_flags & IFF_PROMISC), 9891 from_kuid(&init_user_ns, audit_get_loginuid(current)), 9892 from_kuid(&init_user_ns, uid), 9893 from_kgid(&init_user_ns, gid), 9894 audit_get_sessionid(current)); 9895 } 9896 9897 dev_change_rx_flags(dev, IFF_PROMISC); 9898 } 9899 if (notify) 9900 __dev_notify_flags(dev, old_flags, IFF_PROMISC, 0, NULL); 9901 return 0; 9902 } 9903 9904 int netif_set_promiscuity(struct net_device *dev, int inc) 9905 { 9906 unsigned int old_flags = dev->flags; 9907 int err; 9908 9909 err = __dev_set_promiscuity(dev, inc, true); 9910 if (err < 0) 9911 return err; 9912 if (dev->flags != old_flags) 9913 dev_set_rx_mode(dev); 9914 return err; 9915 } 9916 9917 int netif_set_allmulti(struct net_device *dev, int inc, bool notify) 9918 { 9919 unsigned int old_flags = dev->flags, old_gflags = dev->gflags; 9920 unsigned int allmulti, flags; 9921 9922 netdev_assert_locked_ops_compat(dev); 9923 9924 allmulti = dev->allmulti + inc; 9925 if (allmulti == 0) { 9926 /* 9927 * Avoid overflow. 9928 * If inc causes overflow, untouch allmulti and return error. 9929 */ 9930 if (unlikely(inc > 0)) { 9931 netdev_warn(dev, "allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n"); 9932 return -EOVERFLOW; 9933 } 9934 flags = old_flags & ~IFF_ALLMULTI; 9935 } else { 9936 flags = old_flags | IFF_ALLMULTI; 9937 } 9938 WRITE_ONCE(dev->allmulti, allmulti); 9939 if (flags != old_flags) { 9940 WRITE_ONCE(dev->flags, flags); 9941 netdev_info(dev, "%s allmulticast mode\n", 9942 dev->flags & IFF_ALLMULTI ? "entered" : "left"); 9943 dev_change_rx_flags(dev, IFF_ALLMULTI); 9944 dev_set_rx_mode(dev); 9945 if (notify) 9946 __dev_notify_flags(dev, old_flags, 9947 dev->gflags ^ old_gflags, 0, NULL); 9948 } 9949 return 0; 9950 } 9951 9952 9953 /** 9954 * netif_get_flags() - get flags reported to userspace 9955 * @dev: device 9956 * 9957 * Get the combination of flag bits exported through APIs to userspace. 9958 */ 9959 unsigned int netif_get_flags(const struct net_device *dev) 9960 { 9961 unsigned int flags; 9962 9963 flags = (READ_ONCE(dev->flags) & ~(IFF_PROMISC | 9964 IFF_ALLMULTI | 9965 IFF_RUNNING | 9966 IFF_LOWER_UP | 9967 IFF_DORMANT)) | 9968 (READ_ONCE(dev->gflags) & (IFF_PROMISC | 9969 IFF_ALLMULTI)); 9970 9971 if (netif_running(dev)) { 9972 if (netif_oper_up(dev)) 9973 flags |= IFF_RUNNING; 9974 if (netif_carrier_ok(dev)) 9975 flags |= IFF_LOWER_UP; 9976 if (netif_dormant(dev)) 9977 flags |= IFF_DORMANT; 9978 } 9979 9980 return flags; 9981 } 9982 EXPORT_SYMBOL(netif_get_flags); 9983 9984 int __dev_change_flags(struct net_device *dev, unsigned int flags, 9985 struct netlink_ext_ack *extack) 9986 { 9987 unsigned int old_flags = dev->flags; 9988 int ret; 9989 9990 netdev_assert_locked_ops_compat(dev); 9991 9992 /* 9993 * Set the flags on our device. 9994 */ 9995 9996 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP | 9997 IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL | 9998 IFF_AUTOMEDIA)) | 9999 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC | 10000 IFF_ALLMULTI)); 10001 10002 /* 10003 * Load in the correct multicast list now the flags have changed. 10004 */ 10005 10006 if ((old_flags ^ flags) & IFF_MULTICAST) 10007 dev_change_rx_flags(dev, IFF_MULTICAST); 10008 10009 dev_set_rx_mode(dev); 10010 10011 /* 10012 * Have we downed the interface. We handle IFF_UP ourselves 10013 * according to user attempts to set it, rather than blindly 10014 * setting it. 10015 */ 10016 10017 ret = 0; 10018 if ((old_flags ^ flags) & IFF_UP) { 10019 if (old_flags & IFF_UP) 10020 __dev_close(dev); 10021 else 10022 ret = __dev_open(dev, extack); 10023 } 10024 10025 if ((flags ^ dev->gflags) & IFF_PROMISC) { 10026 int inc = (flags & IFF_PROMISC) ? 1 : -1; 10027 old_flags = dev->flags; 10028 10029 dev->gflags ^= IFF_PROMISC; 10030 10031 if (__dev_set_promiscuity(dev, inc, false) >= 0) 10032 if (dev->flags != old_flags) 10033 dev_set_rx_mode(dev); 10034 } 10035 10036 /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI 10037 * is important. Some (broken) drivers set IFF_PROMISC, when 10038 * IFF_ALLMULTI is requested not asking us and not reporting. 10039 */ 10040 if ((flags ^ dev->gflags) & IFF_ALLMULTI) { 10041 int inc = (flags & IFF_ALLMULTI) ? 1 : -1; 10042 10043 dev->gflags ^= IFF_ALLMULTI; 10044 netif_set_allmulti(dev, inc, false); 10045 } 10046 10047 return ret; 10048 } 10049 10050 void __dev_notify_flags(struct net_device *dev, unsigned int old_flags, 10051 unsigned int gchanges, u32 portid, 10052 const struct nlmsghdr *nlh) 10053 { 10054 unsigned int changes = dev->flags ^ old_flags; 10055 10056 netdev_assert_locked_ops_compat(dev); 10057 10058 if (gchanges) 10059 rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC, portid, nlh); 10060 10061 if (changes & IFF_UP) { 10062 if (dev->flags & IFF_UP) 10063 call_netdevice_notifiers(NETDEV_UP, dev); 10064 else 10065 call_netdevice_notifiers(NETDEV_DOWN, dev); 10066 } 10067 10068 if (dev->flags & IFF_UP && 10069 (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE))) { 10070 struct netdev_notifier_change_info change_info = { 10071 .info = { 10072 .dev = dev, 10073 }, 10074 .flags_changed = changes, 10075 }; 10076 10077 call_netdevice_notifiers_info(NETDEV_CHANGE, &change_info.info); 10078 } 10079 } 10080 10081 int netif_change_flags(struct net_device *dev, unsigned int flags, 10082 struct netlink_ext_ack *extack) 10083 { 10084 int ret; 10085 unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags; 10086 10087 ret = __dev_change_flags(dev, flags, extack); 10088 if (ret < 0) 10089 return ret; 10090 10091 changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags); 10092 __dev_notify_flags(dev, old_flags, changes, 0, NULL); 10093 return ret; 10094 } 10095 EXPORT_SYMBOL(netif_change_flags); 10096 10097 int __netif_set_mtu(struct net_device *dev, int new_mtu) 10098 { 10099 const struct net_device_ops *ops = dev->netdev_ops; 10100 10101 if (ops->ndo_change_mtu) 10102 return ops->ndo_change_mtu(dev, new_mtu); 10103 10104 /* Pairs with all the lockless reads of dev->mtu in the stack */ 10105 WRITE_ONCE(dev->mtu, new_mtu); 10106 return 0; 10107 } 10108 EXPORT_SYMBOL_NS_GPL(__netif_set_mtu, "NETDEV_INTERNAL"); 10109 10110 int dev_validate_mtu(struct net_device *dev, int new_mtu, 10111 struct netlink_ext_ack *extack) 10112 { 10113 /* MTU must be positive, and in range */ 10114 if (new_mtu < 0 || new_mtu < dev->min_mtu) { 10115 NL_SET_ERR_MSG(extack, "mtu less than device minimum"); 10116 return -EINVAL; 10117 } 10118 10119 if (dev->max_mtu > 0 && new_mtu > dev->max_mtu) { 10120 NL_SET_ERR_MSG(extack, "mtu greater than device maximum"); 10121 return -EINVAL; 10122 } 10123 return 0; 10124 } 10125 10126 /** 10127 * netif_set_mtu_ext() - Change maximum transfer unit 10128 * @dev: device 10129 * @new_mtu: new transfer unit 10130 * @extack: netlink extended ack 10131 * 10132 * Change the maximum transfer size of the network device. 10133 * 10134 * Return: 0 on success, -errno on failure. 10135 */ 10136 int netif_set_mtu_ext(struct net_device *dev, int new_mtu, 10137 struct netlink_ext_ack *extack) 10138 { 10139 int err, orig_mtu; 10140 10141 netdev_assert_locked_ops_compat(dev); 10142 10143 if (new_mtu == dev->mtu) 10144 return 0; 10145 10146 err = dev_validate_mtu(dev, new_mtu, extack); 10147 if (err) 10148 return err; 10149 10150 if (!netif_device_present(dev)) 10151 return -ENODEV; 10152 10153 err = call_netdevice_notifiers(NETDEV_PRECHANGEMTU, dev); 10154 err = notifier_to_errno(err); 10155 if (err) 10156 return err; 10157 10158 orig_mtu = dev->mtu; 10159 err = __netif_set_mtu(dev, new_mtu); 10160 10161 if (!err) { 10162 err = call_netdevice_notifiers_mtu(NETDEV_CHANGEMTU, dev, 10163 orig_mtu); 10164 err = notifier_to_errno(err); 10165 if (err) { 10166 /* setting mtu back and notifying everyone again, 10167 * so that they have a chance to revert changes. 10168 */ 10169 __netif_set_mtu(dev, orig_mtu); 10170 call_netdevice_notifiers_mtu(NETDEV_CHANGEMTU, dev, 10171 new_mtu); 10172 } 10173 } 10174 return err; 10175 } 10176 10177 int netif_set_mtu(struct net_device *dev, int new_mtu) 10178 { 10179 struct netlink_ext_ack extack; 10180 int err; 10181 10182 memset(&extack, 0, sizeof(extack)); 10183 err = netif_set_mtu_ext(dev, new_mtu, &extack); 10184 if (err && extack._msg) 10185 net_err_ratelimited("%s: %s\n", dev->name, extack._msg); 10186 return err; 10187 } 10188 EXPORT_SYMBOL(netif_set_mtu); 10189 10190 int netif_change_tx_queue_len(struct net_device *dev, unsigned long new_len) 10191 { 10192 unsigned int orig_len = dev->tx_queue_len; 10193 int res; 10194 10195 if (new_len > S16_MAX) 10196 return -ERANGE; 10197 10198 if (new_len != orig_len) { 10199 WRITE_ONCE(dev->tx_queue_len, new_len); 10200 res = call_netdevice_notifiers(NETDEV_CHANGE_TX_QUEUE_LEN, dev); 10201 res = notifier_to_errno(res); 10202 if (res) 10203 goto err_rollback; 10204 res = dev_qdisc_change_tx_queue_len(dev); 10205 if (res) 10206 goto err_rollback; 10207 } 10208 10209 return 0; 10210 10211 err_rollback: 10212 netdev_err(dev, "refused to change device tx_queue_len\n"); 10213 WRITE_ONCE(dev->tx_queue_len, orig_len); 10214 return res; 10215 } 10216 10217 void netif_set_group(struct net_device *dev, int new_group) 10218 { 10219 dev->group = new_group; 10220 } 10221 10222 /** 10223 * netif_pre_changeaddr_notify() - Call NETDEV_PRE_CHANGEADDR. 10224 * @dev: device 10225 * @addr: new address 10226 * @extack: netlink extended ack 10227 * 10228 * Return: 0 on success, -errno on failure. 10229 */ 10230 int netif_pre_changeaddr_notify(struct net_device *dev, const char *addr, 10231 struct netlink_ext_ack *extack) 10232 { 10233 struct netdev_notifier_pre_changeaddr_info info = { 10234 .info.dev = dev, 10235 .info.extack = extack, 10236 .dev_addr = addr, 10237 }; 10238 int rc; 10239 10240 rc = call_netdevice_notifiers_info(NETDEV_PRE_CHANGEADDR, &info.info); 10241 return notifier_to_errno(rc); 10242 } 10243 EXPORT_SYMBOL_NS_GPL(netif_pre_changeaddr_notify, "NETDEV_INTERNAL"); 10244 10245 int netif_set_mac_address(struct net_device *dev, struct sockaddr_storage *ss, 10246 struct netlink_ext_ack *extack) 10247 { 10248 const struct net_device_ops *ops = dev->netdev_ops; 10249 int err; 10250 10251 if (!ops->ndo_set_mac_address) 10252 return -EOPNOTSUPP; 10253 if (ss->ss_family != dev->type) 10254 return -EINVAL; 10255 if (!netif_device_present(dev)) 10256 return -ENODEV; 10257 err = netif_pre_changeaddr_notify(dev, ss->__data, extack); 10258 if (err) 10259 return err; 10260 if (memcmp(dev->dev_addr, ss->__data, dev->addr_len)) { 10261 err = ops->ndo_set_mac_address(dev, ss); 10262 if (err) 10263 return err; 10264 } 10265 dev->addr_assign_type = NET_ADDR_SET; 10266 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev); 10267 add_device_randomness(dev->dev_addr, dev->addr_len); 10268 return 0; 10269 } 10270 10271 DECLARE_RWSEM(dev_addr_sem); 10272 10273 /* "sa" is a true struct sockaddr with limited "sa_data" member. */ 10274 int netif_get_mac_address(struct sockaddr *sa, struct net *net, char *dev_name) 10275 { 10276 size_t size = sizeof(sa->sa_data); 10277 struct net_device *dev; 10278 int ret = 0; 10279 10280 down_read(&dev_addr_sem); 10281 rcu_read_lock(); 10282 10283 dev = dev_get_by_name_rcu(net, dev_name); 10284 if (!dev) { 10285 ret = -ENODEV; 10286 goto unlock; 10287 } 10288 if (!dev->addr_len) 10289 memset(sa->sa_data, 0, size); 10290 else 10291 memcpy(sa->sa_data, dev->dev_addr, 10292 min_t(size_t, size, dev->addr_len)); 10293 sa->sa_family = dev->type; 10294 10295 unlock: 10296 rcu_read_unlock(); 10297 up_read(&dev_addr_sem); 10298 return ret; 10299 } 10300 EXPORT_SYMBOL_NS_GPL(netif_get_mac_address, "NETDEV_INTERNAL"); 10301 10302 int netif_change_carrier(struct net_device *dev, bool new_carrier) 10303 { 10304 const struct net_device_ops *ops = dev->netdev_ops; 10305 10306 if (!ops->ndo_change_carrier) 10307 return -EOPNOTSUPP; 10308 if (!netif_device_present(dev)) 10309 return -ENODEV; 10310 return ops->ndo_change_carrier(dev, new_carrier); 10311 } 10312 10313 /** 10314 * dev_get_phys_port_id - Get device physical port ID 10315 * @dev: device 10316 * @ppid: port ID 10317 * 10318 * Get device physical port ID 10319 */ 10320 int dev_get_phys_port_id(struct net_device *dev, 10321 struct netdev_phys_item_id *ppid) 10322 { 10323 const struct net_device_ops *ops = dev->netdev_ops; 10324 10325 if (!ops->ndo_get_phys_port_id) 10326 return -EOPNOTSUPP; 10327 return ops->ndo_get_phys_port_id(dev, ppid); 10328 } 10329 10330 /** 10331 * dev_get_phys_port_name - Get device physical port name 10332 * @dev: device 10333 * @name: port name 10334 * @len: limit of bytes to copy to name 10335 * 10336 * Get device physical port name 10337 */ 10338 int dev_get_phys_port_name(struct net_device *dev, 10339 char *name, size_t len) 10340 { 10341 const struct net_device_ops *ops = dev->netdev_ops; 10342 int err; 10343 10344 if (ops->ndo_get_phys_port_name) { 10345 err = ops->ndo_get_phys_port_name(dev, name, len); 10346 if (err != -EOPNOTSUPP) 10347 return err; 10348 } 10349 return devlink_compat_phys_port_name_get(dev, name, len); 10350 } 10351 10352 /** 10353 * netif_get_port_parent_id() - Get the device's port parent identifier 10354 * @dev: network device 10355 * @ppid: pointer to a storage for the port's parent identifier 10356 * @recurse: allow/disallow recursion to lower devices 10357 * 10358 * Get the devices's port parent identifier. 10359 * 10360 * Return: 0 on success, -errno on failure. 10361 */ 10362 int netif_get_port_parent_id(struct net_device *dev, 10363 struct netdev_phys_item_id *ppid, bool recurse) 10364 { 10365 const struct net_device_ops *ops = dev->netdev_ops; 10366 struct netdev_phys_item_id first = { }; 10367 struct net_device *lower_dev; 10368 struct list_head *iter; 10369 int err; 10370 10371 if (ops->ndo_get_port_parent_id) { 10372 err = ops->ndo_get_port_parent_id(dev, ppid); 10373 if (err != -EOPNOTSUPP) 10374 return err; 10375 } 10376 10377 err = devlink_compat_switch_id_get(dev, ppid); 10378 if (!recurse || err != -EOPNOTSUPP) 10379 return err; 10380 10381 netdev_for_each_lower_dev(dev, lower_dev, iter) { 10382 err = netif_get_port_parent_id(lower_dev, ppid, true); 10383 if (err) 10384 break; 10385 if (!first.id_len) 10386 first = *ppid; 10387 else if (memcmp(&first, ppid, sizeof(*ppid))) 10388 return -EOPNOTSUPP; 10389 } 10390 10391 return err; 10392 } 10393 EXPORT_SYMBOL(netif_get_port_parent_id); 10394 10395 /** 10396 * netdev_port_same_parent_id - Indicate if two network devices have 10397 * the same port parent identifier 10398 * @a: first network device 10399 * @b: second network device 10400 */ 10401 bool netdev_port_same_parent_id(struct net_device *a, struct net_device *b) 10402 { 10403 struct netdev_phys_item_id a_id = { }; 10404 struct netdev_phys_item_id b_id = { }; 10405 10406 if (netif_get_port_parent_id(a, &a_id, true) || 10407 netif_get_port_parent_id(b, &b_id, true)) 10408 return false; 10409 10410 return netdev_phys_item_id_same(&a_id, &b_id); 10411 } 10412 EXPORT_SYMBOL(netdev_port_same_parent_id); 10413 10414 static struct net_device *dev_get_iflink_dev(struct net_device *dev) 10415 { 10416 struct net *net; 10417 10418 ASSERT_RTNL(); 10419 10420 if (!dev->netdev_ops->ndo_get_iflink || !dev->rtnl_link_ops || 10421 !dev->rtnl_link_ops->get_link_net) 10422 return dev; 10423 10424 net = dev->rtnl_link_ops->get_link_net(dev); 10425 return __dev_get_by_index(net, dev_get_iflink(dev)); 10426 } 10427 10428 int netif_change_proto_down(struct net_device *dev, bool proto_down) 10429 { 10430 struct net_device *iflink_dev; 10431 10432 if (!dev->change_proto_down) 10433 return -EOPNOTSUPP; 10434 if (!netif_device_present(dev)) 10435 return -ENODEV; 10436 iflink_dev = dev_get_iflink_dev(dev); 10437 if (!iflink_dev) 10438 return -ENODEV; 10439 WRITE_ONCE(dev->proto_down, proto_down); 10440 if (proto_down) 10441 netif_carrier_off(dev); 10442 else if (dev == iflink_dev || netif_carrier_ok(iflink_dev)) 10443 netif_carrier_on(dev); 10444 return 0; 10445 } 10446 10447 /** 10448 * netdev_change_proto_down_reason_locked - proto down reason 10449 * 10450 * @dev: device 10451 * @mask: proto down mask 10452 * @value: proto down value 10453 */ 10454 void netdev_change_proto_down_reason_locked(struct net_device *dev, 10455 unsigned long mask, u32 value) 10456 { 10457 u32 proto_down_reason; 10458 int b; 10459 10460 if (!mask) { 10461 proto_down_reason = value; 10462 } else { 10463 proto_down_reason = dev->proto_down_reason; 10464 for_each_set_bit(b, &mask, 32) { 10465 if (value & (1 << b)) 10466 proto_down_reason |= BIT(b); 10467 else 10468 proto_down_reason &= ~BIT(b); 10469 } 10470 } 10471 WRITE_ONCE(dev->proto_down_reason, proto_down_reason); 10472 } 10473 10474 struct bpf_xdp_link { 10475 struct bpf_link link; 10476 struct net_device *dev; /* protected by rtnl_lock, no refcnt held */ 10477 int flags; 10478 }; 10479 10480 static enum bpf_xdp_mode dev_xdp_mode(struct net_device *dev, u32 flags) 10481 { 10482 if (flags & XDP_FLAGS_HW_MODE) 10483 return XDP_MODE_HW; 10484 if (flags & XDP_FLAGS_DRV_MODE) 10485 return XDP_MODE_DRV; 10486 if (flags & XDP_FLAGS_SKB_MODE) 10487 return XDP_MODE_SKB; 10488 return dev->netdev_ops->ndo_bpf ? XDP_MODE_DRV : XDP_MODE_SKB; 10489 } 10490 10491 static bpf_op_t dev_xdp_bpf_op(struct net_device *dev, enum bpf_xdp_mode mode) 10492 { 10493 switch (mode) { 10494 case XDP_MODE_SKB: 10495 return generic_xdp_install; 10496 case XDP_MODE_DRV: 10497 case XDP_MODE_HW: 10498 return dev->netdev_ops->ndo_bpf; 10499 default: 10500 return NULL; 10501 } 10502 } 10503 10504 static struct bpf_xdp_link *dev_xdp_link(struct net_device *dev, 10505 enum bpf_xdp_mode mode) 10506 { 10507 return dev->xdp_state[mode].link; 10508 } 10509 10510 static struct bpf_prog *dev_xdp_prog(struct net_device *dev, 10511 enum bpf_xdp_mode mode) 10512 { 10513 struct bpf_xdp_link *link = dev_xdp_link(dev, mode); 10514 10515 if (link) 10516 return link->link.prog; 10517 return dev->xdp_state[mode].prog; 10518 } 10519 10520 u8 dev_xdp_prog_count(struct net_device *dev) 10521 { 10522 u8 count = 0; 10523 int i; 10524 10525 for (i = 0; i < __MAX_XDP_MODE; i++) 10526 if (dev->xdp_state[i].prog || dev->xdp_state[i].link) 10527 count++; 10528 return count; 10529 } 10530 EXPORT_SYMBOL_GPL(dev_xdp_prog_count); 10531 10532 u8 dev_xdp_sb_prog_count(struct net_device *dev) 10533 { 10534 struct bpf_prog *prog; 10535 u8 count = 0; 10536 int i; 10537 10538 for (i = 0; i < __MAX_XDP_MODE; i++) { 10539 prog = dev_xdp_prog(dev, i); 10540 if (prog && !prog->aux->xdp_has_frags) 10541 count++; 10542 } 10543 return count; 10544 } 10545 10546 int netif_xdp_propagate(struct net_device *dev, struct netdev_bpf *bpf) 10547 { 10548 if (!dev->netdev_ops->ndo_bpf) 10549 return -EOPNOTSUPP; 10550 10551 if (dev->cfg->hds_config == ETHTOOL_TCP_DATA_SPLIT_ENABLED && 10552 bpf->command == XDP_SETUP_PROG && 10553 bpf->prog && !bpf->prog->aux->xdp_has_frags) { 10554 NL_SET_ERR_MSG(bpf->extack, 10555 "unable to propagate XDP to device using tcp-data-split"); 10556 return -EBUSY; 10557 } 10558 10559 if (dev_get_min_mp_channel_count(dev)) { 10560 NL_SET_ERR_MSG(bpf->extack, "unable to propagate XDP to device using memory provider"); 10561 return -EBUSY; 10562 } 10563 10564 return dev->netdev_ops->ndo_bpf(dev, bpf); 10565 } 10566 EXPORT_SYMBOL_GPL(netif_xdp_propagate); 10567 10568 u32 dev_xdp_prog_id(struct net_device *dev, enum bpf_xdp_mode mode) 10569 { 10570 struct bpf_prog *prog = dev_xdp_prog(dev, mode); 10571 10572 return prog ? prog->aux->id : 0; 10573 } 10574 10575 static void dev_xdp_set_link(struct net_device *dev, enum bpf_xdp_mode mode, 10576 struct bpf_xdp_link *link) 10577 { 10578 dev->xdp_state[mode].link = link; 10579 dev->xdp_state[mode].prog = NULL; 10580 } 10581 10582 static void dev_xdp_set_prog(struct net_device *dev, enum bpf_xdp_mode mode, 10583 struct bpf_prog *prog) 10584 { 10585 dev->xdp_state[mode].link = NULL; 10586 dev->xdp_state[mode].prog = prog; 10587 } 10588 10589 static int dev_xdp_install(struct net_device *dev, enum bpf_xdp_mode mode, 10590 bpf_op_t bpf_op, struct netlink_ext_ack *extack, 10591 u32 flags, struct bpf_prog *prog) 10592 { 10593 struct netdev_bpf xdp; 10594 int err; 10595 10596 netdev_assert_locked_ops_compat(dev); 10597 10598 if (prog) { 10599 enum bpf_xdp_mode other_mode = mode == XDP_MODE_SKB 10600 ? XDP_MODE_DRV : XDP_MODE_SKB; 10601 bool offload = mode == XDP_MODE_HW; 10602 10603 if (!offload && dev_xdp_prog(dev, other_mode)) { 10604 NL_SET_ERR_MSG(extack, "Native and generic XDP can't be active at the same time"); 10605 return -EEXIST; 10606 } 10607 if (!offload && bpf_prog_is_offloaded(prog->aux)) { 10608 NL_SET_ERR_MSG(extack, "Using offloaded program without HW_MODE flag is not supported"); 10609 return -EINVAL; 10610 } 10611 if (bpf_prog_is_dev_bound(prog->aux) && !bpf_offload_dev_match(prog, dev)) { 10612 NL_SET_ERR_MSG(extack, "Program bound to different device"); 10613 return -EINVAL; 10614 } 10615 if (bpf_prog_is_dev_bound(prog->aux) && mode == XDP_MODE_SKB) { 10616 NL_SET_ERR_MSG(extack, "Can't attach device-bound programs in generic mode"); 10617 return -EINVAL; 10618 } 10619 if (prog->expected_attach_type == BPF_XDP_DEVMAP) { 10620 NL_SET_ERR_MSG(extack, "BPF_XDP_DEVMAP programs can not be attached to a device"); 10621 return -EINVAL; 10622 } 10623 if (prog->expected_attach_type == BPF_XDP_CPUMAP) { 10624 NL_SET_ERR_MSG(extack, "BPF_XDP_CPUMAP programs can not be attached to a device"); 10625 return -EINVAL; 10626 } 10627 } 10628 10629 if (dev->cfg->hds_config == ETHTOOL_TCP_DATA_SPLIT_ENABLED && 10630 prog && !prog->aux->xdp_has_frags) { 10631 NL_SET_ERR_MSG(extack, "unable to install XDP to device using tcp-data-split"); 10632 return -EBUSY; 10633 } 10634 10635 if (dev_get_min_mp_channel_count(dev)) { 10636 NL_SET_ERR_MSG(extack, "unable to install XDP to device using memory provider"); 10637 return -EBUSY; 10638 } 10639 10640 memset(&xdp, 0, sizeof(xdp)); 10641 xdp.command = mode == XDP_MODE_HW ? XDP_SETUP_PROG_HW : XDP_SETUP_PROG; 10642 xdp.extack = extack; 10643 xdp.flags = flags; 10644 xdp.prog = prog; 10645 10646 /* Drivers assume refcnt is already incremented (i.e, prog pointer is 10647 * "moved" into driver), so they don't increment it on their own, but 10648 * they do decrement refcnt when program is detached or replaced. 10649 * Given net_device also owns link/prog, we need to bump refcnt here 10650 * to prevent drivers from underflowing it. 10651 */ 10652 if (prog) 10653 bpf_prog_inc(prog); 10654 err = bpf_op(dev, &xdp); 10655 if (err) { 10656 if (prog) 10657 bpf_prog_put(prog); 10658 return err; 10659 } 10660 10661 if (mode != XDP_MODE_HW) 10662 bpf_prog_change_xdp(dev_xdp_prog(dev, mode), prog); 10663 10664 return 0; 10665 } 10666 10667 static void dev_xdp_uninstall(struct net_device *dev) 10668 { 10669 struct bpf_xdp_link *link; 10670 struct bpf_prog *prog; 10671 enum bpf_xdp_mode mode; 10672 bpf_op_t bpf_op; 10673 10674 ASSERT_RTNL(); 10675 10676 for (mode = XDP_MODE_SKB; mode < __MAX_XDP_MODE; mode++) { 10677 prog = dev_xdp_prog(dev, mode); 10678 if (!prog) 10679 continue; 10680 10681 bpf_op = dev_xdp_bpf_op(dev, mode); 10682 if (!bpf_op) 10683 continue; 10684 10685 WARN_ON(dev_xdp_install(dev, mode, bpf_op, NULL, 0, NULL)); 10686 10687 /* auto-detach link from net device */ 10688 link = dev_xdp_link(dev, mode); 10689 if (link) 10690 link->dev = NULL; 10691 else 10692 bpf_prog_put(prog); 10693 10694 dev_xdp_set_link(dev, mode, NULL); 10695 } 10696 } 10697 10698 static int dev_xdp_attach(struct net_device *dev, struct netlink_ext_ack *extack, 10699 struct bpf_xdp_link *link, struct bpf_prog *new_prog, 10700 struct bpf_prog *old_prog, u32 flags) 10701 { 10702 unsigned int num_modes = hweight32(flags & XDP_FLAGS_MODES); 10703 struct bpf_prog *cur_prog; 10704 struct net_device *upper; 10705 struct list_head *iter; 10706 enum bpf_xdp_mode mode; 10707 bpf_op_t bpf_op; 10708 int err; 10709 10710 ASSERT_RTNL(); 10711 10712 /* either link or prog attachment, never both */ 10713 if (link && (new_prog || old_prog)) 10714 return -EINVAL; 10715 /* link supports only XDP mode flags */ 10716 if (link && (flags & ~XDP_FLAGS_MODES)) { 10717 NL_SET_ERR_MSG(extack, "Invalid XDP flags for BPF link attachment"); 10718 return -EINVAL; 10719 } 10720 /* just one XDP mode bit should be set, zero defaults to drv/skb mode */ 10721 if (num_modes > 1) { 10722 NL_SET_ERR_MSG(extack, "Only one XDP mode flag can be set"); 10723 return -EINVAL; 10724 } 10725 /* avoid ambiguity if offload + drv/skb mode progs are both loaded */ 10726 if (!num_modes && dev_xdp_prog_count(dev) > 1) { 10727 NL_SET_ERR_MSG(extack, 10728 "More than one program loaded, unset mode is ambiguous"); 10729 return -EINVAL; 10730 } 10731 /* old_prog != NULL implies XDP_FLAGS_REPLACE is set */ 10732 if (old_prog && !(flags & XDP_FLAGS_REPLACE)) { 10733 NL_SET_ERR_MSG(extack, "XDP_FLAGS_REPLACE is not specified"); 10734 return -EINVAL; 10735 } 10736 10737 mode = dev_xdp_mode(dev, flags); 10738 /* can't replace attached link */ 10739 if (dev_xdp_link(dev, mode)) { 10740 NL_SET_ERR_MSG(extack, "Can't replace active BPF XDP link"); 10741 return -EBUSY; 10742 } 10743 10744 /* don't allow if an upper device already has a program */ 10745 netdev_for_each_upper_dev_rcu(dev, upper, iter) { 10746 if (dev_xdp_prog_count(upper) > 0) { 10747 NL_SET_ERR_MSG(extack, "Cannot attach when an upper device already has a program"); 10748 return -EEXIST; 10749 } 10750 } 10751 10752 cur_prog = dev_xdp_prog(dev, mode); 10753 /* can't replace attached prog with link */ 10754 if (link && cur_prog) { 10755 NL_SET_ERR_MSG(extack, "Can't replace active XDP program with BPF link"); 10756 return -EBUSY; 10757 } 10758 if ((flags & XDP_FLAGS_REPLACE) && cur_prog != old_prog) { 10759 NL_SET_ERR_MSG(extack, "Active program does not match expected"); 10760 return -EEXIST; 10761 } 10762 10763 /* put effective new program into new_prog */ 10764 if (link) 10765 new_prog = link->link.prog; 10766 10767 if (new_prog) { 10768 if ((flags & XDP_FLAGS_UPDATE_IF_NOEXIST) && cur_prog) { 10769 NL_SET_ERR_MSG(extack, "XDP program already attached"); 10770 return -EBUSY; 10771 } 10772 } 10773 10774 /* don't call drivers if the effective program didn't change */ 10775 if (new_prog != cur_prog) { 10776 bpf_op = dev_xdp_bpf_op(dev, mode); 10777 if (!bpf_op) { 10778 NL_SET_ERR_MSG(extack, "Underlying driver does not support XDP in native mode"); 10779 return -EOPNOTSUPP; 10780 } 10781 10782 err = dev_xdp_install(dev, mode, bpf_op, extack, flags, new_prog); 10783 if (err) 10784 return err; 10785 } 10786 10787 if (link) 10788 dev_xdp_set_link(dev, mode, link); 10789 else 10790 dev_xdp_set_prog(dev, mode, new_prog); 10791 if (cur_prog) 10792 bpf_prog_put(cur_prog); 10793 10794 return 0; 10795 } 10796 10797 static int dev_xdp_attach_link(struct net_device *dev, 10798 struct netlink_ext_ack *extack, 10799 struct bpf_xdp_link *link) 10800 { 10801 return dev_xdp_attach(dev, extack, link, NULL, NULL, link->flags); 10802 } 10803 10804 static int dev_xdp_detach_link(struct net_device *dev, 10805 struct netlink_ext_ack *extack, 10806 struct bpf_xdp_link *link) 10807 { 10808 enum bpf_xdp_mode mode; 10809 bpf_op_t bpf_op; 10810 10811 ASSERT_RTNL(); 10812 10813 mode = dev_xdp_mode(dev, link->flags); 10814 if (dev_xdp_link(dev, mode) != link) 10815 return -EINVAL; 10816 10817 bpf_op = dev_xdp_bpf_op(dev, mode); 10818 WARN_ON(dev_xdp_install(dev, mode, bpf_op, NULL, 0, NULL)); 10819 dev_xdp_set_link(dev, mode, NULL); 10820 return 0; 10821 } 10822 10823 static void bpf_xdp_link_release(struct bpf_link *link) 10824 { 10825 struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link); 10826 10827 rtnl_lock(); 10828 10829 /* if racing with net_device's tear down, xdp_link->dev might be 10830 * already NULL, in which case link was already auto-detached 10831 */ 10832 if (xdp_link->dev) { 10833 netdev_lock_ops(xdp_link->dev); 10834 WARN_ON(dev_xdp_detach_link(xdp_link->dev, NULL, xdp_link)); 10835 netdev_unlock_ops(xdp_link->dev); 10836 xdp_link->dev = NULL; 10837 } 10838 10839 rtnl_unlock(); 10840 } 10841 10842 static int bpf_xdp_link_detach(struct bpf_link *link) 10843 { 10844 bpf_xdp_link_release(link); 10845 return 0; 10846 } 10847 10848 static void bpf_xdp_link_dealloc(struct bpf_link *link) 10849 { 10850 struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link); 10851 10852 kfree(xdp_link); 10853 } 10854 10855 static void bpf_xdp_link_show_fdinfo(const struct bpf_link *link, 10856 struct seq_file *seq) 10857 { 10858 struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link); 10859 u32 ifindex = 0; 10860 10861 rtnl_lock(); 10862 if (xdp_link->dev) 10863 ifindex = xdp_link->dev->ifindex; 10864 rtnl_unlock(); 10865 10866 seq_printf(seq, "ifindex:\t%u\n", ifindex); 10867 } 10868 10869 static int bpf_xdp_link_fill_link_info(const struct bpf_link *link, 10870 struct bpf_link_info *info) 10871 { 10872 struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link); 10873 u32 ifindex = 0; 10874 10875 rtnl_lock(); 10876 if (xdp_link->dev) 10877 ifindex = xdp_link->dev->ifindex; 10878 rtnl_unlock(); 10879 10880 info->xdp.ifindex = ifindex; 10881 return 0; 10882 } 10883 10884 static int bpf_xdp_link_update(struct bpf_link *link, struct bpf_prog *new_prog, 10885 struct bpf_prog *old_prog) 10886 { 10887 struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link); 10888 enum bpf_xdp_mode mode; 10889 bpf_op_t bpf_op; 10890 int err = 0; 10891 10892 rtnl_lock(); 10893 10894 /* link might have been auto-released already, so fail */ 10895 if (!xdp_link->dev) { 10896 err = -ENOLINK; 10897 goto out_unlock; 10898 } 10899 10900 if (old_prog && link->prog != old_prog) { 10901 err = -EPERM; 10902 goto out_unlock; 10903 } 10904 old_prog = link->prog; 10905 if (old_prog->type != new_prog->type || 10906 old_prog->expected_attach_type != new_prog->expected_attach_type) { 10907 err = -EINVAL; 10908 goto out_unlock; 10909 } 10910 10911 if (old_prog == new_prog) { 10912 /* no-op, don't disturb drivers */ 10913 bpf_prog_put(new_prog); 10914 goto out_unlock; 10915 } 10916 10917 netdev_lock_ops(xdp_link->dev); 10918 mode = dev_xdp_mode(xdp_link->dev, xdp_link->flags); 10919 bpf_op = dev_xdp_bpf_op(xdp_link->dev, mode); 10920 err = dev_xdp_install(xdp_link->dev, mode, bpf_op, NULL, 10921 xdp_link->flags, new_prog); 10922 if (!err) 10923 old_prog = xchg(&link->prog, new_prog); 10924 netdev_unlock_ops(xdp_link->dev); 10925 if (err) 10926 goto out_unlock; 10927 10928 bpf_prog_put(old_prog); 10929 10930 out_unlock: 10931 rtnl_unlock(); 10932 return err; 10933 } 10934 10935 static const struct bpf_link_ops bpf_xdp_link_lops = { 10936 .release = bpf_xdp_link_release, 10937 .dealloc = bpf_xdp_link_dealloc, 10938 .detach = bpf_xdp_link_detach, 10939 .show_fdinfo = bpf_xdp_link_show_fdinfo, 10940 .fill_link_info = bpf_xdp_link_fill_link_info, 10941 .update_prog = bpf_xdp_link_update, 10942 }; 10943 10944 int bpf_xdp_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) 10945 { 10946 struct net *net = current->nsproxy->net_ns; 10947 struct bpf_link_primer link_primer; 10948 struct netlink_ext_ack extack = {}; 10949 struct bpf_xdp_link *link; 10950 struct net_device *dev; 10951 int err, fd; 10952 10953 rtnl_lock(); 10954 dev = dev_get_by_index(net, attr->link_create.target_ifindex); 10955 if (!dev) { 10956 rtnl_unlock(); 10957 return -EINVAL; 10958 } 10959 10960 link = kzalloc_obj(*link, GFP_USER); 10961 if (!link) { 10962 err = -ENOMEM; 10963 goto unlock; 10964 } 10965 10966 bpf_link_init(&link->link, BPF_LINK_TYPE_XDP, &bpf_xdp_link_lops, prog, 10967 attr->link_create.attach_type); 10968 link->dev = dev; 10969 link->flags = attr->link_create.flags; 10970 10971 err = bpf_link_prime(&link->link, &link_primer); 10972 if (err) { 10973 kfree(link); 10974 goto unlock; 10975 } 10976 10977 netdev_lock_ops(dev); 10978 err = dev_xdp_attach_link(dev, &extack, link); 10979 netdev_unlock_ops(dev); 10980 rtnl_unlock(); 10981 10982 if (err) { 10983 link->dev = NULL; 10984 bpf_link_cleanup(&link_primer); 10985 trace_bpf_xdp_link_attach_failed(extack._msg); 10986 goto out_put_dev; 10987 } 10988 10989 fd = bpf_link_settle(&link_primer); 10990 /* link itself doesn't hold dev's refcnt to not complicate shutdown */ 10991 dev_put(dev); 10992 return fd; 10993 10994 unlock: 10995 rtnl_unlock(); 10996 10997 out_put_dev: 10998 dev_put(dev); 10999 return err; 11000 } 11001 11002 /** 11003 * dev_change_xdp_fd - set or clear a bpf program for a device rx path 11004 * @dev: device 11005 * @extack: netlink extended ack 11006 * @fd: new program fd or negative value to clear 11007 * @expected_fd: old program fd that userspace expects to replace or clear 11008 * @flags: xdp-related flags 11009 * 11010 * Set or clear a bpf program for a device 11011 */ 11012 int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack, 11013 int fd, int expected_fd, u32 flags) 11014 { 11015 enum bpf_xdp_mode mode = dev_xdp_mode(dev, flags); 11016 struct bpf_prog *new_prog = NULL, *old_prog = NULL; 11017 int err; 11018 11019 ASSERT_RTNL(); 11020 11021 if (fd >= 0) { 11022 new_prog = bpf_prog_get_type_dev(fd, BPF_PROG_TYPE_XDP, 11023 mode != XDP_MODE_SKB); 11024 if (IS_ERR(new_prog)) 11025 return PTR_ERR(new_prog); 11026 } 11027 11028 if (expected_fd >= 0) { 11029 old_prog = bpf_prog_get_type_dev(expected_fd, BPF_PROG_TYPE_XDP, 11030 mode != XDP_MODE_SKB); 11031 if (IS_ERR(old_prog)) { 11032 err = PTR_ERR(old_prog); 11033 old_prog = NULL; 11034 goto err_out; 11035 } 11036 } 11037 11038 err = dev_xdp_attach(dev, extack, NULL, new_prog, old_prog, flags); 11039 11040 err_out: 11041 if (err && new_prog) 11042 bpf_prog_put(new_prog); 11043 if (old_prog) 11044 bpf_prog_put(old_prog); 11045 return err; 11046 } 11047 11048 u32 dev_get_min_mp_channel_count(const struct net_device *dev) 11049 { 11050 int i; 11051 11052 netdev_assert_locked_ops_compat(dev); 11053 11054 for (i = dev->real_num_rx_queues - 1; i >= 0; i--) 11055 if (dev->_rx[i].mp_params.mp_priv) 11056 /* The channel count is the idx plus 1. */ 11057 return i + 1; 11058 11059 return 0; 11060 } 11061 11062 /** 11063 * dev_index_reserve() - allocate an ifindex in a namespace 11064 * @net: the applicable net namespace 11065 * @ifindex: requested ifindex, pass %0 to get one allocated 11066 * 11067 * Allocate a ifindex for a new device. Caller must either use the ifindex 11068 * to store the device (via list_netdevice()) or call dev_index_release() 11069 * to give the index up. 11070 * 11071 * Return: a suitable unique value for a new device interface number or -errno. 11072 */ 11073 static int dev_index_reserve(struct net *net, u32 ifindex) 11074 { 11075 int err; 11076 11077 if (ifindex > INT_MAX) { 11078 DEBUG_NET_WARN_ON_ONCE(1); 11079 return -EINVAL; 11080 } 11081 11082 if (!ifindex) 11083 err = xa_alloc_cyclic(&net->dev_by_index, &ifindex, NULL, 11084 xa_limit_31b, &net->ifindex, GFP_KERNEL); 11085 else 11086 err = xa_insert(&net->dev_by_index, ifindex, NULL, GFP_KERNEL); 11087 if (err < 0) 11088 return err; 11089 11090 return ifindex; 11091 } 11092 11093 static void dev_index_release(struct net *net, int ifindex) 11094 { 11095 /* Expect only unused indexes, unlist_netdevice() removes the used */ 11096 WARN_ON(xa_erase(&net->dev_by_index, ifindex)); 11097 } 11098 11099 static bool from_cleanup_net(void) 11100 { 11101 #ifdef CONFIG_NET_NS 11102 return current == READ_ONCE(cleanup_net_task); 11103 #else 11104 return false; 11105 #endif 11106 } 11107 11108 /* Delayed registration/unregisteration */ 11109 LIST_HEAD(net_todo_list); 11110 DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq); 11111 atomic_t dev_unreg_count = ATOMIC_INIT(0); 11112 11113 static void net_set_todo(struct net_device *dev) 11114 { 11115 list_add_tail(&dev->todo_list, &net_todo_list); 11116 atomic_inc(&dev_net(dev)->dev_unreg_pending); 11117 } 11118 11119 /* Paired with net_set_todo(). @net must not be used after this call. */ 11120 static void net_unreg_done(struct net *net) 11121 { 11122 if (atomic_dec_and_test(&net->dev_unreg_pending)) 11123 wake_up(&netdev_unregistering_wq); 11124 } 11125 11126 static netdev_features_t netdev_sync_upper_features(struct net_device *lower, 11127 struct net_device *upper, netdev_features_t features) 11128 { 11129 netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES; 11130 netdev_features_t feature; 11131 int feature_bit; 11132 11133 for_each_netdev_feature(upper_disables, feature_bit) { 11134 feature = __NETIF_F_BIT(feature_bit); 11135 if (!(upper->wanted_features & feature) 11136 && (features & feature)) { 11137 netdev_dbg(lower, "Dropping feature %pNF, upper dev %s has it off.\n", 11138 &feature, upper->name); 11139 features &= ~feature; 11140 } 11141 } 11142 11143 return features; 11144 } 11145 11146 static void netdev_sync_lower_features(struct net_device *upper, 11147 struct net_device *lower, netdev_features_t features) 11148 { 11149 netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES; 11150 netdev_features_t feature; 11151 int feature_bit; 11152 11153 for_each_netdev_feature(upper_disables, feature_bit) { 11154 feature = __NETIF_F_BIT(feature_bit); 11155 if (!(features & feature) && (lower->features & feature)) { 11156 netdev_dbg(upper, "Disabling feature %pNF on lower dev %s.\n", 11157 &feature, lower->name); 11158 netdev_lock_ops(lower); 11159 lower->wanted_features &= ~feature; 11160 __netdev_update_features(lower); 11161 11162 if (unlikely(lower->features & feature)) 11163 netdev_WARN(upper, "failed to disable %pNF on %s!\n", 11164 &feature, lower->name); 11165 else 11166 netdev_features_change(lower); 11167 netdev_unlock_ops(lower); 11168 } 11169 } 11170 } 11171 11172 static bool netdev_has_ip_or_hw_csum(netdev_features_t features) 11173 { 11174 netdev_features_t ip_csum_mask = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM; 11175 bool ip_csum = (features & ip_csum_mask) == ip_csum_mask; 11176 bool hw_csum = features & NETIF_F_HW_CSUM; 11177 11178 return ip_csum || hw_csum; 11179 } 11180 11181 static netdev_features_t netdev_fix_features(struct net_device *dev, 11182 netdev_features_t features) 11183 { 11184 /* Fix illegal checksum combinations */ 11185 if ((features & NETIF_F_HW_CSUM) && 11186 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) { 11187 netdev_warn(dev, "mixed HW and IP checksum settings.\n"); 11188 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM); 11189 } 11190 11191 /* TSO requires that SG is present as well. */ 11192 if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) { 11193 netdev_dbg(dev, "Dropping TSO features since no SG feature.\n"); 11194 features &= ~NETIF_F_ALL_TSO; 11195 } 11196 11197 if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) && 11198 !(features & NETIF_F_IP_CSUM)) { 11199 netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n"); 11200 features &= ~NETIF_F_TSO; 11201 features &= ~NETIF_F_TSO_ECN; 11202 } 11203 11204 if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) && 11205 !(features & NETIF_F_IPV6_CSUM)) { 11206 netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n"); 11207 features &= ~NETIF_F_TSO6; 11208 } 11209 11210 /* TSO with IPv4 ID mangling requires IPv4 TSO be enabled */ 11211 if ((features & NETIF_F_TSO_MANGLEID) && !(features & NETIF_F_TSO)) 11212 features &= ~NETIF_F_TSO_MANGLEID; 11213 11214 /* TSO ECN requires that TSO is present as well. */ 11215 if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN) 11216 features &= ~NETIF_F_TSO_ECN; 11217 11218 /* Software GSO depends on SG. */ 11219 if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) { 11220 netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n"); 11221 features &= ~NETIF_F_GSO; 11222 } 11223 11224 /* GSO partial features require GSO partial be set */ 11225 if ((features & dev->gso_partial_features) && 11226 !(features & NETIF_F_GSO_PARTIAL)) { 11227 netdev_dbg(dev, 11228 "Dropping partially supported GSO features since no GSO partial.\n"); 11229 features &= ~dev->gso_partial_features; 11230 } 11231 11232 if (!(features & NETIF_F_RXCSUM)) { 11233 /* NETIF_F_GRO_HW implies doing RXCSUM since every packet 11234 * successfully merged by hardware must also have the 11235 * checksum verified by hardware. If the user does not 11236 * want to enable RXCSUM, logically, we should disable GRO_HW. 11237 */ 11238 if (features & NETIF_F_GRO_HW) { 11239 netdev_dbg(dev, "Dropping NETIF_F_GRO_HW since no RXCSUM feature.\n"); 11240 features &= ~NETIF_F_GRO_HW; 11241 } 11242 } 11243 11244 /* LRO/HW-GRO features cannot be combined with RX-FCS */ 11245 if (features & NETIF_F_RXFCS) { 11246 if (features & NETIF_F_LRO) { 11247 netdev_dbg(dev, "Dropping LRO feature since RX-FCS is requested.\n"); 11248 features &= ~NETIF_F_LRO; 11249 } 11250 11251 if (features & NETIF_F_GRO_HW) { 11252 netdev_dbg(dev, "Dropping HW-GRO feature since RX-FCS is requested.\n"); 11253 features &= ~NETIF_F_GRO_HW; 11254 } 11255 } 11256 11257 if ((features & NETIF_F_GRO_HW) && (features & NETIF_F_LRO)) { 11258 netdev_dbg(dev, "Dropping LRO feature since HW-GRO is requested.\n"); 11259 features &= ~NETIF_F_LRO; 11260 } 11261 11262 if ((features & NETIF_F_HW_TLS_TX) && !netdev_has_ip_or_hw_csum(features)) { 11263 netdev_dbg(dev, "Dropping TLS TX HW offload feature since no CSUM feature.\n"); 11264 features &= ~NETIF_F_HW_TLS_TX; 11265 } 11266 11267 if ((features & NETIF_F_HW_TLS_RX) && !(features & NETIF_F_RXCSUM)) { 11268 netdev_dbg(dev, "Dropping TLS RX HW offload feature since no RXCSUM feature.\n"); 11269 features &= ~NETIF_F_HW_TLS_RX; 11270 } 11271 11272 if ((features & NETIF_F_GSO_UDP_L4) && !netdev_has_ip_or_hw_csum(features)) { 11273 netdev_dbg(dev, "Dropping USO feature since no CSUM feature.\n"); 11274 features &= ~NETIF_F_GSO_UDP_L4; 11275 } 11276 11277 return features; 11278 } 11279 11280 int __netdev_update_features(struct net_device *dev) 11281 { 11282 struct net_device *upper, *lower; 11283 netdev_features_t features; 11284 struct list_head *iter; 11285 int err = -1; 11286 11287 ASSERT_RTNL(); 11288 netdev_assert_locked_ops_compat(dev); 11289 11290 features = netdev_get_wanted_features(dev); 11291 11292 if (dev->netdev_ops->ndo_fix_features) 11293 features = dev->netdev_ops->ndo_fix_features(dev, features); 11294 11295 /* driver might be less strict about feature dependencies */ 11296 features = netdev_fix_features(dev, features); 11297 11298 /* some features can't be enabled if they're off on an upper device */ 11299 netdev_for_each_upper_dev_rcu(dev, upper, iter) 11300 features = netdev_sync_upper_features(dev, upper, features); 11301 11302 if (dev->features == features) 11303 goto sync_lower; 11304 11305 netdev_dbg(dev, "Features changed: %pNF -> %pNF\n", 11306 &dev->features, &features); 11307 11308 if (dev->netdev_ops->ndo_set_features) 11309 err = dev->netdev_ops->ndo_set_features(dev, features); 11310 else 11311 err = 0; 11312 11313 if (unlikely(err < 0)) { 11314 netdev_err(dev, 11315 "set_features() failed (%d); wanted %pNF, left %pNF\n", 11316 err, &features, &dev->features); 11317 /* return non-0 since some features might have changed and 11318 * it's better to fire a spurious notification than miss it 11319 */ 11320 return -1; 11321 } 11322 11323 sync_lower: 11324 /* some features must be disabled on lower devices when disabled 11325 * on an upper device (think: bonding master or bridge) 11326 */ 11327 netdev_for_each_lower_dev(dev, lower, iter) 11328 netdev_sync_lower_features(dev, lower, features); 11329 11330 if (!err) { 11331 netdev_features_t diff = features ^ dev->features; 11332 11333 if (diff & NETIF_F_RX_UDP_TUNNEL_PORT) { 11334 /* udp_tunnel_{get,drop}_rx_info both need 11335 * NETIF_F_RX_UDP_TUNNEL_PORT enabled on the 11336 * device, or they won't do anything. 11337 * Thus we need to update dev->features 11338 * *before* calling udp_tunnel_get_rx_info, 11339 * but *after* calling udp_tunnel_drop_rx_info. 11340 */ 11341 udp_tunnel_nic_lock(dev); 11342 if (features & NETIF_F_RX_UDP_TUNNEL_PORT) { 11343 dev->features = features; 11344 udp_tunnel_get_rx_info(dev); 11345 } else { 11346 udp_tunnel_drop_rx_info(dev); 11347 } 11348 udp_tunnel_nic_unlock(dev); 11349 } 11350 11351 if (diff & NETIF_F_HW_VLAN_CTAG_FILTER) { 11352 if (features & NETIF_F_HW_VLAN_CTAG_FILTER) { 11353 dev->features = features; 11354 err |= vlan_get_rx_ctag_filter_info(dev); 11355 } else { 11356 vlan_drop_rx_ctag_filter_info(dev); 11357 } 11358 } 11359 11360 if (diff & NETIF_F_HW_VLAN_STAG_FILTER) { 11361 if (features & NETIF_F_HW_VLAN_STAG_FILTER) { 11362 dev->features = features; 11363 err |= vlan_get_rx_stag_filter_info(dev); 11364 } else { 11365 vlan_drop_rx_stag_filter_info(dev); 11366 } 11367 } 11368 11369 dev->features = features; 11370 } 11371 11372 return err < 0 ? 0 : 1; 11373 } 11374 11375 /** 11376 * netdev_update_features - recalculate device features 11377 * @dev: the device to check 11378 * 11379 * Recalculate dev->features set and send notifications if it 11380 * has changed. Should be called after driver or hardware dependent 11381 * conditions might have changed that influence the features. 11382 */ 11383 void netdev_update_features(struct net_device *dev) 11384 { 11385 if (__netdev_update_features(dev)) 11386 netdev_features_change(dev); 11387 } 11388 EXPORT_SYMBOL(netdev_update_features); 11389 11390 /** 11391 * netdev_change_features - recalculate device features 11392 * @dev: the device to check 11393 * 11394 * Recalculate dev->features set and send notifications even 11395 * if they have not changed. Should be called instead of 11396 * netdev_update_features() if also dev->vlan_features might 11397 * have changed to allow the changes to be propagated to stacked 11398 * VLAN devices. 11399 */ 11400 void netdev_change_features(struct net_device *dev) 11401 { 11402 __netdev_update_features(dev); 11403 netdev_features_change(dev); 11404 } 11405 EXPORT_SYMBOL(netdev_change_features); 11406 11407 /** 11408 * netif_stacked_transfer_operstate - transfer operstate 11409 * @rootdev: the root or lower level device to transfer state from 11410 * @dev: the device to transfer operstate to 11411 * 11412 * Transfer operational state from root to device. This is normally 11413 * called when a stacking relationship exists between the root 11414 * device and the device(a leaf device). 11415 */ 11416 void netif_stacked_transfer_operstate(const struct net_device *rootdev, 11417 struct net_device *dev) 11418 { 11419 if (rootdev->operstate == IF_OPER_DORMANT) 11420 netif_dormant_on(dev); 11421 else 11422 netif_dormant_off(dev); 11423 11424 if (rootdev->operstate == IF_OPER_TESTING) 11425 netif_testing_on(dev); 11426 else 11427 netif_testing_off(dev); 11428 11429 if (netif_carrier_ok(rootdev)) 11430 netif_carrier_on(dev); 11431 else 11432 netif_carrier_off(dev); 11433 } 11434 EXPORT_SYMBOL(netif_stacked_transfer_operstate); 11435 11436 static int netif_alloc_rx_queues(struct net_device *dev) 11437 { 11438 unsigned int i, count = dev->num_rx_queues; 11439 struct netdev_rx_queue *rx; 11440 size_t sz = count * sizeof(*rx); 11441 int err = 0; 11442 11443 BUG_ON(count < 1); 11444 11445 rx = kvzalloc(sz, GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL); 11446 if (!rx) 11447 return -ENOMEM; 11448 11449 dev->_rx = rx; 11450 11451 for (i = 0; i < count; i++) { 11452 rx[i].dev = dev; 11453 11454 /* XDP RX-queue setup */ 11455 err = xdp_rxq_info_reg(&rx[i].xdp_rxq, dev, i, 0); 11456 if (err < 0) 11457 goto err_rxq_info; 11458 } 11459 return 0; 11460 11461 err_rxq_info: 11462 /* Rollback successful reg's and free other resources */ 11463 while (i--) 11464 xdp_rxq_info_unreg(&rx[i].xdp_rxq); 11465 kvfree(dev->_rx); 11466 dev->_rx = NULL; 11467 return err; 11468 } 11469 11470 static void netif_free_rx_queues(struct net_device *dev) 11471 { 11472 unsigned int i, count = dev->num_rx_queues; 11473 11474 /* netif_alloc_rx_queues alloc failed, resources have been unreg'ed */ 11475 if (!dev->_rx) 11476 return; 11477 11478 for (i = 0; i < count; i++) 11479 xdp_rxq_info_unreg(&dev->_rx[i].xdp_rxq); 11480 11481 kvfree(dev->_rx); 11482 } 11483 11484 static void netdev_init_one_queue(struct net_device *dev, 11485 struct netdev_queue *queue, void *_unused) 11486 { 11487 /* Initialize queue lock */ 11488 spin_lock_init(&queue->_xmit_lock); 11489 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type); 11490 queue->xmit_lock_owner = -1; 11491 netdev_queue_numa_node_write(queue, NUMA_NO_NODE); 11492 queue->dev = dev; 11493 #ifdef CONFIG_BQL 11494 dql_init(&queue->dql, HZ); 11495 #endif 11496 } 11497 11498 static void netif_free_tx_queues(struct net_device *dev) 11499 { 11500 kvfree(dev->_tx); 11501 } 11502 11503 static int netif_alloc_netdev_queues(struct net_device *dev) 11504 { 11505 unsigned int count = dev->num_tx_queues; 11506 struct netdev_queue *tx; 11507 size_t sz = count * sizeof(*tx); 11508 11509 if (count < 1 || count > 0xffff) 11510 return -EINVAL; 11511 11512 tx = kvzalloc(sz, GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL); 11513 if (!tx) 11514 return -ENOMEM; 11515 11516 dev->_tx = tx; 11517 11518 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL); 11519 spin_lock_init(&dev->tx_global_lock); 11520 spin_lock_init(&dev->watchdog_lock); 11521 dev->watchdog_ref_held = false; 11522 return 0; 11523 } 11524 11525 void netif_tx_stop_all_queues(struct net_device *dev) 11526 { 11527 unsigned int i; 11528 11529 for (i = 0; i < dev->num_tx_queues; i++) { 11530 struct netdev_queue *txq = netdev_get_tx_queue(dev, i); 11531 11532 netif_tx_stop_queue(txq); 11533 } 11534 } 11535 EXPORT_SYMBOL(netif_tx_stop_all_queues); 11536 11537 static int netdev_do_alloc_pcpu_stats(struct net_device *dev) 11538 { 11539 void __percpu *v; 11540 11541 /* Drivers implementing ndo_get_peer_dev must support tstat 11542 * accounting, so that skb_do_redirect() can bump the dev's 11543 * RX stats upon network namespace switch. 11544 */ 11545 if (dev->netdev_ops->ndo_get_peer_dev && 11546 dev->pcpu_stat_type != NETDEV_PCPU_STAT_TSTATS) 11547 return -EOPNOTSUPP; 11548 11549 switch (dev->pcpu_stat_type) { 11550 case NETDEV_PCPU_STAT_NONE: 11551 return 0; 11552 case NETDEV_PCPU_STAT_LSTATS: 11553 v = dev->lstats = netdev_alloc_pcpu_stats(struct pcpu_lstats); 11554 break; 11555 case NETDEV_PCPU_STAT_TSTATS: 11556 v = dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats); 11557 break; 11558 case NETDEV_PCPU_STAT_DSTATS: 11559 v = dev->dstats = netdev_alloc_pcpu_stats(struct pcpu_dstats); 11560 break; 11561 default: 11562 return -EINVAL; 11563 } 11564 11565 return v ? 0 : -ENOMEM; 11566 } 11567 11568 static void netdev_do_free_pcpu_stats(struct net_device *dev) 11569 { 11570 switch (dev->pcpu_stat_type) { 11571 case NETDEV_PCPU_STAT_NONE: 11572 return; 11573 case NETDEV_PCPU_STAT_LSTATS: 11574 free_percpu(dev->lstats); 11575 break; 11576 case NETDEV_PCPU_STAT_TSTATS: 11577 free_percpu(dev->tstats); 11578 break; 11579 case NETDEV_PCPU_STAT_DSTATS: 11580 free_percpu(dev->dstats); 11581 break; 11582 } 11583 } 11584 11585 static void netdev_free_phy_link_topology(struct net_device *dev) 11586 { 11587 struct phy_link_topology *topo = dev->link_topo; 11588 11589 if (IS_ENABLED(CONFIG_PHYLIB) && topo) { 11590 xa_destroy(&topo->phys); 11591 xa_destroy(&topo->ports); 11592 kfree(topo); 11593 dev->link_topo = NULL; 11594 } 11595 } 11596 11597 static int netdev_check_ops(struct net_device *dev) 11598 { 11599 const struct net_device_ops *ops = dev->netdev_ops; 11600 11601 if (((dev->hw_features | dev->features) & 11602 NETIF_F_HW_VLAN_CTAG_FILTER) && 11603 (!ops->ndo_vlan_rx_add_vid || !ops->ndo_vlan_rx_kill_vid)) { 11604 netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n"); 11605 return -EINVAL; 11606 } 11607 11608 if (!ops->ndo_hwtstamp_get != !ops->ndo_hwtstamp_set) { 11609 netdev_WARN(dev, "driver implements only one hwtstamp NDO\n"); 11610 return -EINVAL; 11611 } 11612 11613 if (netdev_need_ops_lock(dev) && ops->ndo_set_rx_mode && 11614 !ops->ndo_set_rx_mode_async) { 11615 netdev_WARN(dev, "ops-locked drivers should use ndo_set_rx_mode_async\n"); 11616 return -EINVAL; 11617 } 11618 11619 return 0; 11620 } 11621 11622 /** 11623 * register_netdevice() - register a network device 11624 * @dev: device to register 11625 * 11626 * Take a prepared network device structure and make it externally accessible. 11627 * A %NETDEV_REGISTER message is sent to the netdev notifier chain. 11628 * Callers must hold the rtnl lock - you may want register_netdev() 11629 * instead of this. 11630 */ 11631 int register_netdevice(struct net_device *dev) 11632 { 11633 int ret; 11634 struct net *net = dev_net(dev); 11635 11636 BUILD_BUG_ON(sizeof(netdev_features_t) * BITS_PER_BYTE < 11637 NETDEV_FEATURE_COUNT); 11638 BUG_ON(dev_boot_phase); 11639 ASSERT_RTNL(); 11640 11641 might_sleep(); 11642 11643 /* When net_device's are persistent, this will be fatal. */ 11644 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED); 11645 BUG_ON(!net); 11646 11647 ret = ethtool_check_ops(dev->ethtool_ops); 11648 if (ret) 11649 return ret; 11650 11651 /* rss ctx ID 0 is reserved for the default context, start from 1 */ 11652 xa_init_flags(&dev->ethtool->rss_ctx, XA_FLAGS_ALLOC1); 11653 mutex_init(&dev->ethtool->rss_lock); 11654 11655 spin_lock_init(&dev->addr_list_lock); 11656 netdev_set_addr_lockdep_class(dev); 11657 11658 ret = dev_get_valid_name(net, dev, dev->name); 11659 if (ret < 0) 11660 goto out; 11661 11662 ret = -ENOMEM; 11663 dev->name_node = netdev_name_node_head_alloc(dev); 11664 if (!dev->name_node) 11665 goto out; 11666 11667 /* Init, if this function is available */ 11668 if (dev->netdev_ops->ndo_init) { 11669 ret = dev->netdev_ops->ndo_init(dev); 11670 if (ret) { 11671 if (ret > 0) 11672 ret = -EIO; 11673 goto err_free_name; 11674 } 11675 } 11676 11677 ret = netdev_check_ops(dev); 11678 if (ret) 11679 goto err_uninit; 11680 11681 ret = netdev_do_alloc_pcpu_stats(dev); 11682 if (ret) 11683 goto err_uninit; 11684 11685 ret = dev_index_reserve(net, dev->ifindex); 11686 if (ret < 0) 11687 goto err_free_pcpu; 11688 dev->ifindex = ret; 11689 11690 /* Transfer changeable features to wanted_features and enable 11691 * software offloads (GSO and GRO). 11692 */ 11693 dev->hw_features |= (NETIF_F_SOFT_FEATURES | NETIF_F_SOFT_FEATURES_OFF); 11694 dev->features |= NETIF_F_SOFT_FEATURES; 11695 11696 if (dev->udp_tunnel_nic_info) { 11697 dev->features |= NETIF_F_RX_UDP_TUNNEL_PORT; 11698 dev->hw_features |= NETIF_F_RX_UDP_TUNNEL_PORT; 11699 } 11700 11701 dev->wanted_features = dev->features & dev->hw_features; 11702 11703 if (!(dev->flags & IFF_LOOPBACK)) 11704 dev->hw_features |= NETIF_F_NOCACHE_COPY; 11705 11706 /* If IPv4 TCP segmentation offload is supported we should also 11707 * allow the device to enable segmenting the frame with the option 11708 * of ignoring a static IP ID value. This doesn't enable the 11709 * feature itself but allows the user to enable it later. 11710 */ 11711 if (dev->hw_features & NETIF_F_TSO) 11712 dev->hw_features |= NETIF_F_TSO_MANGLEID; 11713 if (dev->vlan_features & NETIF_F_TSO) 11714 dev->vlan_features |= NETIF_F_TSO_MANGLEID; 11715 if (dev->mpls_features & NETIF_F_TSO) 11716 dev->mpls_features |= NETIF_F_TSO_MANGLEID; 11717 if (dev->hw_enc_features & NETIF_F_TSO) 11718 dev->hw_enc_features |= NETIF_F_TSO_MANGLEID; 11719 11720 /* TSO_MANGLEID belongs in mangleid_features by definition */ 11721 dev->mangleid_features |= NETIF_F_TSO_MANGLEID; 11722 11723 /* Make NETIF_F_HIGHDMA inheritable to VLAN devices. 11724 */ 11725 dev->vlan_features |= NETIF_F_HIGHDMA; 11726 11727 /* Make NETIF_F_SG inheritable to tunnel devices. 11728 */ 11729 dev->hw_enc_features |= NETIF_F_SG | NETIF_F_GSO_PARTIAL; 11730 11731 /* Make NETIF_F_SG inheritable to MPLS. 11732 */ 11733 dev->mpls_features |= NETIF_F_SG; 11734 11735 ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev); 11736 ret = notifier_to_errno(ret); 11737 if (ret) 11738 goto err_ifindex_release; 11739 11740 ret = netdev_register_kobject(dev); 11741 11742 netdev_lock(dev); 11743 WRITE_ONCE(dev->reg_state, ret ? NETREG_UNREGISTERED : NETREG_REGISTERED); 11744 netdev_unlock(dev); 11745 11746 if (ret) 11747 goto err_uninit_notify; 11748 11749 netdev_lock_ops(dev); 11750 __netdev_update_features(dev); 11751 netdev_unlock_ops(dev); 11752 11753 /* 11754 * Default initial state at registry is that the 11755 * device is present. 11756 */ 11757 11758 set_bit(__LINK_STATE_PRESENT, &dev->state); 11759 11760 linkwatch_init_dev(dev); 11761 11762 dev_init_scheduler(dev); 11763 11764 netdev_hold(dev, &dev->dev_registered_tracker, GFP_KERNEL); 11765 list_netdevice(dev); 11766 11767 add_device_randomness(dev->dev_addr, dev->addr_len); 11768 11769 /* If the device has permanent device address, driver should 11770 * set dev_addr and also addr_assign_type should be set to 11771 * NET_ADDR_PERM (default value). 11772 */ 11773 if (dev->addr_assign_type == NET_ADDR_PERM) 11774 memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len); 11775 11776 /* Notify protocols, that a new device appeared. */ 11777 netdev_lock_ops(dev); 11778 ret = call_netdevice_notifiers(NETDEV_REGISTER, dev); 11779 netdev_unlock_ops(dev); 11780 ret = notifier_to_errno(ret); 11781 if (ret) { 11782 /* Expect explicit free_netdev() on failure */ 11783 dev->needs_free_netdev = false; 11784 unregister_netdevice_queue(dev, NULL); 11785 goto out; 11786 } 11787 /* 11788 * Prevent userspace races by waiting until the network 11789 * device is fully setup before sending notifications. 11790 */ 11791 netdev_uevent_add(dev); 11792 if (!(dev->rtnl_link_ops && dev->rtnl_link_initializing)) 11793 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL, 0, NULL); 11794 11795 out: 11796 return ret; 11797 11798 err_uninit_notify: 11799 call_netdevice_notifiers(NETDEV_PRE_UNINIT, dev); 11800 err_ifindex_release: 11801 dev_index_release(net, dev->ifindex); 11802 err_free_pcpu: 11803 netdev_do_free_pcpu_stats(dev); 11804 err_uninit: 11805 if (dev->netdev_ops->ndo_uninit) 11806 dev->netdev_ops->ndo_uninit(dev); 11807 if (dev->priv_destructor) 11808 dev->priv_destructor(dev); 11809 err_free_name: 11810 netdev_name_node_free(dev->name_node); 11811 goto out; 11812 } 11813 EXPORT_SYMBOL(register_netdevice); 11814 11815 /* Initialize the core of a dummy net device. 11816 * The setup steps dummy netdevs need which normal netdevs get by going 11817 * through register_netdevice(). 11818 */ 11819 static void init_dummy_netdev(struct net_device *dev) 11820 { 11821 /* make sure we BUG if trying to hit standard 11822 * register/unregister code path 11823 */ 11824 dev->reg_state = NETREG_DUMMY; 11825 11826 /* a dummy interface is started by default */ 11827 set_bit(__LINK_STATE_PRESENT, &dev->state); 11828 set_bit(__LINK_STATE_START, &dev->state); 11829 11830 /* Note : We dont allocate pcpu_refcnt for dummy devices, 11831 * because users of this 'device' dont need to change 11832 * its refcount. 11833 */ 11834 } 11835 11836 /** 11837 * register_netdev - register a network device 11838 * @dev: device to register 11839 * 11840 * Take a completed network device structure and add it to the kernel 11841 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier 11842 * chain. 0 is returned on success. A negative errno code is returned 11843 * on a failure to set up the device, or if the name is a duplicate. 11844 * 11845 * This is a wrapper around register_netdevice that takes the rtnl semaphore 11846 * and expands the device name if you passed a format string to 11847 * alloc_netdev. 11848 */ 11849 int register_netdev(struct net_device *dev) 11850 { 11851 struct net *net = dev_net(dev); 11852 int err; 11853 11854 if (rtnl_net_lock_killable(net)) 11855 return -EINTR; 11856 11857 err = register_netdevice(dev); 11858 11859 rtnl_net_unlock(net); 11860 11861 return err; 11862 } 11863 EXPORT_SYMBOL(register_netdev); 11864 11865 int netdev_refcnt_read(const struct net_device *dev) 11866 { 11867 #ifdef CONFIG_PCPU_DEV_REFCNT 11868 int i, refcnt = 0; 11869 11870 for_each_possible_cpu(i) 11871 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i); 11872 return refcnt; 11873 #else 11874 return refcount_read(&dev->dev_refcnt); 11875 #endif 11876 } 11877 EXPORT_SYMBOL(netdev_refcnt_read); 11878 11879 int netdev_unregister_timeout_secs __read_mostly = 10; 11880 11881 #define WAIT_REFS_MIN_MSECS 1 11882 #define WAIT_REFS_MAX_MSECS 250 11883 /** 11884 * netdev_wait_allrefs_any - wait until all references are gone. 11885 * @list: list of net_devices to wait on 11886 * 11887 * This is called when unregistering network devices. 11888 * 11889 * Any protocol or device that holds a reference should register 11890 * for netdevice notification, and cleanup and put back the 11891 * reference if they receive an UNREGISTER event. 11892 * We can get stuck here if buggy protocols don't correctly 11893 * call dev_put. 11894 */ 11895 static struct net_device *netdev_wait_allrefs_any(struct list_head *list) 11896 { 11897 unsigned long rebroadcast_time, warning_time; 11898 struct net_device *dev; 11899 int wait = 0; 11900 11901 rebroadcast_time = warning_time = jiffies; 11902 11903 list_for_each_entry(dev, list, todo_list) 11904 if (netdev_refcnt_read(dev) == 1) 11905 return dev; 11906 11907 while (true) { 11908 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) { 11909 rtnl_lock(); 11910 11911 /* Rebroadcast unregister notification */ 11912 list_for_each_entry(dev, list, todo_list) { 11913 struct net *net = dev_net(dev); 11914 11915 __rtnl_net_lock(net); 11916 call_netdevice_notifiers(NETDEV_UNREGISTER, dev); 11917 __rtnl_net_unlock(net); 11918 } 11919 11920 __rtnl_unlock(); 11921 rcu_barrier(); 11922 rtnl_lock(); 11923 11924 list_for_each_entry(dev, list, todo_list) 11925 if (test_bit(__LINK_STATE_LINKWATCH_PENDING, 11926 &dev->state)) { 11927 /* We must not have linkwatch events 11928 * pending on unregister. If this 11929 * happens, we simply run the queue 11930 * unscheduled, resulting in a noop 11931 * for this device. 11932 */ 11933 linkwatch_run_queue(); 11934 break; 11935 } 11936 11937 __rtnl_unlock(); 11938 11939 rebroadcast_time = jiffies; 11940 } 11941 11942 rcu_barrier(); 11943 11944 if (!wait) { 11945 wait = WAIT_REFS_MIN_MSECS; 11946 } else { 11947 msleep(wait); 11948 wait = min(wait << 1, WAIT_REFS_MAX_MSECS); 11949 } 11950 11951 list_for_each_entry(dev, list, todo_list) 11952 if (netdev_refcnt_read(dev) == 1) 11953 return dev; 11954 11955 if (time_after(jiffies, warning_time + 11956 READ_ONCE(netdev_unregister_timeout_secs) * HZ)) { 11957 list_for_each_entry(dev, list, todo_list) { 11958 pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n", 11959 dev->name, netdev_refcnt_read(dev)); 11960 ref_tracker_dir_print(&dev->refcnt_tracker, 10); 11961 } 11962 11963 warning_time = jiffies; 11964 } 11965 } 11966 } 11967 11968 /* The sequence is: 11969 * 11970 * rtnl_lock(); 11971 * ... 11972 * register_netdevice(x1); 11973 * register_netdevice(x2); 11974 * ... 11975 * unregister_netdevice(y1); 11976 * unregister_netdevice(y2); 11977 * ... 11978 * rtnl_unlock(); 11979 * free_netdev(y1); 11980 * free_netdev(y2); 11981 * 11982 * We are invoked by rtnl_unlock(). 11983 * This allows us to deal with problems: 11984 * 1) We can delete sysfs objects which invoke hotplug 11985 * without deadlocking with linkwatch via keventd. 11986 * 2) Since we run with the RTNL semaphore not held, we can sleep 11987 * safely in order to wait for the netdev refcnt to drop to zero. 11988 * 11989 * We must not return until all unregister events added during 11990 * the interval the lock was held have been completed. 11991 */ 11992 void netdev_run_todo(void) 11993 { 11994 struct net_device *dev, *tmp; 11995 struct list_head list; 11996 int cnt; 11997 #ifdef CONFIG_LOCKDEP 11998 struct list_head unlink_list; 11999 12000 list_replace_init(&net_unlink_list, &unlink_list); 12001 12002 while (!list_empty(&unlink_list)) { 12003 dev = list_first_entry(&unlink_list, struct net_device, 12004 unlink_list); 12005 list_del_init(&dev->unlink_list); 12006 dev->nested_level = dev->lower_level - 1; 12007 } 12008 #endif 12009 12010 /* Snapshot list, allow later requests */ 12011 list_replace_init(&net_todo_list, &list); 12012 12013 __rtnl_unlock(); 12014 12015 /* Wait for rcu callbacks to finish before next phase */ 12016 if (!list_empty(&list)) 12017 rcu_barrier(); 12018 12019 list_for_each_entry_safe(dev, tmp, &list, todo_list) { 12020 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) { 12021 netdev_WARN(dev, "run_todo but not unregistering\n"); 12022 list_del(&dev->todo_list); 12023 net_unreg_done(dev_net(dev)); 12024 continue; 12025 } 12026 12027 netdev_lock(dev); 12028 WRITE_ONCE(dev->reg_state, NETREG_UNREGISTERED); 12029 netdev_unlock(dev); 12030 linkwatch_sync_dev(dev); 12031 } 12032 12033 if (!list_empty(&list)) { 12034 rt_flush_dev(NULL); 12035 rt6_uncached_list_flush_dev(NULL); 12036 } 12037 12038 cnt = 0; 12039 while (!list_empty(&list)) { 12040 struct net *net; 12041 12042 dev = netdev_wait_allrefs_any(&list); 12043 list_del(&dev->todo_list); 12044 net = dev_net(dev); 12045 12046 /* paranoia */ 12047 BUG_ON(netdev_refcnt_read(dev) != 1); 12048 BUG_ON(!list_empty(&dev->ptype_all)); 12049 BUG_ON(!list_empty(&dev->ptype_specific)); 12050 WARN_ON(rcu_access_pointer(dev->ip_ptr)); 12051 WARN_ON(rcu_access_pointer(dev->ip6_ptr)); 12052 12053 netdev_name_node_alt_flush(dev); 12054 netdev_name_node_free(dev->name_node); 12055 netdev_do_free_pcpu_stats(dev); 12056 if (dev->priv_destructor) 12057 dev->priv_destructor(dev); 12058 if (dev->needs_free_netdev) 12059 free_netdev(dev); 12060 12061 cnt++; 12062 12063 /* Free network device */ 12064 kobject_put(&dev->dev.kobj); 12065 12066 net_unreg_done(net); 12067 } 12068 if (cnt && atomic_sub_and_test(cnt, &dev_unreg_count)) 12069 wake_up(&netdev_unregistering_wq); 12070 } 12071 12072 /* Collate per-cpu network dstats statistics 12073 * 12074 * Read per-cpu network statistics from dev->dstats and populate the related 12075 * fields in @s. 12076 */ 12077 static void dev_fetch_dstats(struct rtnl_link_stats64 *s, 12078 const struct pcpu_dstats __percpu *dstats) 12079 { 12080 int cpu; 12081 12082 for_each_possible_cpu(cpu) { 12083 u64 rx_packets, rx_bytes, rx_drops; 12084 u64 tx_packets, tx_bytes, tx_drops; 12085 const struct pcpu_dstats *stats; 12086 unsigned int start; 12087 12088 stats = per_cpu_ptr(dstats, cpu); 12089 do { 12090 start = u64_stats_fetch_begin(&stats->syncp); 12091 rx_packets = u64_stats_read(&stats->rx_packets); 12092 rx_bytes = u64_stats_read(&stats->rx_bytes); 12093 rx_drops = u64_stats_read(&stats->rx_drops); 12094 tx_packets = u64_stats_read(&stats->tx_packets); 12095 tx_bytes = u64_stats_read(&stats->tx_bytes); 12096 tx_drops = u64_stats_read(&stats->tx_drops); 12097 } while (u64_stats_fetch_retry(&stats->syncp, start)); 12098 12099 s->rx_packets += rx_packets; 12100 s->rx_bytes += rx_bytes; 12101 s->rx_dropped += rx_drops; 12102 s->tx_packets += tx_packets; 12103 s->tx_bytes += tx_bytes; 12104 s->tx_dropped += tx_drops; 12105 } 12106 } 12107 12108 /* ndo_get_stats64 implementation for dtstats-based accounting. 12109 * 12110 * Populate @s from dev->stats and dev->dstats. This is used internally by the 12111 * core for NETDEV_PCPU_STAT_DSTAT-type stats collection. 12112 */ 12113 static void dev_get_dstats64(const struct net_device *dev, 12114 struct rtnl_link_stats64 *s) 12115 { 12116 netdev_stats_to_stats64(s, &dev->stats); 12117 dev_fetch_dstats(s, dev->dstats); 12118 } 12119 12120 /* Convert net_device_stats to rtnl_link_stats64. rtnl_link_stats64 has 12121 * all the same fields in the same order as net_device_stats, with only 12122 * the type differing, but rtnl_link_stats64 may have additional fields 12123 * at the end for newer counters. 12124 */ 12125 void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64, 12126 const struct net_device_stats *netdev_stats) 12127 { 12128 size_t i, n = sizeof(*netdev_stats) / sizeof(atomic_long_t); 12129 const atomic_long_t *src = (atomic_long_t *)netdev_stats; 12130 u64 *dst = (u64 *)stats64; 12131 12132 BUILD_BUG_ON(n > sizeof(*stats64) / sizeof(u64)); 12133 for (i = 0; i < n; i++) 12134 dst[i] = (unsigned long)atomic_long_read(&src[i]); 12135 /* zero out counters that only exist in rtnl_link_stats64 */ 12136 memset((char *)stats64 + n * sizeof(u64), 0, 12137 sizeof(*stats64) - n * sizeof(u64)); 12138 } 12139 EXPORT_SYMBOL(netdev_stats_to_stats64); 12140 12141 static __cold struct net_device_core_stats __percpu *netdev_core_stats_alloc( 12142 struct net_device *dev) 12143 { 12144 struct net_device_core_stats __percpu *p; 12145 12146 p = alloc_percpu_gfp(struct net_device_core_stats, 12147 GFP_ATOMIC | __GFP_NOWARN); 12148 12149 if (p && cmpxchg(&dev->core_stats, NULL, p)) 12150 free_percpu(p); 12151 12152 /* This READ_ONCE() pairs with the cmpxchg() above */ 12153 return READ_ONCE(dev->core_stats); 12154 } 12155 12156 noinline void netdev_core_stats_inc(struct net_device *dev, u32 offset) 12157 { 12158 /* This READ_ONCE() pairs with the write in netdev_core_stats_alloc() */ 12159 struct net_device_core_stats __percpu *p = READ_ONCE(dev->core_stats); 12160 unsigned long __percpu *field; 12161 12162 if (unlikely(!p)) { 12163 p = netdev_core_stats_alloc(dev); 12164 if (!p) 12165 return; 12166 } 12167 12168 field = (unsigned long __percpu *)((void __percpu *)p + offset); 12169 this_cpu_inc(*field); 12170 } 12171 EXPORT_SYMBOL_GPL(netdev_core_stats_inc); 12172 12173 /** 12174 * dev_get_stats - get network device statistics 12175 * @dev: device to get statistics from 12176 * @storage: place to store stats 12177 * 12178 * Get network statistics from device. Return @storage. 12179 * The device driver may provide its own method by setting 12180 * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats; 12181 * otherwise the internal statistics structure is used. 12182 */ 12183 struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev, 12184 struct rtnl_link_stats64 *storage) 12185 { 12186 const struct net_device_ops *ops = dev->netdev_ops; 12187 const struct net_device_core_stats __percpu *p; 12188 12189 /* 12190 * IPv{4,6} and udp tunnels share common stat helpers and use 12191 * different stat type (NETDEV_PCPU_STAT_TSTATS vs 12192 * NETDEV_PCPU_STAT_DSTATS). Ensure the accounting is consistent. 12193 */ 12194 BUILD_BUG_ON(offsetof(struct pcpu_sw_netstats, rx_bytes) != 12195 offsetof(struct pcpu_dstats, rx_bytes)); 12196 BUILD_BUG_ON(offsetof(struct pcpu_sw_netstats, rx_packets) != 12197 offsetof(struct pcpu_dstats, rx_packets)); 12198 BUILD_BUG_ON(offsetof(struct pcpu_sw_netstats, tx_bytes) != 12199 offsetof(struct pcpu_dstats, tx_bytes)); 12200 BUILD_BUG_ON(offsetof(struct pcpu_sw_netstats, tx_packets) != 12201 offsetof(struct pcpu_dstats, tx_packets)); 12202 12203 if (ops->ndo_get_stats64) { 12204 memset(storage, 0, sizeof(*storage)); 12205 ops->ndo_get_stats64(dev, storage); 12206 } else if (ops->ndo_get_stats) { 12207 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev)); 12208 } else if (dev->pcpu_stat_type == NETDEV_PCPU_STAT_TSTATS) { 12209 dev_get_tstats64(dev, storage); 12210 } else if (dev->pcpu_stat_type == NETDEV_PCPU_STAT_DSTATS) { 12211 dev_get_dstats64(dev, storage); 12212 } else { 12213 netdev_stats_to_stats64(storage, &dev->stats); 12214 } 12215 12216 /* This READ_ONCE() pairs with the write in netdev_core_stats_alloc() */ 12217 p = READ_ONCE(dev->core_stats); 12218 if (p) { 12219 const struct net_device_core_stats *core_stats; 12220 int i; 12221 12222 for_each_possible_cpu(i) { 12223 core_stats = per_cpu_ptr(p, i); 12224 storage->rx_dropped += READ_ONCE(core_stats->rx_dropped); 12225 storage->tx_dropped += READ_ONCE(core_stats->tx_dropped); 12226 storage->rx_nohandler += READ_ONCE(core_stats->rx_nohandler); 12227 storage->rx_otherhost_dropped += READ_ONCE(core_stats->rx_otherhost_dropped); 12228 } 12229 } 12230 return storage; 12231 } 12232 EXPORT_SYMBOL(dev_get_stats); 12233 12234 /** 12235 * dev_fetch_sw_netstats - get per-cpu network device statistics 12236 * @s: place to store stats 12237 * @netstats: per-cpu network stats to read from 12238 * 12239 * Read per-cpu network statistics and populate the related fields in @s. 12240 */ 12241 void dev_fetch_sw_netstats(struct rtnl_link_stats64 *s, 12242 const struct pcpu_sw_netstats __percpu *netstats) 12243 { 12244 int cpu; 12245 12246 for_each_possible_cpu(cpu) { 12247 u64 rx_packets, rx_bytes, tx_packets, tx_bytes; 12248 const struct pcpu_sw_netstats *stats; 12249 unsigned int start; 12250 12251 stats = per_cpu_ptr(netstats, cpu); 12252 do { 12253 start = u64_stats_fetch_begin(&stats->syncp); 12254 rx_packets = u64_stats_read(&stats->rx_packets); 12255 rx_bytes = u64_stats_read(&stats->rx_bytes); 12256 tx_packets = u64_stats_read(&stats->tx_packets); 12257 tx_bytes = u64_stats_read(&stats->tx_bytes); 12258 } while (u64_stats_fetch_retry(&stats->syncp, start)); 12259 12260 s->rx_packets += rx_packets; 12261 s->rx_bytes += rx_bytes; 12262 s->tx_packets += tx_packets; 12263 s->tx_bytes += tx_bytes; 12264 } 12265 } 12266 EXPORT_SYMBOL_GPL(dev_fetch_sw_netstats); 12267 12268 /** 12269 * dev_get_tstats64 - ndo_get_stats64 implementation 12270 * @dev: device to get statistics from 12271 * @s: place to store stats 12272 * 12273 * Populate @s from dev->stats and dev->tstats. Can be used as 12274 * ndo_get_stats64() callback. 12275 */ 12276 void dev_get_tstats64(struct net_device *dev, struct rtnl_link_stats64 *s) 12277 { 12278 netdev_stats_to_stats64(s, &dev->stats); 12279 dev_fetch_sw_netstats(s, dev->tstats); 12280 } 12281 EXPORT_SYMBOL_GPL(dev_get_tstats64); 12282 12283 struct netdev_queue *dev_ingress_queue_create(struct net_device *dev) 12284 { 12285 struct netdev_queue *queue = dev_ingress_queue(dev); 12286 12287 #ifdef CONFIG_NET_CLS_ACT 12288 if (queue) 12289 return queue; 12290 queue = kzalloc_obj(*queue); 12291 if (!queue) 12292 return NULL; 12293 netdev_init_one_queue(dev, queue, NULL); 12294 RCU_INIT_POINTER(queue->qdisc, &noop_qdisc); 12295 RCU_INIT_POINTER(queue->qdisc_sleeping, &noop_qdisc); 12296 rcu_assign_pointer(dev->ingress_queue, queue); 12297 #endif 12298 return queue; 12299 } 12300 12301 static const struct ethtool_ops default_ethtool_ops; 12302 12303 void netdev_set_default_ethtool_ops(struct net_device *dev, 12304 const struct ethtool_ops *ops) 12305 { 12306 if (dev->ethtool_ops == &default_ethtool_ops) 12307 dev->ethtool_ops = ops; 12308 } 12309 EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops); 12310 12311 /** 12312 * netdev_sw_irq_coalesce_default_on() - enable SW IRQ coalescing by default 12313 * @dev: netdev to enable the IRQ coalescing on 12314 * 12315 * Sets a conservative default for SW IRQ coalescing. Users can use 12316 * sysfs attributes to override the default values. 12317 */ 12318 void netdev_sw_irq_coalesce_default_on(struct net_device *dev) 12319 { 12320 WARN_ON(dev->reg_state == NETREG_REGISTERED); 12321 12322 if (!IS_ENABLED(CONFIG_PREEMPT_RT)) { 12323 netdev_set_gro_flush_timeout(dev, 20000); 12324 netdev_set_defer_hard_irqs(dev, 1); 12325 } 12326 } 12327 EXPORT_SYMBOL_GPL(netdev_sw_irq_coalesce_default_on); 12328 12329 /** 12330 * alloc_netdev_mqs - allocate network device 12331 * @sizeof_priv: size of private data to allocate space for 12332 * @name: device name format string 12333 * @name_assign_type: origin of device name 12334 * @setup: callback to initialize device 12335 * @txqs: the number of TX subqueues to allocate 12336 * @rxqs: the number of RX subqueues to allocate 12337 * 12338 * Allocates a struct net_device with private data area for driver use 12339 * and performs basic initialization. Also allocates subqueue structs 12340 * for each queue on the device. 12341 */ 12342 struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name, 12343 unsigned char name_assign_type, 12344 void (*setup)(struct net_device *), 12345 unsigned int txqs, unsigned int rxqs) 12346 { 12347 struct net_device *dev; 12348 size_t napi_config_sz; 12349 unsigned int maxqs; 12350 12351 BUG_ON(strlen(name) >= sizeof(dev->name)); 12352 12353 if (txqs < 1) { 12354 pr_err("alloc_netdev: Unable to allocate device with zero queues\n"); 12355 return NULL; 12356 } 12357 12358 if (rxqs < 1) { 12359 pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n"); 12360 return NULL; 12361 } 12362 12363 maxqs = max(txqs, rxqs); 12364 12365 dev = kvzalloc_flex(*dev, priv, sizeof_priv, 12366 GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL); 12367 if (!dev) 12368 return NULL; 12369 12370 dev->priv_len = sizeof_priv; 12371 12372 ref_tracker_dir_init(&dev->refcnt_tracker, 128, "netdev"); 12373 #ifdef CONFIG_PCPU_DEV_REFCNT 12374 dev->pcpu_refcnt = alloc_percpu(int); 12375 if (!dev->pcpu_refcnt) 12376 goto free_dev; 12377 __dev_hold(dev); 12378 #else 12379 refcount_set(&dev->dev_refcnt, 1); 12380 #endif 12381 12382 if (dev_addr_init(dev)) 12383 goto free_pcpu; 12384 12385 dev_mc_init(dev); 12386 dev_uc_init(dev); 12387 12388 dev_net_set(dev, &init_net); 12389 12390 dev->gso_max_size = GSO_LEGACY_MAX_SIZE; 12391 dev->xdp_zc_max_segs = 1; 12392 dev->gso_max_segs = GSO_MAX_SEGS; 12393 dev->gro_max_size = GRO_LEGACY_MAX_SIZE; 12394 dev->gso_ipv4_max_size = GSO_LEGACY_MAX_SIZE; 12395 dev->gro_ipv4_max_size = GRO_LEGACY_MAX_SIZE; 12396 dev->tso_max_size = TSO_LEGACY_MAX_SIZE; 12397 dev->tso_max_segs = TSO_MAX_SEGS; 12398 dev->upper_level = 1; 12399 dev->lower_level = 1; 12400 #ifdef CONFIG_LOCKDEP 12401 dev->nested_level = 0; 12402 INIT_LIST_HEAD(&dev->unlink_list); 12403 #endif 12404 12405 INIT_LIST_HEAD(&dev->napi_list); 12406 INIT_LIST_HEAD(&dev->unreg_list); 12407 #ifdef CONFIG_DEBUG_NET_SMALL_RTNL 12408 INIT_LIST_HEAD(&dev->unreg_list_net); 12409 #endif 12410 INIT_LIST_HEAD(&dev->close_list); 12411 INIT_LIST_HEAD(&dev->link_watch_list); 12412 INIT_LIST_HEAD(&dev->adj_list.upper); 12413 INIT_LIST_HEAD(&dev->adj_list.lower); 12414 INIT_LIST_HEAD(&dev->ptype_all); 12415 INIT_LIST_HEAD(&dev->ptype_specific); 12416 INIT_LIST_HEAD(&dev->net_notifier_list); 12417 INIT_LIST_HEAD(&dev->work_node); 12418 #ifdef CONFIG_NET_SCHED 12419 hash_init(dev->qdisc_hash); 12420 #endif 12421 12422 mutex_init(&dev->lock); 12423 /* see also netdev_set_instance_lock_class() */ 12424 lock_set_cmp_fn(&dev->lock, netdev_lock_cmp_fn, NULL); 12425 netif_rx_mode_init(dev); 12426 12427 dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM; 12428 setup(dev); 12429 12430 if (!dev->tx_queue_len) { 12431 dev->priv_flags |= IFF_NO_QUEUE; 12432 dev->tx_queue_len = DEFAULT_TX_QUEUE_LEN; 12433 } 12434 12435 dev->num_tx_queues = txqs; 12436 dev->real_num_tx_queues = txqs; 12437 if (netif_alloc_netdev_queues(dev)) 12438 goto free_all; 12439 12440 dev->num_rx_queues = rxqs; 12441 dev->real_num_rx_queues = rxqs; 12442 if (netif_alloc_rx_queues(dev)) 12443 goto free_all; 12444 dev->ethtool = kzalloc_obj(*dev->ethtool, GFP_KERNEL_ACCOUNT); 12445 if (!dev->ethtool) 12446 goto free_all; 12447 12448 if (netdev_alloc_config(dev)) 12449 goto free_all; 12450 12451 dev->num_napi_configs = maxqs; 12452 napi_config_sz = array_size(maxqs, sizeof(*dev->napi_config)); 12453 dev->napi_config = kvzalloc(napi_config_sz, GFP_KERNEL_ACCOUNT); 12454 if (!dev->napi_config) 12455 goto free_all; 12456 12457 strscpy(dev->name, name); 12458 dev->name_assign_type = name_assign_type; 12459 dev->group = INIT_NETDEV_GROUP; 12460 if (!dev->ethtool_ops) 12461 dev->ethtool_ops = &default_ethtool_ops; 12462 12463 nf_hook_netdev_init(dev); 12464 12465 return dev; 12466 12467 free_all: 12468 free_netdev(dev); 12469 return NULL; 12470 12471 free_pcpu: 12472 #ifdef CONFIG_PCPU_DEV_REFCNT 12473 free_percpu(dev->pcpu_refcnt); 12474 free_dev: 12475 #endif 12476 ref_tracker_dir_exit(&dev->refcnt_tracker); 12477 kvfree(dev); 12478 return NULL; 12479 } 12480 EXPORT_SYMBOL(alloc_netdev_mqs); 12481 12482 static void netdev_napi_exit(struct net_device *dev) 12483 { 12484 if (!list_empty(&dev->napi_list)) { 12485 struct napi_struct *p, *n; 12486 12487 netdev_lock(dev); 12488 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list) 12489 __netif_napi_del_locked(p); 12490 netdev_unlock(dev); 12491 12492 synchronize_net(); 12493 } 12494 12495 kvfree(dev->napi_config); 12496 } 12497 12498 /** 12499 * free_netdev - free network device 12500 * @dev: device 12501 * 12502 * This function does the last stage of destroying an allocated device 12503 * interface. The reference to the device object is released. If this 12504 * is the last reference then it will be freed.Must be called in process 12505 * context. 12506 */ 12507 void free_netdev(struct net_device *dev) 12508 { 12509 might_sleep(); 12510 12511 /* When called immediately after register_netdevice() failed the unwind 12512 * handling may still be dismantling the device. Handle that case by 12513 * deferring the free. 12514 */ 12515 if (dev->reg_state == NETREG_UNREGISTERING) { 12516 ASSERT_RTNL(); 12517 dev->needs_free_netdev = true; 12518 return; 12519 } 12520 12521 netdev_free_config(dev); 12522 kfree(dev->ethtool); 12523 netif_free_tx_queues(dev); 12524 netif_free_rx_queues(dev); 12525 12526 kfree(rcu_dereference_protected(dev->ingress_queue, 1)); 12527 12528 __hw_addr_flush(&dev->rx_mode_addr_cache); 12529 12530 /* Flush device addresses */ 12531 dev_addr_flush(dev); 12532 12533 netdev_napi_exit(dev); 12534 12535 netif_del_cpu_rmap(dev); 12536 12537 ref_tracker_dir_exit(&dev->refcnt_tracker); 12538 #ifdef CONFIG_PCPU_DEV_REFCNT 12539 free_percpu(dev->pcpu_refcnt); 12540 dev->pcpu_refcnt = NULL; 12541 #endif 12542 free_percpu(dev->core_stats); 12543 dev->core_stats = NULL; 12544 free_percpu(dev->xdp_bulkq); 12545 dev->xdp_bulkq = NULL; 12546 12547 netdev_free_phy_link_topology(dev); 12548 12549 mutex_destroy(&dev->lock); 12550 12551 /* Compatibility with error handling in drivers */ 12552 if (dev->reg_state == NETREG_UNINITIALIZED || 12553 dev->reg_state == NETREG_DUMMY) { 12554 kvfree(dev); 12555 return; 12556 } 12557 12558 BUG_ON(dev->reg_state != NETREG_UNREGISTERED); 12559 WRITE_ONCE(dev->reg_state, NETREG_RELEASED); 12560 12561 /* will free via device release */ 12562 put_device(&dev->dev); 12563 } 12564 EXPORT_SYMBOL(free_netdev); 12565 12566 /** 12567 * alloc_netdev_dummy - Allocate and initialize a dummy net device. 12568 * @sizeof_priv: size of private data to allocate space for 12569 * 12570 * Return: the allocated net_device on success, NULL otherwise 12571 */ 12572 struct net_device *alloc_netdev_dummy(int sizeof_priv) 12573 { 12574 return alloc_netdev(sizeof_priv, "dummy#", NET_NAME_UNKNOWN, 12575 init_dummy_netdev); 12576 } 12577 EXPORT_SYMBOL_GPL(alloc_netdev_dummy); 12578 12579 /** 12580 * synchronize_net - Synchronize with packet receive processing 12581 * 12582 * Wait for packets currently being received to be done. 12583 * Does not block later packets from starting. 12584 */ 12585 void synchronize_net(void) 12586 { 12587 might_sleep(); 12588 if (from_cleanup_net() || rtnl_is_locked()) 12589 synchronize_rcu_expedited(); 12590 else 12591 synchronize_rcu(); 12592 } 12593 EXPORT_SYMBOL(synchronize_net); 12594 12595 static void netdev_rss_contexts_free(struct net_device *dev) 12596 { 12597 struct ethtool_rxfh_context *ctx; 12598 unsigned long context; 12599 12600 mutex_lock(&dev->ethtool->rss_lock); 12601 xa_for_each(&dev->ethtool->rss_ctx, context, ctx) { 12602 xa_erase(&dev->ethtool->rss_ctx, context); 12603 dev->ethtool_ops->remove_rxfh_context(dev, ctx, context, NULL); 12604 kfree(ctx); 12605 } 12606 xa_destroy(&dev->ethtool->rss_ctx); 12607 mutex_unlock(&dev->ethtool->rss_lock); 12608 } 12609 12610 /** 12611 * unregister_netdevice_queue - remove device from the kernel 12612 * @dev: device 12613 * @head: list 12614 * 12615 * This function shuts down a device interface and removes it 12616 * from the kernel tables. 12617 * If head not NULL, device is queued to be unregistered later. 12618 * 12619 * Callers must hold the rtnl semaphore. You may want 12620 * unregister_netdev() instead of this. 12621 */ 12622 12623 void unregister_netdevice_queue(struct net_device *dev, struct list_head *head) 12624 { 12625 ASSERT_RTNL(); 12626 12627 #ifdef CONFIG_DEBUG_NET_SMALL_RTNL 12628 DEBUG_NET_WARN_ON_ONCE(!list_empty(&dev->unreg_list_net)); 12629 #endif 12630 12631 if (head) { 12632 list_move_tail(&dev->unreg_list, head); 12633 } else { 12634 LIST_HEAD(single); 12635 12636 list_add(&dev->unreg_list, &single); 12637 unregister_netdevice_many(&single); 12638 } 12639 } 12640 EXPORT_SYMBOL(unregister_netdevice_queue); 12641 12642 static void dev_memory_provider_uninstall(struct net_device *dev) 12643 { 12644 unsigned int i; 12645 12646 for (i = 0; i < dev->real_num_rx_queues; i++) { 12647 struct netdev_rx_queue *rxq = &dev->_rx[i]; 12648 12649 __netif_mp_uninstall_rxq(rxq, &rxq->mp_params); 12650 } 12651 } 12652 12653 /* devices must be UP and netdev_lock()'d */ 12654 static void netif_close_many_and_unlock(struct list_head *close_head) 12655 { 12656 struct net_device *dev, *tmp; 12657 12658 netif_close_many(close_head, false); 12659 12660 /* ... now unlock them */ 12661 list_for_each_entry_safe(dev, tmp, close_head, close_list) { 12662 netdev_unlock(dev); 12663 list_del_init(&dev->close_list); 12664 } 12665 } 12666 12667 /* Handle one class of ops-locked devices. Since close requires the lock 12668 * we need to be careful about which classes we allow to nest. 12669 */ 12670 static void netdev_lock_ops_close_many(struct list_head *head, 12671 struct list_head *close_head, 12672 bool leasing) 12673 { 12674 struct net_device *dev; 12675 12676 list_for_each_entry(dev, head, unreg_list) { 12677 if (!(dev->flags & IFF_UP) || !netdev_need_ops_lock(dev) || 12678 netdev_can_create_queue(dev, NULL) != leasing) 12679 continue; 12680 list_add_tail(&dev->close_list, close_head); 12681 netdev_lock(dev); 12682 12683 #ifdef CONFIG_LOCKDEP 12684 /* We can only track up to MAX_LOCK_DEPTH locks per task. 12685 * Reserve half the available slots for additional locks 12686 * possibly taken by notifiers and (soft)irqs. 12687 */ 12688 if (lockdep_depth(current) > MAX_LOCK_DEPTH / 2) 12689 netif_close_many_and_unlock(close_head); 12690 #endif 12691 } 12692 } 12693 12694 bool unregister_netdevice_queued(const struct net_device *dev) 12695 { 12696 ASSERT_RTNL(); 12697 return !list_empty(&dev->unreg_list); 12698 } 12699 12700 void unregister_netdevice_many_notify(struct list_head *head, 12701 u32 portid, const struct nlmsghdr *nlh) 12702 { 12703 struct net_device *dev, *tmp; 12704 LIST_HEAD(close_head); 12705 int cnt = 0; 12706 12707 BUG_ON(dev_boot_phase); 12708 ASSERT_RTNL(); 12709 12710 if (list_empty(head)) 12711 return; 12712 12713 list_for_each_entry_safe(dev, tmp, head, unreg_list) { 12714 /* Some devices call without registering 12715 * for initialization unwind. Remove those 12716 * devices and proceed with the remaining. 12717 */ 12718 if (dev->reg_state == NETREG_UNINITIALIZED) { 12719 pr_debug("unregister_netdevice: device %s/%p never was registered\n", 12720 dev->name, dev); 12721 12722 WARN_ON(1); 12723 list_del(&dev->unreg_list); 12724 continue; 12725 } 12726 dev->dismantle = true; 12727 BUG_ON(dev->reg_state != NETREG_REGISTERED); 12728 } 12729 12730 /* If device is running, close it first. Start with ops locked... */ 12731 netdev_lock_ops_close_many(head, &close_head, true); /* queue leasing */ 12732 netdev_lock_ops_close_many(head, &close_head, false); /* the rest */ 12733 netif_close_many_and_unlock(&close_head); 12734 /* ... now go over the rest. */ 12735 list_for_each_entry(dev, head, unreg_list) { 12736 if (!netdev_need_ops_lock(dev)) 12737 list_add_tail(&dev->close_list, &close_head); 12738 } 12739 netif_close_many(&close_head, true); 12740 12741 list_for_each_entry(dev, head, unreg_list) { 12742 /* And unlink it from device chain. */ 12743 unlist_netdevice(dev); 12744 netdev_lock(dev); 12745 WRITE_ONCE(dev->reg_state, NETREG_UNREGISTERING); 12746 netdev_unlock(dev); 12747 } 12748 flush_all_backlogs(); 12749 12750 synchronize_net(); 12751 12752 list_for_each_entry(dev, head, unreg_list) { 12753 struct sk_buff *skb = NULL; 12754 12755 /* Shutdown queueing discipline. */ 12756 netdev_lock_ops(dev); 12757 dev_shutdown(dev); 12758 dev_tcx_uninstall(dev); 12759 dev_xdp_uninstall(dev); 12760 dev_memory_provider_uninstall(dev); 12761 netdev_work_cancel_all(dev); 12762 netdev_unlock_ops(dev); 12763 bpf_dev_bound_netdev_unregister(dev); 12764 12765 netdev_offload_xstats_disable_all(dev); 12766 12767 /* Notify protocols, that we are about to destroy 12768 * this device. They should clean all the things. 12769 */ 12770 call_netdevice_notifiers(NETDEV_UNREGISTER, dev); 12771 12772 if (!(dev->rtnl_link_ops && dev->rtnl_link_initializing)) 12773 skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U, 0, 12774 GFP_KERNEL, NULL, 0, 12775 portid, nlh); 12776 12777 /* 12778 * Flush the unicast and multicast chains 12779 */ 12780 dev_uc_flush(dev); 12781 dev_mc_flush(dev); 12782 12783 12784 netdev_rss_contexts_free(dev); 12785 12786 call_netdevice_notifiers(NETDEV_PRE_UNINIT, dev); 12787 12788 if (dev->netdev_ops->ndo_uninit) 12789 dev->netdev_ops->ndo_uninit(dev); 12790 12791 mutex_destroy(&dev->ethtool->rss_lock); 12792 12793 net_shaper_flush_netdev(dev); 12794 12795 if (skb) 12796 rtmsg_ifinfo_send(skb, dev, GFP_KERNEL, portid, nlh); 12797 12798 /* Notifier chain MUST detach us all upper devices. */ 12799 WARN_ON(netdev_has_any_upper_dev(dev)); 12800 WARN_ON(netdev_has_any_lower_dev(dev)); 12801 12802 /* Remove entries from kobject tree */ 12803 netdev_unregister_kobject(dev); 12804 #ifdef CONFIG_XPS 12805 /* Remove XPS queueing entries */ 12806 netif_reset_xps_queues_gt(dev, 0); 12807 #endif 12808 } 12809 12810 synchronize_net(); 12811 12812 list_for_each_entry(dev, head, unreg_list) { 12813 #ifdef CONFIG_DEBUG_NET_SMALL_RTNL 12814 struct net *net = dev_net(dev); 12815 12816 /* spin_lock() can be moved outside of the loop 12817 * once the per-netns RTNL conversion completes. 12818 */ 12819 spin_lock(&net->dev_unreg_lock); 12820 list_del(&dev->unreg_list_net); 12821 spin_unlock(&net->dev_unreg_lock); 12822 #endif 12823 netdev_put(dev, &dev->dev_registered_tracker); 12824 net_set_todo(dev); 12825 cnt++; 12826 } 12827 atomic_add(cnt, &dev_unreg_count); 12828 12829 list_del(head); 12830 } 12831 12832 /** 12833 * unregister_netdevice_many - unregister many devices 12834 * @head: list of devices 12835 * 12836 * Note: As most callers use a stack allocated list_head, 12837 * we force a list_del() to make sure stack won't be corrupted later. 12838 */ 12839 void unregister_netdevice_many(struct list_head *head) 12840 { 12841 unregister_netdevice_many_notify(head, 0, NULL); 12842 } 12843 EXPORT_SYMBOL(unregister_netdevice_many); 12844 12845 #ifdef CONFIG_DEBUG_NET_SMALL_RTNL 12846 void unregister_netdevice_queue_net(struct net *net, struct net_device *dev, 12847 struct list_head *head) 12848 { 12849 netdev_lock(dev); 12850 12851 if (net_eq(dev_net(dev), net)) { 12852 netdev_unlock(dev); 12853 unregister_netdevice_queue(dev, head); 12854 return; 12855 } 12856 12857 net = dev_net(dev); 12858 12859 spin_lock(&net->dev_unreg_lock); 12860 12861 DEBUG_NET_WARN_ON_ONCE(!list_empty(&dev->unreg_list)); 12862 DEBUG_NET_WARN_ON_ONCE(!list_empty(&dev->unreg_list_net)); 12863 12864 list_add_tail(&dev->unreg_list_net, &net->dev_unreg_head); 12865 rtnl_net_queue_work(net); 12866 12867 spin_unlock(&net->dev_unreg_lock); 12868 12869 netdev_unlock(dev); 12870 } 12871 EXPORT_SYMBOL(unregister_netdevice_queue_net); 12872 12873 void unregister_netdevice_queue_many_net(struct net *net, struct list_head *head) 12874 { 12875 struct net_device *dev, *tmp; 12876 12877 spin_lock(&net->dev_unreg_lock); 12878 list_for_each_entry_safe(dev, tmp, head, unreg_list) { 12879 /* Once all cross-netns unregister_netdevice_queue() is 12880 * converted to _net() (or for debugging), remove this check. 12881 */ 12882 if (!net_eq(dev_net(dev), net)) 12883 continue; 12884 12885 DEBUG_NET_WARN_ONCE(!net_eq(dev_net(dev), net), 12886 "%s was unregistered from a different netns.\n", 12887 dev->name); 12888 12889 list_del_init(&dev->unreg_list); 12890 list_move_tail(&dev->unreg_list_net, &net->dev_unreg_head); 12891 } 12892 spin_unlock(&net->dev_unreg_lock); 12893 } 12894 12895 static void unregister_netdevice_move_net(struct net *net_old, 12896 struct net *net, 12897 struct net_device *dev) 12898 { 12899 if (net_old > net) { 12900 spin_lock(&net->dev_unreg_lock); 12901 spin_lock_nested(&net_old->dev_unreg_lock, SINGLE_DEPTH_NESTING); 12902 } else { 12903 spin_lock(&net_old->dev_unreg_lock); 12904 spin_lock_nested(&net->dev_unreg_lock, SINGLE_DEPTH_NESTING); 12905 } 12906 12907 if (!list_empty(&dev->unreg_list_net)) { 12908 list_del(&dev->unreg_list_net); 12909 list_add_tail(&dev->unreg_list_net, &net->dev_unreg_head); 12910 } 12911 12912 spin_unlock(&net_old->dev_unreg_lock); 12913 spin_unlock(&net->dev_unreg_lock); 12914 } 12915 12916 void unregister_netdevice_many_net(struct net *net) 12917 { 12918 struct net_device *dev, *tmp; 12919 LIST_HEAD(unreg_head_net); 12920 LIST_HEAD(unreg_head); 12921 12922 spin_lock(&net->dev_unreg_lock); 12923 list_splice_init(&net->dev_unreg_head, &unreg_head_net); 12924 spin_unlock(&net->dev_unreg_lock); 12925 12926 list_for_each_entry_safe(dev, tmp, &unreg_head_net, unreg_list_net) { 12927 list_del_init(&dev->unreg_list_net); 12928 list_add_tail(&dev->unreg_list, &unreg_head); 12929 } 12930 12931 unregister_netdevice_many(&unreg_head); 12932 } 12933 #endif 12934 12935 /** 12936 * unregister_netdev - remove device from the kernel 12937 * @dev: device 12938 * 12939 * This function shuts down a device interface and removes it 12940 * from the kernel tables. 12941 * 12942 * This is just a wrapper for unregister_netdevice that takes 12943 * the rtnl semaphore. In general you want to use this and not 12944 * unregister_netdevice. 12945 */ 12946 void unregister_netdev(struct net_device *dev) 12947 { 12948 rtnl_net_dev_lock(dev); 12949 unregister_netdevice(dev); 12950 rtnl_net_dev_unlock(dev); 12951 } 12952 EXPORT_SYMBOL(unregister_netdev); 12953 12954 int __dev_change_net_namespace(struct net_device *dev, struct net *net, 12955 const char *pat, int new_ifindex, 12956 struct netlink_ext_ack *extack) 12957 { 12958 struct netdev_name_node *name_node, *tmp; 12959 struct net *net_old = dev_net(dev); 12960 char new_name[IFNAMSIZ] = {}; 12961 int err, new_nsid; 12962 12963 ASSERT_RTNL(); 12964 12965 /* Don't allow namespace local devices to be moved. */ 12966 err = -EINVAL; 12967 if (dev->netns_immutable) { 12968 NL_SET_ERR_MSG(extack, "The interface netns is immutable"); 12969 goto out; 12970 } 12971 12972 /* Ensure the device has been registered */ 12973 if (dev->reg_state != NETREG_REGISTERED) { 12974 NL_SET_ERR_MSG(extack, "The interface isn't registered"); 12975 goto out; 12976 } 12977 12978 /* Get out if there is nothing todo */ 12979 err = 0; 12980 if (net_eq(net_old, net)) 12981 goto out; 12982 12983 /* Pick the destination device name, and ensure 12984 * we can use it in the destination network namespace. 12985 */ 12986 err = -EEXIST; 12987 if (netdev_name_in_use(net, dev->name)) { 12988 /* We get here if we can't use the current device name */ 12989 if (!pat) { 12990 NL_SET_ERR_MSG(extack, 12991 "An interface with the same name exists in the target netns"); 12992 goto out; 12993 } 12994 err = dev_prep_valid_name(net, dev, pat, new_name, EEXIST); 12995 if (err < 0) { 12996 NL_SET_ERR_MSG_FMT(extack, 12997 "Unable to use '%s' for the new interface name in the target netns", 12998 pat); 12999 goto out; 13000 } 13001 } 13002 /* Check that none of the altnames conflicts. */ 13003 err = -EEXIST; 13004 netdev_for_each_altname_safe(dev, name_node, tmp) { 13005 if (!netdev_name_in_use(net, name_node->name)) 13006 continue; 13007 13008 if (!check_net(net_old)) { 13009 __netdev_name_node_alt_destroy(name_node); 13010 continue; 13011 } 13012 13013 NL_SET_ERR_MSG_FMT(extack, 13014 "An interface with the altname %s exists in the target netns", 13015 name_node->name); 13016 goto out; 13017 } 13018 13019 /* Check that new_ifindex isn't used yet. */ 13020 if (new_ifindex) { 13021 err = dev_index_reserve(net, new_ifindex); 13022 if (err < 0) { 13023 NL_SET_ERR_MSG_FMT(extack, 13024 "The ifindex %d is not available in the target netns", 13025 new_ifindex); 13026 goto out; 13027 } 13028 } else { 13029 /* If there is an ifindex conflict assign a new one */ 13030 err = dev_index_reserve(net, dev->ifindex); 13031 if (err == -EBUSY) 13032 err = dev_index_reserve(net, 0); 13033 if (err < 0) { 13034 NL_SET_ERR_MSG(extack, 13035 "Unable to allocate a new ifindex in the target netns"); 13036 goto out; 13037 } 13038 new_ifindex = err; 13039 } 13040 13041 /* 13042 * And now a mini version of register_netdevice unregister_netdevice. 13043 */ 13044 13045 netdev_lock_ops(dev); 13046 /* If device is running close it first. */ 13047 netif_close(dev); 13048 /* And unlink it from device chain */ 13049 unlist_netdevice(dev); 13050 13051 if (!netdev_need_ops_lock(dev)) 13052 netdev_lock(dev); 13053 dev->moving_ns = true; 13054 netdev_unlock(dev); 13055 13056 synchronize_net(); 13057 13058 /* Shutdown queueing discipline. */ 13059 netdev_lock_ops(dev); 13060 dev_shutdown(dev); 13061 netdev_unlock_ops(dev); 13062 13063 /* Notify protocols, that we are about to destroy 13064 * this device. They should clean all the things. 13065 * 13066 * Note that dev->reg_state stays at NETREG_REGISTERED. 13067 * This is wanted because this way 8021q and macvlan know 13068 * the device is just moving and can keep their slaves up. 13069 */ 13070 call_netdevice_notifiers(NETDEV_UNREGISTER, dev); 13071 rcu_barrier(); 13072 13073 new_nsid = peernet2id_alloc(dev_net(dev), net, GFP_KERNEL); 13074 13075 rtmsg_ifinfo_newnet(RTM_DELLINK, dev, ~0U, GFP_KERNEL, &new_nsid, 13076 new_ifindex); 13077 13078 /* 13079 * Flush the unicast and multicast chains 13080 */ 13081 dev_uc_flush(dev); 13082 dev_mc_flush(dev); 13083 13084 /* Send a netdev-removed uevent to the old namespace */ 13085 kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE); 13086 netdev_adjacent_del_links(dev); 13087 13088 /* Move per-net netdevice notifiers that are following the netdevice */ 13089 move_netdevice_notifiers_dev_net(dev, net); 13090 13091 /* Actually switch the network namespace */ 13092 netdev_lock(dev); 13093 dev_net_set(dev, net); 13094 netdev_unlock(dev); 13095 dev->ifindex = new_ifindex; 13096 13097 #ifdef CONFIG_DEBUG_NET_SMALL_RTNL 13098 unregister_netdevice_move_net(net_old, net, dev); 13099 #endif 13100 13101 if (new_name[0]) { 13102 /* Rename the netdev to prepared name */ 13103 write_seqlock_bh(&netdev_rename_lock); 13104 strscpy(dev->name, new_name, IFNAMSIZ); 13105 write_sequnlock_bh(&netdev_rename_lock); 13106 } 13107 13108 /* Fixup kobjects */ 13109 dev_set_uevent_suppress(&dev->dev, 1); 13110 err = device_rename(&dev->dev, dev->name); 13111 dev_set_uevent_suppress(&dev->dev, 0); 13112 WARN_ON(err); 13113 13114 /* Send a netdev-add uevent to the new namespace */ 13115 kobject_uevent(&dev->dev.kobj, KOBJ_ADD); 13116 netdev_adjacent_add_links(dev); 13117 13118 /* Adapt owner in case owning user namespace of target network 13119 * namespace is different from the original one. 13120 */ 13121 err = netdev_change_owner(dev, net_old, net); 13122 WARN_ON(err); 13123 13124 netdev_lock(dev); 13125 dev->moving_ns = false; 13126 if (!netdev_need_ops_lock(dev)) 13127 netdev_unlock(dev); 13128 13129 /* Add the device back in the hashes */ 13130 list_netdevice(dev); 13131 /* Notify protocols, that a new device appeared. */ 13132 call_netdevice_notifiers(NETDEV_REGISTER, dev); 13133 netdev_unlock_ops(dev); 13134 13135 /* 13136 * Prevent userspace races by waiting until the network 13137 * device is fully setup before sending notifications. 13138 */ 13139 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL, 0, NULL); 13140 13141 synchronize_net(); 13142 err = 0; 13143 out: 13144 return err; 13145 } 13146 13147 static int dev_cpu_dead(unsigned int oldcpu) 13148 { 13149 struct sk_buff **list_skb; 13150 struct sk_buff *skb; 13151 unsigned int cpu; 13152 int node; 13153 struct softnet_data *sd, *oldsd, *remsd = NULL; 13154 13155 local_irq_disable(); 13156 cpu = smp_processor_id(); 13157 sd = &per_cpu(softnet_data, cpu); 13158 oldsd = &per_cpu(softnet_data, oldcpu); 13159 13160 /* Find end of our completion_queue. */ 13161 list_skb = &sd->completion_queue; 13162 while (*list_skb) 13163 list_skb = &(*list_skb)->next; 13164 /* Append completion queue from offline CPU. */ 13165 *list_skb = oldsd->completion_queue; 13166 oldsd->completion_queue = NULL; 13167 13168 /* Append output queue from offline CPU. */ 13169 if (oldsd->output_queue) { 13170 *sd->output_queue_tailp = oldsd->output_queue; 13171 sd->output_queue_tailp = oldsd->output_queue_tailp; 13172 oldsd->output_queue = NULL; 13173 oldsd->output_queue_tailp = &oldsd->output_queue; 13174 } 13175 /* Append NAPI poll list from offline CPU, with one exception : 13176 * process_backlog() must be called by cpu owning percpu backlog. 13177 * We properly handle process_queue & input_pkt_queue later. 13178 */ 13179 while (!list_empty(&oldsd->poll_list)) { 13180 struct napi_struct *napi = list_first_entry(&oldsd->poll_list, 13181 struct napi_struct, 13182 poll_list); 13183 13184 list_del_init(&napi->poll_list); 13185 if (napi->poll == process_backlog) 13186 napi->state &= NAPIF_STATE_THREADED; 13187 else 13188 ____napi_schedule(sd, napi); 13189 } 13190 13191 raise_softirq_irqoff(NET_TX_SOFTIRQ); 13192 local_irq_enable(); 13193 13194 if (!use_backlog_threads()) { 13195 #ifdef CONFIG_RPS 13196 remsd = oldsd->rps_ipi_list; 13197 oldsd->rps_ipi_list = NULL; 13198 #endif 13199 /* send out pending IPI's on offline CPU */ 13200 net_rps_send_ipi(remsd); 13201 } 13202 13203 /* Process offline CPU's input_pkt_queue */ 13204 while ((skb = __skb_dequeue(&oldsd->process_queue))) { 13205 netif_rx(skb); 13206 rps_input_queue_head_incr(oldsd); 13207 } 13208 while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) { 13209 netif_rx(skb); 13210 rps_input_queue_head_incr(oldsd); 13211 } 13212 13213 for_each_node(node) 13214 skb_defer_node_flush(per_cpu_ptr(net_hotdata.skb_defer_nodes, 13215 oldcpu) + node); 13216 node = cpu_to_node(oldcpu); 13217 if (node_possible(node) && 13218 !cpumask_intersects(cpumask_of_node(node), cpu_online_mask)) { 13219 for_each_possible_cpu(cpu) 13220 skb_defer_node_flush(per_cpu_ptr(net_hotdata.skb_defer_nodes, 13221 cpu) + node); 13222 } 13223 13224 return 0; 13225 } 13226 13227 /** 13228 * netdev_increment_features - increment feature set by one 13229 * @all: current feature set 13230 * @one: new feature set 13231 * @mask: mask feature set 13232 * 13233 * Computes a new feature set after adding a device with feature set 13234 * @one to the master device with current feature set @all. Will not 13235 * enable anything that is off in @mask. Returns the new feature set. 13236 */ 13237 netdev_features_t netdev_increment_features(netdev_features_t all, 13238 netdev_features_t one, netdev_features_t mask) 13239 { 13240 if (mask & NETIF_F_HW_CSUM) 13241 mask |= NETIF_F_CSUM_MASK; 13242 mask |= NETIF_F_VLAN_CHALLENGED; 13243 13244 all |= one & (NETIF_F_ONE_FOR_ALL | NETIF_F_CSUM_MASK) & mask; 13245 all &= one | ~NETIF_F_ALL_FOR_ALL; 13246 13247 /* If one device supports hw checksumming, set for all. */ 13248 if (all & NETIF_F_HW_CSUM) 13249 all &= ~(NETIF_F_CSUM_MASK & ~NETIF_F_HW_CSUM); 13250 13251 return all; 13252 } 13253 EXPORT_SYMBOL(netdev_increment_features); 13254 13255 /** 13256 * netdev_compute_master_upper_features - compute feature from lowers 13257 * @dev: the upper device 13258 * @update_header: whether to update upper device's header_len/headroom/tailroom 13259 * 13260 * Recompute the upper device's feature based on all lower devices. 13261 */ 13262 void netdev_compute_master_upper_features(struct net_device *dev, bool update_header) 13263 { 13264 unsigned int dst_release_flag = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM; 13265 netdev_features_t gso_partial_features = MASTER_UPPER_DEV_GSO_PARTIAL_FEATURES; 13266 netdev_features_t xfrm_features = MASTER_UPPER_DEV_XFRM_FEATURES; 13267 netdev_features_t mpls_features = MASTER_UPPER_DEV_MPLS_FEATURES; 13268 netdev_features_t vlan_features = MASTER_UPPER_DEV_VLAN_FEATURES; 13269 netdev_features_t enc_features = MASTER_UPPER_DEV_ENC_FEATURES; 13270 unsigned short max_header_len = ETH_HLEN; 13271 unsigned int tso_max_size = TSO_MAX_SIZE; 13272 unsigned short max_headroom = 0; 13273 unsigned short max_tailroom = 0; 13274 u16 tso_max_segs = TSO_MAX_SEGS; 13275 struct net_device *lower_dev; 13276 struct list_head *iter; 13277 13278 mpls_features = netdev_base_features(mpls_features); 13279 vlan_features = netdev_base_features(vlan_features); 13280 enc_features = netdev_base_features(enc_features); 13281 13282 netdev_for_each_lower_dev(dev, lower_dev, iter) { 13283 gso_partial_features = netdev_increment_features(gso_partial_features, 13284 lower_dev->gso_partial_features, 13285 MASTER_UPPER_DEV_GSO_PARTIAL_FEATURES); 13286 13287 vlan_features = netdev_increment_features(vlan_features, 13288 lower_dev->vlan_features, 13289 MASTER_UPPER_DEV_VLAN_FEATURES); 13290 13291 enc_features = netdev_increment_features(enc_features, 13292 lower_dev->hw_enc_features, 13293 MASTER_UPPER_DEV_ENC_FEATURES); 13294 13295 if (IS_ENABLED(CONFIG_XFRM_OFFLOAD)) 13296 xfrm_features = netdev_increment_features(xfrm_features, 13297 lower_dev->hw_enc_features, 13298 MASTER_UPPER_DEV_XFRM_FEATURES); 13299 13300 mpls_features = netdev_increment_features(mpls_features, 13301 lower_dev->mpls_features, 13302 MASTER_UPPER_DEV_MPLS_FEATURES); 13303 13304 dst_release_flag &= lower_dev->priv_flags; 13305 13306 if (update_header) { 13307 max_header_len = max(max_header_len, lower_dev->hard_header_len); 13308 max_headroom = max(max_headroom, lower_dev->needed_headroom); 13309 max_tailroom = max(max_tailroom, lower_dev->needed_tailroom); 13310 } 13311 13312 tso_max_size = min(tso_max_size, lower_dev->tso_max_size); 13313 tso_max_segs = min(tso_max_segs, lower_dev->tso_max_segs); 13314 } 13315 13316 dev->gso_partial_features = gso_partial_features; 13317 dev->vlan_features = vlan_features; 13318 dev->hw_enc_features = enc_features | NETIF_F_GSO_ENCAP_ALL | 13319 NETIF_F_HW_VLAN_CTAG_TX | 13320 NETIF_F_HW_VLAN_STAG_TX; 13321 if (IS_ENABLED(CONFIG_XFRM_OFFLOAD)) 13322 dev->hw_enc_features |= xfrm_features; 13323 dev->mpls_features = mpls_features; 13324 13325 dev->priv_flags &= ~IFF_XMIT_DST_RELEASE; 13326 if ((dev->priv_flags & IFF_XMIT_DST_RELEASE_PERM) && 13327 dst_release_flag == (IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM)) 13328 dev->priv_flags |= IFF_XMIT_DST_RELEASE; 13329 13330 if (update_header) { 13331 dev->hard_header_len = max_header_len; 13332 dev->needed_headroom = max_headroom; 13333 dev->needed_tailroom = max_tailroom; 13334 } 13335 13336 netif_set_tso_max_segs(dev, tso_max_segs); 13337 netif_set_tso_max_size(dev, tso_max_size); 13338 13339 netdev_change_features(dev); 13340 } 13341 EXPORT_SYMBOL(netdev_compute_master_upper_features); 13342 13343 static struct hlist_head * __net_init netdev_create_hash(void) 13344 { 13345 int i; 13346 struct hlist_head *hash; 13347 13348 hash = kmalloc_objs(*hash, NETDEV_HASHENTRIES); 13349 if (hash != NULL) 13350 for (i = 0; i < NETDEV_HASHENTRIES; i++) 13351 INIT_HLIST_HEAD(&hash[i]); 13352 13353 return hash; 13354 } 13355 13356 /* Initialize per network namespace state */ 13357 static int __net_init netdev_init(struct net *net) 13358 { 13359 BUILD_BUG_ON(GRO_HASH_BUCKETS > 13360 BITS_PER_BYTE * sizeof_field(struct gro_node, bitmask)); 13361 13362 INIT_LIST_HEAD(&net->dev_base_head); 13363 13364 net->dev_name_head = netdev_create_hash(); 13365 if (net->dev_name_head == NULL) 13366 goto err_name; 13367 13368 net->dev_index_head = netdev_create_hash(); 13369 if (net->dev_index_head == NULL) 13370 goto err_idx; 13371 13372 xa_init_flags(&net->dev_by_index, XA_FLAGS_ALLOC1); 13373 13374 RAW_INIT_NOTIFIER_HEAD(&net->netdev_chain); 13375 13376 return 0; 13377 13378 err_idx: 13379 kfree(net->dev_name_head); 13380 err_name: 13381 return -ENOMEM; 13382 } 13383 13384 /** 13385 * netdev_drivername - network driver for the device 13386 * @dev: network device 13387 * 13388 * Determine network driver for device. 13389 */ 13390 const char *netdev_drivername(const struct net_device *dev) 13391 { 13392 const struct device_driver *driver; 13393 const struct device *parent; 13394 const char *empty = ""; 13395 13396 parent = dev->dev.parent; 13397 if (!parent) 13398 return empty; 13399 13400 driver = parent->driver; 13401 if (driver && driver->name) 13402 return driver->name; 13403 return empty; 13404 } 13405 13406 static void __netdev_printk(const char *level, const struct net_device *dev, 13407 struct va_format *vaf) 13408 { 13409 if (dev && dev->dev.parent) { 13410 dev_printk_emit(level[1] - '0', 13411 dev->dev.parent, 13412 "%s %s %s%s: %pV", 13413 dev_driver_string(dev->dev.parent), 13414 dev_name(dev->dev.parent), 13415 netdev_name(dev), netdev_reg_state(dev), 13416 vaf); 13417 } else if (dev) { 13418 printk("%s%s%s: %pV", 13419 level, netdev_name(dev), netdev_reg_state(dev), vaf); 13420 } else { 13421 printk("%s(NULL net_device): %pV", level, vaf); 13422 } 13423 } 13424 13425 void netdev_printk(const char *level, const struct net_device *dev, 13426 const char *format, ...) 13427 { 13428 struct va_format vaf; 13429 va_list args; 13430 13431 va_start(args, format); 13432 13433 vaf.fmt = format; 13434 vaf.va = &args; 13435 13436 __netdev_printk(level, dev, &vaf); 13437 13438 va_end(args); 13439 } 13440 EXPORT_SYMBOL(netdev_printk); 13441 13442 #define define_netdev_printk_level(func, level) \ 13443 void func(const struct net_device *dev, const char *fmt, ...) \ 13444 { \ 13445 struct va_format vaf; \ 13446 va_list args; \ 13447 \ 13448 va_start(args, fmt); \ 13449 \ 13450 vaf.fmt = fmt; \ 13451 vaf.va = &args; \ 13452 \ 13453 __netdev_printk(level, dev, &vaf); \ 13454 \ 13455 va_end(args); \ 13456 } \ 13457 EXPORT_SYMBOL(func); 13458 13459 define_netdev_printk_level(netdev_emerg, KERN_EMERG); 13460 define_netdev_printk_level(netdev_alert, KERN_ALERT); 13461 define_netdev_printk_level(netdev_crit, KERN_CRIT); 13462 define_netdev_printk_level(netdev_err, KERN_ERR); 13463 define_netdev_printk_level(netdev_warn, KERN_WARNING); 13464 define_netdev_printk_level(netdev_notice, KERN_NOTICE); 13465 define_netdev_printk_level(netdev_info, KERN_INFO); 13466 13467 static void __net_exit netdev_exit(struct net *net) 13468 { 13469 kfree(net->dev_name_head); 13470 kfree(net->dev_index_head); 13471 xa_destroy(&net->dev_by_index); 13472 if (net != &init_net) 13473 WARN_ON_ONCE(!list_empty(&net->dev_base_head)); 13474 } 13475 13476 static struct pernet_operations __net_initdata netdev_net_ops = { 13477 .init = netdev_init, 13478 .exit = netdev_exit, 13479 }; 13480 13481 static void __net_exit default_device_exit_net(struct net *net) 13482 { 13483 struct net_device *dev, *aux; 13484 /* 13485 * Push all migratable network devices back to the 13486 * initial network namespace 13487 */ 13488 13489 for_each_netdev_safe(net, dev, aux) { 13490 int err; 13491 char fb_name[IFNAMSIZ]; 13492 13493 /* Ignore unmoveable devices (i.e. loopback) */ 13494 if (dev->netns_immutable) 13495 continue; 13496 13497 /* Leave virtual devices for the generic cleanup */ 13498 if (dev->rtnl_link_ops && !dev->rtnl_link_ops->netns_refund) 13499 continue; 13500 13501 /* Push remaining network devices to init_net */ 13502 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex); 13503 if (netdev_name_in_use(&init_net, fb_name)) 13504 snprintf(fb_name, IFNAMSIZ, "dev%%d"); 13505 13506 err = dev_change_net_namespace(dev, &init_net, fb_name); 13507 if (err) { 13508 pr_emerg("%s: failed to move %s to init_net: %d\n", 13509 __func__, dev->name, err); 13510 BUG(); 13511 } 13512 } 13513 } 13514 13515 static bool __net_exit netns_dev_unreg_pending(const struct list_head *net_list) 13516 { 13517 struct net *net; 13518 13519 list_for_each_entry(net, net_list, exit_list) { 13520 if (atomic_read(&net->dev_unreg_pending)) 13521 return true; 13522 } 13523 return false; 13524 } 13525 13526 static void __net_exit default_device_exit_batch(struct list_head *net_list) 13527 { 13528 /* At exit all network devices most be removed from a network 13529 * namespace. Do this in the reverse order of registration. 13530 * Do this across as many network namespaces as possible to 13531 * improve batching efficiency. 13532 */ 13533 struct net_device *dev; 13534 struct net *net; 13535 LIST_HEAD(dev_kill_list); 13536 13537 rtnl_lock(); 13538 13539 __rtnl_net_lock(&init_net); 13540 13541 list_for_each_entry(net, net_list, exit_list) { 13542 __rtnl_net_lock(net); 13543 default_device_exit_net(net); 13544 __rtnl_net_unlock(net); 13545 13546 cond_resched(); 13547 } 13548 13549 __rtnl_net_unlock(&init_net); 13550 13551 list_for_each_entry(net, net_list, exit_list) { 13552 __rtnl_net_lock(net); 13553 13554 for_each_netdev_reverse(net, dev) { 13555 if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink) 13556 dev->rtnl_link_ops->dellink(dev, &dev_kill_list); 13557 else 13558 unregister_netdevice_queue(dev, &dev_kill_list); 13559 } 13560 13561 unregister_netdevice_queue_many_net(net, &dev_kill_list); 13562 __rtnl_net_unlock(net); 13563 } 13564 unregister_netdevice_many(&dev_kill_list); 13565 rtnl_unlock(); 13566 13567 /* Devices of these netns unregistered from another context 13568 * (e.g. a veth whose peer was deleted) can still be waiting 13569 * in netdev_run_todo() for their last references, while 13570 * NETDEV_UNREGISTER is rebroadcast for them. Wait for them 13571 * before pernet exit handlers tear down the state used by 13572 * netdevice notifiers, and before the netns is freed. 13573 * No new unregistration can start: all devices are unlisted. 13574 */ 13575 wait_event(netdev_unregistering_wq, !netns_dev_unreg_pending(net_list)); 13576 13577 rtnl_net_flush_workqueue(); 13578 } 13579 13580 static struct pernet_operations __net_initdata default_device_ops = { 13581 .exit_batch = default_device_exit_batch, 13582 }; 13583 13584 static void __init net_dev_struct_check(void) 13585 { 13586 /* TX read-mostly hotpath */ 13587 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, priv_flags_fast); 13588 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, netdev_ops); 13589 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, header_ops); 13590 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, _tx); 13591 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, real_num_tx_queues); 13592 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, gso_max_size); 13593 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, gso_ipv4_max_size); 13594 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, gso_max_segs); 13595 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, gso_partial_features); 13596 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, num_tc); 13597 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, mtu); 13598 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, needed_headroom); 13599 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, tc_to_txq); 13600 #ifdef CONFIG_XPS 13601 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, xps_maps); 13602 #endif 13603 #ifdef CONFIG_NETFILTER_EGRESS 13604 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, nf_hooks_egress); 13605 #endif 13606 #ifdef CONFIG_NET_XGRESS 13607 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, tcx_egress); 13608 #endif 13609 CACHELINE_ASSERT_GROUP_SIZE(struct net_device, net_device_read_tx, 160); 13610 13611 /* TXRX read-mostly hotpath */ 13612 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, lstats); 13613 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, state); 13614 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, flags); 13615 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, hard_header_len); 13616 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, features); 13617 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, ip6_ptr); 13618 CACHELINE_ASSERT_GROUP_SIZE(struct net_device, net_device_read_txrx, 46); 13619 13620 /* RX read-mostly hotpath */ 13621 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, ptype_specific); 13622 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, ifindex); 13623 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, real_num_rx_queues); 13624 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, _rx); 13625 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, gro_max_size); 13626 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, gro_ipv4_max_size); 13627 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, rx_handler); 13628 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, rx_handler_data); 13629 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, nd_net); 13630 #ifdef CONFIG_NETPOLL 13631 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, npinfo); 13632 #endif 13633 #ifdef CONFIG_NET_XGRESS 13634 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, tcx_ingress); 13635 #endif 13636 CACHELINE_ASSERT_GROUP_SIZE(struct net_device, net_device_read_rx, 92); 13637 } 13638 13639 /* 13640 * Initialize the DEV module. At boot time this walks the device list and 13641 * unhooks any devices that fail to initialise (normally hardware not 13642 * present) and leaves us with a valid list of present and active devices. 13643 * 13644 */ 13645 13646 /* We allocate 256 pages for each CPU if PAGE_SHIFT is 12 */ 13647 #define SYSTEM_PERCPU_PAGE_POOL_SIZE ((1 << 20) / PAGE_SIZE) 13648 13649 static int net_page_pool_create(int cpuid) 13650 { 13651 #if IS_ENABLED(CONFIG_PAGE_POOL) 13652 struct page_pool_params page_pool_params = { 13653 .pool_size = SYSTEM_PERCPU_PAGE_POOL_SIZE, 13654 .flags = PP_FLAG_SYSTEM_POOL, 13655 .nid = cpu_to_mem(cpuid), 13656 }; 13657 struct page_pool *pp_ptr; 13658 int err; 13659 13660 pp_ptr = page_pool_create_percpu(&page_pool_params, cpuid); 13661 if (IS_ERR(pp_ptr)) 13662 return -ENOMEM; 13663 13664 err = xdp_reg_page_pool(pp_ptr); 13665 if (err) { 13666 page_pool_destroy(pp_ptr); 13667 return err; 13668 } 13669 13670 per_cpu(system_page_pool.pool, cpuid) = pp_ptr; 13671 #endif 13672 return 0; 13673 } 13674 13675 static int backlog_napi_should_run(unsigned int cpu) 13676 { 13677 struct softnet_data *sd = per_cpu_ptr(&softnet_data, cpu); 13678 struct napi_struct *napi = &sd->backlog; 13679 13680 return test_bit(NAPI_STATE_SCHED_THREADED, &napi->state); 13681 } 13682 13683 static void run_backlog_napi(unsigned int cpu) 13684 { 13685 struct softnet_data *sd = per_cpu_ptr(&softnet_data, cpu); 13686 13687 napi_threaded_poll_loop(&sd->backlog, NULL); 13688 } 13689 13690 static void backlog_napi_setup(unsigned int cpu) 13691 { 13692 struct softnet_data *sd = per_cpu_ptr(&softnet_data, cpu); 13693 struct napi_struct *napi = &sd->backlog; 13694 13695 napi->thread = this_cpu_read(backlog_napi); 13696 set_bit(NAPI_STATE_THREADED, &napi->state); 13697 } 13698 13699 static struct smp_hotplug_thread backlog_threads = { 13700 .store = &backlog_napi, 13701 .thread_should_run = backlog_napi_should_run, 13702 .thread_fn = run_backlog_napi, 13703 .thread_comm = "backlog_napi/%u", 13704 .setup = backlog_napi_setup, 13705 }; 13706 13707 /* 13708 * This is called single threaded during boot, so no need 13709 * to take the rtnl semaphore. 13710 */ 13711 static int __init net_dev_init(void) 13712 { 13713 int i, rc = -ENOMEM; 13714 13715 BUG_ON(!dev_boot_phase); 13716 13717 net_dev_struct_check(); 13718 13719 if (dev_proc_init()) 13720 goto out; 13721 13722 if (netdev_kobject_init()) 13723 goto out; 13724 13725 for (i = 0; i < PTYPE_HASH_SIZE; i++) 13726 INIT_LIST_HEAD(&ptype_base[i]); 13727 13728 if (register_pernet_subsys(&netdev_net_ops)) 13729 goto out; 13730 13731 /* 13732 * Initialise the packet receive queues. 13733 */ 13734 13735 flush_backlogs_fallback = flush_backlogs_alloc(); 13736 if (!flush_backlogs_fallback) 13737 goto out; 13738 13739 for_each_possible_cpu(i) { 13740 struct softnet_data *sd = &per_cpu(softnet_data, i); 13741 13742 skb_queue_head_init(&sd->input_pkt_queue); 13743 skb_queue_head_init(&sd->process_queue); 13744 #ifdef CONFIG_XFRM_OFFLOAD 13745 skb_queue_head_init(&sd->xfrm_backlog); 13746 #endif 13747 INIT_LIST_HEAD(&sd->poll_list); 13748 sd->output_queue_tailp = &sd->output_queue; 13749 #ifdef CONFIG_RPS 13750 INIT_CSD(&sd->csd, rps_trigger_softirq, sd); 13751 sd->cpu = i; 13752 #endif 13753 INIT_CSD(&sd->defer_csd, trigger_rx_softirq, sd); 13754 13755 gro_init(&sd->backlog.gro); 13756 sd->backlog.poll = process_backlog; 13757 sd->backlog.weight = weight_p; 13758 INIT_LIST_HEAD(&sd->backlog.poll_list); 13759 13760 if (net_page_pool_create(i)) 13761 goto out; 13762 } 13763 net_hotdata.skb_defer_nodes = 13764 __alloc_percpu(sizeof(struct skb_defer_node) * nr_node_ids, 13765 __alignof__(struct skb_defer_node)); 13766 if (!net_hotdata.skb_defer_nodes) 13767 goto out; 13768 if (use_backlog_threads()) 13769 smpboot_register_percpu_thread(&backlog_threads); 13770 13771 dev_boot_phase = 0; 13772 13773 /* The loopback device is special if any other network devices 13774 * is present in a network namespace the loopback device must 13775 * be present. Since we now dynamically allocate and free the 13776 * loopback device ensure this invariant is maintained by 13777 * keeping the loopback device as the first device on the 13778 * list of network devices. Ensuring the loopback devices 13779 * is the first device that appears and the last network device 13780 * that disappears. 13781 */ 13782 if (register_pernet_device(&loopback_net_ops)) 13783 goto out; 13784 13785 if (register_pernet_device(&default_device_ops)) 13786 goto out; 13787 13788 open_softirq(NET_TX_SOFTIRQ, net_tx_action); 13789 open_softirq(NET_RX_SOFTIRQ, net_rx_action); 13790 13791 rc = cpuhp_setup_state_nocalls(CPUHP_NET_DEV_DEAD, "net/dev:dead", 13792 NULL, dev_cpu_dead); 13793 WARN_ON(rc < 0); 13794 rc = 0; 13795 13796 /* avoid static key IPIs to isolated CPUs */ 13797 if (housekeeping_enabled(HK_TYPE_MISC)) 13798 net_enable_timestamp(); 13799 out: 13800 if (rc < 0) { 13801 for_each_possible_cpu(i) { 13802 struct page_pool *pp_ptr; 13803 13804 pp_ptr = per_cpu(system_page_pool.pool, i); 13805 if (!pp_ptr) 13806 continue; 13807 13808 xdp_unreg_page_pool(pp_ptr); 13809 page_pool_destroy(pp_ptr); 13810 per_cpu(system_page_pool.pool, i) = NULL; 13811 } 13812 } 13813 13814 return rc; 13815 } 13816 13817 subsys_initcall(net_dev_init); 13818