1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * NET3 Protocol independent device support routines. 4 * 5 * Derived from the non IP parts of dev.c 1.0.19 6 * Authors: Ross Biro 7 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG> 8 * Mark Evans, <evansmp@uhura.aston.ac.uk> 9 * 10 * Additional Authors: 11 * Florian la Roche <rzsfl@rz.uni-sb.de> 12 * Alan Cox <gw4pts@gw4pts.ampr.org> 13 * David Hinds <dahinds@users.sourceforge.net> 14 * Alexey Kuznetsov <kuznet@ms2.inr.ac.ru> 15 * Adam Sulmicki <adam@cfar.umd.edu> 16 * Pekka Riikonen <priikone@poesidon.pspt.fi> 17 * 18 * Changes: 19 * D.J. Barrow : Fixed bug where dev->refcnt gets set 20 * to 2 if register_netdev gets called 21 * before net_dev_init & also removed a 22 * few lines of code in the process. 23 * Alan Cox : device private ioctl copies fields back. 24 * Alan Cox : Transmit queue code does relevant 25 * stunts to keep the queue safe. 26 * Alan Cox : Fixed double lock. 27 * Alan Cox : Fixed promisc NULL pointer trap 28 * ???????? : Support the full private ioctl range 29 * Alan Cox : Moved ioctl permission check into 30 * drivers 31 * Tim Kordas : SIOCADDMULTI/SIOCDELMULTI 32 * Alan Cox : 100 backlog just doesn't cut it when 33 * you start doing multicast video 8) 34 * Alan Cox : Rewrote net_bh and list manager. 35 * Alan Cox : Fix ETH_P_ALL echoback lengths. 36 * Alan Cox : Took out transmit every packet pass 37 * Saved a few bytes in the ioctl handler 38 * Alan Cox : Network driver sets packet type before 39 * calling netif_rx. Saves a function 40 * call a packet. 41 * Alan Cox : Hashed net_bh() 42 * Richard Kooijman: Timestamp fixes. 43 * Alan Cox : Wrong field in SIOCGIFDSTADDR 44 * Alan Cox : Device lock protection. 45 * Alan Cox : Fixed nasty side effect of device close 46 * changes. 47 * Rudi Cilibrasi : Pass the right thing to 48 * set_mac_address() 49 * Dave Miller : 32bit quantity for the device lock to 50 * make it work out on a Sparc. 51 * Bjorn Ekwall : Added KERNELD hack. 52 * Alan Cox : Cleaned up the backlog initialise. 53 * Craig Metz : SIOCGIFCONF fix if space for under 54 * 1 device. 55 * Thomas Bogendoerfer : Return ENODEV for dev_open, if there 56 * is no device open function. 57 * Andi Kleen : Fix error reporting for SIOCGIFCONF 58 * Michael Chastain : Fix signed/unsigned for SIOCGIFCONF 59 * Cyrus Durgin : Cleaned for KMOD 60 * Adam Sulmicki : Bug Fix : Network Device Unload 61 * A network device unload needs to purge 62 * the backlog queue. 63 * Paul Rusty Russell : SIOCSIFNAME 64 * Pekka Riikonen : Netdev boot-time settings code 65 * Andrew Morton : Make unregister_netdevice wait 66 * indefinitely on dev->refcnt 67 * J Hadi Salim : - Backlog queue sampling 68 * - netif_rx() feedback 69 */ 70 71 #include <linux/uaccess.h> 72 #include <linux/bitmap.h> 73 #include <linux/capability.h> 74 #include <linux/cpu.h> 75 #include <linux/types.h> 76 #include <linux/kernel.h> 77 #include <linux/hash.h> 78 #include <linux/slab.h> 79 #include <linux/sched.h> 80 #include <linux/sched/isolation.h> 81 #include <linux/sched/mm.h> 82 #include <linux/smpboot.h> 83 #include <linux/mutex.h> 84 #include <linux/rwsem.h> 85 #include <linux/string.h> 86 #include <linux/mm.h> 87 #include <linux/socket.h> 88 #include <linux/sockios.h> 89 #include <linux/errno.h> 90 #include <linux/interrupt.h> 91 #include <linux/if_ether.h> 92 #include <linux/netdevice.h> 93 #include <linux/etherdevice.h> 94 #include <linux/ethtool.h> 95 #include <linux/ethtool_netlink.h> 96 #include <linux/skbuff.h> 97 #include <linux/kthread.h> 98 #include <linux/bpf.h> 99 #include <linux/bpf_trace.h> 100 #include <net/net_namespace.h> 101 #include <net/sock.h> 102 #include <net/busy_poll.h> 103 #include <linux/rtnetlink.h> 104 #include <linux/stat.h> 105 #include <net/dsa.h> 106 #include <net/dst.h> 107 #include <net/dst_metadata.h> 108 #include <net/gro.h> 109 #include <net/netdev_queues.h> 110 #include <net/pkt_sched.h> 111 #include <net/pkt_cls.h> 112 #include <net/checksum.h> 113 #include <net/xfrm.h> 114 #include <net/tcx.h> 115 #include <linux/highmem.h> 116 #include <linux/init.h> 117 #include <linux/module.h> 118 #include <linux/netpoll.h> 119 #include <linux/rcupdate.h> 120 #include <linux/delay.h> 121 #include <net/iw_handler.h> 122 #include <asm/current.h> 123 #include <linux/audit.h> 124 #include <linux/dmaengine.h> 125 #include <linux/err.h> 126 #include <linux/ctype.h> 127 #include <linux/if_arp.h> 128 #include <linux/if_vlan.h> 129 #include <linux/ip.h> 130 #include <net/ip.h> 131 #include <net/mpls.h> 132 #include <net/route.h> 133 #include <linux/ipv6.h> 134 #include <net/ip6_route.h> 135 #include <linux/in.h> 136 #include <linux/jhash.h> 137 #include <linux/random.h> 138 #include <trace/events/napi.h> 139 #include <trace/events/net.h> 140 #include <trace/events/skb.h> 141 #include <trace/events/qdisc.h> 142 #include <trace/events/xdp.h> 143 #include <linux/inetdevice.h> 144 #include <linux/cpu_rmap.h> 145 #include <linux/static_key.h> 146 #include <linux/hashtable.h> 147 #include <linux/vmalloc.h> 148 #include <linux/if_macvlan.h> 149 #include <linux/errqueue.h> 150 #include <linux/hrtimer.h> 151 #include <linux/netfilter_netdev.h> 152 #include <linux/crash_dump.h> 153 #include <linux/sctp.h> 154 #include <net/udp_tunnel.h> 155 #include <linux/net_namespace.h> 156 #include <linux/indirect_call_wrapper.h> 157 #include <net/devlink.h> 158 #include <linux/pm_runtime.h> 159 #include <linux/prandom.h> 160 #include <linux/once_lite.h> 161 #include <net/netdev_lock.h> 162 #include <net/netdev_rx_queue.h> 163 #include <net/page_pool/types.h> 164 #include <net/page_pool/helpers.h> 165 #include <net/page_pool/memory_provider.h> 166 #include <net/rps.h> 167 #include <linux/phy_link_topology.h> 168 169 #include "dev.h" 170 #include "devmem.h" 171 #include "net-sysfs.h" 172 173 static DEFINE_SPINLOCK(ptype_lock); 174 struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly; 175 176 static int netif_rx_internal(struct sk_buff *skb); 177 static int call_netdevice_notifiers_extack(unsigned long val, 178 struct net_device *dev, 179 struct netlink_ext_ack *extack); 180 181 static DEFINE_MUTEX(ifalias_mutex); 182 183 /* protects napi_hash addition/deletion and napi_gen_id */ 184 static DEFINE_SPINLOCK(napi_hash_lock); 185 186 static unsigned int napi_gen_id = NR_CPUS; 187 static DEFINE_READ_MOSTLY_HASHTABLE(napi_hash, 8); 188 189 static inline void dev_base_seq_inc(struct net *net) 190 { 191 unsigned int val = net->dev_base_seq + 1; 192 193 WRITE_ONCE(net->dev_base_seq, val ?: 1); 194 } 195 196 static inline struct hlist_head *dev_name_hash(struct net *net, const char *name) 197 { 198 unsigned int hash = full_name_hash(net, name, strnlen(name, IFNAMSIZ)); 199 200 return &net->dev_name_head[hash_32(hash, NETDEV_HASHBITS)]; 201 } 202 203 static inline struct hlist_head *dev_index_hash(struct net *net, int ifindex) 204 { 205 return &net->dev_index_head[ifindex & (NETDEV_HASHENTRIES - 1)]; 206 } 207 208 #ifndef CONFIG_PREEMPT_RT 209 210 static DEFINE_STATIC_KEY_FALSE(use_backlog_threads_key); 211 212 static int __init setup_backlog_napi_threads(char *arg) 213 { 214 static_branch_enable(&use_backlog_threads_key); 215 return 0; 216 } 217 early_param("thread_backlog_napi", setup_backlog_napi_threads); 218 219 static bool use_backlog_threads(void) 220 { 221 return static_branch_unlikely(&use_backlog_threads_key); 222 } 223 224 #else 225 226 static bool use_backlog_threads(void) 227 { 228 return true; 229 } 230 231 #endif 232 233 static inline void backlog_lock_irq_save(struct softnet_data *sd, 234 unsigned long *flags) 235 { 236 if (IS_ENABLED(CONFIG_PREEMPT_RT)) { 237 spin_lock_irqsave(&sd->input_pkt_queue.lock, *flags); 238 } else { 239 local_irq_save(*flags); 240 if (IS_ENABLED(CONFIG_RPS) || use_backlog_threads()) 241 spin_lock(&sd->input_pkt_queue.lock); 242 } 243 } 244 245 static inline void backlog_lock_irq_disable(struct softnet_data *sd) 246 { 247 if (IS_ENABLED(CONFIG_RPS) || use_backlog_threads()) 248 spin_lock_irq(&sd->input_pkt_queue.lock); 249 else 250 local_irq_disable(); 251 } 252 253 static inline void backlog_unlock_irq_restore(struct softnet_data *sd, 254 unsigned long flags) 255 { 256 if (IS_ENABLED(CONFIG_PREEMPT_RT)) { 257 spin_unlock_irqrestore(&sd->input_pkt_queue.lock, flags); 258 } else { 259 if (IS_ENABLED(CONFIG_RPS) || use_backlog_threads()) 260 spin_unlock(&sd->input_pkt_queue.lock); 261 local_irq_restore(flags); 262 } 263 } 264 265 static inline void backlog_unlock_irq_enable(struct softnet_data *sd) 266 { 267 if (IS_ENABLED(CONFIG_RPS) || use_backlog_threads()) 268 spin_unlock_irq(&sd->input_pkt_queue.lock); 269 else 270 local_irq_enable(); 271 } 272 273 static struct netdev_name_node *netdev_name_node_alloc(struct net_device *dev, 274 const char *name) 275 { 276 struct netdev_name_node *name_node; 277 278 name_node = kmalloc_obj(*name_node); 279 if (!name_node) 280 return NULL; 281 INIT_HLIST_NODE(&name_node->hlist); 282 name_node->dev = dev; 283 name_node->name = name; 284 return name_node; 285 } 286 287 static struct netdev_name_node * 288 netdev_name_node_head_alloc(struct net_device *dev) 289 { 290 struct netdev_name_node *name_node; 291 292 name_node = netdev_name_node_alloc(dev, dev->name); 293 if (!name_node) 294 return NULL; 295 INIT_LIST_HEAD(&name_node->list); 296 return name_node; 297 } 298 299 static void netdev_name_node_free(struct netdev_name_node *name_node) 300 { 301 kfree(name_node); 302 } 303 304 static void netdev_name_node_add(struct net *net, 305 struct netdev_name_node *name_node) 306 { 307 hlist_add_head_rcu(&name_node->hlist, 308 dev_name_hash(net, name_node->name)); 309 } 310 311 static void netdev_name_node_del(struct netdev_name_node *name_node) 312 { 313 hlist_del_rcu(&name_node->hlist); 314 } 315 316 static struct netdev_name_node *netdev_name_node_lookup(struct net *net, 317 const char *name) 318 { 319 struct hlist_head *head = dev_name_hash(net, name); 320 struct netdev_name_node *name_node; 321 322 hlist_for_each_entry(name_node, head, hlist) 323 if (!strcmp(name_node->name, name)) 324 return name_node; 325 return NULL; 326 } 327 328 static struct netdev_name_node *netdev_name_node_lookup_rcu(struct net *net, 329 const char *name) 330 { 331 struct hlist_head *head = dev_name_hash(net, name); 332 struct netdev_name_node *name_node; 333 334 hlist_for_each_entry_rcu(name_node, head, hlist) 335 if (!strcmp(name_node->name, name)) 336 return name_node; 337 return NULL; 338 } 339 340 bool netdev_name_in_use(struct net *net, const char *name) 341 { 342 return netdev_name_node_lookup(net, name); 343 } 344 EXPORT_SYMBOL(netdev_name_in_use); 345 346 int netdev_name_node_alt_create(struct net_device *dev, const char *name) 347 { 348 struct netdev_name_node *name_node; 349 struct net *net = dev_net(dev); 350 351 name_node = netdev_name_node_lookup(net, name); 352 if (name_node) 353 return -EEXIST; 354 name_node = netdev_name_node_alloc(dev, name); 355 if (!name_node) 356 return -ENOMEM; 357 netdev_name_node_add(net, name_node); 358 /* The node that holds dev->name acts as a head of per-device list. */ 359 list_add_tail_rcu(&name_node->list, &dev->name_node->list); 360 361 return 0; 362 } 363 364 static void netdev_name_node_alt_free(struct rcu_head *head) 365 { 366 struct netdev_name_node *name_node = 367 container_of(head, struct netdev_name_node, rcu); 368 369 kfree(name_node->name); 370 netdev_name_node_free(name_node); 371 } 372 373 static void __netdev_name_node_alt_destroy(struct netdev_name_node *name_node) 374 { 375 netdev_name_node_del(name_node); 376 list_del_rcu(&name_node->list); 377 call_rcu(&name_node->rcu, netdev_name_node_alt_free); 378 } 379 380 int netdev_name_node_alt_destroy(struct net_device *dev, const char *name) 381 { 382 struct netdev_name_node *name_node; 383 struct net *net = dev_net(dev); 384 385 name_node = netdev_name_node_lookup(net, name); 386 if (!name_node) 387 return -ENOENT; 388 /* lookup might have found our primary name or a name belonging 389 * to another device. 390 */ 391 if (name_node == dev->name_node || name_node->dev != dev) 392 return -EINVAL; 393 394 __netdev_name_node_alt_destroy(name_node); 395 return 0; 396 } 397 398 static void netdev_name_node_alt_flush(struct net_device *dev) 399 { 400 struct netdev_name_node *name_node, *tmp; 401 402 list_for_each_entry_safe(name_node, tmp, &dev->name_node->list, list) { 403 list_del(&name_node->list); 404 netdev_name_node_alt_free(&name_node->rcu); 405 } 406 } 407 408 /* Device list insertion */ 409 static void list_netdevice(struct net_device *dev) 410 { 411 struct netdev_name_node *name_node; 412 struct net *net = dev_net(dev); 413 414 ASSERT_RTNL(); 415 416 list_add_tail_rcu(&dev->dev_list, &net->dev_base_head); 417 netdev_name_node_add(net, dev->name_node); 418 hlist_add_head_rcu(&dev->index_hlist, 419 dev_index_hash(net, dev->ifindex)); 420 421 netdev_for_each_altname(dev, name_node) 422 netdev_name_node_add(net, name_node); 423 424 /* We reserved the ifindex, this can't fail */ 425 WARN_ON(xa_store(&net->dev_by_index, dev->ifindex, dev, GFP_KERNEL)); 426 427 dev_base_seq_inc(net); 428 } 429 430 /* Device list removal 431 * caller must respect a RCU grace period before freeing/reusing dev 432 */ 433 static void unlist_netdevice(struct net_device *dev) 434 { 435 struct netdev_name_node *name_node; 436 struct net *net = dev_net(dev); 437 438 ASSERT_RTNL(); 439 440 xa_erase(&net->dev_by_index, dev->ifindex); 441 442 netdev_for_each_altname(dev, name_node) 443 netdev_name_node_del(name_node); 444 445 /* Unlink dev from the device chain */ 446 list_del_rcu(&dev->dev_list); 447 netdev_name_node_del(dev->name_node); 448 hlist_del_rcu(&dev->index_hlist); 449 450 dev_base_seq_inc(dev_net(dev)); 451 } 452 453 /* 454 * Our notifier list 455 */ 456 457 static RAW_NOTIFIER_HEAD(netdev_chain); 458 459 /* 460 * Device drivers call our routines to queue packets here. We empty the 461 * queue in the local softnet handler. 462 */ 463 464 DEFINE_PER_CPU_ALIGNED(struct softnet_data, softnet_data) = { 465 .process_queue_bh_lock = INIT_LOCAL_LOCK(process_queue_bh_lock), 466 }; 467 EXPORT_PER_CPU_SYMBOL(softnet_data); 468 469 /* Page_pool has a lockless array/stack to alloc/recycle pages. 470 * PP consumers must pay attention to run APIs in the appropriate context 471 * (e.g. NAPI context). 472 */ 473 DEFINE_PER_CPU(struct page_pool_bh, system_page_pool) = { 474 .bh_lock = INIT_LOCAL_LOCK(bh_lock), 475 }; 476 477 #ifdef CONFIG_LOCKDEP 478 /* 479 * register_netdevice() inits txq->_xmit_lock and sets lockdep class 480 * according to dev->type 481 */ 482 static const unsigned short netdev_lock_type[] = { 483 ARPHRD_NETROM, ARPHRD_ETHER, ARPHRD_EETHER, ARPHRD_AX25, 484 ARPHRD_PRONET, ARPHRD_CHAOS, ARPHRD_IEEE802, ARPHRD_ARCNET, 485 ARPHRD_APPLETLK, ARPHRD_DLCI, ARPHRD_ATM, ARPHRD_METRICOM, 486 ARPHRD_IEEE1394, ARPHRD_EUI64, ARPHRD_INFINIBAND, ARPHRD_SLIP, 487 ARPHRD_CSLIP, ARPHRD_SLIP6, ARPHRD_CSLIP6, ARPHRD_RSRVD, 488 ARPHRD_ADAPT, ARPHRD_ROSE, ARPHRD_X25, ARPHRD_HWX25, 489 ARPHRD_CAN, ARPHRD_MCTP, 490 ARPHRD_PPP, ARPHRD_CISCO, ARPHRD_LAPB, ARPHRD_DDCMP, 491 ARPHRD_RAWHDLC, ARPHRD_RAWIP, 492 ARPHRD_TUNNEL, ARPHRD_TUNNEL6, ARPHRD_FRAD, 493 ARPHRD_SKIP, ARPHRD_LOOPBACK, ARPHRD_LOCALTLK, ARPHRD_FDDI, 494 ARPHRD_BIF, ARPHRD_SIT, ARPHRD_IPDDP, ARPHRD_IPGRE, 495 ARPHRD_PIMREG, ARPHRD_HIPPI, ARPHRD_ASH, ARPHRD_ECONET, 496 ARPHRD_IRDA, ARPHRD_FCPP, ARPHRD_FCAL, ARPHRD_FCPL, 497 ARPHRD_FCFABRIC, ARPHRD_IEEE80211, ARPHRD_IEEE80211_PRISM, 498 ARPHRD_IEEE80211_RADIOTAP, 499 ARPHRD_IEEE802154, ARPHRD_IEEE802154_MONITOR, 500 ARPHRD_PHONET, ARPHRD_PHONET_PIPE, 501 ARPHRD_CAIF, ARPHRD_IP6GRE, ARPHRD_NETLINK, ARPHRD_6LOWPAN, 502 ARPHRD_VSOCKMON, 503 ARPHRD_VOID, ARPHRD_NONE}; 504 505 static const char *const netdev_lock_name[] = { 506 "_xmit_NETROM", "_xmit_ETHER", "_xmit_EETHER", "_xmit_AX25", 507 "_xmit_PRONET", "_xmit_CHAOS", "_xmit_IEEE802", "_xmit_ARCNET", 508 "_xmit_APPLETLK", "_xmit_DLCI", "_xmit_ATM", "_xmit_METRICOM", 509 "_xmit_IEEE1394", "_xmit_EUI64", "_xmit_INFINIBAND", "_xmit_SLIP", 510 "_xmit_CSLIP", "_xmit_SLIP6", "_xmit_CSLIP6", "_xmit_RSRVD", 511 "_xmit_ADAPT", "_xmit_ROSE", "_xmit_X25", "_xmit_HWX25", 512 "_xmit_CAN", "_xmit_MCTP", 513 "_xmit_PPP", "_xmit_CISCO", "_xmit_LAPB", "_xmit_DDCMP", 514 "_xmit_RAWHDLC", "_xmit_RAWIP", 515 "_xmit_TUNNEL", "_xmit_TUNNEL6", "_xmit_FRAD", 516 "_xmit_SKIP", "_xmit_LOOPBACK", "_xmit_LOCALTLK", "_xmit_FDDI", 517 "_xmit_BIF", "_xmit_SIT", "_xmit_IPDDP", "_xmit_IPGRE", 518 "_xmit_PIMREG", "_xmit_HIPPI", "_xmit_ASH", "_xmit_ECONET", 519 "_xmit_IRDA", "_xmit_FCPP", "_xmit_FCAL", "_xmit_FCPL", 520 "_xmit_FCFABRIC", "_xmit_IEEE80211", "_xmit_IEEE80211_PRISM", 521 "_xmit_IEEE80211_RADIOTAP", 522 "_xmit_IEEE802154", "_xmit_IEEE802154_MONITOR", 523 "_xmit_PHONET", "_xmit_PHONET_PIPE", 524 "_xmit_CAIF", "_xmit_IP6GRE", "_xmit_NETLINK", "_xmit_6LOWPAN", 525 "_xmit_VSOCKMON", 526 "_xmit_VOID", "_xmit_NONE"}; 527 528 static struct lock_class_key netdev_xmit_lock_key[ARRAY_SIZE(netdev_lock_type)]; 529 static struct lock_class_key netdev_addr_lock_key[ARRAY_SIZE(netdev_lock_type)]; 530 531 static inline unsigned short netdev_lock_pos(unsigned short dev_type) 532 { 533 int i; 534 535 for (i = 0; i < ARRAY_SIZE(netdev_lock_type); i++) 536 if (netdev_lock_type[i] == dev_type) 537 return i; 538 /* the last key is used by default */ 539 WARN_ONCE(1, "netdev_lock_pos() could not find dev_type=%u\n", dev_type); 540 return ARRAY_SIZE(netdev_lock_type) - 1; 541 } 542 543 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock, 544 unsigned short dev_type) 545 { 546 int i; 547 548 i = netdev_lock_pos(dev_type); 549 lockdep_set_class_and_name(lock, &netdev_xmit_lock_key[i], 550 netdev_lock_name[i]); 551 } 552 553 static inline void netdev_set_addr_lockdep_class(struct net_device *dev) 554 { 555 int i; 556 557 i = netdev_lock_pos(dev->type); 558 lockdep_set_class_and_name(&dev->addr_list_lock, 559 &netdev_addr_lock_key[i], 560 netdev_lock_name[i]); 561 } 562 #else 563 static inline void netdev_set_xmit_lockdep_class(spinlock_t *lock, 564 unsigned short dev_type) 565 { 566 } 567 568 static inline void netdev_set_addr_lockdep_class(struct net_device *dev) 569 { 570 } 571 #endif 572 573 #ifdef CONFIG_PROVE_LOCKING 574 static int netdev_lock_cmp_fn(const struct lockdep_map *a, 575 const struct lockdep_map *b) 576 { 577 if (a == b) 578 return 0; 579 580 /* @a and @b are of same lock class. 581 * cmp_fn won't be called for devices of different classes. 582 * 583 * For the same class only allow nesting under the protection 584 * of rtnl_lock. Note that we can't use lockdep_rtnl_is_held() 585 * here, it always answers UNKNOWN from within lockdep. 586 */ 587 return rtnl_is_locked() ? -1 : 1; 588 } 589 590 /* A virtual device can be locked before the physical device it leases 591 * queues from, see netdev_nl_queue_create_doit(). Keep the two kinds 592 * in separate classes so the dependency graph enforces the order; 593 * netdev_lock_cmp_fn() then only has to rule on same-class nesting. 594 * Other virtual devices stay in the default class. 595 */ 596 void netdev_set_instance_lock_class(struct net_device *dev) 597 { 598 static struct lock_class_key netdev_virt_instance_lock_key; 599 600 if (!netdev_can_create_queue(dev, NULL)) 601 return; 602 603 lockdep_set_class(&dev->lock, &netdev_virt_instance_lock_key); 604 lock_set_cmp_fn(&dev->lock, netdev_lock_cmp_fn, NULL); 605 } 606 EXPORT_SYMBOL_GPL(netdev_set_instance_lock_class); 607 #endif 608 609 /******************************************************************************* 610 * 611 * Protocol management and registration routines 612 * 613 *******************************************************************************/ 614 615 616 /* 617 * Add a protocol ID to the list. Now that the input handler is 618 * smarter we can dispense with all the messy stuff that used to be 619 * here. 620 * 621 * BEWARE!!! Protocol handlers, mangling input packets, 622 * MUST BE last in hash buckets and checking protocol handlers 623 * MUST start from promiscuous ptype_all chain in net_bh. 624 * It is true now, do not change it. 625 * Explanation follows: if protocol handler, mangling packet, will 626 * be the first on list, it is not able to sense, that packet 627 * is cloned and should be copied-on-write, so that it will 628 * change it and subsequent readers will get broken packet. 629 * --ANK (980803) 630 */ 631 632 static inline struct list_head *ptype_head(const struct packet_type *pt) 633 { 634 if (pt->type == htons(ETH_P_ALL)) { 635 if (!pt->af_packet_net && !pt->dev) 636 return NULL; 637 638 return pt->dev ? &pt->dev->ptype_all : 639 &pt->af_packet_net->ptype_all; 640 } 641 642 if (pt->dev) 643 return &pt->dev->ptype_specific; 644 645 return pt->af_packet_net ? &pt->af_packet_net->ptype_specific : 646 &ptype_base[ntohs(pt->type) & PTYPE_HASH_MASK]; 647 } 648 649 /** 650 * dev_add_pack - add packet handler 651 * @pt: packet type declaration 652 * 653 * Add a protocol handler to the networking stack. The passed &packet_type 654 * is linked into kernel lists and may not be freed until it has been 655 * removed from the kernel lists. 656 * 657 * This call does not sleep therefore it can not 658 * guarantee all CPU's that are in middle of receiving packets 659 * will see the new packet type (until the next received packet). 660 */ 661 662 void dev_add_pack(struct packet_type *pt) 663 { 664 struct list_head *head = ptype_head(pt); 665 666 if (WARN_ON_ONCE(!head)) 667 return; 668 669 spin_lock(&ptype_lock); 670 list_add_rcu(&pt->list, head); 671 spin_unlock(&ptype_lock); 672 } 673 EXPORT_SYMBOL(dev_add_pack); 674 675 /** 676 * __dev_remove_pack - remove packet handler 677 * @pt: packet type declaration 678 * 679 * Remove a protocol handler that was previously added to the kernel 680 * protocol handlers by dev_add_pack(). The passed &packet_type is removed 681 * from the kernel lists and can be freed or reused once this function 682 * returns. 683 * 684 * The packet type might still be in use by receivers 685 * and must not be freed until after all the CPU's have gone 686 * through a quiescent state. 687 */ 688 void __dev_remove_pack(struct packet_type *pt) 689 { 690 struct list_head *head = ptype_head(pt); 691 struct packet_type *pt1; 692 693 if (!head) 694 return; 695 696 spin_lock(&ptype_lock); 697 698 list_for_each_entry(pt1, head, list) { 699 if (pt == pt1) { 700 list_del_rcu(&pt->list); 701 goto out; 702 } 703 } 704 705 pr_warn("dev_remove_pack: %p not found\n", pt); 706 out: 707 spin_unlock(&ptype_lock); 708 } 709 EXPORT_SYMBOL(__dev_remove_pack); 710 711 /** 712 * dev_remove_pack - remove packet handler 713 * @pt: packet type declaration 714 * 715 * Remove a protocol handler that was previously added to the kernel 716 * protocol handlers by dev_add_pack(). The passed &packet_type is removed 717 * from the kernel lists and can be freed or reused once this function 718 * returns. 719 * 720 * This call sleeps to guarantee that no CPU is looking at the packet 721 * type after return. 722 */ 723 void dev_remove_pack(struct packet_type *pt) 724 { 725 __dev_remove_pack(pt); 726 727 synchronize_net(); 728 } 729 EXPORT_SYMBOL(dev_remove_pack); 730 731 732 /******************************************************************************* 733 * 734 * Device Interface Subroutines 735 * 736 *******************************************************************************/ 737 738 /** 739 * dev_get_iflink - get 'iflink' value of a interface 740 * @dev: targeted interface 741 * 742 * Indicates the ifindex the interface is linked to. 743 * Physical interfaces have the same 'ifindex' and 'iflink' values. 744 */ 745 746 int dev_get_iflink(const struct net_device *dev) 747 { 748 if (dev->netdev_ops && dev->netdev_ops->ndo_get_iflink) 749 return dev->netdev_ops->ndo_get_iflink(dev); 750 751 return READ_ONCE(dev->ifindex); 752 } 753 EXPORT_SYMBOL(dev_get_iflink); 754 755 /** 756 * dev_fill_metadata_dst - Retrieve tunnel egress information. 757 * @dev: targeted interface 758 * @skb: The packet. 759 * 760 * For better visibility of tunnel traffic OVS needs to retrieve 761 * egress tunnel information for a packet. Following API allows 762 * user to get this info. 763 */ 764 int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb) 765 { 766 struct ip_tunnel_info *info; 767 768 if (!dev->netdev_ops || !dev->netdev_ops->ndo_fill_metadata_dst) 769 return -EINVAL; 770 771 info = skb_tunnel_info_unclone(skb); 772 if (!info) 773 return -ENOMEM; 774 if (unlikely(!(info->mode & IP_TUNNEL_INFO_TX))) 775 return -EINVAL; 776 777 return dev->netdev_ops->ndo_fill_metadata_dst(dev, skb); 778 } 779 EXPORT_SYMBOL_GPL(dev_fill_metadata_dst); 780 781 static struct net_device_path *dev_fwd_path(struct net_device_path_stack *stack) 782 { 783 if (stack->num_paths + 1 > NET_DEVICE_PATH_STACK_MAX) 784 return NULL; 785 786 return &stack->path[stack->num_paths]; 787 } 788 789 void dev_fill_forward_path_release(struct net_device_path_stack *stack) 790 { 791 struct net_device_path *path; 792 int k; 793 794 if (stack->num_paths == 0) 795 return; 796 797 for (k = stack->num_paths - 1; k >= 0; k--) { 798 path = &stack->path[k]; 799 switch (path->type) { 800 case DEV_PATH_TUN: 801 dst_release(path->tun.dst); 802 break; 803 default: 804 break; 805 } 806 } 807 } 808 EXPORT_SYMBOL_GPL(dev_fill_forward_path_release); 809 810 int dev_fill_forward_path(struct net_device_path_ctx *ctx, 811 struct net_device_path_stack *stack) 812 { 813 const struct net_device *last_dev; 814 struct net_device_path *path; 815 int ret = 0; 816 817 stack->num_paths = 0; 818 while (ctx->dev && ctx->dev->netdev_ops->ndo_fill_forward_path) { 819 last_dev = ctx->dev; 820 path = dev_fwd_path(stack); 821 if (!path) 822 goto err_out; 823 824 memset(path, 0, sizeof(struct net_device_path)); 825 ret = ctx->dev->netdev_ops->ndo_fill_forward_path(ctx, path); 826 if (ret < 0) 827 goto err_out; 828 829 stack->num_paths++; 830 if (last_dev == ctx->dev) 831 goto err_out; 832 } 833 834 if (!ctx->dev) 835 return ret; 836 837 path = dev_fwd_path(stack); 838 if (!path) 839 goto err_out; 840 841 path->type = DEV_PATH_ETHERNET; 842 path->dev = ctx->dev; 843 stack->num_paths++; 844 845 return 0; 846 err_out: 847 dev_fill_forward_path_release(stack); 848 849 return -1; 850 } 851 EXPORT_SYMBOL_GPL(dev_fill_forward_path); 852 853 /* must be called under rcu_read_lock(), as we dont take a reference */ 854 static struct napi_struct *napi_by_id(unsigned int napi_id) 855 { 856 unsigned int hash = napi_id % HASH_SIZE(napi_hash); 857 struct napi_struct *napi; 858 859 hlist_for_each_entry_rcu(napi, &napi_hash[hash], napi_hash_node) 860 if (napi->napi_id == napi_id) 861 return napi; 862 863 return NULL; 864 } 865 866 /* must be called under rcu_read_lock(), as we dont take a reference */ 867 static struct napi_struct * 868 netdev_napi_by_id(struct net *net, unsigned int napi_id) 869 { 870 struct napi_struct *napi; 871 872 napi = napi_by_id(napi_id); 873 if (!napi) 874 return NULL; 875 876 if (WARN_ON_ONCE(!napi->dev)) 877 return NULL; 878 if (!net_eq(net, dev_net(napi->dev))) 879 return NULL; 880 881 return napi; 882 } 883 884 /** 885 * netdev_napi_by_id_lock() - find a device by NAPI ID and lock it 886 * @net: the applicable net namespace 887 * @napi_id: ID of a NAPI of a target device 888 * 889 * Find a NAPI instance with @napi_id. Lock its device. 890 * The device must be in %NETREG_REGISTERED state for lookup to succeed. 891 * netdev_unlock() must be called to release it. 892 * 893 * Return: pointer to NAPI, its device with lock held, NULL if not found. 894 */ 895 struct napi_struct * 896 netdev_napi_by_id_lock(struct net *net, unsigned int napi_id) 897 { 898 struct napi_struct *napi; 899 struct net_device *dev; 900 901 rcu_read_lock(); 902 napi = netdev_napi_by_id(net, napi_id); 903 if (!napi || READ_ONCE(napi->dev->reg_state) != NETREG_REGISTERED) { 904 rcu_read_unlock(); 905 return NULL; 906 } 907 908 dev = napi->dev; 909 dev_hold(dev); 910 rcu_read_unlock(); 911 912 dev = __netdev_put_lock(dev, net); 913 if (!dev) 914 return NULL; 915 916 rcu_read_lock(); 917 napi = netdev_napi_by_id(net, napi_id); 918 if (napi && napi->dev != dev) 919 napi = NULL; 920 rcu_read_unlock(); 921 922 if (!napi) 923 netdev_unlock(dev); 924 return napi; 925 } 926 927 /** 928 * __dev_get_by_name - find a device by its name 929 * @net: the applicable net namespace 930 * @name: name to find 931 * 932 * Find an interface by name. Must be called under RTNL semaphore. 933 * If the name is found a pointer to the device is returned. 934 * If the name is not found then %NULL is returned. The 935 * reference counters are not incremented so the caller must be 936 * careful with locks. 937 */ 938 939 struct net_device *__dev_get_by_name(struct net *net, const char *name) 940 { 941 struct netdev_name_node *node_name; 942 943 node_name = netdev_name_node_lookup(net, name); 944 return node_name ? node_name->dev : NULL; 945 } 946 EXPORT_SYMBOL(__dev_get_by_name); 947 948 /** 949 * dev_get_by_name_rcu - find a device by its name 950 * @net: the applicable net namespace 951 * @name: name to find 952 * 953 * Find an interface by name. 954 * If the name is found a pointer to the device is returned. 955 * If the name is not found then %NULL is returned. 956 * The reference counters are not incremented so the caller must be 957 * careful with locks. The caller must hold RCU lock. 958 */ 959 960 struct net_device *dev_get_by_name_rcu(struct net *net, const char *name) 961 { 962 struct netdev_name_node *node_name; 963 964 node_name = netdev_name_node_lookup_rcu(net, name); 965 return node_name ? node_name->dev : NULL; 966 } 967 EXPORT_SYMBOL(dev_get_by_name_rcu); 968 969 /* Deprecated for new users, call netdev_get_by_name() instead */ 970 struct net_device *dev_get_by_name(struct net *net, const char *name) 971 { 972 struct net_device *dev; 973 974 rcu_read_lock(); 975 dev = dev_get_by_name_rcu(net, name); 976 dev_hold(dev); 977 rcu_read_unlock(); 978 return dev; 979 } 980 EXPORT_SYMBOL(dev_get_by_name); 981 982 /** 983 * netdev_get_by_name() - find a device by its name 984 * @net: the applicable net namespace 985 * @name: name to find 986 * @tracker: tracking object for the acquired reference 987 * @gfp: allocation flags for the tracker 988 * 989 * Find an interface by name. This can be called from any 990 * context and does its own locking. The returned handle has 991 * the usage count incremented and the caller must use netdev_put() to 992 * release it when it is no longer needed. %NULL is returned if no 993 * matching device is found. 994 */ 995 struct net_device *netdev_get_by_name(struct net *net, const char *name, 996 netdevice_tracker *tracker, gfp_t gfp) 997 { 998 struct net_device *dev; 999 1000 dev = dev_get_by_name(net, name); 1001 if (dev) 1002 netdev_tracker_alloc(dev, tracker, gfp); 1003 return dev; 1004 } 1005 EXPORT_SYMBOL(netdev_get_by_name); 1006 1007 /** 1008 * __dev_get_by_index - find a device by its ifindex 1009 * @net: the applicable net namespace 1010 * @ifindex: index of device 1011 * 1012 * Search for an interface by index. Returns %NULL if the device 1013 * is not found or a pointer to the device. The device has not 1014 * had its reference counter increased so the caller must be careful 1015 * about locking. The caller must hold the RTNL semaphore. 1016 */ 1017 1018 struct net_device *__dev_get_by_index(struct net *net, int ifindex) 1019 { 1020 struct net_device *dev; 1021 struct hlist_head *head = dev_index_hash(net, ifindex); 1022 1023 hlist_for_each_entry(dev, head, index_hlist) 1024 if (dev->ifindex == ifindex) 1025 return dev; 1026 1027 return NULL; 1028 } 1029 EXPORT_SYMBOL(__dev_get_by_index); 1030 1031 /** 1032 * dev_get_by_index_rcu - find a device by its ifindex 1033 * @net: the applicable net namespace 1034 * @ifindex: index of device 1035 * 1036 * Search for an interface by index. Returns %NULL if the device 1037 * is not found or a pointer to the device. The device has not 1038 * had its reference counter increased so the caller must be careful 1039 * about locking. The caller must hold RCU lock. 1040 */ 1041 1042 struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex) 1043 { 1044 struct net_device *dev; 1045 struct hlist_head *head = dev_index_hash(net, ifindex); 1046 1047 hlist_for_each_entry_rcu(dev, head, index_hlist) 1048 if (dev->ifindex == ifindex) 1049 return dev; 1050 1051 return NULL; 1052 } 1053 EXPORT_SYMBOL(dev_get_by_index_rcu); 1054 1055 /* Deprecated for new users, call netdev_get_by_index() instead */ 1056 struct net_device *dev_get_by_index(struct net *net, int ifindex) 1057 { 1058 struct net_device *dev; 1059 1060 rcu_read_lock(); 1061 dev = dev_get_by_index_rcu(net, ifindex); 1062 dev_hold(dev); 1063 rcu_read_unlock(); 1064 return dev; 1065 } 1066 EXPORT_SYMBOL(dev_get_by_index); 1067 1068 /** 1069 * netdev_get_by_index() - find a device by its ifindex 1070 * @net: the applicable net namespace 1071 * @ifindex: index of device 1072 * @tracker: tracking object for the acquired reference 1073 * @gfp: allocation flags for the tracker 1074 * 1075 * Search for an interface by index. Returns NULL if the device 1076 * is not found or a pointer to the device. The device returned has 1077 * had a reference added and the pointer is safe until the user calls 1078 * netdev_put() to indicate they have finished with it. 1079 */ 1080 struct net_device *netdev_get_by_index(struct net *net, int ifindex, 1081 netdevice_tracker *tracker, gfp_t gfp) 1082 { 1083 struct net_device *dev; 1084 1085 dev = dev_get_by_index(net, ifindex); 1086 if (dev) 1087 netdev_tracker_alloc(dev, tracker, gfp); 1088 return dev; 1089 } 1090 EXPORT_SYMBOL(netdev_get_by_index); 1091 1092 /** 1093 * dev_get_by_napi_id - find a device by napi_id 1094 * @napi_id: ID of the NAPI struct 1095 * 1096 * Search for an interface by NAPI ID. Returns %NULL if the device 1097 * is not found or a pointer to the device. The device has not had 1098 * its reference counter increased so the caller must be careful 1099 * about locking. The caller must hold RCU lock. 1100 */ 1101 struct net_device *dev_get_by_napi_id(unsigned int napi_id) 1102 { 1103 struct napi_struct *napi; 1104 1105 WARN_ON_ONCE(!rcu_read_lock_held()); 1106 1107 if (!napi_id_valid(napi_id)) 1108 return NULL; 1109 1110 napi = napi_by_id(napi_id); 1111 1112 return napi ? napi->dev : NULL; 1113 } 1114 1115 /* Release the held reference on the net_device, and if the net_device 1116 * is still registered try to lock the instance lock. If device is being 1117 * unregistered NULL will be returned (but the reference has been released, 1118 * either way!) 1119 * 1120 * This helper is intended for locking net_device after it has been looked up 1121 * using a lockless lookup helper. Lock prevents the instance from going away. 1122 */ 1123 struct net_device * 1124 netdev_put_lock(struct net_device *dev, struct net *net, 1125 netdevice_tracker *tracker) 1126 { 1127 netdev_lock(dev); 1128 if (dev->reg_state > NETREG_REGISTERED || 1129 dev->moving_ns || !net_eq(dev_net(dev), net)) { 1130 netdev_unlock(dev); 1131 netdev_put(dev, tracker); 1132 return NULL; 1133 } 1134 netdev_put(dev, tracker); 1135 return dev; 1136 } 1137 1138 static struct net_device * 1139 __netdev_put_lock_ops_compat(struct net_device *dev, struct net *net) 1140 { 1141 netdev_lock_ops_compat(dev); 1142 if (dev->reg_state > NETREG_REGISTERED || 1143 dev->moving_ns || !net_eq(dev_net(dev), net)) { 1144 netdev_unlock_ops_compat(dev); 1145 dev_put(dev); 1146 return NULL; 1147 } 1148 dev_put(dev); 1149 return dev; 1150 } 1151 1152 /** 1153 * netdev_get_by_index_lock() - find a device by its ifindex 1154 * @net: the applicable net namespace 1155 * @ifindex: index of device 1156 * 1157 * Search for an interface by index. If a valid device 1158 * with @ifindex is found it will be returned with netdev->lock held. 1159 * netdev_unlock() must be called to release it. 1160 * 1161 * Return: pointer to a device with lock held, NULL if not found. 1162 */ 1163 struct net_device *netdev_get_by_index_lock(struct net *net, int ifindex) 1164 { 1165 struct net_device *dev; 1166 1167 dev = dev_get_by_index(net, ifindex); 1168 if (!dev) 1169 return NULL; 1170 1171 return __netdev_put_lock(dev, net); 1172 } 1173 1174 struct net_device * 1175 netdev_get_by_index_lock_ops_compat(struct net *net, int ifindex) 1176 { 1177 struct net_device *dev; 1178 1179 dev = dev_get_by_index(net, ifindex); 1180 if (!dev) 1181 return NULL; 1182 1183 return __netdev_put_lock_ops_compat(dev, net); 1184 } 1185 1186 struct net_device * 1187 netdev_xa_find_lock(struct net *net, struct net_device *dev, 1188 unsigned long *index) 1189 { 1190 if (dev) 1191 netdev_unlock(dev); 1192 1193 do { 1194 rcu_read_lock(); 1195 dev = xa_find(&net->dev_by_index, index, ULONG_MAX, XA_PRESENT); 1196 if (!dev) { 1197 rcu_read_unlock(); 1198 return NULL; 1199 } 1200 dev_hold(dev); 1201 rcu_read_unlock(); 1202 1203 dev = __netdev_put_lock(dev, net); 1204 if (dev) 1205 return dev; 1206 1207 (*index)++; 1208 } while (true); 1209 } 1210 1211 struct net_device * 1212 netdev_xa_find_lock_ops_compat(struct net *net, struct net_device *dev, 1213 unsigned long *index) 1214 { 1215 if (dev) 1216 netdev_unlock_ops_compat(dev); 1217 1218 do { 1219 rcu_read_lock(); 1220 dev = xa_find(&net->dev_by_index, index, ULONG_MAX, XA_PRESENT); 1221 if (!dev) { 1222 rcu_read_unlock(); 1223 return NULL; 1224 } 1225 dev_hold(dev); 1226 rcu_read_unlock(); 1227 1228 dev = __netdev_put_lock_ops_compat(dev, net); 1229 if (dev) 1230 return dev; 1231 1232 (*index)++; 1233 } while (true); 1234 } 1235 1236 static DEFINE_SEQLOCK(netdev_rename_lock); 1237 1238 void netdev_copy_name(struct net_device *dev, char *name) 1239 { 1240 unsigned int seq; 1241 1242 do { 1243 seq = read_seqbegin(&netdev_rename_lock); 1244 strscpy(name, dev->name, IFNAMSIZ); 1245 } while (read_seqretry(&netdev_rename_lock, seq)); 1246 } 1247 1248 /** 1249 * netdev_get_name - get a netdevice name, knowing its ifindex. 1250 * @net: network namespace 1251 * @name: a pointer to the buffer where the name will be stored. 1252 * @ifindex: the ifindex of the interface to get the name from. 1253 */ 1254 int netdev_get_name(struct net *net, char *name, int ifindex) 1255 { 1256 struct net_device *dev; 1257 int ret; 1258 1259 rcu_read_lock(); 1260 1261 dev = dev_get_by_index_rcu(net, ifindex); 1262 if (!dev) { 1263 ret = -ENODEV; 1264 goto out; 1265 } 1266 1267 netdev_copy_name(dev, name); 1268 1269 ret = 0; 1270 out: 1271 rcu_read_unlock(); 1272 return ret; 1273 } 1274 1275 static bool dev_addr_cmp(struct net_device *dev, unsigned short type, 1276 const char *ha) 1277 { 1278 return dev->type == type && !memcmp(dev->dev_addr, ha, dev->addr_len); 1279 } 1280 1281 /** 1282 * dev_getbyhwaddr_rcu - find a device by its hardware address 1283 * @net: the applicable net namespace 1284 * @type: media type of device 1285 * @ha: hardware address 1286 * 1287 * Search for an interface by MAC address. Returns NULL if the device 1288 * is not found or a pointer to the device. 1289 * The caller must hold RCU. 1290 * The returned device has not had its ref count increased 1291 * and the caller must therefore be careful about locking 1292 * 1293 */ 1294 1295 struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type, 1296 const char *ha) 1297 { 1298 struct net_device *dev; 1299 1300 for_each_netdev_rcu(net, dev) 1301 if (dev_addr_cmp(dev, type, ha)) 1302 return dev; 1303 1304 return NULL; 1305 } 1306 EXPORT_SYMBOL(dev_getbyhwaddr_rcu); 1307 1308 /** 1309 * dev_getbyhwaddr() - find a device by its hardware address 1310 * @net: the applicable net namespace 1311 * @type: media type of device 1312 * @ha: hardware address 1313 * 1314 * Similar to dev_getbyhwaddr_rcu(), but the owner needs to hold 1315 * rtnl_lock. 1316 * 1317 * Context: rtnl_lock() must be held. 1318 * Return: pointer to the net_device, or NULL if not found 1319 */ 1320 struct net_device *dev_getbyhwaddr(struct net *net, unsigned short type, 1321 const char *ha) 1322 { 1323 struct net_device *dev; 1324 1325 ASSERT_RTNL(); 1326 for_each_netdev(net, dev) 1327 if (dev_addr_cmp(dev, type, ha)) 1328 return dev; 1329 1330 return NULL; 1331 } 1332 EXPORT_SYMBOL(dev_getbyhwaddr); 1333 1334 struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type) 1335 { 1336 struct net_device *dev, *ret = NULL; 1337 1338 rcu_read_lock(); 1339 for_each_netdev_rcu(net, dev) 1340 if (dev->type == type) { 1341 dev_hold(dev); 1342 ret = dev; 1343 break; 1344 } 1345 rcu_read_unlock(); 1346 return ret; 1347 } 1348 EXPORT_SYMBOL(dev_getfirstbyhwtype); 1349 1350 /** 1351 * netdev_get_by_flags_rcu - find any device with given flags 1352 * @net: the applicable net namespace 1353 * @tracker: tracking object for the acquired reference 1354 * @if_flags: IFF_* values 1355 * @mask: bitmask of bits in if_flags to check 1356 * 1357 * Search for any interface with the given flags. 1358 * 1359 * Context: rcu_read_lock() must be held. 1360 * Returns: NULL if a device is not found or a pointer to the device. 1361 */ 1362 struct net_device *netdev_get_by_flags_rcu(struct net *net, netdevice_tracker *tracker, 1363 unsigned short if_flags, unsigned short mask) 1364 { 1365 struct net_device *dev; 1366 1367 for_each_netdev_rcu(net, dev) { 1368 if (((READ_ONCE(dev->flags) ^ if_flags) & mask) == 0) { 1369 netdev_hold(dev, tracker, GFP_ATOMIC); 1370 return dev; 1371 } 1372 } 1373 1374 return NULL; 1375 } 1376 1377 /** 1378 * dev_valid_name - check if name is okay for network device 1379 * @name: name string 1380 * 1381 * Network device names need to be valid file names to 1382 * allow sysfs to work. We also disallow any kind of 1383 * whitespace. 1384 */ 1385 bool dev_valid_name(const char *name) 1386 { 1387 if (*name == '\0') 1388 return false; 1389 if (strnlen(name, IFNAMSIZ) == IFNAMSIZ) 1390 return false; 1391 if (!strcmp(name, ".") || !strcmp(name, "..")) 1392 return false; 1393 1394 while (*name) { 1395 if (*name == '/' || *name == ':' || isspace(*name)) 1396 return false; 1397 name++; 1398 } 1399 return true; 1400 } 1401 EXPORT_SYMBOL(dev_valid_name); 1402 1403 /** 1404 * __dev_alloc_name - allocate a name for a device 1405 * @net: network namespace to allocate the device name in 1406 * @name: name format string 1407 * @res: result name string 1408 * 1409 * Passed a format string - eg "lt%d" it will try and find a suitable 1410 * id. It scans list of devices to build up a free map, then chooses 1411 * the first empty slot. The caller must hold the dev_base or rtnl lock 1412 * while allocating the name and adding the device in order to avoid 1413 * duplicates. 1414 * Limited to bits_per_byte * page size devices (ie 32K on most platforms). 1415 * Returns the number of the unit assigned or a negative errno code. 1416 */ 1417 1418 static int __dev_alloc_name(struct net *net, const char *name, char *res) 1419 { 1420 int i = 0; 1421 const char *p; 1422 const int max_netdevices = 8*PAGE_SIZE; 1423 unsigned long *inuse; 1424 struct net_device *d; 1425 char buf[IFNAMSIZ]; 1426 1427 /* Verify the string as this thing may have come from the user. 1428 * There must be one "%d" and no other "%" characters. 1429 */ 1430 p = strchr(name, '%'); 1431 if (!p || p[1] != 'd' || strchr(p + 2, '%')) 1432 return -EINVAL; 1433 1434 /* Use one page as a bit array of possible slots */ 1435 inuse = bitmap_zalloc(max_netdevices, GFP_ATOMIC); 1436 if (!inuse) 1437 return -ENOMEM; 1438 1439 for_each_netdev(net, d) { 1440 struct netdev_name_node *name_node; 1441 1442 netdev_for_each_altname(d, name_node) { 1443 if (!sscanf(name_node->name, name, &i)) 1444 continue; 1445 if (i < 0 || i >= max_netdevices) 1446 continue; 1447 1448 /* avoid cases where sscanf is not exact inverse of printf */ 1449 snprintf(buf, IFNAMSIZ, name, i); 1450 if (!strncmp(buf, name_node->name, IFNAMSIZ)) 1451 __set_bit(i, inuse); 1452 } 1453 if (!sscanf(d->name, name, &i)) 1454 continue; 1455 if (i < 0 || i >= max_netdevices) 1456 continue; 1457 1458 /* avoid cases where sscanf is not exact inverse of printf */ 1459 snprintf(buf, IFNAMSIZ, name, i); 1460 if (!strncmp(buf, d->name, IFNAMSIZ)) 1461 __set_bit(i, inuse); 1462 } 1463 1464 i = find_first_zero_bit(inuse, max_netdevices); 1465 bitmap_free(inuse); 1466 if (i == max_netdevices) 1467 return -ENFILE; 1468 1469 /* 'res' and 'name' could overlap, use 'buf' as an intermediate buffer */ 1470 strscpy(buf, name, IFNAMSIZ); 1471 snprintf(res, IFNAMSIZ, buf, i); 1472 return i; 1473 } 1474 1475 /* Returns negative errno or allocated unit id (see __dev_alloc_name()) */ 1476 static int dev_prep_valid_name(struct net *net, struct net_device *dev, 1477 const char *want_name, char *out_name, 1478 int dup_errno) 1479 { 1480 if (!dev_valid_name(want_name)) 1481 return -EINVAL; 1482 1483 if (strchr(want_name, '%')) 1484 return __dev_alloc_name(net, want_name, out_name); 1485 1486 if (netdev_name_in_use(net, want_name)) 1487 return -dup_errno; 1488 if (out_name != want_name) 1489 strscpy(out_name, want_name, IFNAMSIZ); 1490 return 0; 1491 } 1492 1493 /** 1494 * dev_alloc_name - allocate a name for a device 1495 * @dev: device 1496 * @name: name format string 1497 * 1498 * Passed a format string - eg "lt%d" it will try and find a suitable 1499 * id. It scans list of devices to build up a free map, then chooses 1500 * the first empty slot. The caller must hold the dev_base or rtnl lock 1501 * while allocating the name and adding the device in order to avoid 1502 * duplicates. 1503 * Limited to bits_per_byte * page size devices (ie 32K on most platforms). 1504 * Returns the number of the unit assigned or a negative errno code. 1505 */ 1506 1507 int dev_alloc_name(struct net_device *dev, const char *name) 1508 { 1509 return dev_prep_valid_name(dev_net(dev), dev, name, dev->name, ENFILE); 1510 } 1511 EXPORT_SYMBOL(dev_alloc_name); 1512 1513 static int dev_get_valid_name(struct net *net, struct net_device *dev, 1514 const char *name) 1515 { 1516 int ret; 1517 1518 ret = dev_prep_valid_name(net, dev, name, dev->name, EEXIST); 1519 return ret < 0 ? ret : 0; 1520 } 1521 1522 int netif_change_name(struct net_device *dev, const char *newname) 1523 { 1524 struct net *net = dev_net(dev); 1525 unsigned char old_assign_type; 1526 char oldname[IFNAMSIZ]; 1527 int err = 0; 1528 int ret; 1529 1530 ASSERT_RTNL_NET(net); 1531 1532 if (!strncmp(newname, dev->name, IFNAMSIZ)) 1533 return 0; 1534 1535 memcpy(oldname, dev->name, IFNAMSIZ); 1536 1537 write_seqlock_bh(&netdev_rename_lock); 1538 err = dev_get_valid_name(net, dev, newname); 1539 write_sequnlock_bh(&netdev_rename_lock); 1540 1541 if (err < 0) 1542 return err; 1543 1544 if (oldname[0] && !strchr(oldname, '%')) 1545 netdev_info(dev, "renamed from %s%s\n", oldname, 1546 dev->flags & IFF_UP ? " (while UP)" : ""); 1547 1548 old_assign_type = dev->name_assign_type; 1549 WRITE_ONCE(dev->name_assign_type, NET_NAME_RENAMED); 1550 1551 rollback: 1552 ret = device_rename(&dev->dev, dev->name); 1553 if (ret) { 1554 write_seqlock_bh(&netdev_rename_lock); 1555 memcpy(dev->name, oldname, IFNAMSIZ); 1556 write_sequnlock_bh(&netdev_rename_lock); 1557 WRITE_ONCE(dev->name_assign_type, old_assign_type); 1558 return ret; 1559 } 1560 1561 netdev_adjacent_rename_links(dev, oldname); 1562 1563 netdev_name_node_del(dev->name_node); 1564 1565 synchronize_net(); 1566 1567 netdev_name_node_add(net, dev->name_node); 1568 1569 ret = call_netdevice_notifiers(NETDEV_CHANGENAME, dev); 1570 ret = notifier_to_errno(ret); 1571 1572 if (ret) { 1573 /* err >= 0 after dev_alloc_name() or stores the first errno */ 1574 if (err >= 0) { 1575 err = ret; 1576 write_seqlock_bh(&netdev_rename_lock); 1577 memcpy(dev->name, oldname, IFNAMSIZ); 1578 write_sequnlock_bh(&netdev_rename_lock); 1579 memcpy(oldname, newname, IFNAMSIZ); 1580 WRITE_ONCE(dev->name_assign_type, old_assign_type); 1581 old_assign_type = NET_NAME_RENAMED; 1582 goto rollback; 1583 } else { 1584 netdev_err(dev, "name change rollback failed: %d\n", 1585 ret); 1586 } 1587 } 1588 1589 return err; 1590 } 1591 1592 int netif_set_alias(struct net_device *dev, const char *alias, size_t len) 1593 { 1594 struct dev_ifalias *new_alias = NULL; 1595 1596 if (len >= IFALIASZ) 1597 return -EINVAL; 1598 1599 if (len) { 1600 new_alias = kmalloc(sizeof(*new_alias) + len + 1, GFP_KERNEL); 1601 if (!new_alias) 1602 return -ENOMEM; 1603 1604 memcpy(new_alias->ifalias, alias, len); 1605 new_alias->ifalias[len] = 0; 1606 } 1607 1608 mutex_lock(&ifalias_mutex); 1609 new_alias = rcu_replace_pointer(dev->ifalias, new_alias, 1610 mutex_is_locked(&ifalias_mutex)); 1611 mutex_unlock(&ifalias_mutex); 1612 1613 if (new_alias) 1614 kfree_rcu(new_alias, rcuhead); 1615 1616 return len; 1617 } 1618 1619 /** 1620 * dev_get_alias - get ifalias of a device 1621 * @dev: device 1622 * @name: buffer to store name of ifalias 1623 * @len: size of buffer 1624 * 1625 * get ifalias for a device. Caller must make sure dev cannot go 1626 * away, e.g. rcu read lock or own a reference count to device. 1627 */ 1628 int dev_get_alias(const struct net_device *dev, char *name, size_t len) 1629 { 1630 const struct dev_ifalias *alias; 1631 int ret = 0; 1632 1633 rcu_read_lock(); 1634 alias = rcu_dereference(dev->ifalias); 1635 if (alias) 1636 ret = snprintf(name, len, "%s", alias->ifalias); 1637 rcu_read_unlock(); 1638 1639 return ret; 1640 } 1641 1642 /** 1643 * netdev_features_change - device changes features 1644 * @dev: device to cause notification 1645 * 1646 * Called to indicate a device has changed features. 1647 */ 1648 void netdev_features_change(struct net_device *dev) 1649 { 1650 call_netdevice_notifiers(NETDEV_FEAT_CHANGE, dev); 1651 } 1652 EXPORT_SYMBOL(netdev_features_change); 1653 1654 void netif_state_change(struct net_device *dev) 1655 { 1656 netdev_assert_locked_ops_compat_or_invisible(dev); 1657 1658 if (dev->flags & IFF_UP) { 1659 struct netdev_notifier_change_info change_info = { 1660 .info.dev = dev, 1661 }; 1662 1663 call_netdevice_notifiers_info(NETDEV_CHANGE, 1664 &change_info.info); 1665 rtmsg_ifinfo(RTM_NEWLINK, dev, 0, GFP_KERNEL, 0, NULL); 1666 } 1667 } 1668 1669 /** 1670 * __netdev_notify_peers - notify network peers about existence of @dev, 1671 * to be called when rtnl lock is already held. 1672 * @dev: network device 1673 * 1674 * Generate traffic such that interested network peers are aware of 1675 * @dev, such as by generating a gratuitous ARP. This may be used when 1676 * a device wants to inform the rest of the network about some sort of 1677 * reconfiguration such as a failover event or virtual machine 1678 * migration. 1679 */ 1680 void __netdev_notify_peers(struct net_device *dev) 1681 { 1682 ASSERT_RTNL(); 1683 call_netdevice_notifiers(NETDEV_NOTIFY_PEERS, dev); 1684 call_netdevice_notifiers(NETDEV_RESEND_IGMP, dev); 1685 } 1686 EXPORT_SYMBOL(__netdev_notify_peers); 1687 1688 /** 1689 * netdev_notify_peers - notify network peers about existence of @dev 1690 * @dev: network device 1691 * 1692 * Generate traffic such that interested network peers are aware of 1693 * @dev, such as by generating a gratuitous ARP. This may be used when 1694 * a device wants to inform the rest of the network about some sort of 1695 * reconfiguration such as a failover event or virtual machine 1696 * migration. 1697 */ 1698 void netdev_notify_peers(struct net_device *dev) 1699 { 1700 rtnl_lock(); 1701 __netdev_notify_peers(dev); 1702 rtnl_unlock(); 1703 } 1704 EXPORT_SYMBOL(netdev_notify_peers); 1705 1706 static int napi_threaded_poll(void *data); 1707 1708 static int napi_kthread_create(struct napi_struct *n) 1709 { 1710 int err = 0; 1711 1712 /* Create and wake up the kthread once to put it in 1713 * TASK_INTERRUPTIBLE mode to avoid the blocked task 1714 * warning and work with loadavg. 1715 */ 1716 n->thread = kthread_run(napi_threaded_poll, n, "napi/%s-%d", 1717 n->dev->name, n->napi_id); 1718 if (IS_ERR(n->thread)) { 1719 err = PTR_ERR(n->thread); 1720 pr_err("kthread_run failed with err %d\n", err); 1721 n->thread = NULL; 1722 } 1723 1724 return err; 1725 } 1726 1727 static int __dev_open(struct net_device *dev, struct netlink_ext_ack *extack) 1728 { 1729 const struct net_device_ops *ops = dev->netdev_ops; 1730 int ret; 1731 1732 ASSERT_RTNL(); 1733 dev_addr_check(dev); 1734 1735 if (!netif_device_present(dev)) { 1736 /* may be detached because parent is runtime-suspended */ 1737 if (dev->dev.parent) 1738 pm_runtime_resume(dev->dev.parent); 1739 if (!netif_device_present(dev)) 1740 return -ENODEV; 1741 } 1742 1743 /* Block netpoll from trying to do any rx path servicing. 1744 * If we don't do this there is a chance ndo_poll_controller 1745 * or ndo_poll may be running while we open the device 1746 */ 1747 netpoll_poll_disable(dev); 1748 1749 ret = call_netdevice_notifiers_extack(NETDEV_PRE_UP, dev, extack); 1750 ret = notifier_to_errno(ret); 1751 if (ret) 1752 return ret; 1753 1754 set_bit(__LINK_STATE_START, &dev->state); 1755 1756 netdev_assert_locked_ops_compat(dev); 1757 1758 if (ops->ndo_validate_addr) 1759 ret = ops->ndo_validate_addr(dev); 1760 1761 if (!ret && ops->ndo_open) 1762 ret = ops->ndo_open(dev); 1763 1764 netpoll_poll_enable(dev); 1765 1766 if (ret) 1767 clear_bit(__LINK_STATE_START, &dev->state); 1768 else { 1769 netif_set_up(dev, true); 1770 dev_set_rx_mode(dev); 1771 dev_activate(dev); 1772 add_device_randomness(dev->dev_addr, dev->addr_len); 1773 } 1774 1775 return ret; 1776 } 1777 1778 int netif_open(struct net_device *dev, struct netlink_ext_ack *extack) 1779 { 1780 int ret; 1781 1782 if (dev->flags & IFF_UP) 1783 return 0; 1784 1785 ret = __dev_open(dev, extack); 1786 if (ret < 0) 1787 return ret; 1788 1789 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP | IFF_RUNNING, GFP_KERNEL, 0, NULL); 1790 call_netdevice_notifiers(NETDEV_UP, dev); 1791 1792 return ret; 1793 } 1794 EXPORT_SYMBOL(netif_open); 1795 1796 static void __dev_close_many(struct list_head *head) 1797 { 1798 struct net_device *dev; 1799 1800 ASSERT_RTNL(); 1801 might_sleep(); 1802 1803 list_for_each_entry(dev, head, close_list) { 1804 /* Temporarily disable netpoll until the interface is down */ 1805 netpoll_poll_disable(dev); 1806 1807 call_netdevice_notifiers(NETDEV_GOING_DOWN, dev); 1808 1809 clear_bit(__LINK_STATE_START, &dev->state); 1810 1811 /* Synchronize to scheduled poll. We cannot touch poll list, it 1812 * can be even on different cpu. So just clear netif_running(). 1813 * 1814 * dev->stop() will invoke napi_disable() on all of it's 1815 * napi_struct instances on this device. 1816 */ 1817 smp_mb__after_atomic(); /* Commit netif_running(). */ 1818 } 1819 1820 dev_deactivate_many(head, true); 1821 1822 list_for_each_entry(dev, head, close_list) { 1823 const struct net_device_ops *ops = dev->netdev_ops; 1824 1825 /* 1826 * Call the device specific close. This cannot fail. 1827 * Only if device is UP 1828 * 1829 * We allow it to be called even after a DETACH hot-plug 1830 * event. 1831 */ 1832 1833 netdev_assert_locked_ops_compat(dev); 1834 1835 if (ops->ndo_stop) 1836 ops->ndo_stop(dev); 1837 1838 netif_rx_mode_cancel_retry(dev); 1839 netif_set_up(dev, false); 1840 netpoll_poll_enable(dev); 1841 } 1842 } 1843 1844 static void __dev_close(struct net_device *dev) 1845 { 1846 LIST_HEAD(single); 1847 1848 list_add(&dev->close_list, &single); 1849 __dev_close_many(&single); 1850 list_del(&single); 1851 } 1852 1853 void netif_close_many(struct list_head *head, bool unlink) 1854 { 1855 struct net_device *dev, *tmp; 1856 1857 /* Remove the devices that don't need to be closed */ 1858 list_for_each_entry_safe(dev, tmp, head, close_list) 1859 if (!(dev->flags & IFF_UP)) 1860 list_del_init(&dev->close_list); 1861 1862 __dev_close_many(head); 1863 1864 list_for_each_entry_safe(dev, tmp, head, close_list) { 1865 netdev_assert_locked_ops_compat(dev); 1866 rtmsg_ifinfo(RTM_NEWLINK, dev, IFF_UP | IFF_RUNNING, GFP_KERNEL, 0, NULL); 1867 call_netdevice_notifiers(NETDEV_DOWN, dev); 1868 if (unlink) 1869 list_del_init(&dev->close_list); 1870 } 1871 } 1872 EXPORT_SYMBOL_NS_GPL(netif_close_many, "NETDEV_INTERNAL"); 1873 1874 void netif_close(struct net_device *dev) 1875 { 1876 if (dev->flags & IFF_UP) { 1877 LIST_HEAD(single); 1878 1879 list_add(&dev->close_list, &single); 1880 netif_close_many(&single, true); 1881 list_del(&single); 1882 } 1883 } 1884 EXPORT_SYMBOL(netif_close); 1885 1886 void netif_disable_lro(struct net_device *dev) 1887 { 1888 struct net_device *lower_dev; 1889 struct list_head *iter; 1890 1891 dev->wanted_features &= ~NETIF_F_LRO; 1892 netdev_update_features(dev); 1893 1894 if (unlikely(dev->features & NETIF_F_LRO)) 1895 netdev_WARN(dev, "failed to disable LRO!\n"); 1896 1897 netdev_for_each_lower_dev(dev, lower_dev, iter) { 1898 netdev_lock_ops(lower_dev); 1899 netif_disable_lro(lower_dev); 1900 netdev_unlock_ops(lower_dev); 1901 } 1902 } 1903 1904 /** 1905 * dev_disable_gro_hw - disable HW Generic Receive Offload on a device 1906 * @dev: device 1907 * 1908 * Disable HW Generic Receive Offload (GRO_HW) on a net device. Must be 1909 * called under RTNL. This is needed if Generic XDP is installed on 1910 * the device. 1911 */ 1912 static void dev_disable_gro_hw(struct net_device *dev) 1913 { 1914 dev->wanted_features &= ~NETIF_F_GRO_HW; 1915 netdev_update_features(dev); 1916 1917 if (unlikely(dev->features & NETIF_F_GRO_HW)) 1918 netdev_WARN(dev, "failed to disable GRO_HW!\n"); 1919 } 1920 1921 const char *netdev_cmd_to_name(enum netdev_cmd cmd) 1922 { 1923 #define N(val) \ 1924 case NETDEV_##val: \ 1925 return "NETDEV_" __stringify(val); 1926 switch (cmd) { 1927 N(UP) N(DOWN) N(REBOOT) N(CHANGE) N(REGISTER) N(UNREGISTER) 1928 N(CHANGEMTU) N(CHANGEADDR) N(GOING_DOWN) N(CHANGENAME) N(FEAT_CHANGE) 1929 N(BONDING_FAILOVER) N(PRE_UP) N(PRE_TYPE_CHANGE) N(POST_TYPE_CHANGE) 1930 N(POST_INIT) N(PRE_UNINIT) N(RELEASE) N(NOTIFY_PEERS) N(JOIN) 1931 N(CHANGEUPPER) N(RESEND_IGMP) N(PRECHANGEMTU) N(CHANGEINFODATA) 1932 N(BONDING_INFO) N(PRECHANGEUPPER) N(CHANGELOWERSTATE) 1933 N(UDP_TUNNEL_PUSH_INFO) N(UDP_TUNNEL_DROP_INFO) N(CHANGE_TX_QUEUE_LEN) 1934 N(CVLAN_FILTER_PUSH_INFO) N(CVLAN_FILTER_DROP_INFO) 1935 N(SVLAN_FILTER_PUSH_INFO) N(SVLAN_FILTER_DROP_INFO) 1936 N(PRE_CHANGEADDR) N(OFFLOAD_XSTATS_ENABLE) N(OFFLOAD_XSTATS_DISABLE) 1937 N(OFFLOAD_XSTATS_REPORT_USED) N(OFFLOAD_XSTATS_REPORT_DELTA) 1938 N(XDP_FEAT_CHANGE) 1939 } 1940 #undef N 1941 return "UNKNOWN_NETDEV_EVENT"; 1942 } 1943 EXPORT_SYMBOL_GPL(netdev_cmd_to_name); 1944 1945 static int call_netdevice_notifier(struct notifier_block *nb, unsigned long val, 1946 struct net_device *dev) 1947 { 1948 struct netdev_notifier_info info = { 1949 .dev = dev, 1950 }; 1951 1952 return nb->notifier_call(nb, val, &info); 1953 } 1954 1955 static int call_netdevice_register_notifiers(struct notifier_block *nb, 1956 struct net_device *dev) 1957 { 1958 int err; 1959 1960 err = call_netdevice_notifier(nb, NETDEV_REGISTER, dev); 1961 err = notifier_to_errno(err); 1962 if (err) 1963 return err; 1964 1965 if (!(dev->flags & IFF_UP)) 1966 return 0; 1967 1968 call_netdevice_notifier(nb, NETDEV_UP, dev); 1969 return 0; 1970 } 1971 1972 static void call_netdevice_unregister_notifiers(struct notifier_block *nb, 1973 struct net_device *dev) 1974 { 1975 if (dev->flags & IFF_UP) { 1976 netdev_lock_ops(dev); 1977 call_netdevice_notifier(nb, NETDEV_GOING_DOWN, 1978 dev); 1979 call_netdevice_notifier(nb, NETDEV_DOWN, dev); 1980 netdev_unlock_ops(dev); 1981 } 1982 call_netdevice_notifier(nb, NETDEV_UNREGISTER, dev); 1983 } 1984 1985 static int call_netdevice_register_net_notifiers(struct notifier_block *nb, 1986 struct net *net) 1987 { 1988 struct net_device *dev; 1989 int err; 1990 1991 for_each_netdev(net, dev) { 1992 netdev_lock_ops(dev); 1993 err = call_netdevice_register_notifiers(nb, dev); 1994 netdev_unlock_ops(dev); 1995 if (err) 1996 goto rollback; 1997 } 1998 return 0; 1999 2000 rollback: 2001 for_each_netdev_continue_reverse(net, dev) 2002 call_netdevice_unregister_notifiers(nb, dev); 2003 return err; 2004 } 2005 2006 static void call_netdevice_unregister_net_notifiers(struct notifier_block *nb, 2007 struct net *net) 2008 { 2009 struct net_device *dev; 2010 2011 for_each_netdev(net, dev) 2012 call_netdevice_unregister_notifiers(nb, dev); 2013 } 2014 2015 static int dev_boot_phase = 1; 2016 2017 /** 2018 * register_netdevice_notifier - register a network notifier block 2019 * @nb: notifier 2020 * 2021 * Register a notifier to be called when network device events occur. 2022 * The notifier passed is linked into the kernel structures and must 2023 * not be reused until it has been unregistered. A negative errno code 2024 * is returned on a failure. 2025 * 2026 * When registered all registration and up events are replayed 2027 * to the new notifier to allow device to have a race free 2028 * view of the network device list. 2029 */ 2030 2031 int register_netdevice_notifier(struct notifier_block *nb) 2032 { 2033 struct net *net; 2034 int err; 2035 2036 /* Close race with setup_net() and cleanup_net() */ 2037 down_write(&pernet_ops_rwsem); 2038 2039 /* When RTNL is removed, we need protection for netdev_chain. */ 2040 rtnl_lock(); 2041 2042 err = raw_notifier_chain_register(&netdev_chain, nb); 2043 if (err) 2044 goto unlock; 2045 if (dev_boot_phase) 2046 goto unlock; 2047 for_each_net(net) { 2048 __rtnl_net_lock(net); 2049 err = call_netdevice_register_net_notifiers(nb, net); 2050 __rtnl_net_unlock(net); 2051 if (err) 2052 goto rollback; 2053 } 2054 2055 unlock: 2056 rtnl_unlock(); 2057 up_write(&pernet_ops_rwsem); 2058 return err; 2059 2060 rollback: 2061 for_each_net_continue_reverse(net) { 2062 __rtnl_net_lock(net); 2063 call_netdevice_unregister_net_notifiers(nb, net); 2064 __rtnl_net_unlock(net); 2065 } 2066 2067 raw_notifier_chain_unregister(&netdev_chain, nb); 2068 goto unlock; 2069 } 2070 EXPORT_SYMBOL(register_netdevice_notifier); 2071 2072 /** 2073 * unregister_netdevice_notifier - unregister a network notifier block 2074 * @nb: notifier 2075 * 2076 * Unregister a notifier previously registered by 2077 * register_netdevice_notifier(). The notifier is unlinked into the 2078 * kernel structures and may then be reused. A negative errno code 2079 * is returned on a failure. 2080 * 2081 * After unregistering unregister and down device events are synthesized 2082 * for all devices on the device list to the removed notifier to remove 2083 * the need for special case cleanup code. 2084 */ 2085 2086 int unregister_netdevice_notifier(struct notifier_block *nb) 2087 { 2088 struct net *net; 2089 int err; 2090 2091 /* Close race with setup_net() and cleanup_net() */ 2092 down_write(&pernet_ops_rwsem); 2093 rtnl_lock(); 2094 err = raw_notifier_chain_unregister(&netdev_chain, nb); 2095 if (err) 2096 goto unlock; 2097 2098 for_each_net(net) { 2099 __rtnl_net_lock(net); 2100 call_netdevice_unregister_net_notifiers(nb, net); 2101 __rtnl_net_unlock(net); 2102 } 2103 2104 unlock: 2105 rtnl_unlock(); 2106 up_write(&pernet_ops_rwsem); 2107 return err; 2108 } 2109 EXPORT_SYMBOL(unregister_netdevice_notifier); 2110 2111 static int __register_netdevice_notifier_net(struct net *net, 2112 struct notifier_block *nb, 2113 bool ignore_call_fail) 2114 { 2115 int err; 2116 2117 err = raw_notifier_chain_register(&net->netdev_chain, nb); 2118 if (err) 2119 return err; 2120 if (dev_boot_phase) 2121 return 0; 2122 2123 err = call_netdevice_register_net_notifiers(nb, net); 2124 if (err && !ignore_call_fail) 2125 goto chain_unregister; 2126 2127 return 0; 2128 2129 chain_unregister: 2130 raw_notifier_chain_unregister(&net->netdev_chain, nb); 2131 return err; 2132 } 2133 2134 static int __unregister_netdevice_notifier_net(struct net *net, 2135 struct notifier_block *nb) 2136 { 2137 int err; 2138 2139 err = raw_notifier_chain_unregister(&net->netdev_chain, nb); 2140 if (err) 2141 return err; 2142 2143 call_netdevice_unregister_net_notifiers(nb, net); 2144 return 0; 2145 } 2146 2147 /** 2148 * register_netdevice_notifier_net - register a per-netns network notifier block 2149 * @net: network namespace 2150 * @nb: notifier 2151 * 2152 * Register a notifier to be called when network device events occur. 2153 * The notifier passed is linked into the kernel structures and must 2154 * not be reused until it has been unregistered. A negative errno code 2155 * is returned on a failure. 2156 * 2157 * When registered all registration and up events are replayed 2158 * to the new notifier to allow device to have a race free 2159 * view of the network device list. 2160 */ 2161 2162 int register_netdevice_notifier_net(struct net *net, struct notifier_block *nb) 2163 { 2164 int err; 2165 2166 rtnl_net_lock(net); 2167 err = __register_netdevice_notifier_net(net, nb, false); 2168 rtnl_net_unlock(net); 2169 2170 return err; 2171 } 2172 EXPORT_SYMBOL(register_netdevice_notifier_net); 2173 2174 /** 2175 * unregister_netdevice_notifier_net - unregister a per-netns 2176 * network notifier block 2177 * @net: network namespace 2178 * @nb: notifier 2179 * 2180 * Unregister a notifier previously registered by 2181 * register_netdevice_notifier_net(). The notifier is unlinked from the 2182 * kernel structures and may then be reused. A negative errno code 2183 * is returned on a failure. 2184 * 2185 * After unregistering unregister and down device events are synthesized 2186 * for all devices on the device list to the removed notifier to remove 2187 * the need for special case cleanup code. 2188 */ 2189 2190 int unregister_netdevice_notifier_net(struct net *net, 2191 struct notifier_block *nb) 2192 { 2193 int err; 2194 2195 rtnl_net_lock(net); 2196 err = __unregister_netdevice_notifier_net(net, nb); 2197 rtnl_net_unlock(net); 2198 2199 return err; 2200 } 2201 EXPORT_SYMBOL(unregister_netdevice_notifier_net); 2202 2203 static void __move_netdevice_notifier_net(struct net *src_net, 2204 struct net *dst_net, 2205 struct notifier_block *nb) 2206 { 2207 __unregister_netdevice_notifier_net(src_net, nb); 2208 __register_netdevice_notifier_net(dst_net, nb, true); 2209 } 2210 2211 static void rtnl_net_dev_lock(struct net_device *dev) 2212 { 2213 bool again; 2214 2215 do { 2216 struct net *net; 2217 2218 again = false; 2219 2220 /* netns might be being dismantled. */ 2221 rcu_read_lock(); 2222 net = dev_net_rcu(dev); 2223 net_passive_inc(net); 2224 rcu_read_unlock(); 2225 2226 rtnl_net_lock(net); 2227 2228 #ifdef CONFIG_NET_NS 2229 /* dev might have been moved to another netns. */ 2230 if (!net_eq(net, rcu_access_pointer(dev->nd_net.net))) { 2231 rtnl_net_unlock(net); 2232 net_passive_dec(net); 2233 again = true; 2234 } 2235 #endif 2236 } while (again); 2237 } 2238 2239 static void rtnl_net_dev_unlock(struct net_device *dev) 2240 { 2241 struct net *net = dev_net(dev); 2242 2243 rtnl_net_unlock(net); 2244 net_passive_dec(net); 2245 } 2246 2247 int register_netdevice_notifier_dev_net(struct net_device *dev, 2248 struct notifier_block *nb, 2249 struct netdev_net_notifier *nn) 2250 { 2251 int err; 2252 2253 rtnl_net_dev_lock(dev); 2254 err = __register_netdevice_notifier_net(dev_net(dev), nb, false); 2255 if (!err) { 2256 nn->nb = nb; 2257 list_add(&nn->list, &dev->net_notifier_list); 2258 } 2259 rtnl_net_dev_unlock(dev); 2260 2261 return err; 2262 } 2263 EXPORT_SYMBOL(register_netdevice_notifier_dev_net); 2264 2265 int unregister_netdevice_notifier_dev_net(struct net_device *dev, 2266 struct notifier_block *nb, 2267 struct netdev_net_notifier *nn) 2268 { 2269 int err; 2270 2271 rtnl_net_dev_lock(dev); 2272 list_del(&nn->list); 2273 err = __unregister_netdevice_notifier_net(dev_net(dev), nb); 2274 rtnl_net_dev_unlock(dev); 2275 2276 return err; 2277 } 2278 EXPORT_SYMBOL(unregister_netdevice_notifier_dev_net); 2279 2280 static void move_netdevice_notifiers_dev_net(struct net_device *dev, 2281 struct net *net) 2282 { 2283 struct netdev_net_notifier *nn; 2284 2285 list_for_each_entry(nn, &dev->net_notifier_list, list) 2286 __move_netdevice_notifier_net(dev_net(dev), net, nn->nb); 2287 } 2288 2289 /** 2290 * call_netdevice_notifiers_info - call all network notifier blocks 2291 * @val: value passed unmodified to notifier function 2292 * @info: notifier information data 2293 * 2294 * Call all network notifier blocks. Parameters and return value 2295 * are as for raw_notifier_call_chain(). 2296 */ 2297 2298 int call_netdevice_notifiers_info(unsigned long val, 2299 struct netdev_notifier_info *info) 2300 { 2301 struct net *net = dev_net(info->dev); 2302 int ret; 2303 2304 ASSERT_RTNL(); 2305 2306 /* Run per-netns notifier block chain first, then run the global one. 2307 * Hopefully, one day, the global one is going to be removed after 2308 * all notifier block registrators get converted to be per-netns. 2309 */ 2310 ret = raw_notifier_call_chain(&net->netdev_chain, val, info); 2311 if (ret & NOTIFY_STOP_MASK) 2312 return ret; 2313 return raw_notifier_call_chain(&netdev_chain, val, info); 2314 } 2315 2316 /** 2317 * call_netdevice_notifiers_info_robust - call per-netns notifier blocks 2318 * for and rollback on error 2319 * @val_up: value passed unmodified to notifier function 2320 * @val_down: value passed unmodified to the notifier function when 2321 * recovering from an error on @val_up 2322 * @info: notifier information data 2323 * 2324 * Call all per-netns network notifier blocks, but not notifier blocks on 2325 * the global notifier chain. Parameters and return value are as for 2326 * raw_notifier_call_chain_robust(). 2327 */ 2328 2329 static int 2330 call_netdevice_notifiers_info_robust(unsigned long val_up, 2331 unsigned long val_down, 2332 struct netdev_notifier_info *info) 2333 { 2334 struct net *net = dev_net(info->dev); 2335 2336 ASSERT_RTNL(); 2337 2338 return raw_notifier_call_chain_robust(&net->netdev_chain, 2339 val_up, val_down, info); 2340 } 2341 2342 static int call_netdevice_notifiers_extack(unsigned long val, 2343 struct net_device *dev, 2344 struct netlink_ext_ack *extack) 2345 { 2346 struct netdev_notifier_info info = { 2347 .dev = dev, 2348 .extack = extack, 2349 }; 2350 2351 return call_netdevice_notifiers_info(val, &info); 2352 } 2353 2354 /** 2355 * call_netdevice_notifiers - call all network notifier blocks 2356 * @val: value passed unmodified to notifier function 2357 * @dev: net_device pointer passed unmodified to notifier function 2358 * 2359 * Call all network notifier blocks. Parameters and return value 2360 * are as for raw_notifier_call_chain(). 2361 */ 2362 2363 int call_netdevice_notifiers(unsigned long val, struct net_device *dev) 2364 { 2365 return call_netdevice_notifiers_extack(val, dev, NULL); 2366 } 2367 EXPORT_SYMBOL(call_netdevice_notifiers); 2368 2369 /** 2370 * call_netdevice_notifiers_mtu - call all network notifier blocks 2371 * @val: value passed unmodified to notifier function 2372 * @dev: net_device pointer passed unmodified to notifier function 2373 * @arg: additional u32 argument passed to the notifier function 2374 * 2375 * Call all network notifier blocks. Parameters and return value 2376 * are as for raw_notifier_call_chain(). 2377 */ 2378 static int call_netdevice_notifiers_mtu(unsigned long val, 2379 struct net_device *dev, u32 arg) 2380 { 2381 struct netdev_notifier_info_ext info = { 2382 .info.dev = dev, 2383 .ext.mtu = arg, 2384 }; 2385 2386 BUILD_BUG_ON(offsetof(struct netdev_notifier_info_ext, info) != 0); 2387 2388 return call_netdevice_notifiers_info(val, &info.info); 2389 } 2390 2391 #ifdef CONFIG_NET_INGRESS 2392 static DEFINE_STATIC_KEY_FALSE(ingress_needed_key); 2393 2394 void net_inc_ingress_queue(void) 2395 { 2396 static_branch_inc(&ingress_needed_key); 2397 } 2398 EXPORT_SYMBOL_GPL(net_inc_ingress_queue); 2399 2400 void net_dec_ingress_queue(void) 2401 { 2402 static_branch_dec(&ingress_needed_key); 2403 } 2404 EXPORT_SYMBOL_GPL(net_dec_ingress_queue); 2405 #endif 2406 2407 #ifdef CONFIG_NET_EGRESS 2408 static DEFINE_STATIC_KEY_FALSE(egress_needed_key); 2409 2410 void net_inc_egress_queue(void) 2411 { 2412 static_branch_inc(&egress_needed_key); 2413 } 2414 EXPORT_SYMBOL_GPL(net_inc_egress_queue); 2415 2416 void net_dec_egress_queue(void) 2417 { 2418 static_branch_dec(&egress_needed_key); 2419 } 2420 EXPORT_SYMBOL_GPL(net_dec_egress_queue); 2421 #endif 2422 2423 #ifdef CONFIG_NET_CLS_ACT 2424 DEFINE_STATIC_KEY_FALSE(tcf_sw_enabled_key); 2425 EXPORT_SYMBOL(tcf_sw_enabled_key); 2426 #endif 2427 2428 DEFINE_STATIC_KEY_FALSE(netstamp_needed_key); 2429 EXPORT_SYMBOL(netstamp_needed_key); 2430 #ifdef CONFIG_JUMP_LABEL 2431 static atomic_t netstamp_needed_deferred; 2432 static atomic_t netstamp_wanted; 2433 static void netstamp_clear(struct work_struct *work) 2434 { 2435 int deferred = atomic_xchg(&netstamp_needed_deferred, 0); 2436 int wanted; 2437 2438 wanted = atomic_add_return(deferred, &netstamp_wanted); 2439 if (wanted > 0) 2440 static_branch_enable(&netstamp_needed_key); 2441 else 2442 static_branch_disable(&netstamp_needed_key); 2443 } 2444 static DECLARE_WORK(netstamp_work, netstamp_clear); 2445 #endif 2446 2447 void net_enable_timestamp(void) 2448 { 2449 #ifdef CONFIG_JUMP_LABEL 2450 int wanted = atomic_read(&netstamp_wanted); 2451 2452 while (wanted > 0) { 2453 if (atomic_try_cmpxchg(&netstamp_wanted, &wanted, wanted + 1)) 2454 return; 2455 } 2456 atomic_inc(&netstamp_needed_deferred); 2457 schedule_work(&netstamp_work); 2458 #else 2459 static_branch_inc(&netstamp_needed_key); 2460 #endif 2461 } 2462 EXPORT_SYMBOL(net_enable_timestamp); 2463 2464 void net_disable_timestamp(void) 2465 { 2466 #ifdef CONFIG_JUMP_LABEL 2467 int wanted = atomic_read(&netstamp_wanted); 2468 2469 while (wanted > 1) { 2470 if (atomic_try_cmpxchg(&netstamp_wanted, &wanted, wanted - 1)) 2471 return; 2472 } 2473 atomic_dec(&netstamp_needed_deferred); 2474 schedule_work(&netstamp_work); 2475 #else 2476 static_branch_dec(&netstamp_needed_key); 2477 #endif 2478 } 2479 EXPORT_SYMBOL(net_disable_timestamp); 2480 2481 static inline void net_timestamp_set(struct sk_buff *skb) 2482 { 2483 skb->tstamp = 0; 2484 skb->tstamp_type = SKB_CLOCK_REALTIME; 2485 if (static_branch_unlikely(&netstamp_needed_key)) 2486 skb->tstamp = ktime_get_real(); 2487 } 2488 2489 #define net_timestamp_check(COND, SKB) \ 2490 if (static_branch_unlikely(&netstamp_needed_key)) { \ 2491 if ((COND) && !(SKB)->tstamp) \ 2492 (SKB)->tstamp = ktime_get_real(); \ 2493 } \ 2494 2495 bool is_skb_forwardable(const struct net_device *dev, const struct sk_buff *skb) 2496 { 2497 return __is_skb_forwardable(dev, skb, true); 2498 } 2499 EXPORT_SYMBOL_GPL(is_skb_forwardable); 2500 2501 static int __dev_forward_skb2(struct net_device *dev, struct sk_buff *skb, 2502 bool check_mtu) 2503 { 2504 enum skb_drop_reason reason; 2505 int ret; 2506 2507 ret = ____dev_forward_skb(dev, skb, check_mtu); 2508 if (unlikely(ret)) 2509 return ret; 2510 2511 reason = eth_type_trans_careful(skb, dev); 2512 if (unlikely(reason)) { 2513 dev_core_stats_rx_dropped_inc(dev); 2514 kfree_skb_reason(skb, reason); 2515 return NET_RX_DROP; 2516 } 2517 skb_postpull_rcsum(skb, eth_hdr(skb), ETH_HLEN); 2518 2519 return 0; 2520 } 2521 2522 int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb) 2523 { 2524 return __dev_forward_skb2(dev, skb, true); 2525 } 2526 EXPORT_SYMBOL_GPL(__dev_forward_skb); 2527 2528 /** 2529 * dev_forward_skb - loopback an skb to another netif 2530 * 2531 * @dev: destination network device 2532 * @skb: buffer to forward 2533 * 2534 * return values: 2535 * NET_RX_SUCCESS (no congestion) 2536 * NET_RX_DROP (packet was dropped, but freed) 2537 * 2538 * dev_forward_skb can be used for injecting an skb from the 2539 * start_xmit function of one device into the receive queue 2540 * of another device. 2541 * 2542 * The receiving device may be in another namespace, so 2543 * we have to clear all information in the skb that could 2544 * impact namespace isolation. 2545 */ 2546 int dev_forward_skb(struct net_device *dev, struct sk_buff *skb) 2547 { 2548 return __dev_forward_skb(dev, skb) ?: netif_rx_internal(skb); 2549 } 2550 EXPORT_SYMBOL_GPL(dev_forward_skb); 2551 2552 int dev_forward_skb_nomtu(struct net_device *dev, struct sk_buff *skb) 2553 { 2554 return __dev_forward_skb2(dev, skb, false) ?: netif_rx_internal(skb); 2555 } 2556 2557 static int deliver_skb(struct sk_buff *skb, 2558 struct packet_type *pt_prev, 2559 struct net_device *orig_dev) 2560 { 2561 if (unlikely(skb_orphan_frags_rx(skb, GFP_ATOMIC))) 2562 return -ENOMEM; 2563 refcount_inc(&skb->users); 2564 return pt_prev->func(skb, skb->dev, pt_prev, orig_dev); 2565 } 2566 2567 static inline void deliver_ptype_list_skb(struct sk_buff *skb, 2568 struct packet_type **pt, 2569 struct net_device *orig_dev, 2570 __be16 type, 2571 struct list_head *ptype_list) 2572 { 2573 struct packet_type *ptype, *pt_prev = *pt; 2574 2575 list_for_each_entry_rcu(ptype, ptype_list, list) { 2576 if (ptype->type != type) 2577 continue; 2578 if (unlikely(pt_prev)) 2579 deliver_skb(skb, pt_prev, orig_dev); 2580 pt_prev = ptype; 2581 } 2582 *pt = pt_prev; 2583 } 2584 2585 static inline bool skb_loop_sk(struct packet_type *ptype, struct sk_buff *skb) 2586 { 2587 if (!ptype->af_packet_priv || !skb->sk) 2588 return false; 2589 2590 if (ptype->id_match) 2591 return ptype->id_match(ptype, skb->sk); 2592 else if ((struct sock *)ptype->af_packet_priv == skb->sk) 2593 return true; 2594 2595 return false; 2596 } 2597 2598 /** 2599 * dev_nit_active_rcu - return true if any network interface taps are in use 2600 * 2601 * The caller must hold the RCU lock 2602 * 2603 * @dev: network device to check for the presence of taps 2604 */ 2605 bool dev_nit_active_rcu(const struct net_device *dev) 2606 { 2607 /* Callers may hold either RCU or RCU BH lock */ 2608 WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held()); 2609 2610 return !list_empty(&dev_net(dev)->ptype_all) || 2611 !list_empty(&dev->ptype_all); 2612 } 2613 EXPORT_SYMBOL_GPL(dev_nit_active_rcu); 2614 2615 /* 2616 * Support routine. Sends outgoing frames to any network 2617 * taps currently in use. 2618 */ 2619 2620 void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev) 2621 { 2622 struct packet_type *ptype, *pt_prev = NULL; 2623 struct list_head *ptype_list; 2624 struct sk_buff *skb2 = NULL; 2625 2626 rcu_read_lock(); 2627 ptype_list = &dev_net_rcu(dev)->ptype_all; 2628 again: 2629 list_for_each_entry_rcu(ptype, ptype_list, list) { 2630 if (READ_ONCE(ptype->ignore_outgoing)) 2631 continue; 2632 2633 /* Never send packets back to the socket 2634 * they originated from - MvS (miquels@drinkel.ow.org) 2635 */ 2636 if (skb_loop_sk(ptype, skb)) 2637 continue; 2638 2639 if (unlikely(pt_prev)) { 2640 deliver_skb(skb2, pt_prev, skb->dev); 2641 pt_prev = ptype; 2642 continue; 2643 } 2644 2645 /* need to clone skb, done only once */ 2646 skb2 = skb_clone(skb, GFP_ATOMIC); 2647 if (!skb2) 2648 goto out_unlock; 2649 2650 net_timestamp_set(skb2); 2651 2652 /* skb->nh should be correctly 2653 * set by sender, so that the second statement is 2654 * just protection against buggy protocols. 2655 */ 2656 skb_reset_mac_header(skb2); 2657 2658 if (skb_network_header(skb2) < skb2->data || 2659 skb_network_header(skb2) > skb_tail_pointer(skb2)) { 2660 net_crit_ratelimited("protocol %04x is buggy, dev %s\n", 2661 ntohs(skb2->protocol), 2662 dev->name); 2663 skb_reset_network_header(skb2); 2664 } 2665 2666 skb2->transport_header = skb2->network_header; 2667 skb2->pkt_type = PACKET_OUTGOING; 2668 pt_prev = ptype; 2669 } 2670 2671 if (ptype_list != &dev->ptype_all) { 2672 ptype_list = &dev->ptype_all; 2673 goto again; 2674 } 2675 out_unlock: 2676 if (pt_prev) { 2677 if (!skb_orphan_frags_rx(skb2, GFP_ATOMIC)) 2678 pt_prev->func(skb2, skb->dev, pt_prev, skb->dev); 2679 else 2680 kfree_skb(skb2); 2681 } 2682 rcu_read_unlock(); 2683 } 2684 EXPORT_SYMBOL_GPL(dev_queue_xmit_nit); 2685 2686 /** 2687 * netif_setup_tc - Handle tc mappings on real_num_tx_queues change 2688 * @dev: Network device 2689 * @txq: number of queues available 2690 * 2691 * If real_num_tx_queues is changed the tc mappings may no longer be 2692 * valid. To resolve this verify the tc mapping remains valid and if 2693 * not NULL the mapping. With no priorities mapping to this 2694 * offset/count pair it will no longer be used. In the worst case TC0 2695 * is invalid nothing can be done so disable priority mappings. If is 2696 * expected that drivers will fix this mapping if they can before 2697 * calling netif_set_real_num_tx_queues. 2698 */ 2699 static void netif_setup_tc(struct net_device *dev, unsigned int txq) 2700 { 2701 struct netdev_tc_txq res; 2702 int i; 2703 2704 res.combined = READ_ONCE(dev->tc_to_txq[0].combined); 2705 2706 /* If TC0 is invalidated disable TC mapping */ 2707 if (res.offset + res.count > txq) { 2708 netdev_warn(dev, "Number of in use tx queues changed invalidating tc mappings. Priority traffic classification disabled!\n"); 2709 WRITE_ONCE(dev->num_tc, 0); 2710 return; 2711 } 2712 2713 /* Invalidated prio to tc mappings set to TC0 */ 2714 for (i = 1; i < TC_BITMASK + 1; i++) { 2715 int q = netdev_get_prio_tc_map(dev, i); 2716 2717 res.combined = READ_ONCE(dev->tc_to_txq[q].combined); 2718 if (res.offset + res.count > txq) { 2719 netdev_warn(dev, "Number of in use tx queues changed. Priority %i to tc mapping %i is no longer valid. Setting map to 0\n", 2720 i, q); 2721 netdev_set_prio_tc_map(dev, i, 0); 2722 } 2723 } 2724 } 2725 2726 int netdev_txq_to_tc(struct net_device *dev, unsigned int txq) 2727 { 2728 if (READ_ONCE(dev->num_tc)) { 2729 struct netdev_tc_txq *tc = &dev->tc_to_txq[0]; 2730 int i; 2731 2732 /* walk through the TCs and see if it falls into any of them */ 2733 for (i = 0; i < TC_MAX_QUEUE; i++, tc++) { 2734 struct netdev_tc_txq res; 2735 2736 res.combined = READ_ONCE(tc->combined); 2737 if ((txq - res.offset) < res.count) 2738 return i; 2739 } 2740 2741 /* didn't find it, just return -1 to indicate no match */ 2742 return -1; 2743 } 2744 2745 return 0; 2746 } 2747 EXPORT_SYMBOL(netdev_txq_to_tc); 2748 2749 #ifdef CONFIG_XPS 2750 static struct static_key xps_needed __read_mostly; 2751 static struct static_key xps_rxqs_needed __read_mostly; 2752 static DEFINE_MUTEX(xps_map_mutex); 2753 #define xmap_dereference(P) \ 2754 rcu_dereference_protected((P), lockdep_is_held(&xps_map_mutex)) 2755 2756 static bool remove_xps_queue(struct xps_dev_maps *dev_maps, 2757 struct xps_dev_maps *old_maps, int tci, u16 index) 2758 { 2759 struct xps_map *map = NULL; 2760 int pos; 2761 2762 map = xmap_dereference(dev_maps->attr_map[tci]); 2763 if (!map) 2764 return false; 2765 2766 for (pos = map->len; pos--;) { 2767 if (map->queues[pos] != index) 2768 continue; 2769 2770 if (map->len > 1) { 2771 map->queues[pos] = map->queues[--map->len]; 2772 break; 2773 } 2774 2775 if (old_maps) 2776 RCU_INIT_POINTER(old_maps->attr_map[tci], NULL); 2777 RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL); 2778 kfree_rcu(map, rcu); 2779 return false; 2780 } 2781 2782 return true; 2783 } 2784 2785 static bool remove_xps_queue_cpu(struct net_device *dev, 2786 struct xps_dev_maps *dev_maps, 2787 int cpu, u16 offset, u16 count) 2788 { 2789 int num_tc = dev_maps->num_tc; 2790 bool active = false; 2791 int tci; 2792 2793 for (tci = cpu * num_tc; num_tc--; tci++) { 2794 int i, j; 2795 2796 for (i = count, j = offset; i--; j++) { 2797 if (!remove_xps_queue(dev_maps, NULL, tci, j)) 2798 break; 2799 } 2800 2801 active |= i < 0; 2802 } 2803 2804 return active; 2805 } 2806 2807 static void reset_xps_maps(struct net_device *dev, 2808 struct xps_dev_maps *dev_maps, 2809 enum xps_map_type type) 2810 { 2811 static_key_slow_dec_cpuslocked(&xps_needed); 2812 if (type == XPS_RXQS) 2813 static_key_slow_dec_cpuslocked(&xps_rxqs_needed); 2814 2815 RCU_INIT_POINTER(dev->xps_maps[type], NULL); 2816 2817 kfree_rcu(dev_maps, rcu); 2818 } 2819 2820 static void clean_xps_maps(struct net_device *dev, enum xps_map_type type, 2821 u16 offset, u16 count) 2822 { 2823 struct xps_dev_maps *dev_maps; 2824 bool active = false; 2825 int i, j; 2826 2827 dev_maps = xmap_dereference(dev->xps_maps[type]); 2828 if (!dev_maps) 2829 return; 2830 2831 for (j = 0; j < dev_maps->nr_ids; j++) 2832 active |= remove_xps_queue_cpu(dev, dev_maps, j, offset, count); 2833 if (!active) 2834 reset_xps_maps(dev, dev_maps, type); 2835 2836 if (type == XPS_CPUS) { 2837 for (i = offset + (count - 1); count--; i--) 2838 netdev_queue_numa_node_write( 2839 netdev_get_tx_queue(dev, i), NUMA_NO_NODE); 2840 } 2841 } 2842 2843 static void netif_reset_xps_queues(struct net_device *dev, u16 offset, 2844 u16 count) 2845 { 2846 if (!static_key_false(&xps_needed)) 2847 return; 2848 2849 cpus_read_lock(); 2850 mutex_lock(&xps_map_mutex); 2851 2852 if (static_key_false(&xps_rxqs_needed)) 2853 clean_xps_maps(dev, XPS_RXQS, offset, count); 2854 2855 clean_xps_maps(dev, XPS_CPUS, offset, count); 2856 2857 mutex_unlock(&xps_map_mutex); 2858 cpus_read_unlock(); 2859 } 2860 2861 static void netif_reset_xps_queues_gt(struct net_device *dev, u16 index) 2862 { 2863 netif_reset_xps_queues(dev, index, dev->num_tx_queues - index); 2864 } 2865 2866 static struct xps_map *expand_xps_map(struct xps_map *map, int attr_index, 2867 u16 index, bool is_rxqs_map) 2868 { 2869 struct xps_map *new_map; 2870 int alloc_len = XPS_MIN_MAP_ALLOC; 2871 int i, pos; 2872 2873 for (pos = 0; map && pos < map->len; pos++) { 2874 if (map->queues[pos] != index) 2875 continue; 2876 return map; 2877 } 2878 2879 /* Need to add tx-queue to this CPU's/rx-queue's existing map */ 2880 if (map) { 2881 if (pos < map->alloc_len) 2882 return map; 2883 2884 alloc_len = map->alloc_len * 2; 2885 } 2886 2887 /* Need to allocate new map to store tx-queue on this CPU's/rx-queue's 2888 * map 2889 */ 2890 if (is_rxqs_map) 2891 new_map = kzalloc(XPS_MAP_SIZE(alloc_len), GFP_KERNEL); 2892 else 2893 new_map = kzalloc_node(XPS_MAP_SIZE(alloc_len), GFP_KERNEL, 2894 cpu_to_node(attr_index)); 2895 if (!new_map) 2896 return NULL; 2897 2898 for (i = 0; i < pos; i++) 2899 new_map->queues[i] = map->queues[i]; 2900 new_map->alloc_len = alloc_len; 2901 new_map->len = pos; 2902 2903 return new_map; 2904 } 2905 2906 /* Copy xps maps at a given index */ 2907 static void xps_copy_dev_maps(struct xps_dev_maps *dev_maps, 2908 struct xps_dev_maps *new_dev_maps, int index, 2909 int tc, bool skip_tc) 2910 { 2911 int i, tci = index * dev_maps->num_tc; 2912 struct xps_map *map; 2913 2914 /* copy maps belonging to foreign traffic classes */ 2915 for (i = 0; i < dev_maps->num_tc; i++, tci++) { 2916 if (i == tc && skip_tc) 2917 continue; 2918 2919 /* fill in the new device map from the old device map */ 2920 map = xmap_dereference(dev_maps->attr_map[tci]); 2921 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map); 2922 } 2923 } 2924 2925 /* Must be called under cpus_read_lock */ 2926 int __netif_set_xps_queue(struct net_device *dev, const unsigned long *mask, 2927 u16 index, enum xps_map_type type) 2928 { 2929 struct xps_dev_maps *dev_maps, *new_dev_maps = NULL, *old_dev_maps = NULL; 2930 int maps_sz, num_tc = 1, tc = 0, dev_num_tc; 2931 const unsigned long *online_mask = NULL; 2932 bool active = false, copy = false; 2933 int i, j, tci, numa_node_id = -2; 2934 struct xps_map *map, *new_map; 2935 unsigned int nr_ids; 2936 2937 WARN_ON_ONCE(index >= dev->num_tx_queues); 2938 2939 dev_num_tc = READ_ONCE(dev->num_tc); 2940 if (dev_num_tc) { 2941 /* Do not allow XPS on subordinate device directly */ 2942 num_tc = dev_num_tc; 2943 if (num_tc < 0) 2944 return -EINVAL; 2945 2946 /* If queue belongs to subordinate dev use its map */ 2947 dev = netdev_get_tx_queue(dev, index)->sb_dev ? : dev; 2948 2949 tc = netdev_txq_to_tc(dev, index); 2950 if (tc < 0 || tc >= num_tc) 2951 return -EINVAL; 2952 } 2953 2954 mutex_lock(&xps_map_mutex); 2955 2956 dev_maps = xmap_dereference(dev->xps_maps[type]); 2957 if (type == XPS_RXQS) { 2958 maps_sz = XPS_RXQ_DEV_MAPS_SIZE(num_tc, dev->num_rx_queues); 2959 nr_ids = dev->num_rx_queues; 2960 } else { 2961 maps_sz = XPS_CPU_DEV_MAPS_SIZE(num_tc); 2962 if (num_possible_cpus() > 1) 2963 online_mask = cpumask_bits(cpu_online_mask); 2964 nr_ids = nr_cpu_ids; 2965 } 2966 2967 if (maps_sz < L1_CACHE_BYTES) 2968 maps_sz = L1_CACHE_BYTES; 2969 2970 /* The old dev_maps could be larger or smaller than the one we're 2971 * setting up now, as dev->num_tc or nr_ids could have been updated in 2972 * between. We could try to be smart, but let's be safe instead and only 2973 * copy foreign traffic classes if the two map sizes match. 2974 */ 2975 if (dev_maps && 2976 dev_maps->num_tc == num_tc && dev_maps->nr_ids == nr_ids) 2977 copy = true; 2978 2979 /* allocate memory for queue storage */ 2980 for (j = -1; j = netif_attrmask_next_and(j, online_mask, mask, nr_ids), 2981 j < nr_ids;) { 2982 if (!new_dev_maps) { 2983 new_dev_maps = kzalloc(maps_sz, GFP_KERNEL); 2984 if (!new_dev_maps) { 2985 mutex_unlock(&xps_map_mutex); 2986 return -ENOMEM; 2987 } 2988 2989 new_dev_maps->nr_ids = nr_ids; 2990 new_dev_maps->num_tc = num_tc; 2991 } 2992 2993 tci = j * num_tc + tc; 2994 map = copy ? xmap_dereference(dev_maps->attr_map[tci]) : NULL; 2995 2996 map = expand_xps_map(map, j, index, type == XPS_RXQS); 2997 if (!map) 2998 goto error; 2999 3000 RCU_INIT_POINTER(new_dev_maps->attr_map[tci], map); 3001 } 3002 3003 if (!new_dev_maps) 3004 goto out_no_new_maps; 3005 3006 if (!dev_maps) { 3007 /* Increment static keys at most once per type */ 3008 static_key_slow_inc_cpuslocked(&xps_needed); 3009 if (type == XPS_RXQS) 3010 static_key_slow_inc_cpuslocked(&xps_rxqs_needed); 3011 } 3012 3013 for (j = 0; j < nr_ids; j++) { 3014 bool skip_tc = false; 3015 3016 tci = j * num_tc + tc; 3017 if (netif_attr_test_mask(j, mask, nr_ids) && 3018 netif_attr_test_online(j, online_mask, nr_ids)) { 3019 /* add tx-queue to CPU/rx-queue maps */ 3020 int pos = 0; 3021 3022 skip_tc = true; 3023 3024 map = xmap_dereference(new_dev_maps->attr_map[tci]); 3025 while ((pos < map->len) && (map->queues[pos] != index)) 3026 pos++; 3027 3028 if (pos == map->len) 3029 map->queues[map->len++] = index; 3030 #ifdef CONFIG_NUMA 3031 if (type == XPS_CPUS) { 3032 if (numa_node_id == -2) 3033 numa_node_id = cpu_to_node(j); 3034 else if (numa_node_id != cpu_to_node(j)) 3035 numa_node_id = -1; 3036 } 3037 #endif 3038 } 3039 3040 if (copy) 3041 xps_copy_dev_maps(dev_maps, new_dev_maps, j, tc, 3042 skip_tc); 3043 } 3044 3045 rcu_assign_pointer(dev->xps_maps[type], new_dev_maps); 3046 3047 /* Cleanup old maps */ 3048 if (!dev_maps) 3049 goto out_no_old_maps; 3050 3051 for (j = 0; j < dev_maps->nr_ids; j++) { 3052 for (i = dev_maps->num_tc, tci = j * dev_maps->num_tc; i--; tci++) { 3053 map = xmap_dereference(dev_maps->attr_map[tci]); 3054 if (!map) 3055 continue; 3056 3057 if (copy) { 3058 new_map = xmap_dereference(new_dev_maps->attr_map[tci]); 3059 if (map == new_map) 3060 continue; 3061 } 3062 3063 RCU_INIT_POINTER(dev_maps->attr_map[tci], NULL); 3064 kfree_rcu(map, rcu); 3065 } 3066 } 3067 3068 old_dev_maps = dev_maps; 3069 3070 out_no_old_maps: 3071 dev_maps = new_dev_maps; 3072 active = true; 3073 3074 out_no_new_maps: 3075 if (type == XPS_CPUS) 3076 /* update Tx queue numa node */ 3077 netdev_queue_numa_node_write(netdev_get_tx_queue(dev, index), 3078 (numa_node_id >= 0) ? 3079 numa_node_id : NUMA_NO_NODE); 3080 3081 if (!dev_maps) 3082 goto out_no_maps; 3083 3084 /* removes tx-queue from unused CPUs/rx-queues */ 3085 for (j = 0; j < dev_maps->nr_ids; j++) { 3086 tci = j * dev_maps->num_tc; 3087 3088 for (i = 0; i < dev_maps->num_tc; i++, tci++) { 3089 if (i == tc && 3090 netif_attr_test_mask(j, mask, dev_maps->nr_ids) && 3091 netif_attr_test_online(j, online_mask, dev_maps->nr_ids)) 3092 continue; 3093 3094 active |= remove_xps_queue(dev_maps, 3095 copy ? old_dev_maps : NULL, 3096 tci, index); 3097 } 3098 } 3099 3100 if (old_dev_maps) 3101 kfree_rcu(old_dev_maps, rcu); 3102 3103 /* free map if not active */ 3104 if (!active) 3105 reset_xps_maps(dev, dev_maps, type); 3106 3107 out_no_maps: 3108 mutex_unlock(&xps_map_mutex); 3109 3110 return 0; 3111 error: 3112 /* remove any maps that we added */ 3113 for (j = 0; j < nr_ids; j++) { 3114 for (i = num_tc, tci = j * num_tc; i--; tci++) { 3115 new_map = xmap_dereference(new_dev_maps->attr_map[tci]); 3116 map = copy ? 3117 xmap_dereference(dev_maps->attr_map[tci]) : 3118 NULL; 3119 if (new_map && new_map != map) 3120 kfree(new_map); 3121 } 3122 } 3123 3124 mutex_unlock(&xps_map_mutex); 3125 3126 kfree(new_dev_maps); 3127 return -ENOMEM; 3128 } 3129 EXPORT_SYMBOL_GPL(__netif_set_xps_queue); 3130 3131 int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask, 3132 u16 index) 3133 { 3134 int ret; 3135 3136 cpus_read_lock(); 3137 ret = __netif_set_xps_queue(dev, cpumask_bits(mask), index, XPS_CPUS); 3138 cpus_read_unlock(); 3139 3140 return ret; 3141 } 3142 EXPORT_SYMBOL(netif_set_xps_queue); 3143 3144 #endif 3145 static void netdev_unbind_all_sb_channels(struct net_device *dev) 3146 { 3147 struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues]; 3148 3149 /* Unbind any subordinate channels */ 3150 while (txq-- != &dev->_tx[0]) { 3151 if (txq->sb_dev) 3152 netdev_unbind_sb_channel(dev, txq->sb_dev); 3153 } 3154 } 3155 3156 void netdev_reset_tc(struct net_device *dev) 3157 { 3158 int i; 3159 3160 #ifdef CONFIG_XPS 3161 netif_reset_xps_queues_gt(dev, 0); 3162 #endif 3163 netdev_unbind_all_sb_channels(dev); 3164 3165 /* Reset TC configuration of device */ 3166 WRITE_ONCE(dev->num_tc, 0); 3167 for (i = 0; i < TC_MAX_QUEUE; i++) 3168 WRITE_ONCE(dev->tc_to_txq[i].combined, 0); 3169 for (i = 0; i <= TC_BITMASK; i++) 3170 WRITE_ONCE(dev->prio_tc_map[i], 0); 3171 } 3172 EXPORT_SYMBOL(netdev_reset_tc); 3173 3174 int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset) 3175 { 3176 struct netdev_tc_txq res = { 3177 .count = count, 3178 .offset = offset, 3179 }; 3180 3181 if (tc >= READ_ONCE(dev->num_tc)) 3182 return -EINVAL; 3183 3184 #ifdef CONFIG_XPS 3185 netif_reset_xps_queues(dev, offset, count); 3186 #endif 3187 WRITE_ONCE(dev->tc_to_txq[tc].combined, res.combined); 3188 return 0; 3189 } 3190 EXPORT_SYMBOL(netdev_set_tc_queue); 3191 3192 int netdev_set_num_tc(struct net_device *dev, u8 num_tc) 3193 { 3194 if (num_tc > TC_MAX_QUEUE) 3195 return -EINVAL; 3196 3197 #ifdef CONFIG_XPS 3198 netif_reset_xps_queues_gt(dev, 0); 3199 #endif 3200 netdev_unbind_all_sb_channels(dev); 3201 3202 WRITE_ONCE(dev->num_tc, num_tc); 3203 return 0; 3204 } 3205 EXPORT_SYMBOL(netdev_set_num_tc); 3206 3207 void netdev_unbind_sb_channel(struct net_device *dev, 3208 struct net_device *sb_dev) 3209 { 3210 struct netdev_queue *txq = &dev->_tx[dev->num_tx_queues]; 3211 int i; 3212 3213 #ifdef CONFIG_XPS 3214 netif_reset_xps_queues_gt(sb_dev, 0); 3215 #endif 3216 for (i = 0; i < TC_MAX_QUEUE; i++) 3217 WRITE_ONCE(sb_dev->tc_to_txq[i].combined, 0); 3218 for (i = 0; i <= TC_BITMASK; i++) 3219 WRITE_ONCE(sb_dev->prio_tc_map[i], 0); 3220 3221 while (txq-- != &dev->_tx[0]) { 3222 if (txq->sb_dev == sb_dev) 3223 txq->sb_dev = NULL; 3224 } 3225 } 3226 EXPORT_SYMBOL(netdev_unbind_sb_channel); 3227 3228 int netdev_bind_sb_channel_queue(struct net_device *dev, 3229 struct net_device *sb_dev, 3230 u8 tc, u16 count, u16 offset) 3231 { 3232 /* Make certain the sb_dev and dev are already configured */ 3233 if (READ_ONCE(sb_dev->num_tc) >= 0 || tc >= READ_ONCE(dev->num_tc)) 3234 return -EINVAL; 3235 3236 /* We cannot hand out queues we don't have */ 3237 if ((offset + count) > dev->real_num_tx_queues) 3238 return -EINVAL; 3239 3240 /* Record the mapping */ 3241 struct netdev_tc_txq res = { 3242 .count = count, 3243 .offset = offset, 3244 }; 3245 3246 WRITE_ONCE(sb_dev->tc_to_txq[tc].combined, res.combined); 3247 3248 /* Provide a way for Tx queue to find the tc_to_txq map or 3249 * XPS map for itself. 3250 */ 3251 while (count--) 3252 netdev_get_tx_queue(dev, count + offset)->sb_dev = sb_dev; 3253 3254 return 0; 3255 } 3256 EXPORT_SYMBOL(netdev_bind_sb_channel_queue); 3257 3258 int netdev_set_sb_channel(struct net_device *dev, u16 channel) 3259 { 3260 /* Do not use a multiqueue device to represent a subordinate channel */ 3261 if (netif_is_multiqueue(dev)) 3262 return -ENODEV; 3263 3264 /* We allow channels 1 - 32767 to be used for subordinate channels. 3265 * Channel 0 is meant to be "native" mode and used only to represent 3266 * the main root device. We allow writing 0 to reset the device back 3267 * to normal mode after being used as a subordinate channel. 3268 */ 3269 if (channel > S16_MAX) 3270 return -EINVAL; 3271 3272 WRITE_ONCE(dev->num_tc, -channel); 3273 3274 return 0; 3275 } 3276 EXPORT_SYMBOL(netdev_set_sb_channel); 3277 3278 /* 3279 * Routine to help set real_num_tx_queues. To avoid skbs mapped to queues 3280 * greater than real_num_tx_queues stale skbs on the qdisc must be flushed. 3281 */ 3282 int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq) 3283 { 3284 bool disabling; 3285 int rc; 3286 3287 disabling = txq < dev->real_num_tx_queues; 3288 3289 if (txq < 1 || txq > dev->num_tx_queues) 3290 return -EINVAL; 3291 3292 if (dev->reg_state == NETREG_REGISTERED || 3293 dev->reg_state == NETREG_UNREGISTERING) { 3294 netdev_assert_locked_ops_compat(dev); 3295 3296 rc = netdev_queue_update_kobjects(dev, dev->real_num_tx_queues, 3297 txq); 3298 if (rc) 3299 return rc; 3300 3301 if (READ_ONCE(dev->num_tc)) 3302 netif_setup_tc(dev, txq); 3303 3304 net_shaper_set_real_num_tx_queues(dev, txq); 3305 3306 dev_qdisc_change_real_num_tx(dev, txq); 3307 3308 dev->real_num_tx_queues = txq; 3309 3310 if (disabling) { 3311 synchronize_net(); 3312 qdisc_reset_all_tx_gt(dev, txq); 3313 #ifdef CONFIG_XPS 3314 netif_reset_xps_queues_gt(dev, txq); 3315 #endif 3316 } 3317 } else { 3318 dev->real_num_tx_queues = txq; 3319 } 3320 3321 return 0; 3322 } 3323 EXPORT_SYMBOL(netif_set_real_num_tx_queues); 3324 3325 /** 3326 * netif_set_real_num_rx_queues - set actual number of RX queues used 3327 * @dev: Network device 3328 * @rxq: Actual number of RX queues 3329 * 3330 * This must be called either with the rtnl_lock held or before 3331 * registration of the net device. Returns 0 on success, or a 3332 * negative error code. If called before registration, it always 3333 * succeeds. 3334 */ 3335 int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq) 3336 { 3337 int rc; 3338 3339 if (rxq < 1 || rxq > dev->num_rx_queues) 3340 return -EINVAL; 3341 3342 if (dev->reg_state == NETREG_REGISTERED) { 3343 netdev_assert_locked_ops_compat(dev); 3344 3345 rc = net_rx_queue_update_kobjects(dev, dev->real_num_rx_queues, 3346 rxq); 3347 if (rc) 3348 return rc; 3349 } 3350 3351 dev->real_num_rx_queues = rxq; 3352 return 0; 3353 } 3354 EXPORT_SYMBOL(netif_set_real_num_rx_queues); 3355 3356 /** 3357 * netif_set_real_num_queues - set actual number of RX and TX queues used 3358 * @dev: Network device 3359 * @txq: Actual number of TX queues 3360 * @rxq: Actual number of RX queues 3361 * 3362 * Set the real number of both TX and RX queues. 3363 * Does nothing if the number of queues is already correct. 3364 */ 3365 int netif_set_real_num_queues(struct net_device *dev, 3366 unsigned int txq, unsigned int rxq) 3367 { 3368 unsigned int old_rxq = dev->real_num_rx_queues; 3369 int err; 3370 3371 if (txq < 1 || txq > dev->num_tx_queues || 3372 rxq < 1 || rxq > dev->num_rx_queues) 3373 return -EINVAL; 3374 3375 /* Start from increases, so the error path only does decreases - 3376 * decreases can't fail. 3377 */ 3378 if (rxq > dev->real_num_rx_queues) { 3379 err = netif_set_real_num_rx_queues(dev, rxq); 3380 if (err) 3381 return err; 3382 } 3383 if (txq > dev->real_num_tx_queues) { 3384 err = netif_set_real_num_tx_queues(dev, txq); 3385 if (err) 3386 goto undo_rx; 3387 } 3388 if (rxq < dev->real_num_rx_queues) 3389 WARN_ON(netif_set_real_num_rx_queues(dev, rxq)); 3390 if (txq < dev->real_num_tx_queues) 3391 WARN_ON(netif_set_real_num_tx_queues(dev, txq)); 3392 3393 return 0; 3394 undo_rx: 3395 WARN_ON(netif_set_real_num_rx_queues(dev, old_rxq)); 3396 return err; 3397 } 3398 EXPORT_SYMBOL(netif_set_real_num_queues); 3399 3400 /** 3401 * netif_set_tso_max_size() - set the max size of TSO frames supported 3402 * @dev: netdev to update 3403 * @size: max skb->len of a TSO frame 3404 * 3405 * Set the limit on the size of TSO super-frames the device can handle. 3406 * Unless explicitly set the stack will assume the value of 3407 * %GSO_LEGACY_MAX_SIZE. 3408 */ 3409 void netif_set_tso_max_size(struct net_device *dev, unsigned int size) 3410 { 3411 dev->tso_max_size = min(GSO_MAX_SIZE, size); 3412 if (size < READ_ONCE(dev->gso_max_size)) 3413 netif_set_gso_max_size(dev, size); 3414 if (size < READ_ONCE(dev->gso_ipv4_max_size)) 3415 netif_set_gso_ipv4_max_size(dev, size); 3416 } 3417 EXPORT_SYMBOL(netif_set_tso_max_size); 3418 3419 /** 3420 * netif_set_tso_max_segs() - set the max number of segs supported for TSO 3421 * @dev: netdev to update 3422 * @segs: max number of TCP segments 3423 * 3424 * Set the limit on the number of TCP segments the device can generate from 3425 * a single TSO super-frame. 3426 * Unless explicitly set the stack will assume the value of %GSO_MAX_SEGS. 3427 */ 3428 void netif_set_tso_max_segs(struct net_device *dev, unsigned int segs) 3429 { 3430 dev->tso_max_segs = segs; 3431 if (segs < READ_ONCE(dev->gso_max_segs)) 3432 netif_set_gso_max_segs(dev, segs); 3433 } 3434 EXPORT_SYMBOL(netif_set_tso_max_segs); 3435 3436 /** 3437 * netif_inherit_tso_max() - copy all TSO limits from a lower device to an upper 3438 * @to: netdev to update 3439 * @from: netdev from which to copy the limits 3440 */ 3441 void netif_inherit_tso_max(struct net_device *to, const struct net_device *from) 3442 { 3443 netif_set_tso_max_size(to, from->tso_max_size); 3444 netif_set_tso_max_segs(to, from->tso_max_segs); 3445 } 3446 EXPORT_SYMBOL(netif_inherit_tso_max); 3447 3448 /** 3449 * netif_get_num_default_rss_queues - default number of RSS queues 3450 * 3451 * Default value is the number of physical cores if there are only 1 or 2, or 3452 * divided by 2 if there are more. 3453 */ 3454 int netif_get_num_default_rss_queues(void) 3455 { 3456 cpumask_var_t cpus; 3457 int cpu, count = 0; 3458 3459 if (unlikely(is_kdump_kernel() || !zalloc_cpumask_var(&cpus, GFP_KERNEL))) 3460 return 1; 3461 3462 cpumask_copy(cpus, cpu_online_mask); 3463 for_each_cpu(cpu, cpus) { 3464 ++count; 3465 cpumask_andnot(cpus, cpus, topology_sibling_cpumask(cpu)); 3466 } 3467 free_cpumask_var(cpus); 3468 3469 return count > 2 ? DIV_ROUND_UP(count, 2) : count; 3470 } 3471 EXPORT_SYMBOL(netif_get_num_default_rss_queues); 3472 3473 static void __netif_reschedule(struct Qdisc *q) 3474 { 3475 struct softnet_data *sd; 3476 unsigned long flags; 3477 3478 local_irq_save(flags); 3479 sd = this_cpu_ptr(&softnet_data); 3480 q->next_sched = NULL; 3481 *sd->output_queue_tailp = q; 3482 sd->output_queue_tailp = &q->next_sched; 3483 raise_softirq_irqoff(NET_TX_SOFTIRQ); 3484 local_irq_restore(flags); 3485 } 3486 3487 void __netif_schedule(struct Qdisc *q) 3488 { 3489 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) 3490 return; 3491 3492 /* If q->defer_list is not empty, at least one thread is 3493 * in __dev_xmit_skb() before llist_del_all(&q->defer_list). 3494 * This thread will attempt to run the queue. 3495 */ 3496 if (!llist_empty(&q->defer_list)) 3497 return; 3498 3499 if (!test_and_set_bit(__QDISC_STATE_SCHED, &q->state)) 3500 __netif_reschedule(q); 3501 } 3502 EXPORT_SYMBOL(__netif_schedule); 3503 3504 struct dev_kfree_skb_cb { 3505 enum skb_drop_reason reason; 3506 }; 3507 3508 static struct dev_kfree_skb_cb *get_kfree_skb_cb(const struct sk_buff *skb) 3509 { 3510 return (struct dev_kfree_skb_cb *)skb->cb; 3511 } 3512 3513 void netif_schedule_queue(struct netdev_queue *txq) 3514 { 3515 rcu_read_lock(); 3516 if (!netif_xmit_stopped(txq)) { 3517 struct Qdisc *q = rcu_dereference(txq->qdisc); 3518 3519 __netif_schedule(q); 3520 } 3521 rcu_read_unlock(); 3522 } 3523 EXPORT_SYMBOL(netif_schedule_queue); 3524 3525 void netif_tx_wake_queue(struct netdev_queue *dev_queue) 3526 { 3527 if (test_and_clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state)) { 3528 struct Qdisc *q; 3529 3530 rcu_read_lock(); 3531 q = rcu_dereference(dev_queue->qdisc); 3532 __netif_schedule(q); 3533 rcu_read_unlock(); 3534 } 3535 } 3536 EXPORT_SYMBOL(netif_tx_wake_queue); 3537 3538 void dev_kfree_skb_irq_reason(struct sk_buff *skb, enum skb_drop_reason reason) 3539 { 3540 unsigned long flags; 3541 3542 if (unlikely(!skb)) 3543 return; 3544 3545 if (likely(refcount_read(&skb->users) == 1)) { 3546 smp_rmb(); 3547 refcount_set(&skb->users, 0); 3548 } else if (likely(!refcount_dec_and_test(&skb->users))) { 3549 return; 3550 } 3551 get_kfree_skb_cb(skb)->reason = reason; 3552 local_irq_save(flags); 3553 skb->next = __this_cpu_read(softnet_data.completion_queue); 3554 __this_cpu_write(softnet_data.completion_queue, skb); 3555 raise_softirq_irqoff(NET_TX_SOFTIRQ); 3556 local_irq_restore(flags); 3557 } 3558 EXPORT_SYMBOL(dev_kfree_skb_irq_reason); 3559 3560 void dev_kfree_skb_any_reason(struct sk_buff *skb, enum skb_drop_reason reason) 3561 { 3562 if (in_hardirq() || irqs_disabled()) 3563 dev_kfree_skb_irq_reason(skb, reason); 3564 else 3565 kfree_skb_reason(skb, reason); 3566 } 3567 EXPORT_SYMBOL(dev_kfree_skb_any_reason); 3568 3569 3570 /** 3571 * netif_device_detach - mark device as removed 3572 * @dev: network device 3573 * 3574 * Mark device as removed from system and therefore no longer available. 3575 */ 3576 void netif_device_detach(struct net_device *dev) 3577 { 3578 if (test_and_clear_bit(__LINK_STATE_PRESENT, &dev->state) && 3579 netif_running(dev)) { 3580 netif_tx_stop_all_queues(dev); 3581 } 3582 } 3583 EXPORT_SYMBOL(netif_device_detach); 3584 3585 /** 3586 * netif_device_attach - mark device as attached 3587 * @dev: network device 3588 * 3589 * Mark device as attached from system and restart if needed. 3590 */ 3591 void netif_device_attach(struct net_device *dev) 3592 { 3593 if (!test_and_set_bit(__LINK_STATE_PRESENT, &dev->state) && 3594 netif_running(dev)) { 3595 netif_tx_wake_all_queues(dev); 3596 netdev_watchdog_up(dev); 3597 } 3598 } 3599 EXPORT_SYMBOL(netif_device_attach); 3600 3601 /* 3602 * Returns a Tx hash based on the given packet descriptor a Tx queues' number 3603 * to be used as a distribution range. 3604 */ 3605 static u16 skb_tx_hash(const struct net_device *dev, 3606 const struct net_device *sb_dev, 3607 struct sk_buff *skb) 3608 { 3609 u32 hash; 3610 u16 qoffset = 0; 3611 u16 qcount = dev->real_num_tx_queues; 3612 3613 if (READ_ONCE(dev->num_tc)) { 3614 u8 tc = netdev_get_prio_tc_map(dev, skb->priority); 3615 struct netdev_tc_txq res; 3616 3617 res.combined = READ_ONCE(sb_dev->tc_to_txq[tc].combined); 3618 qoffset = res.offset; 3619 qcount = res.count; 3620 if (unlikely(!qcount)) { 3621 net_warn_ratelimited("%s: invalid qcount, qoffset %u for tc %u\n", 3622 sb_dev->name, qoffset, tc); 3623 qoffset = 0; 3624 qcount = dev->real_num_tx_queues; 3625 } 3626 } 3627 3628 if (skb_rx_queue_recorded(skb)) { 3629 DEBUG_NET_WARN_ON_ONCE(qcount == 0); 3630 hash = skb_get_rx_queue(skb); 3631 if (hash >= qoffset) 3632 hash -= qoffset; 3633 while (unlikely(hash >= qcount)) 3634 hash -= qcount; 3635 return hash + qoffset; 3636 } 3637 3638 return (u16) reciprocal_scale(skb_get_hash(skb), qcount) + qoffset; 3639 } 3640 3641 void skb_warn_bad_offload(const struct sk_buff *skb) 3642 { 3643 static const netdev_features_t null_features; 3644 struct net_device *dev = skb->dev; 3645 const char *name = ""; 3646 3647 if (!net_ratelimit()) 3648 return; 3649 3650 if (dev) { 3651 if (dev->dev.parent) 3652 name = dev_driver_string(dev->dev.parent); 3653 else 3654 name = netdev_name(dev); 3655 } 3656 skb_dump(KERN_WARNING, skb, false); 3657 WARN(1, "%s: caps=(%pNF, %pNF)\n", 3658 name, dev ? &dev->features : &null_features, 3659 skb->sk ? &skb->sk->sk_route_caps : &null_features); 3660 } 3661 3662 /* 3663 * Invalidate hardware checksum when packet is to be mangled, and 3664 * complete checksum manually on outgoing path. 3665 */ 3666 int skb_checksum_help(struct sk_buff *skb) 3667 { 3668 __wsum csum; 3669 int ret = 0, offset; 3670 3671 if (skb->ip_summed == CHECKSUM_COMPLETE) 3672 goto out_set_summed; 3673 3674 if (unlikely(skb_is_gso(skb))) { 3675 skb_warn_bad_offload(skb); 3676 return -EINVAL; 3677 } 3678 3679 if (!skb_frags_readable(skb)) { 3680 return -EFAULT; 3681 } 3682 3683 /* Before computing a checksum, we should make sure no frag could 3684 * be modified by an external entity : checksum could be wrong. 3685 */ 3686 if (skb_has_shared_frag(skb)) { 3687 ret = __skb_linearize(skb); 3688 if (ret) 3689 goto out; 3690 } 3691 3692 offset = skb_checksum_start_offset(skb); 3693 ret = -EINVAL; 3694 if (unlikely(offset >= skb_headlen(skb))) { 3695 DO_ONCE_LITE(skb_dump, KERN_ERR, skb, false); 3696 WARN_ONCE(true, "offset (%d) >= skb_headlen() (%u)\n", 3697 offset, skb_headlen(skb)); 3698 goto out; 3699 } 3700 csum = skb_checksum(skb, offset, skb->len - offset, 0); 3701 3702 offset += skb->csum_offset; 3703 if (unlikely(offset + sizeof(__sum16) > skb_headlen(skb))) { 3704 DO_ONCE_LITE(skb_dump, KERN_ERR, skb, false); 3705 WARN_ONCE(true, "offset+2 (%zu) > skb_headlen() (%u)\n", 3706 offset + sizeof(__sum16), skb_headlen(skb)); 3707 goto out; 3708 } 3709 ret = skb_ensure_writable(skb, offset + sizeof(__sum16)); 3710 if (ret) 3711 goto out; 3712 3713 *(__sum16 *)(skb->data + offset) = csum_fold(csum) ?: CSUM_MANGLED_0; 3714 out_set_summed: 3715 skb->ip_summed = CHECKSUM_NONE; 3716 out: 3717 return ret; 3718 } 3719 EXPORT_SYMBOL(skb_checksum_help); 3720 3721 #ifdef CONFIG_NET_CRC32C 3722 int skb_crc32c_csum_help(struct sk_buff *skb) 3723 { 3724 u32 crc; 3725 int ret = 0, offset, start; 3726 3727 if (skb->ip_summed != CHECKSUM_PARTIAL) 3728 goto out; 3729 3730 if (unlikely(skb_is_gso(skb))) 3731 goto out; 3732 3733 /* Before computing a checksum, we should make sure no frag could 3734 * be modified by an external entity : checksum could be wrong. 3735 */ 3736 if (unlikely(skb_has_shared_frag(skb))) { 3737 ret = __skb_linearize(skb); 3738 if (ret) 3739 goto out; 3740 } 3741 start = skb_checksum_start_offset(skb); 3742 offset = start + offsetof(struct sctphdr, checksum); 3743 if (WARN_ON_ONCE(offset >= skb_headlen(skb))) { 3744 ret = -EINVAL; 3745 goto out; 3746 } 3747 3748 ret = skb_ensure_writable(skb, offset + sizeof(__le32)); 3749 if (ret) 3750 goto out; 3751 3752 crc = ~skb_crc32c(skb, start, skb->len - start, ~0); 3753 *(__le32 *)(skb->data + offset) = cpu_to_le32(crc); 3754 skb_reset_csum_not_inet(skb); 3755 out: 3756 return ret; 3757 } 3758 EXPORT_SYMBOL(skb_crc32c_csum_help); 3759 #endif /* CONFIG_NET_CRC32C */ 3760 3761 __be16 skb_network_protocol(struct sk_buff *skb, int *depth) 3762 { 3763 __be16 type = skb->protocol; 3764 3765 /* Tunnel gso handlers can set protocol to ethernet. */ 3766 if (type == htons(ETH_P_TEB)) { 3767 struct ethhdr *eth; 3768 3769 if (unlikely(!pskb_may_pull(skb, sizeof(struct ethhdr)))) 3770 return 0; 3771 3772 eth = (struct ethhdr *)skb->data; 3773 type = eth->h_proto; 3774 } 3775 3776 return vlan_get_protocol_and_depth(skb, type, depth); 3777 } 3778 3779 3780 /* Take action when hardware reception checksum errors are detected. */ 3781 #ifdef CONFIG_BUG 3782 static void do_netdev_rx_csum_fault(struct net_device *dev, struct sk_buff *skb) 3783 { 3784 netdev_err(dev, "hw csum failure\n"); 3785 skb_dump(KERN_ERR, skb, true); 3786 dump_stack(); 3787 } 3788 3789 void netdev_rx_csum_fault(struct net_device *dev, struct sk_buff *skb) 3790 { 3791 DO_ONCE_LITE(do_netdev_rx_csum_fault, dev, skb); 3792 } 3793 EXPORT_SYMBOL(netdev_rx_csum_fault); 3794 #endif 3795 3796 /* XXX: check that highmem exists at all on the given machine. */ 3797 static int illegal_highdma(struct net_device *dev, struct sk_buff *skb) 3798 { 3799 #ifdef CONFIG_HIGHMEM 3800 int i; 3801 3802 if (!(dev->features & NETIF_F_HIGHDMA)) { 3803 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 3804 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 3805 struct page *page = skb_frag_page(frag); 3806 3807 if (page && PageHighMem(page)) 3808 return 1; 3809 } 3810 } 3811 #endif 3812 return 0; 3813 } 3814 3815 /* If MPLS offload request, verify we are testing hardware MPLS features 3816 * instead of standard features for the netdev. 3817 */ 3818 #if IS_ENABLED(CONFIG_NET_MPLS_GSO) 3819 static netdev_features_t net_mpls_features(struct sk_buff *skb, 3820 netdev_features_t features, 3821 __be16 type) 3822 { 3823 if (eth_p_mpls(type)) 3824 features &= skb->dev->mpls_features; 3825 3826 return features; 3827 } 3828 #else 3829 static netdev_features_t net_mpls_features(struct sk_buff *skb, 3830 netdev_features_t features, 3831 __be16 type) 3832 { 3833 return features; 3834 } 3835 #endif 3836 3837 static netdev_features_t harmonize_features(struct sk_buff *skb, 3838 netdev_features_t features) 3839 { 3840 __be16 type; 3841 3842 type = skb_network_protocol(skb, NULL); 3843 features = net_mpls_features(skb, features, type); 3844 3845 if (skb->ip_summed != CHECKSUM_NONE && 3846 !can_checksum_protocol(features, type)) { 3847 features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 3848 } 3849 if (illegal_highdma(skb->dev, skb)) 3850 features &= ~NETIF_F_SG; 3851 3852 return features; 3853 } 3854 3855 netdev_features_t passthru_features_check(struct sk_buff *skb, 3856 struct net_device *dev, 3857 netdev_features_t features) 3858 { 3859 return features; 3860 } 3861 EXPORT_SYMBOL(passthru_features_check); 3862 3863 static netdev_features_t dflt_features_check(struct sk_buff *skb, 3864 struct net_device *dev, 3865 netdev_features_t features) 3866 { 3867 return vlan_features_check(skb, features); 3868 } 3869 3870 static bool __skb_has_ipv6_ext_hdr(const struct sk_buff *skb, int nhoff) 3871 { 3872 const struct ipv6hdr *ip6h; 3873 struct ipv6hdr _ip6h; 3874 3875 ip6h = skb_header_pointer(skb, nhoff, sizeof(_ip6h), &_ip6h); 3876 return ip6h && ip6h->version == 6 && ipv6_ext_hdr(ip6h->nexthdr); 3877 } 3878 3879 static bool skb_has_ipv6_extension_hdr(const struct sk_buff *skb) 3880 { 3881 if (vlan_get_protocol(skb) == htons(ETH_P_IPV6)) { 3882 if (__skb_has_ipv6_ext_hdr(skb, skb_network_offset(skb))) 3883 return true; 3884 3885 /* Catch tunnels without skb->encapsulation (e.g., virtio). */ 3886 if (!skb->encapsulation && 3887 skb_transport_header_was_set(skb) && 3888 skb_network_header_len(skb) != sizeof(struct ipv6hdr)) 3889 return true; 3890 } 3891 3892 /* Tunnels without an inner network header, such as SCTP-in-UDP or 3893 * PSP, have no inner IP header and thus no inner extension header. 3894 */ 3895 if (skb->encapsulation && skb_inner_network_header_was_set(skb) && 3896 __skb_has_ipv6_ext_hdr(skb, skb_inner_network_offset(skb))) 3897 return true; 3898 3899 return false; 3900 } 3901 3902 static unsigned int 3903 netif_get_gso_max_size(const struct net_device *dev, const struct sk_buff *skb) 3904 { 3905 /* pairs with WRITE_ONCE() in netif_set_gso(_ipv4)_max_size() */ 3906 return vlan_get_protocol(skb) == htons(ETH_P_IPV6) ? 3907 READ_ONCE(dev->gso_max_size) : 3908 READ_ONCE(dev->gso_ipv4_max_size); 3909 } 3910 3911 static bool 3912 gso_within_dev_limits(const struct sk_buff *skb, const struct net_device *dev) 3913 { 3914 if (skb_shinfo(skb)->gso_segs > READ_ONCE(dev->gso_max_segs)) 3915 return false; 3916 3917 if (unlikely(skb->len >= netif_get_gso_max_size(dev, skb))) 3918 return false; 3919 3920 return true; 3921 } 3922 3923 static netdev_features_t gso_features_check(const struct sk_buff *skb, 3924 struct net_device *dev, 3925 netdev_features_t features, 3926 bool check_limits) 3927 { 3928 if (check_limits && !gso_within_dev_limits(skb, dev)) 3929 return features & ~NETIF_F_GSO_MASK; 3930 3931 if (!skb_shinfo(skb)->gso_type) { 3932 skb_warn_bad_offload(skb); 3933 return features & ~NETIF_F_GSO_MASK; 3934 } 3935 3936 /* Support for GSO partial features requires software 3937 * intervention before we can actually process the packets 3938 * so we need to strip support for any partial features now 3939 * and we can pull them back in after we have partially 3940 * segmented the frame. 3941 */ 3942 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL)) 3943 features &= ~dev->gso_partial_features; 3944 3945 /* Make sure to clear the IPv4 ID mangling feature if the IPv4 header 3946 * has the potential to be fragmented so that TSO does not generate 3947 * segments with the same ID. For encapsulated packets, the ID mangling 3948 * feature is guaranteed not to use the same ID for the outer IPv4 3949 * headers of the generated segments if the headers have the potential 3950 * to be fragmented, so there is no need to clear the IPv4 ID mangling 3951 * feature (see the section about NETIF_F_TSO_MANGLEID in 3952 * segmentation-offloads.rst). 3953 */ 3954 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) { 3955 const struct iphdr *iph; 3956 struct iphdr _iph; 3957 int nhoff = skb->encapsulation ? 3958 skb_inner_network_offset(skb) : 3959 skb_network_offset(skb); 3960 3961 iph = skb_header_pointer(skb, nhoff, sizeof(_iph), &_iph); 3962 3963 if (!iph || !(iph->frag_off & htons(IP_DF))) 3964 features &= ~dev->mangleid_features; 3965 } 3966 3967 /* NETIF_F_IPV6_CSUM does not support IPv6 extension headers, 3968 * so neither does TSO that depends on it. 3969 */ 3970 if (features & NETIF_F_IPV6_CSUM && 3971 skb_has_ipv6_extension_hdr(skb)) 3972 features &= ~(NETIF_F_IPV6_CSUM | NETIF_F_TSO6 | NETIF_F_GSO_UDP_L4); 3973 3974 return features; 3975 } 3976 3977 netdev_features_t 3978 __netif_skb_features(struct sk_buff *skb, bool check_gso_limits) 3979 { 3980 struct net_device *dev = skb->dev; 3981 netdev_features_t features = dev->features; 3982 3983 if (skb_is_gso(skb)) 3984 features = gso_features_check(skb, dev, features, 3985 check_gso_limits); 3986 3987 /* If encapsulation offload request, verify we are testing 3988 * hardware encapsulation features instead of standard 3989 * features for the netdev 3990 */ 3991 if (skb->encapsulation) 3992 features &= dev->hw_enc_features; 3993 3994 if (skb_vlan_tagged(skb)) 3995 features = netdev_intersect_features(features, 3996 dev->vlan_features | 3997 NETIF_F_HW_VLAN_CTAG_TX | 3998 NETIF_F_HW_VLAN_STAG_TX); 3999 4000 if (dev->netdev_ops->ndo_features_check) 4001 features &= dev->netdev_ops->ndo_features_check(skb, dev, 4002 features); 4003 else 4004 features &= dflt_features_check(skb, dev, features); 4005 4006 return harmonize_features(skb, features); 4007 } 4008 EXPORT_SYMBOL(__netif_skb_features); 4009 4010 static unsigned int 4011 skb_gso_output_max_segs(struct sk_buff *skb, struct net_device *dev) 4012 { 4013 unsigned int mss = skb_shinfo(skb)->gso_size; 4014 unsigned int gso_max_size, hdr_len, max_segs; 4015 netdev_features_t features; 4016 struct tcphdr _tcph, *th; 4017 4018 if (!skb_is_gso(skb) || !skb_is_gso_tcp(skb) || 4019 skb->encapsulation || skb_has_frag_list(skb)) 4020 return 0; 4021 4022 /* 4023 * The transport offset can be unset or stale, so hdr_len is only taken 4024 * from a header at the checksum start. 4025 */ 4026 if (!skb_transport_header_was_set(skb) || 4027 skb_checksum_start(skb) != skb_transport_header(skb)) 4028 return 0; 4029 4030 th = skb_header_pointer(skb, skb_transport_offset(skb), sizeof(_tcph), 4031 &_tcph); 4032 if (!th || __tcp_hdrlen(th) < sizeof(*th)) 4033 return 0; 4034 4035 hdr_len = skb_transport_header(skb) - skb_mac_header(skb) + 4036 __tcp_hdrlen(th); 4037 4038 /* The output is still a GSO skb: recheck without the limit checks */ 4039 features = __netif_skb_features(skb, false) | NETIF_F_GSO_ROBUST; 4040 if (!net_gso_ok(features, skb_shinfo(skb)->gso_type)) 4041 return 0; 4042 4043 gso_max_size = netif_get_gso_max_size(dev, skb); 4044 gso_max_size = min(gso_max_size, GSO_LEGACY_MAX_SIZE); 4045 4046 /* 4047 * gso_within_dev_limits() accepts gso_segs == gso_max_segs but 4048 * rejects skb->len >= gso_max_size, so only the size limit needs the 4049 * - 1; the inner min() keeps that subtraction from wrapping. 4050 */ 4051 max_segs = (gso_max_size - min(gso_max_size, hdr_len + 1)) / mss; 4052 max_segs = min(max_segs, READ_ONCE(dev->gso_max_segs)); 4053 4054 return max_segs; 4055 } 4056 4057 static int xmit_one(struct sk_buff *skb, struct net_device *dev, 4058 struct netdev_queue *txq, bool more) 4059 { 4060 unsigned int len; 4061 int rc; 4062 4063 if (dev_nit_active_rcu(dev)) 4064 dev_queue_xmit_nit(skb, dev); 4065 4066 len = skb->len; 4067 trace_net_dev_start_xmit(skb, dev); 4068 rc = netdev_start_xmit(skb, dev, txq, more); 4069 trace_net_dev_xmit(skb, rc, dev, len); 4070 4071 return rc; 4072 } 4073 4074 struct sk_buff *dev_hard_start_xmit(struct sk_buff *first, struct net_device *dev, 4075 struct netdev_queue *txq, int *ret) 4076 { 4077 struct sk_buff *skb = first; 4078 int rc = NETDEV_TX_OK; 4079 4080 while (skb) { 4081 struct sk_buff *next = skb->next; 4082 4083 skb_mark_not_on_list(skb); 4084 rc = xmit_one(skb, dev, txq, next != NULL); 4085 if (unlikely(!dev_xmit_complete(rc))) { 4086 skb->next = next; 4087 goto out; 4088 } 4089 4090 skb = next; 4091 if (netif_tx_queue_stopped(txq) && skb) { 4092 rc = NETDEV_TX_BUSY; 4093 break; 4094 } 4095 } 4096 4097 out: 4098 *ret = rc; 4099 return skb; 4100 } 4101 4102 static struct sk_buff *validate_xmit_vlan(struct sk_buff *skb, 4103 netdev_features_t features) 4104 { 4105 if (skb_vlan_tag_present(skb) && 4106 !vlan_hw_offload_capable(features, skb->vlan_proto)) 4107 skb = __vlan_hwaccel_push_inside(skb); 4108 return skb; 4109 } 4110 4111 int skb_csum_hwoffload_help(struct sk_buff *skb, 4112 const netdev_features_t features) 4113 { 4114 if (unlikely(skb_csum_is_sctp(skb))) 4115 return !!(features & NETIF_F_SCTP_CRC) ? 0 : 4116 skb_crc32c_csum_help(skb); 4117 4118 if (features & NETIF_F_HW_CSUM) 4119 return 0; 4120 4121 if (features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) { 4122 if (skb_has_ipv6_extension_hdr(skb)) 4123 goto sw_checksum; 4124 4125 switch (skb->csum_offset) { 4126 case offsetof(struct tcphdr, check): 4127 case offsetof(struct udphdr, check): 4128 return 0; 4129 } 4130 } 4131 4132 sw_checksum: 4133 return skb_checksum_help(skb); 4134 } 4135 EXPORT_SYMBOL(skb_csum_hwoffload_help); 4136 4137 /* Checks if this SKB belongs to an HW offloaded socket 4138 * and whether any SW fallbacks are required based on dev. 4139 * Check decrypted mark in case skb_orphan() cleared socket. 4140 */ 4141 static struct sk_buff *sk_validate_xmit_skb(struct sk_buff *skb, 4142 struct net_device *dev) 4143 { 4144 #ifdef CONFIG_SOCK_VALIDATE_XMIT 4145 struct sk_buff *(*sk_validate)(struct sock *sk, struct net_device *dev, 4146 struct sk_buff *skb); 4147 struct sock *sk = skb->sk; 4148 4149 sk_validate = NULL; 4150 if (sk) { 4151 if (sk_fullsock(sk)) 4152 sk_validate = sk->sk_validate_xmit_skb; 4153 else if (sk_is_inet(sk) && sk->sk_state == TCP_TIME_WAIT) 4154 sk_validate = inet_twsk(sk)->tw_validate_xmit_skb; 4155 } 4156 4157 if (sk_validate) { 4158 skb = sk_validate(sk, dev, skb); 4159 } else if (unlikely(skb_is_decrypted(skb))) { 4160 pr_warn_ratelimited("unencrypted skb with no associated socket - dropping\n"); 4161 kfree_skb(skb); 4162 skb = NULL; 4163 } 4164 #endif 4165 4166 return skb; 4167 } 4168 4169 static struct sk_buff *validate_xmit_unreadable_skb(struct sk_buff *skb, 4170 struct net_device *dev) 4171 { 4172 struct skb_shared_info *shinfo; 4173 struct net_iov *niov; 4174 4175 if (likely(skb_frags_readable(skb) || 4176 dev->netmem_tx == NETMEM_TX_NO_DMA)) 4177 goto out; 4178 4179 if (dev->netmem_tx == NETMEM_TX_NONE) 4180 goto out_free; 4181 4182 shinfo = skb_shinfo(skb); 4183 4184 if (shinfo->nr_frags > 0) { 4185 niov = netmem_to_net_iov(skb_frag_netmem(&shinfo->frags[0])); 4186 if (net_is_devmem_iov(niov) && 4187 READ_ONCE(net_devmem_iov_binding(niov)->dev) != dev) 4188 goto out_free; 4189 } 4190 4191 out: 4192 return skb; 4193 4194 out_free: 4195 kfree_skb(skb); 4196 return NULL; 4197 } 4198 4199 /* Returns the skb on success, NULL if dropped, or ERR_PTR(-EINPROGRESS) 4200 * if stolen by async xfrm crypto (delivered via xfrm_dev_resume()). 4201 */ 4202 static struct sk_buff *validate_xmit_skb(struct sk_buff *skb, struct net_device *dev, bool *again) 4203 { 4204 netdev_features_t features; 4205 4206 skb = validate_xmit_unreadable_skb(skb, dev); 4207 if (unlikely(!skb)) 4208 goto out_null; 4209 4210 /* Only fraglist GSO, whose segments already carry valid checksums, 4211 * can be handed to a device as CHECKSUM_UNNECESSARY. Any other GSO 4212 * skb has no csum_start for the device to segment from. NONE means 4213 * the same on transmit, but makes netif_needs_gso() send it to the 4214 * GSO code, which turns TCP into CHECKSUM_PARTIAL for TSO and 4215 * segments anything the device cannot offload in software. 4216 */ 4217 if (unlikely(skb->ip_summed == CHECKSUM_UNNECESSARY && 4218 skb_is_gso(skb) && 4219 !(skb_shinfo(skb)->gso_type & SKB_GSO_FRAGLIST))) 4220 skb->ip_summed = CHECKSUM_NONE; 4221 4222 features = netif_skb_features(skb); 4223 skb = validate_xmit_vlan(skb, features); 4224 if (unlikely(!skb)) 4225 goto out_null; 4226 4227 skb = sk_validate_xmit_skb(skb, dev); 4228 if (unlikely(!skb)) 4229 goto out_null; 4230 4231 if (netif_needs_gso(skb, features)) { 4232 unsigned int max_segs = 0; 4233 struct sk_buff *segs; 4234 4235 if (unlikely(!gso_within_dev_limits(skb, dev))) 4236 max_segs = skb_gso_output_max_segs(skb, dev); 4237 4238 segs = __skb_gso_segment(skb, features, true, max_segs); 4239 if (IS_ERR(segs)) { 4240 goto out_kfree_skb; 4241 } else if (segs) { 4242 consume_skb(skb); 4243 skb = segs; 4244 } 4245 } else { 4246 if (skb_needs_linearize(skb, features) && 4247 __skb_linearize(skb)) 4248 goto out_kfree_skb; 4249 4250 /* If packet is not checksummed and device does not 4251 * support checksumming for this protocol, complete 4252 * checksumming here. 4253 */ 4254 if (skb->ip_summed == CHECKSUM_PARTIAL) { 4255 if (skb->encapsulation) 4256 skb_set_inner_transport_header(skb, 4257 skb_checksum_start_offset(skb)); 4258 else 4259 skb_set_transport_header(skb, 4260 skb_checksum_start_offset(skb)); 4261 if (skb_csum_hwoffload_help(skb, features)) 4262 goto out_kfree_skb; 4263 } 4264 } 4265 4266 skb = validate_xmit_xfrm(skb, features, again); 4267 4268 return skb; 4269 4270 out_kfree_skb: 4271 kfree_skb(skb); 4272 out_null: 4273 dev_core_stats_tx_dropped_inc(dev); 4274 return NULL; 4275 } 4276 4277 struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev, bool *again) 4278 { 4279 struct sk_buff *next, *head = NULL, *tail; 4280 4281 for (; skb != NULL; skb = next) { 4282 next = skb->next; 4283 skb_mark_not_on_list(skb); 4284 4285 /* in case skb won't be segmented, point to itself */ 4286 skb->prev = skb; 4287 4288 skb = validate_xmit_skb(skb, dev, again); 4289 if (IS_ERR_OR_NULL(skb)) 4290 continue; 4291 4292 if (!head) 4293 head = skb; 4294 else 4295 tail->next = skb; 4296 /* If skb was segmented, skb->prev points to 4297 * the last segment. If not, it still contains skb. 4298 */ 4299 tail = skb->prev; 4300 } 4301 return head; 4302 } 4303 EXPORT_SYMBOL_GPL(validate_xmit_skb_list); 4304 4305 static enum skb_drop_reason qdisc_pkt_len_segs_init(struct sk_buff *skb) 4306 { 4307 struct skb_shared_info *shinfo = skb_shinfo(skb); 4308 unsigned int hdr_len, tlen; 4309 u32 gso_segs; 4310 4311 qdisc_skb_cb(skb)->pkt_len = skb->len; 4312 if (!shinfo->gso_size) { 4313 qdisc_skb_cb(skb)->pkt_segs = 1; 4314 return SKB_NOT_DROPPED_YET; 4315 } 4316 4317 qdisc_skb_cb(skb)->pkt_segs = gso_segs = shinfo->gso_segs; 4318 4319 /* To get more precise estimation of bytes sent on wire, 4320 * we add to pkt_len the headers size of all segments 4321 */ 4322 4323 /* mac layer + network layer */ 4324 if (!skb->encapsulation) { 4325 if (unlikely(!skb_transport_header_was_set(skb))) 4326 return SKB_NOT_DROPPED_YET; 4327 hdr_len = skb_transport_offset(skb); 4328 } else { 4329 hdr_len = skb_inner_transport_offset(skb); 4330 } 4331 /* + transport layer */ 4332 if (likely(shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))) { 4333 const struct tcphdr *th; 4334 4335 /* A header in front of skb->data gives a negative offset. */ 4336 if ((int)hdr_len < 0 || 4337 !pskb_may_pull(skb, hdr_len + sizeof(struct tcphdr))) 4338 return SKB_DROP_REASON_SKB_BAD_GSO; 4339 4340 th = (const struct tcphdr *)(skb->data + hdr_len); 4341 tlen = __tcp_hdrlen(th); 4342 if (tlen < sizeof(*th)) 4343 return SKB_DROP_REASON_SKB_BAD_GSO; 4344 hdr_len += tlen; 4345 if (!pskb_may_pull(skb, hdr_len)) 4346 return SKB_DROP_REASON_SKB_BAD_GSO; 4347 } else if (shinfo->gso_type & SKB_GSO_UDP_L4) { 4348 if (!pskb_may_pull(skb, hdr_len + sizeof(struct udphdr))) 4349 return SKB_DROP_REASON_SKB_BAD_GSO; 4350 hdr_len += sizeof(struct udphdr); 4351 } 4352 4353 /* prior pskb_may_pull() might have changed skb->head. */ 4354 shinfo = skb_shinfo(skb); 4355 if (unlikely(shinfo->gso_type & SKB_GSO_DODGY)) { 4356 int payload = skb->len - hdr_len; 4357 4358 /* Malicious packet. */ 4359 if (payload <= 0) 4360 return SKB_DROP_REASON_SKB_BAD_GSO; 4361 gso_segs = DIV_ROUND_UP(payload, shinfo->gso_size); 4362 /* shinfo->gso_segs is a 16bit field. 4363 * Also make sure untrusted gso_size/hdr_len can not inflate 4364 * qdisc_pkt_len() beyond QDISC_PKT_LEN_MAX, or deficit 4365 * schedulers could spin in their refill loops. 4366 */ 4367 if (unlikely(gso_segs > GSO_MAX_SEGS || 4368 skb->len + (u64)(gso_segs - 1) * hdr_len > 4369 QDISC_PKT_LEN_MAX)) 4370 return SKB_DROP_REASON_SKB_BAD_GSO; 4371 shinfo->gso_segs = gso_segs; 4372 qdisc_skb_cb(skb)->pkt_segs = gso_segs; 4373 } 4374 qdisc_skb_cb(skb)->pkt_len += (gso_segs - 1) * hdr_len; 4375 return SKB_NOT_DROPPED_YET; 4376 } 4377 4378 static int dev_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *q, 4379 struct sk_buff **to_free, 4380 struct netdev_queue *txq) 4381 { 4382 int rc; 4383 4384 rc = q->enqueue(skb, q, to_free) & NET_XMIT_MASK; 4385 if (rc == NET_XMIT_SUCCESS) 4386 trace_qdisc_enqueue(q, txq, skb); 4387 return rc; 4388 } 4389 4390 static inline int __dev_xmit_skb(struct sk_buff *skb, struct Qdisc *q, 4391 struct net_device *dev, 4392 struct netdev_queue *txq) 4393 { 4394 struct sk_buff *next, *to_free = NULL, *to_free2 = NULL; 4395 spinlock_t *root_lock = qdisc_lock(q); 4396 struct llist_node *ll_list, *first_n; 4397 unsigned long defer_count = 0; 4398 int rc; 4399 4400 qdisc_calculate_pkt_len(skb, q); 4401 4402 tcf_set_qdisc_drop_reason(skb, QDISC_DROP_GENERIC); 4403 4404 if (q->flags & TCQ_F_NOLOCK) { 4405 if (q->flags & TCQ_F_CAN_BYPASS && nolock_qdisc_is_empty(q) && 4406 qdisc_run_begin(q)) { 4407 /* Retest nolock_qdisc_is_empty() within the protection 4408 * of q->seqlock to protect from racing with requeuing. 4409 */ 4410 if (unlikely(!nolock_qdisc_is_empty(q))) { 4411 rc = dev_qdisc_enqueue(skb, q, &to_free, txq); 4412 __qdisc_run(q); 4413 to_free2 = qdisc_run_end(q); 4414 4415 goto free_skbs; 4416 } 4417 4418 qdisc_bstats_cpu_update(q, skb); 4419 if (sch_direct_xmit(skb, q, dev, txq, NULL, true) && 4420 !nolock_qdisc_is_empty(q)) 4421 __qdisc_run(q); 4422 4423 to_free2 = qdisc_run_end(q); 4424 rc = NET_XMIT_SUCCESS; 4425 goto free_skbs; 4426 } 4427 4428 rc = dev_qdisc_enqueue(skb, q, &to_free, txq); 4429 to_free2 = qdisc_run(q); 4430 goto free_skbs; 4431 } 4432 4433 /* Open code llist_add(&skb->ll_node, &q->defer_list) + queue limit. 4434 * In the try_cmpxchg() loop, we want to increment q->defer_count 4435 * at most once to limit the number of skbs in defer_list. 4436 * We perform the defer_count increment only if the list is not empty, 4437 * because some arches have slow atomic_long_inc_return(). 4438 */ 4439 first_n = READ_ONCE(q->defer_list.first); 4440 do { 4441 if (first_n && !defer_count) { 4442 defer_count = atomic_long_inc_return(&q->defer_count); 4443 if (unlikely(defer_count > READ_ONCE(net_hotdata.qdisc_max_burst))) { 4444 kfree_skb_reason(skb, SKB_DROP_REASON_QDISC_BURST_DROP); 4445 return NET_XMIT_DROP; 4446 } 4447 } 4448 skb->ll_node.next = first_n; 4449 } while (!try_cmpxchg(&q->defer_list.first, &first_n, &skb->ll_node)); 4450 4451 /* If defer_list was not empty, we know the cpu which queued 4452 * the first skb will process the whole list for us. 4453 */ 4454 if (first_n) 4455 return NET_XMIT_SUCCESS; 4456 4457 spin_lock(root_lock); 4458 4459 ll_list = llist_del_all(&q->defer_list); 4460 /* There is a small race because we clear defer_count not atomically 4461 * with the prior llist_del_all(). This means defer_list could grow 4462 * over qdisc_max_burst. 4463 */ 4464 atomic_long_set(&q->defer_count, 0); 4465 4466 ll_list = llist_reverse_order(ll_list); 4467 4468 if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, &q->state))) { 4469 llist_for_each_entry_safe(skb, next, ll_list, ll_node) 4470 __qdisc_drop(skb, &to_free); 4471 rc = NET_XMIT_DROP; 4472 goto unlock; 4473 } 4474 if ((q->flags & TCQ_F_CAN_BYPASS) && !qdisc_qlen(q) && 4475 !llist_next(ll_list) && qdisc_run_begin(q)) { 4476 /* 4477 * This is a work-conserving queue; there are no old skbs 4478 * waiting to be sent out; and the qdisc is not running - 4479 * xmit the skb directly. 4480 */ 4481 4482 DEBUG_NET_WARN_ON_ONCE(skb != llist_entry(ll_list, 4483 struct sk_buff, 4484 ll_node)); 4485 qdisc_bstats_update(q, skb); 4486 if (sch_direct_xmit(skb, q, dev, txq, root_lock, true)) 4487 __qdisc_run(q); 4488 to_free2 = qdisc_run_end(q); 4489 rc = NET_XMIT_SUCCESS; 4490 } else { 4491 int count = 0; 4492 4493 llist_for_each_entry_safe(skb, next, ll_list, ll_node) { 4494 if (next) { 4495 prefetch(next); 4496 prefetch(&next->priority); 4497 skb_mark_not_on_list(skb); 4498 } 4499 rc = dev_qdisc_enqueue(skb, q, &to_free, txq); 4500 count++; 4501 } 4502 to_free2 = qdisc_run(q); 4503 if (count != 1) 4504 rc = NET_XMIT_SUCCESS; 4505 } 4506 unlock: 4507 spin_unlock(root_lock); 4508 4509 free_skbs: 4510 tcf_kfree_skb_list(to_free, q, txq, dev); 4511 tcf_kfree_skb_list(to_free2, q, txq, dev); 4512 return rc; 4513 } 4514 4515 #if IS_ENABLED(CONFIG_CGROUP_NET_PRIO) 4516 static void skb_update_prio(struct sk_buff *skb) 4517 { 4518 const struct netprio_map *map; 4519 const struct sock *sk; 4520 unsigned int prioidx; 4521 4522 if (skb->priority) 4523 return; 4524 map = rcu_dereference_bh(skb->dev->priomap); 4525 if (!map) 4526 return; 4527 sk = skb_to_full_sk(skb); 4528 if (!sk) 4529 return; 4530 4531 prioidx = sock_cgroup_prioidx(&sk->sk_cgrp_data); 4532 4533 if (prioidx < map->priomap_len) 4534 skb->priority = map->priomap[prioidx]; 4535 } 4536 #else 4537 #define skb_update_prio(skb) 4538 #endif 4539 4540 /** 4541 * dev_loopback_xmit - loop back @skb 4542 * @net: network namespace this loopback is happening in 4543 * @sk: sk needed to be a netfilter okfn 4544 * @skb: buffer to transmit 4545 */ 4546 int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *skb) 4547 { 4548 skb_reset_mac_header(skb); 4549 __skb_pull(skb, skb_network_offset(skb)); 4550 skb->pkt_type = PACKET_LOOPBACK; 4551 if (skb->ip_summed == CHECKSUM_NONE) 4552 skb->ip_summed = CHECKSUM_UNNECESSARY; 4553 DEBUG_NET_WARN_ON_ONCE(!skb_dst(skb)); 4554 skb_dst_force(skb); 4555 netif_rx(skb); 4556 return 0; 4557 } 4558 EXPORT_SYMBOL(dev_loopback_xmit); 4559 4560 #ifdef CONFIG_NET_EGRESS 4561 static struct netdev_queue * 4562 netdev_tx_queue_mapping(struct net_device *dev, struct sk_buff *skb) 4563 { 4564 int queue = skb_get_queue_mapping(skb); 4565 int capped; 4566 4567 capped = netdev_cap_txqueue(dev, queue); 4568 if (unlikely(capped != queue)) 4569 skb_set_queue_mapping(skb, capped); 4570 4571 return netdev_get_tx_queue(dev, capped); 4572 } 4573 4574 #ifndef CONFIG_PREEMPT_RT 4575 static bool netdev_xmit_txqueue_skipped(void) 4576 { 4577 return __this_cpu_read(softnet_data.xmit.skip_txqueue); 4578 } 4579 4580 bool netdev_xmit_skip_txqueue(bool skip) 4581 { 4582 bool prev = netdev_xmit_txqueue_skipped(); 4583 4584 __this_cpu_write(softnet_data.xmit.skip_txqueue, skip); 4585 4586 return prev; 4587 } 4588 EXPORT_SYMBOL_GPL(netdev_xmit_skip_txqueue); 4589 4590 #else 4591 static bool netdev_xmit_txqueue_skipped(void) 4592 { 4593 return current->net_xmit.skip_txqueue; 4594 } 4595 4596 bool netdev_xmit_skip_txqueue(bool skip) 4597 { 4598 bool prev = netdev_xmit_txqueue_skipped(); 4599 4600 current->net_xmit.skip_txqueue = skip; 4601 4602 return prev; 4603 } 4604 EXPORT_SYMBOL_GPL(netdev_xmit_skip_txqueue); 4605 #endif 4606 #endif /* CONFIG_NET_EGRESS */ 4607 4608 #ifdef CONFIG_NET_XGRESS 4609 static int tc_run(struct tcx_entry *entry, struct sk_buff *skb, 4610 enum skb_drop_reason *drop_reason) 4611 { 4612 int ret = TC_ACT_UNSPEC; 4613 #ifdef CONFIG_NET_CLS_ACT 4614 struct mini_Qdisc *miniq = rcu_dereference_bh(entry->miniq); 4615 struct tcf_result res; 4616 4617 if (!miniq) 4618 return ret; 4619 4620 /* Global bypass */ 4621 if (!static_branch_likely(&tcf_sw_enabled_key)) 4622 return ret; 4623 4624 /* Block-wise bypass */ 4625 if (tcf_block_bypass_sw(miniq->block)) 4626 return ret; 4627 4628 tc_skb_cb(skb)->mru = 0; 4629 qdisc_skb_cb(skb)->post_ct = false; 4630 tcf_set_drop_reason(skb, *drop_reason); 4631 4632 mini_qdisc_bstats_cpu_update(miniq, skb); 4633 ret = tcf_classify(skb, miniq->block, miniq->filter_list, &res, false); 4634 /* Only tcf related quirks below. */ 4635 switch (ret) { 4636 case TC_ACT_SHOT: 4637 *drop_reason = tcf_get_drop_reason(skb); 4638 mini_qdisc_qstats_cpu_drop(miniq); 4639 break; 4640 case TC_ACT_OK: 4641 case TC_ACT_RECLASSIFY: 4642 skb->tc_index = TC_H_MIN(res.classid); 4643 break; 4644 } 4645 #endif /* CONFIG_NET_CLS_ACT */ 4646 return ret; 4647 } 4648 4649 static DEFINE_STATIC_KEY_FALSE(tcx_needed_key); 4650 4651 void tcx_inc(void) 4652 { 4653 static_branch_inc(&tcx_needed_key); 4654 } 4655 4656 void tcx_dec(void) 4657 { 4658 static_branch_dec(&tcx_needed_key); 4659 } 4660 4661 static __always_inline enum tcx_action_base 4662 tcx_run(const struct bpf_mprog_entry *entry, struct sk_buff *skb, 4663 const bool needs_mac) 4664 { 4665 const struct bpf_mprog_fp *fp; 4666 const struct bpf_prog *prog; 4667 int ret = TCX_NEXT; 4668 4669 if (needs_mac) 4670 __skb_push(skb, skb->mac_len); 4671 bpf_mprog_foreach_prog(entry, fp, prog) { 4672 bpf_compute_data_pointers(skb); 4673 ret = bpf_prog_run(prog, skb); 4674 if (ret != TCX_NEXT) 4675 break; 4676 } 4677 if (needs_mac) 4678 __skb_pull(skb, skb->mac_len); 4679 return tcx_action_code(skb, ret); 4680 } 4681 4682 static __always_inline struct sk_buff * 4683 sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret, 4684 struct net_device *orig_dev, bool *another) 4685 { 4686 struct bpf_mprog_entry *entry = rcu_dereference_bh(skb->dev->tcx_ingress); 4687 enum skb_drop_reason drop_reason = SKB_DROP_REASON_TC_INGRESS; 4688 struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx; 4689 int sch_ret; 4690 4691 if (!entry) 4692 return skb; 4693 4694 bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx); 4695 if (unlikely(*pt_prev)) { 4696 *ret = deliver_skb(skb, *pt_prev, orig_dev); 4697 *pt_prev = NULL; 4698 } 4699 4700 qdisc_pkt_len_segs_init(skb); 4701 tcx_set_ingress(skb, true); 4702 4703 if (static_branch_unlikely(&tcx_needed_key)) { 4704 sch_ret = tcx_run(entry, skb, true); 4705 if (sch_ret != TC_ACT_UNSPEC) 4706 goto ingress_verdict; 4707 } 4708 sch_ret = tc_run(tcx_entry(entry), skb, &drop_reason); 4709 ingress_verdict: 4710 switch (sch_ret) { 4711 case TC_ACT_REDIRECT: 4712 /* skb_mac_header check was done by BPF, so we can safely 4713 * push the L2 header back before redirecting to another 4714 * netdev. 4715 */ 4716 __skb_push(skb, skb->mac_len); 4717 if (skb_do_redirect(skb) == -EAGAIN) { 4718 __skb_pull(skb, skb->mac_len); 4719 *another = true; 4720 break; 4721 } 4722 *ret = NET_RX_SUCCESS; 4723 bpf_net_ctx_clear(bpf_net_ctx); 4724 return NULL; 4725 case TC_ACT_SHOT: 4726 kfree_skb_reason(skb, drop_reason); 4727 *ret = NET_RX_DROP; 4728 bpf_net_ctx_clear(bpf_net_ctx); 4729 return NULL; 4730 /* used by tc_run */ 4731 case TC_ACT_STOLEN: 4732 case TC_ACT_QUEUED: 4733 case TC_ACT_TRAP: 4734 consume_skb(skb); 4735 fallthrough; 4736 case TC_ACT_CONSUMED: 4737 *ret = NET_RX_SUCCESS; 4738 bpf_net_ctx_clear(bpf_net_ctx); 4739 return NULL; 4740 } 4741 bpf_net_ctx_clear(bpf_net_ctx); 4742 4743 return skb; 4744 } 4745 4746 static __always_inline struct sk_buff * 4747 sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev) 4748 { 4749 struct bpf_mprog_entry *entry = rcu_dereference_bh(dev->tcx_egress); 4750 enum skb_drop_reason drop_reason = SKB_DROP_REASON_TC_EGRESS; 4751 struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx; 4752 int sch_ret; 4753 4754 if (!entry) 4755 return skb; 4756 4757 bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx); 4758 4759 /* qdisc_skb_cb(skb)->pkt_len & tcx_set_ingress() was 4760 * already set by the caller. 4761 */ 4762 if (static_branch_unlikely(&tcx_needed_key)) { 4763 sch_ret = tcx_run(entry, skb, false); 4764 if (sch_ret != TC_ACT_UNSPEC) 4765 goto egress_verdict; 4766 } 4767 sch_ret = tc_run(tcx_entry(entry), skb, &drop_reason); 4768 egress_verdict: 4769 switch (sch_ret) { 4770 case TC_ACT_REDIRECT: 4771 /* No need to push/pop skb's mac_header here on egress! */ 4772 skb_do_redirect(skb); 4773 *ret = NET_XMIT_SUCCESS; 4774 bpf_net_ctx_clear(bpf_net_ctx); 4775 return NULL; 4776 case TC_ACT_SHOT: 4777 kfree_skb_reason(skb, drop_reason); 4778 *ret = NET_XMIT_DROP; 4779 bpf_net_ctx_clear(bpf_net_ctx); 4780 return NULL; 4781 /* used by tc_run */ 4782 case TC_ACT_STOLEN: 4783 case TC_ACT_QUEUED: 4784 case TC_ACT_TRAP: 4785 consume_skb(skb); 4786 fallthrough; 4787 case TC_ACT_CONSUMED: 4788 *ret = NET_XMIT_SUCCESS; 4789 bpf_net_ctx_clear(bpf_net_ctx); 4790 return NULL; 4791 } 4792 bpf_net_ctx_clear(bpf_net_ctx); 4793 4794 return skb; 4795 } 4796 #else 4797 static __always_inline struct sk_buff * 4798 sch_handle_ingress(struct sk_buff *skb, struct packet_type **pt_prev, int *ret, 4799 struct net_device *orig_dev, bool *another) 4800 { 4801 return skb; 4802 } 4803 4804 static __always_inline struct sk_buff * 4805 sch_handle_egress(struct sk_buff *skb, int *ret, struct net_device *dev) 4806 { 4807 return skb; 4808 } 4809 #endif /* CONFIG_NET_XGRESS */ 4810 4811 #ifdef CONFIG_XPS 4812 static int __get_xps_queue_idx(struct net_device *dev, struct sk_buff *skb, 4813 struct xps_dev_maps *dev_maps, unsigned int tci) 4814 { 4815 int tc = netdev_get_prio_tc_map(dev, skb->priority); 4816 struct xps_map *map; 4817 int queue_index = -1; 4818 4819 if (tc >= dev_maps->num_tc || tci >= dev_maps->nr_ids) 4820 return queue_index; 4821 4822 tci *= dev_maps->num_tc; 4823 tci += tc; 4824 4825 map = rcu_dereference(dev_maps->attr_map[tci]); 4826 if (map) { 4827 if (map->len == 1) 4828 queue_index = map->queues[0]; 4829 else 4830 queue_index = map->queues[reciprocal_scale( 4831 skb_get_hash(skb), map->len)]; 4832 if (unlikely(queue_index >= dev->real_num_tx_queues)) 4833 queue_index = -1; 4834 } 4835 return queue_index; 4836 } 4837 #endif 4838 4839 static int get_xps_queue(struct net_device *dev, struct net_device *sb_dev, 4840 struct sk_buff *skb) 4841 { 4842 #ifdef CONFIG_XPS 4843 struct xps_dev_maps *dev_maps; 4844 struct sock *sk = skb->sk; 4845 int queue_index = -1; 4846 4847 if (!static_key_false(&xps_needed)) 4848 return -1; 4849 4850 rcu_read_lock(); 4851 if (!static_key_false(&xps_rxqs_needed)) 4852 goto get_cpus_map; 4853 4854 dev_maps = rcu_dereference(sb_dev->xps_maps[XPS_RXQS]); 4855 if (dev_maps) { 4856 int tci = sk_rx_queue_get(sk); 4857 4858 if (tci >= 0) 4859 queue_index = __get_xps_queue_idx(dev, skb, dev_maps, 4860 tci); 4861 } 4862 4863 get_cpus_map: 4864 if (queue_index < 0) { 4865 dev_maps = rcu_dereference(sb_dev->xps_maps[XPS_CPUS]); 4866 if (dev_maps) { 4867 unsigned int tci = skb->sender_cpu - 1; 4868 4869 queue_index = __get_xps_queue_idx(dev, skb, dev_maps, 4870 tci); 4871 } 4872 } 4873 rcu_read_unlock(); 4874 4875 return queue_index; 4876 #else 4877 return -1; 4878 #endif 4879 } 4880 4881 u16 dev_pick_tx_zero(struct net_device *dev, struct sk_buff *skb, 4882 struct net_device *sb_dev) 4883 { 4884 return 0; 4885 } 4886 EXPORT_SYMBOL(dev_pick_tx_zero); 4887 4888 int sk_tx_queue_get(const struct sock *sk) 4889 { 4890 int resel, val; 4891 4892 if (!sk) 4893 return -1; 4894 /* Paired with WRITE_ONCE() in sk_tx_queue_clear() 4895 * and sk_tx_queue_set(). 4896 */ 4897 val = READ_ONCE(sk->sk_tx_queue_mapping); 4898 4899 if (val == NO_QUEUE_MAPPING) 4900 return -1; 4901 4902 if (!sk_fullsock(sk)) 4903 return val; 4904 4905 resel = READ_ONCE(sock_net(sk)->core.sysctl_txq_reselection); 4906 if (resel && time_is_before_jiffies( 4907 READ_ONCE(sk->sk_tx_queue_mapping_jiffies) + resel)) 4908 return -1; 4909 4910 return val; 4911 } 4912 EXPORT_SYMBOL(sk_tx_queue_get); 4913 4914 u16 netdev_pick_tx(struct net_device *dev, struct sk_buff *skb, 4915 struct net_device *sb_dev) 4916 { 4917 struct sock *sk = skb->sk; 4918 int queue_index = sk_tx_queue_get(sk); 4919 4920 sb_dev = sb_dev ? : dev; 4921 4922 if (queue_index < 0 || skb->ooo_okay || 4923 queue_index >= dev->real_num_tx_queues) { 4924 int new_index = get_xps_queue(dev, sb_dev, skb); 4925 4926 if (new_index < 0) 4927 new_index = skb_tx_hash(dev, sb_dev, skb); 4928 4929 if (sk && sk_fullsock(sk) && 4930 rcu_access_pointer(sk->sk_dst_cache)) 4931 sk_tx_queue_set(sk, new_index); 4932 4933 queue_index = new_index; 4934 } 4935 4936 return queue_index; 4937 } 4938 EXPORT_SYMBOL(netdev_pick_tx); 4939 4940 struct netdev_queue *netdev_core_pick_tx(struct net_device *dev, 4941 struct sk_buff *skb, 4942 struct net_device *sb_dev) 4943 { 4944 int queue_index = 0; 4945 4946 #ifdef CONFIG_XPS 4947 u32 sender_cpu = skb->sender_cpu - 1; 4948 4949 if (sender_cpu >= (u32)NR_CPUS) 4950 skb->sender_cpu = raw_smp_processor_id() + 1; 4951 #endif 4952 4953 if (dev->real_num_tx_queues != 1) { 4954 const struct net_device_ops *ops = dev->netdev_ops; 4955 4956 if (ops->ndo_select_queue) 4957 queue_index = ops->ndo_select_queue(dev, skb, sb_dev); 4958 else 4959 queue_index = netdev_pick_tx(dev, skb, sb_dev); 4960 4961 queue_index = netdev_cap_txqueue(dev, queue_index); 4962 } 4963 4964 skb_set_queue_mapping(skb, queue_index); 4965 return netdev_get_tx_queue(dev, queue_index); 4966 } 4967 4968 /** 4969 * __dev_queue_xmit() - transmit a buffer 4970 * @skb: buffer to transmit 4971 * @sb_dev: suboordinate device used for L2 forwarding offload 4972 * 4973 * Queue a buffer for transmission to a network device. The caller must 4974 * have set the device and priority and built the buffer before calling 4975 * this function. The function can be called from an interrupt. 4976 * 4977 * When calling this method, interrupts MUST be enabled. This is because 4978 * the BH enable code must have IRQs enabled so that it will not deadlock. 4979 * 4980 * Regardless of the return value, the skb is consumed, so it is currently 4981 * difficult to retry a send to this method. (You can bump the ref count 4982 * before sending to hold a reference for retry if you are careful.) 4983 * 4984 * Return: 4985 * * 0 - buffer successfully transmitted 4986 * * positive qdisc return code - NET_XMIT_DROP etc. 4987 * * negative errno - other errors 4988 */ 4989 int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev) 4990 { 4991 struct net_device *dev = skb->dev; 4992 struct netdev_queue *txq = NULL; 4993 enum skb_drop_reason reason; 4994 int cpu, rc = -ENOMEM; 4995 bool again = false; 4996 struct Qdisc *q; 4997 4998 skb_reset_mac_header(skb); 4999 skb_assert_len(skb); 5000 5001 if (unlikely(skb_shinfo(skb)->tx_flags & 5002 (SKBTX_SCHED_TSTAMP | SKBTX_BPF))) 5003 __skb_tstamp_tx(skb, NULL, NULL, skb->sk, SCM_TSTAMP_SCHED); 5004 5005 reason = qdisc_pkt_len_segs_init(skb); 5006 if (unlikely(reason)) { 5007 dev_core_stats_tx_dropped_inc(dev); 5008 kfree_skb_reason(skb, reason); 5009 return -EINVAL; 5010 } 5011 /* Disable soft irqs for various locks below. Also 5012 * stops preemption for RCU. 5013 */ 5014 rcu_read_lock_bh(); 5015 5016 skb_update_prio(skb); 5017 5018 tcx_set_ingress(skb, false); 5019 #ifdef CONFIG_NET_EGRESS 5020 if (static_branch_unlikely(&egress_needed_key)) { 5021 bool skip_txq; 5022 5023 if (nf_hook_egress_active()) { 5024 skb = nf_hook_egress(skb, &rc, dev); 5025 if (!skb) 5026 goto out; 5027 } 5028 5029 skip_txq = netdev_xmit_skip_txqueue(false); 5030 5031 nf_skip_egress(skb, true); 5032 skb = sch_handle_egress(skb, &rc, dev); 5033 if (!skb) { 5034 netdev_xmit_skip_txqueue(skip_txq); 5035 goto out; 5036 } 5037 nf_skip_egress(skb, false); 5038 5039 if (netdev_xmit_skip_txqueue(skip_txq)) 5040 txq = netdev_tx_queue_mapping(dev, skb); 5041 } 5042 #endif 5043 /* If device/qdisc don't need skb->dst, release it right now while 5044 * its hot in this cpu cache. 5045 */ 5046 if (dev->priv_flags & IFF_XMIT_DST_RELEASE) 5047 skb_dst_drop(skb); 5048 else 5049 skb_dst_force(skb); 5050 5051 if (!txq) 5052 txq = netdev_core_pick_tx(dev, skb, sb_dev); 5053 5054 q = rcu_dereference_bh(txq->qdisc); 5055 5056 trace_net_dev_queue(skb); 5057 if (q->enqueue) { 5058 rc = __dev_xmit_skb(skb, q, dev, txq); 5059 goto out; 5060 } 5061 5062 /* The device has no queue. Common case for software devices: 5063 * loopback, all the sorts of tunnels... 5064 5065 * Really, it is unlikely that netif_tx_lock protection is necessary 5066 * here. (f.e. loopback and IP tunnels are clean ignoring statistics 5067 * counters.) 5068 * However, it is possible, that they rely on protection 5069 * made by us here. 5070 5071 * Check this and shot the lock. It is not prone from deadlocks. 5072 *Either shot noqueue qdisc, it is even simpler 8) 5073 */ 5074 if (unlikely(!(dev->flags & IFF_UP))) { 5075 reason = SKB_DROP_REASON_DEV_READY; 5076 goto drop; 5077 } 5078 5079 cpu = smp_processor_id(); /* ok because BHs are off */ 5080 5081 if (likely(!netif_tx_owned(txq, cpu))) { 5082 bool is_list = false; 5083 5084 if (dev_xmit_recursion()) 5085 goto recursion_alert; 5086 5087 skb = validate_xmit_skb(skb, dev, &again); 5088 if (IS_ERR_OR_NULL(skb)) { 5089 if (PTR_ERR(skb) == -EINPROGRESS) 5090 rc = NET_XMIT_SUCCESS; 5091 goto out; 5092 } 5093 5094 HARD_TX_LOCK(dev, txq, cpu); 5095 5096 if (!netif_xmit_stopped(txq)) { 5097 is_list = !!skb->next; 5098 5099 dev_xmit_recursion_inc(); 5100 skb = dev_hard_start_xmit(skb, dev, txq, &rc); 5101 dev_xmit_recursion_dec(); 5102 5103 /* GSO segments a single SKB into a list of frames. 5104 * TCP expects error to mean none of the data was sent. 5105 */ 5106 if (is_list) 5107 rc = NETDEV_TX_OK; 5108 } 5109 HARD_TX_UNLOCK(dev, txq); 5110 if (!skb) /* xmit completed */ 5111 goto out; 5112 5113 net_crit_ratelimited("Virtual device %s asks to queue packet!\n", 5114 dev->name); 5115 /* NETDEV_TX_BUSY or queue was stopped */ 5116 if (!is_list) 5117 rc = -ENETDOWN; 5118 } else { 5119 /* Recursion is detected! It is possible unfortunately. */ 5120 recursion_alert: 5121 net_crit_ratelimited("Dead loop on virtual device %s (net %llu), fix it urgently!\n", 5122 dev->name, dev_net(dev)->net_cookie); 5123 5124 rc = -ENETDOWN; 5125 } 5126 5127 reason = SKB_DROP_REASON_RECURSION_LIMIT; 5128 drop: 5129 rcu_read_unlock_bh(); 5130 5131 dev_core_stats_tx_dropped_inc(dev); 5132 kfree_skb_list_reason(skb, reason); 5133 return rc; 5134 out: 5135 rcu_read_unlock_bh(); 5136 return rc; 5137 } 5138 EXPORT_SYMBOL(__dev_queue_xmit); 5139 5140 int __dev_direct_xmit(struct sk_buff *skb, u16 queue_id) 5141 { 5142 struct net_device *dev = skb->dev; 5143 struct sk_buff *orig_skb = skb; 5144 struct netdev_queue *txq; 5145 int ret = NETDEV_TX_BUSY; 5146 bool again = false; 5147 5148 if (unlikely(!netif_running(dev) || 5149 !netif_carrier_ok(dev))) 5150 goto drop; 5151 5152 skb = validate_xmit_skb_list(skb, dev, &again); 5153 if (skb != orig_skb) 5154 goto drop; 5155 5156 skb_set_queue_mapping(skb, queue_id); 5157 txq = skb_get_tx_queue(dev, skb); 5158 5159 local_bh_disable(); 5160 5161 dev_xmit_recursion_inc(); 5162 HARD_TX_LOCK(dev, txq, smp_processor_id()); 5163 if (!netif_xmit_frozen_or_drv_stopped(txq)) 5164 ret = netdev_start_xmit(skb, dev, txq, false); 5165 HARD_TX_UNLOCK(dev, txq); 5166 dev_xmit_recursion_dec(); 5167 5168 local_bh_enable(); 5169 return ret; 5170 drop: 5171 dev_core_stats_tx_dropped_inc(dev); 5172 kfree_skb_list(skb); 5173 return NET_XMIT_DROP; 5174 } 5175 EXPORT_SYMBOL(__dev_direct_xmit); 5176 5177 /************************************************************************* 5178 * Receiver routines 5179 *************************************************************************/ 5180 static DEFINE_PER_CPU(struct task_struct *, backlog_napi); 5181 5182 int weight_p __read_mostly = 64; /* old backlog weight */ 5183 int dev_weight_rx_bias __read_mostly = 1; /* bias for backlog weight */ 5184 int dev_weight_tx_bias __read_mostly = 1; /* bias for output_queue quota */ 5185 5186 /* Called with irq disabled */ 5187 static inline void ____napi_schedule(struct softnet_data *sd, 5188 struct napi_struct *napi) 5189 { 5190 struct task_struct *thread; 5191 5192 lockdep_assert_irqs_disabled(); 5193 5194 if (test_bit(NAPI_STATE_THREADED, &napi->state)) { 5195 /* Paired with smp_mb__before_atomic() in 5196 * napi_enable()/netif_set_threaded(). 5197 * Use READ_ONCE() to guarantee a complete 5198 * read on napi->thread. Only call 5199 * wake_up_process() when it's not NULL. 5200 */ 5201 thread = READ_ONCE(napi->thread); 5202 if (thread) { 5203 if (use_backlog_threads() && thread == raw_cpu_read(backlog_napi)) 5204 goto use_local_napi; 5205 5206 set_bit(NAPI_STATE_SCHED_THREADED, &napi->state); 5207 wake_up_process(thread); 5208 return; 5209 } 5210 } 5211 5212 use_local_napi: 5213 DEBUG_NET_WARN_ON_ONCE(!list_empty(&napi->poll_list)); 5214 list_add_tail(&napi->poll_list, &sd->poll_list); 5215 WRITE_ONCE(napi->list_owner, smp_processor_id()); 5216 /* If not called from net_rx_action() 5217 * we have to raise NET_RX_SOFTIRQ. 5218 */ 5219 if (!sd->in_net_rx_action) 5220 raise_softirq_irqoff(NET_RX_SOFTIRQ); 5221 } 5222 5223 #ifdef CONFIG_RPS 5224 5225 struct static_key_false rps_needed __read_mostly; 5226 EXPORT_SYMBOL(rps_needed); 5227 struct static_key_false rfs_needed __read_mostly; 5228 EXPORT_SYMBOL(rfs_needed); 5229 5230 static u32 rfs_slot(u32 hash, rps_tag_ptr tag_ptr) 5231 { 5232 return hash_32(hash, rps_tag_to_log(tag_ptr)); 5233 } 5234 5235 #ifdef CONFIG_RFS_ACCEL 5236 /** 5237 * rps_flow_is_active - check whether the flow is recently active. 5238 * @rflow: Specific flow to check activity. 5239 * @log: ilog2(hashsize). 5240 * @cpu: CPU saved in @rflow. 5241 * 5242 * If the CPU has processed many packets since the flow's last activity 5243 * (beyond 10 times the table size), the flow is considered stale. 5244 * 5245 * Return: true if flow was recently active. 5246 */ 5247 static bool rps_flow_is_active(struct rps_dev_flow *rflow, 5248 u8 log, 5249 unsigned int cpu) 5250 { 5251 unsigned int flow_last_active; 5252 unsigned int sd_input_head; 5253 5254 if (cpu >= nr_cpu_ids) 5255 return false; 5256 5257 sd_input_head = READ_ONCE(per_cpu(softnet_data, cpu).input_queue_head); 5258 flow_last_active = READ_ONCE(rflow->last_qtail); 5259 5260 return (int)(sd_input_head - flow_last_active) < 5261 (int)(10 << log); 5262 } 5263 #endif 5264 5265 static struct rps_dev_flow * 5266 set_rps_cpu(struct net_device *dev, struct sk_buff *skb, 5267 struct rps_dev_flow *rflow, u16 next_cpu, u32 hash) 5268 { 5269 if (next_cpu < nr_cpu_ids) { 5270 u32 head; 5271 #ifdef CONFIG_RFS_ACCEL 5272 struct netdev_rx_queue *rxqueue; 5273 struct rps_dev_flow *flow_table; 5274 struct rps_dev_flow *old_rflow; 5275 struct rps_dev_flow *tmp_rflow; 5276 rps_tag_ptr q_tag_ptr; 5277 unsigned int tmp_cpu; 5278 u16 rxq_index; 5279 u32 flow_id; 5280 int rc; 5281 5282 /* Should we steer this flow to a different hardware queue? */ 5283 if (!skb_rx_queue_recorded(skb) || !dev->rx_cpu_rmap || 5284 !(dev->features & NETIF_F_NTUPLE)) 5285 goto out; 5286 rxq_index = cpu_rmap_lookup_index(dev->rx_cpu_rmap, next_cpu); 5287 if (rxq_index == skb_get_rx_queue(skb)) 5288 goto out; 5289 5290 rxqueue = dev->_rx + rxq_index; 5291 q_tag_ptr = READ_ONCE(rxqueue->rps_flow_table); 5292 if (!q_tag_ptr) 5293 goto out; 5294 5295 flow_id = rfs_slot(hash, q_tag_ptr); 5296 flow_table = rps_tag_to_table(q_tag_ptr); 5297 tmp_rflow = flow_table + flow_id; 5298 tmp_cpu = READ_ONCE(tmp_rflow->cpu); 5299 5300 if (READ_ONCE(tmp_rflow->filter) != RPS_NO_FILTER) { 5301 if (rps_flow_is_active(tmp_rflow, 5302 rps_tag_to_log(q_tag_ptr), 5303 tmp_cpu)) { 5304 if (hash != READ_ONCE(tmp_rflow->hash) || 5305 next_cpu == tmp_cpu) 5306 goto out; 5307 } 5308 } 5309 5310 rc = dev->netdev_ops->ndo_rx_flow_steer(dev, skb, 5311 rxq_index, flow_id); 5312 if (rc < 0) 5313 goto out; 5314 5315 old_rflow = rflow; 5316 rflow = tmp_rflow; 5317 WRITE_ONCE(rflow->filter, rc); 5318 WRITE_ONCE(rflow->hash, hash); 5319 5320 if (old_rflow->filter == rc) 5321 WRITE_ONCE(old_rflow->filter, RPS_NO_FILTER); 5322 out: 5323 #endif 5324 head = READ_ONCE(per_cpu(softnet_data, next_cpu).input_queue_head); 5325 rps_input_queue_tail_save(&rflow->last_qtail, head); 5326 } 5327 5328 WRITE_ONCE(rflow->cpu, next_cpu); 5329 return rflow; 5330 } 5331 5332 /* 5333 * get_rps_cpu is called from netif_receive_skb and returns the target 5334 * CPU from the RPS map of the receiving queue for a given skb. 5335 * rcu_read_lock must be held on entry. 5336 */ 5337 static int get_rps_cpu(struct net_device *dev, struct sk_buff *skb, 5338 struct rps_dev_flow **rflowp) 5339 { 5340 struct netdev_rx_queue *rxqueue = dev->_rx; 5341 rps_tag_ptr global_tag_ptr, q_tag_ptr; 5342 struct rps_map *map; 5343 int cpu = -1; 5344 u32 tcpu; 5345 u32 hash; 5346 5347 if (skb_rx_queue_recorded(skb)) { 5348 u16 index = skb_get_rx_queue(skb); 5349 5350 if (unlikely(index >= dev->real_num_rx_queues)) { 5351 WARN_ONCE(dev->real_num_rx_queues > 1, 5352 "%s received packet on queue %u, but number " 5353 "of RX queues is %u\n", 5354 dev->name, index, dev->real_num_rx_queues); 5355 goto done; 5356 } 5357 rxqueue += index; 5358 } 5359 5360 /* Avoid computing hash if RFS/RPS is not active for this rxqueue */ 5361 5362 q_tag_ptr = READ_ONCE(rxqueue->rps_flow_table); 5363 map = rcu_dereference(rxqueue->rps_map); 5364 if (!q_tag_ptr && !map) 5365 goto done; 5366 5367 skb_reset_network_header(skb); 5368 hash = skb_get_hash(skb); 5369 if (!hash) 5370 goto done; 5371 5372 global_tag_ptr = READ_ONCE(net_hotdata.rps_sock_flow_table); 5373 if (q_tag_ptr && global_tag_ptr) { 5374 struct rps_sock_flow_table *sock_flow_table; 5375 struct rps_dev_flow *flow_table; 5376 struct rps_dev_flow *rflow; 5377 u32 next_cpu; 5378 u32 flow_id; 5379 u32 ident; 5380 5381 /* First check into global flow table if there is a match. 5382 * This READ_ONCE() pairs with WRITE_ONCE() from rps_record_sock_flow(). 5383 */ 5384 flow_id = hash & rps_tag_to_mask(global_tag_ptr); 5385 sock_flow_table = rps_tag_to_table(global_tag_ptr); 5386 ident = READ_ONCE(sock_flow_table[flow_id].ent); 5387 if ((ident ^ hash) & ~net_hotdata.rps_cpu_mask) 5388 goto try_rps; 5389 5390 next_cpu = ident & net_hotdata.rps_cpu_mask; 5391 5392 /* OK, now we know there is a match, 5393 * we can look at the local (per receive queue) flow table 5394 */ 5395 flow_id = rfs_slot(hash, q_tag_ptr); 5396 flow_table = rps_tag_to_table(q_tag_ptr); 5397 rflow = flow_table + flow_id; 5398 tcpu = rflow->cpu; 5399 5400 /* 5401 * If the desired CPU (where last recvmsg was done) is 5402 * different from current CPU (one in the rx-queue flow 5403 * table entry), switch if one of the following holds: 5404 * - Current CPU is unset (>= nr_cpu_ids). 5405 * - Current CPU is offline. 5406 * - The current CPU's queue tail has advanced beyond the 5407 * last packet that was enqueued using this table entry. 5408 * This guarantees that all previous packets for the flow 5409 * have been dequeued, thus preserving in order delivery. 5410 */ 5411 if (unlikely(tcpu != next_cpu) && 5412 (tcpu >= nr_cpu_ids || !cpu_online(tcpu) || 5413 ((int)(READ_ONCE(per_cpu(softnet_data, tcpu).input_queue_head) - 5414 rflow->last_qtail)) >= 0)) { 5415 tcpu = next_cpu; 5416 rflow = set_rps_cpu(dev, skb, rflow, next_cpu, hash); 5417 } 5418 5419 if (tcpu < nr_cpu_ids && cpu_online(tcpu)) { 5420 *rflowp = rflow; 5421 cpu = tcpu; 5422 goto done; 5423 } 5424 } 5425 5426 try_rps: 5427 5428 if (map) { 5429 tcpu = map->cpus[reciprocal_scale(hash, map->len)]; 5430 if (cpu_online(tcpu)) { 5431 cpu = tcpu; 5432 goto done; 5433 } 5434 } 5435 5436 done: 5437 return cpu; 5438 } 5439 5440 #ifdef CONFIG_RFS_ACCEL 5441 5442 /** 5443 * rps_may_expire_flow - check whether an RFS hardware filter may be removed 5444 * @dev: Device on which the filter was set 5445 * @rxq_index: RX queue index 5446 * @flow_id: Flow ID passed to ndo_rx_flow_steer() 5447 * @filter_id: Filter ID returned by ndo_rx_flow_steer() 5448 * 5449 * Drivers that implement ndo_rx_flow_steer() should periodically call 5450 * this function for each installed filter and remove the filters for 5451 * which it returns %true. 5452 */ 5453 bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index, 5454 u32 flow_id, u16 filter_id) 5455 { 5456 struct netdev_rx_queue *rxqueue = dev->_rx + rxq_index; 5457 struct rps_dev_flow *flow_table; 5458 struct rps_dev_flow *rflow; 5459 rps_tag_ptr q_tag_ptr; 5460 bool expire = true; 5461 u8 log; 5462 5463 rcu_read_lock(); 5464 q_tag_ptr = READ_ONCE(rxqueue->rps_flow_table); 5465 log = rps_tag_to_log(q_tag_ptr); 5466 if (q_tag_ptr && flow_id < (1UL << log)) { 5467 unsigned int cpu; 5468 5469 flow_table = rps_tag_to_table(q_tag_ptr); 5470 rflow = flow_table + flow_id; 5471 cpu = READ_ONCE(rflow->cpu); 5472 if (READ_ONCE(rflow->filter) == filter_id && 5473 rps_flow_is_active(rflow, log, cpu)) 5474 expire = false; 5475 } 5476 rcu_read_unlock(); 5477 return expire; 5478 } 5479 EXPORT_SYMBOL(rps_may_expire_flow); 5480 5481 #endif /* CONFIG_RFS_ACCEL */ 5482 5483 /* Called from hardirq (IPI) context */ 5484 static void rps_trigger_softirq(void *data) 5485 { 5486 struct softnet_data *sd = data; 5487 5488 ____napi_schedule(sd, &sd->backlog); 5489 /* Pairs with READ_ONCE() in softnet_seq_show() */ 5490 WRITE_ONCE(sd->received_rps, sd->received_rps + 1); 5491 } 5492 5493 #endif /* CONFIG_RPS */ 5494 5495 /* Called from hardirq (IPI) context */ 5496 static void trigger_rx_softirq(void *data) 5497 { 5498 struct softnet_data *sd = data; 5499 5500 __raise_softirq_irqoff(NET_RX_SOFTIRQ); 5501 smp_store_release(&sd->defer_ipi_scheduled, 0); 5502 } 5503 5504 /* 5505 * After we queued a packet into sd->input_pkt_queue, 5506 * we need to make sure this queue is serviced soon. 5507 * 5508 * - If this is another cpu queue, link it to our rps_ipi_list, 5509 * and make sure we will process rps_ipi_list from net_rx_action(). 5510 * 5511 * - If this is our own queue, NAPI schedule our backlog. 5512 * Note that this also raises NET_RX_SOFTIRQ. 5513 */ 5514 static void napi_schedule_rps(struct softnet_data *sd) 5515 { 5516 struct softnet_data *mysd = this_cpu_ptr(&softnet_data); 5517 5518 #ifdef CONFIG_RPS 5519 if (sd != mysd) { 5520 if (use_backlog_threads()) { 5521 __napi_schedule_irqoff(&sd->backlog); 5522 return; 5523 } 5524 5525 sd->rps_ipi_next = mysd->rps_ipi_list; 5526 mysd->rps_ipi_list = sd; 5527 5528 /* If not called from net_rx_action() or napi_threaded_poll() 5529 * we have to raise NET_RX_SOFTIRQ. 5530 */ 5531 if (!mysd->in_net_rx_action && !mysd->in_napi_threaded_poll) 5532 __raise_softirq_irqoff(NET_RX_SOFTIRQ); 5533 return; 5534 } 5535 #endif /* CONFIG_RPS */ 5536 __napi_schedule_irqoff(&mysd->backlog); 5537 } 5538 5539 void kick_defer_list_purge(unsigned int cpu) 5540 { 5541 struct softnet_data *sd = &per_cpu(softnet_data, cpu); 5542 unsigned long flags; 5543 5544 if (use_backlog_threads()) { 5545 backlog_lock_irq_save(sd, &flags); 5546 5547 if (!__test_and_set_bit(NAPI_STATE_SCHED, &sd->backlog.state)) 5548 __napi_schedule_irqoff(&sd->backlog); 5549 5550 backlog_unlock_irq_restore(sd, flags); 5551 5552 } else if (!cmpxchg(&sd->defer_ipi_scheduled, 0, 1)) { 5553 if (smp_call_function_single_async(cpu, &sd->defer_csd)) 5554 WRITE_ONCE(sd->defer_ipi_scheduled, 0); 5555 } 5556 } 5557 5558 #ifdef CONFIG_NET_FLOW_LIMIT 5559 int netdev_flow_limit_table_len __read_mostly = (1 << 12); 5560 #endif 5561 5562 static bool skb_flow_limit(struct sk_buff *skb, unsigned int qlen, 5563 int max_backlog) 5564 { 5565 #ifdef CONFIG_NET_FLOW_LIMIT 5566 unsigned int old_flow, new_flow; 5567 const struct softnet_data *sd; 5568 struct sd_flow_limit *fl; 5569 5570 if (likely(qlen < (max_backlog >> 1))) 5571 return false; 5572 5573 sd = this_cpu_ptr(&softnet_data); 5574 5575 rcu_read_lock(); 5576 fl = rcu_dereference(sd->flow_limit); 5577 if (fl) { 5578 new_flow = hash_32(skb_get_hash(skb), fl->log_buckets); 5579 old_flow = fl->history[fl->history_head]; 5580 fl->history[fl->history_head] = new_flow; 5581 5582 fl->history_head++; 5583 fl->history_head &= FLOW_LIMIT_HISTORY - 1; 5584 5585 if (likely(fl->buckets[old_flow])) 5586 fl->buckets[old_flow]--; 5587 5588 if (++fl->buckets[new_flow] > (FLOW_LIMIT_HISTORY >> 1)) { 5589 /* Pairs with READ_ONCE() in softnet_seq_show() */ 5590 WRITE_ONCE(fl->count, fl->count + 1); 5591 rcu_read_unlock(); 5592 return true; 5593 } 5594 } 5595 rcu_read_unlock(); 5596 #endif 5597 return false; 5598 } 5599 5600 /* 5601 * enqueue_to_backlog is called to queue an skb to a per CPU backlog 5602 * queue (may be a remote CPU queue). 5603 */ 5604 static int enqueue_to_backlog(struct sk_buff *skb, int cpu, 5605 unsigned int *qtail) 5606 { 5607 enum skb_drop_reason reason; 5608 struct softnet_data *sd; 5609 unsigned long flags; 5610 unsigned int qlen; 5611 int max_backlog; 5612 u32 tail; 5613 5614 reason = SKB_DROP_REASON_DEV_READY; 5615 5616 sd = &per_cpu(softnet_data, cpu); 5617 5618 qlen = skb_queue_len_lockless(&sd->input_pkt_queue); 5619 max_backlog = READ_ONCE(net_hotdata.max_backlog); 5620 if (unlikely(qlen > max_backlog) || 5621 skb_flow_limit(skb, qlen, max_backlog)) 5622 goto cpu_backlog_drop; 5623 backlog_lock_irq_save(sd, &flags); 5624 qlen = skb_queue_len(&sd->input_pkt_queue); 5625 if (likely(qlen <= max_backlog)) { 5626 if (unlikely(!netif_running(skb->dev))) { 5627 backlog_unlock_irq_restore(sd, flags); 5628 goto bad_dev; 5629 } 5630 if (!qlen) { 5631 /* Schedule NAPI for backlog device. We can use 5632 * non atomic operation as we own the queue lock. 5633 */ 5634 if (!__test_and_set_bit(NAPI_STATE_SCHED, 5635 &sd->backlog.state)) 5636 napi_schedule_rps(sd); 5637 } 5638 __skb_queue_tail(&sd->input_pkt_queue, skb); 5639 tail = rps_input_queue_tail_incr(sd); 5640 backlog_unlock_irq_restore(sd, flags); 5641 5642 /* save the tail outside of the critical section */ 5643 rps_input_queue_tail_save(qtail, tail); 5644 return NET_RX_SUCCESS; 5645 } 5646 5647 backlog_unlock_irq_restore(sd, flags); 5648 5649 cpu_backlog_drop: 5650 reason = SKB_DROP_REASON_CPU_BACKLOG; 5651 numa_drop_add(&sd->drop_counters, 1); 5652 bad_dev: 5653 dev_core_stats_rx_dropped_inc(skb->dev); 5654 kfree_skb_reason(skb, reason); 5655 return NET_RX_DROP; 5656 } 5657 5658 static struct netdev_rx_queue *netif_get_rxqueue(struct sk_buff *skb) 5659 { 5660 struct net_device *dev = skb->dev; 5661 struct netdev_rx_queue *rxqueue; 5662 5663 rxqueue = dev->_rx; 5664 5665 if (skb_rx_queue_recorded(skb)) { 5666 u16 index = skb_get_rx_queue(skb); 5667 5668 if (unlikely(index >= dev->real_num_rx_queues)) { 5669 WARN_ONCE(dev->real_num_rx_queues > 1, 5670 "%s received packet on queue %u, but number " 5671 "of RX queues is %u\n", 5672 dev->name, index, dev->real_num_rx_queues); 5673 5674 return rxqueue; /* Return first rxqueue */ 5675 } 5676 rxqueue += index; 5677 } 5678 return rxqueue; 5679 } 5680 5681 u32 bpf_prog_run_generic_xdp(struct sk_buff *skb, struct xdp_buff *xdp, 5682 const struct bpf_prog *xdp_prog) 5683 { 5684 void *orig_data, *orig_data_end, *hard_start; 5685 struct netdev_rx_queue *rxqueue; 5686 bool orig_bcast, orig_host; 5687 u32 mac_len, frame_sz; 5688 __be16 orig_eth_type; 5689 struct ethhdr *eth; 5690 u32 metalen, act; 5691 int off; 5692 5693 /* The XDP program wants to see the packet starting at the MAC 5694 * header. 5695 */ 5696 mac_len = skb->data - skb_mac_header(skb); 5697 hard_start = skb->data - skb_headroom(skb); 5698 5699 /* SKB "head" area always have tailroom for skb_shared_info */ 5700 frame_sz = (void *)skb_end_pointer(skb) - hard_start; 5701 frame_sz += SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 5702 5703 rxqueue = netif_get_rxqueue(skb); 5704 xdp_init_buff(xdp, frame_sz, &rxqueue->xdp_rxq); 5705 xdp_prepare_buff(xdp, hard_start, skb_headroom(skb) - mac_len, 5706 skb_headlen(skb) + mac_len, true); 5707 if (skb_is_nonlinear(skb)) { 5708 skb_shinfo(skb)->xdp_frags_size = skb->data_len; 5709 xdp_buff_set_frags_flag(xdp); 5710 } else { 5711 xdp_buff_clear_frags_flag(xdp); 5712 } 5713 5714 orig_data_end = xdp->data_end; 5715 orig_data = xdp->data; 5716 eth = (struct ethhdr *)xdp->data; 5717 orig_host = ether_addr_equal_64bits(eth->h_dest, skb->dev->dev_addr); 5718 orig_bcast = is_multicast_ether_addr_64bits(eth->h_dest); 5719 orig_eth_type = eth->h_proto; 5720 5721 act = bpf_prog_run_xdp(xdp_prog, xdp); 5722 5723 /* check if bpf_xdp_adjust_head was used */ 5724 off = xdp->data - orig_data; 5725 if (off) { 5726 if (off > 0) 5727 __skb_pull(skb, off); 5728 else if (off < 0) 5729 __skb_push(skb, -off); 5730 5731 skb->mac_header += off; 5732 skb_reset_network_header(skb); 5733 } 5734 5735 /* check if bpf_xdp_adjust_tail was used */ 5736 off = xdp->data_end - orig_data_end; 5737 if (off != 0) { 5738 skb_set_tail_pointer(skb, xdp->data_end - xdp->data); 5739 skb->len += off; /* positive on grow, negative on shrink */ 5740 } 5741 5742 /* XDP frag metadata (e.g. nr_frags) are updated in eBPF helpers 5743 * (e.g. bpf_xdp_adjust_tail). Remove the old fragment contribution 5744 * from skb->len before updating data_len, then add the new one back. 5745 */ 5746 skb->len -= skb->data_len; 5747 if (xdp_buff_has_frags(xdp)) { 5748 skb->data_len = skb_shinfo(skb)->xdp_frags_size; 5749 skb->len += skb->data_len; 5750 } else { 5751 skb->data_len = 0; 5752 } 5753 5754 /* check if XDP changed eth hdr such SKB needs update */ 5755 eth = (struct ethhdr *)xdp->data; 5756 if ((orig_eth_type != eth->h_proto) || 5757 (orig_host != ether_addr_equal_64bits(eth->h_dest, 5758 skb->dev->dev_addr)) || 5759 (orig_bcast != is_multicast_ether_addr_64bits(eth->h_dest))) { 5760 __skb_push(skb, ETH_HLEN); 5761 skb->pkt_type = PACKET_HOST; 5762 skb->protocol = eth_type_trans(skb, skb->dev); 5763 } 5764 5765 /* Redirect/Tx gives L2 packet, code that will reuse skb must __skb_pull 5766 * before calling us again on redirect path. We do not call do_redirect 5767 * as we leave that up to the caller. 5768 * 5769 * Caller is responsible for managing lifetime of skb (i.e. calling 5770 * kfree_skb in response to actions it cannot handle/XDP_DROP). 5771 */ 5772 switch (act) { 5773 case XDP_REDIRECT: 5774 case XDP_TX: 5775 __skb_push(skb, mac_len); 5776 break; 5777 case XDP_PASS: 5778 metalen = xdp->data - xdp->data_meta; 5779 if (metalen) 5780 skb_metadata_set(skb, metalen); 5781 break; 5782 } 5783 5784 return act; 5785 } 5786 5787 static int 5788 netif_skb_check_for_xdp(struct sk_buff **pskb, const struct bpf_prog *prog) 5789 { 5790 struct sk_buff *skb = *pskb; 5791 int err, hroom, troom; 5792 5793 local_lock_nested_bh(&system_page_pool.bh_lock); 5794 err = skb_cow_data_for_xdp(this_cpu_read(system_page_pool.pool), pskb, prog); 5795 local_unlock_nested_bh(&system_page_pool.bh_lock); 5796 if (!err) 5797 return 0; 5798 5799 /* In case we have to go down the path and also linearize, 5800 * then lets do the pskb_expand_head() work just once here. 5801 */ 5802 hroom = XDP_PACKET_HEADROOM - skb_headroom(skb); 5803 troom = skb->tail + skb->data_len - skb->end; 5804 err = pskb_expand_head(skb, 5805 hroom > 0 ? ALIGN(hroom, NET_SKB_PAD) : 0, 5806 troom > 0 ? troom + 128 : 0, GFP_ATOMIC); 5807 if (err) 5808 return err; 5809 5810 return skb_linearize(skb); 5811 } 5812 5813 static u32 netif_receive_generic_xdp(struct sk_buff **pskb, 5814 struct xdp_buff *xdp, 5815 const struct bpf_prog *xdp_prog) 5816 { 5817 struct sk_buff *skb = *pskb; 5818 u32 mac_len, act = XDP_DROP; 5819 5820 /* Reinjected packets coming from act_mirred or similar should 5821 * not get XDP generic processing. 5822 */ 5823 if (skb_is_redirected(skb)) 5824 return XDP_PASS; 5825 5826 /* XDP packets must have sufficient headroom of XDP_PACKET_HEADROOM 5827 * bytes. This is the guarantee that also native XDP provides, 5828 * thus we need to do it here as well. 5829 */ 5830 mac_len = skb->data - skb_mac_header(skb); 5831 __skb_push(skb, mac_len); 5832 5833 if (skb_cloned(skb) || skb_is_nonlinear(skb) || 5834 skb_headroom(skb) < XDP_PACKET_HEADROOM) { 5835 if (netif_skb_check_for_xdp(pskb, xdp_prog)) 5836 goto do_drop; 5837 } 5838 5839 __skb_pull(*pskb, mac_len); 5840 5841 act = bpf_prog_run_generic_xdp(*pskb, xdp, xdp_prog); 5842 switch (act) { 5843 case XDP_REDIRECT: 5844 case XDP_TX: 5845 case XDP_PASS: 5846 break; 5847 default: 5848 bpf_warn_invalid_xdp_action((*pskb)->dev, xdp_prog, act); 5849 fallthrough; 5850 case XDP_ABORTED: 5851 trace_xdp_exception((*pskb)->dev, xdp_prog, act); 5852 fallthrough; 5853 case XDP_DROP: 5854 do_drop: 5855 kfree_skb(*pskb); 5856 break; 5857 } 5858 5859 return act; 5860 } 5861 5862 /* When doing generic XDP we have to bypass the qdisc layer and the 5863 * network taps in order to match in-driver-XDP behavior. This also means 5864 * that XDP packets are able to starve other packets going through a qdisc, 5865 * and DDOS attacks will be more effective. In-driver-XDP use dedicated TX 5866 * queues, so they do not have this starvation issue. 5867 */ 5868 void generic_xdp_tx(struct sk_buff *skb, const struct bpf_prog *xdp_prog) 5869 { 5870 struct net_device *dev = skb->dev; 5871 struct netdev_queue *txq; 5872 bool free_skb = true; 5873 int cpu, rc; 5874 5875 txq = netdev_core_pick_tx(dev, skb, NULL); 5876 cpu = smp_processor_id(); 5877 HARD_TX_LOCK(dev, txq, cpu); 5878 if (!netif_xmit_frozen_or_drv_stopped(txq)) { 5879 rc = netdev_start_xmit(skb, dev, txq, 0); 5880 if (dev_xmit_complete(rc)) 5881 free_skb = false; 5882 } 5883 HARD_TX_UNLOCK(dev, txq); 5884 if (free_skb) { 5885 trace_xdp_exception(dev, xdp_prog, XDP_TX); 5886 dev_core_stats_tx_dropped_inc(dev); 5887 kfree_skb(skb); 5888 } 5889 } 5890 5891 static DEFINE_STATIC_KEY_FALSE(generic_xdp_needed_key); 5892 5893 int do_xdp_generic(const struct bpf_prog *xdp_prog, struct sk_buff **pskb) 5894 { 5895 struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx; 5896 5897 if (xdp_prog) { 5898 struct xdp_buff xdp; 5899 u32 act; 5900 int err; 5901 5902 bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx); 5903 act = netif_receive_generic_xdp(pskb, &xdp, xdp_prog); 5904 if (act != XDP_PASS) { 5905 switch (act) { 5906 case XDP_REDIRECT: 5907 err = xdp_do_generic_redirect((*pskb)->dev, *pskb, 5908 &xdp, xdp_prog); 5909 if (err) 5910 goto out_redir; 5911 break; 5912 case XDP_TX: 5913 generic_xdp_tx(*pskb, xdp_prog); 5914 break; 5915 } 5916 bpf_net_ctx_clear(bpf_net_ctx); 5917 return XDP_DROP; 5918 } 5919 bpf_net_ctx_clear(bpf_net_ctx); 5920 } 5921 return XDP_PASS; 5922 out_redir: 5923 bpf_net_ctx_clear(bpf_net_ctx); 5924 kfree_skb_reason(*pskb, SKB_DROP_REASON_XDP); 5925 return XDP_DROP; 5926 } 5927 EXPORT_SYMBOL_GPL(do_xdp_generic); 5928 5929 static int netif_rx_internal(struct sk_buff *skb) 5930 { 5931 int ret; 5932 5933 net_timestamp_check(READ_ONCE(net_hotdata.tstamp_prequeue), skb); 5934 5935 trace_netif_rx(skb); 5936 5937 #ifdef CONFIG_RPS 5938 if (static_branch_unlikely(&rps_needed)) { 5939 struct rps_dev_flow voidflow, *rflow = &voidflow; 5940 int cpu; 5941 5942 rcu_read_lock(); 5943 5944 cpu = get_rps_cpu(skb->dev, skb, &rflow); 5945 if (cpu < 0) 5946 cpu = smp_processor_id(); 5947 5948 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail); 5949 5950 rcu_read_unlock(); 5951 } else 5952 #endif 5953 { 5954 unsigned int qtail; 5955 5956 ret = enqueue_to_backlog(skb, smp_processor_id(), &qtail); 5957 } 5958 return ret; 5959 } 5960 5961 /** 5962 * __netif_rx - Slightly optimized version of netif_rx 5963 * @skb: buffer to post 5964 * 5965 * This behaves as netif_rx except that it does not disable bottom halves. 5966 * As a result this function may only be invoked from the interrupt context 5967 * (either hard or soft interrupt). 5968 */ 5969 int __netif_rx(struct sk_buff *skb) 5970 { 5971 int ret; 5972 5973 lockdep_assert_once(hardirq_count() | softirq_count()); 5974 5975 trace_netif_rx_entry(skb); 5976 ret = netif_rx_internal(skb); 5977 trace_netif_rx_exit(ret); 5978 return ret; 5979 } 5980 EXPORT_SYMBOL(__netif_rx); 5981 5982 /** 5983 * netif_rx - post buffer to the network code 5984 * @skb: buffer to post 5985 * 5986 * This function receives a packet from a device driver and queues it for 5987 * the upper (protocol) levels to process via the backlog NAPI device. It 5988 * always succeeds. The buffer may be dropped during processing for 5989 * congestion control or by the protocol layers. 5990 * The network buffer is passed via the backlog NAPI device. Modern NIC 5991 * driver should use NAPI and GRO. 5992 * This function can used from interrupt and from process context. The 5993 * caller from process context must not disable interrupts before invoking 5994 * this function. 5995 * 5996 * return values: 5997 * NET_RX_SUCCESS (no congestion) 5998 * NET_RX_DROP (packet was dropped) 5999 * 6000 */ 6001 int netif_rx(struct sk_buff *skb) 6002 { 6003 bool need_bh_off = !(hardirq_count() | softirq_count()); 6004 int ret; 6005 6006 if (need_bh_off) 6007 local_bh_disable(); 6008 trace_netif_rx_entry(skb); 6009 ret = netif_rx_internal(skb); 6010 trace_netif_rx_exit(ret); 6011 if (need_bh_off) 6012 local_bh_enable(); 6013 return ret; 6014 } 6015 EXPORT_SYMBOL(netif_rx); 6016 6017 static __latent_entropy void net_tx_action(void) 6018 { 6019 struct softnet_data *sd = this_cpu_ptr(&softnet_data); 6020 6021 if (sd->completion_queue) { 6022 struct sk_buff *clist; 6023 6024 local_irq_disable(); 6025 clist = sd->completion_queue; 6026 sd->completion_queue = NULL; 6027 local_irq_enable(); 6028 6029 while (clist) { 6030 struct sk_buff *skb = clist; 6031 6032 clist = clist->next; 6033 6034 WARN_ON(refcount_read(&skb->users)); 6035 if (likely(get_kfree_skb_cb(skb)->reason == SKB_CONSUMED)) 6036 trace_consume_skb(skb, net_tx_action); 6037 else 6038 trace_kfree_skb(skb, net_tx_action, 6039 get_kfree_skb_cb(skb)->reason, NULL); 6040 6041 if (skb->fclone != SKB_FCLONE_UNAVAILABLE) 6042 __kfree_skb(skb); 6043 else 6044 __napi_kfree_skb(skb, 6045 get_kfree_skb_cb(skb)->reason); 6046 } 6047 } 6048 6049 if (sd->output_queue) { 6050 struct Qdisc *head; 6051 6052 local_irq_disable(); 6053 head = sd->output_queue; 6054 sd->output_queue = NULL; 6055 sd->output_queue_tailp = &sd->output_queue; 6056 local_irq_enable(); 6057 6058 rcu_read_lock(); 6059 6060 while (head) { 6061 spinlock_t *root_lock = NULL; 6062 struct sk_buff *to_free; 6063 struct Qdisc *q = head; 6064 6065 head = head->next_sched; 6066 6067 /* We need to make sure head->next_sched is read 6068 * before clearing __QDISC_STATE_SCHED 6069 */ 6070 smp_mb__before_atomic(); 6071 6072 if (!(q->flags & TCQ_F_NOLOCK)) { 6073 root_lock = qdisc_lock(q); 6074 spin_lock(root_lock); 6075 } else if (unlikely(test_bit(__QDISC_STATE_DEACTIVATED, 6076 &q->state))) { 6077 /* There is a synchronize_net() between 6078 * STATE_DEACTIVATED flag being set and 6079 * qdisc_reset()/some_qdisc_is_busy() in 6080 * dev_deactivate(), so we can safely bail out 6081 * early here to avoid data race between 6082 * qdisc_deactivate() and some_qdisc_is_busy() 6083 * for lockless qdisc. 6084 */ 6085 clear_bit(__QDISC_STATE_SCHED, &q->state); 6086 continue; 6087 } 6088 6089 clear_bit(__QDISC_STATE_SCHED, &q->state); 6090 to_free = qdisc_run(q); 6091 if (root_lock) 6092 spin_unlock(root_lock); 6093 tcf_kfree_skb_list(to_free, q, NULL, qdisc_dev(q)); 6094 } 6095 6096 rcu_read_unlock(); 6097 } 6098 6099 xfrm_dev_backlog(sd); 6100 } 6101 6102 /** 6103 * netdev_is_rx_handler_busy - check if receive handler is registered 6104 * @dev: device to check 6105 * 6106 * Check if a receive handler is already registered for a given device. 6107 * Return true if there one. 6108 * 6109 * The caller must hold the rtnl_mutex. 6110 */ 6111 bool netdev_is_rx_handler_busy(struct net_device *dev) 6112 { 6113 ASSERT_RTNL(); 6114 return dev && rtnl_dereference(dev->rx_handler); 6115 } 6116 EXPORT_SYMBOL_GPL(netdev_is_rx_handler_busy); 6117 6118 /** 6119 * netdev_rx_handler_register - register receive handler 6120 * @dev: device to register a handler for 6121 * @rx_handler: receive handler to register 6122 * @rx_handler_data: data pointer that is used by rx handler 6123 * 6124 * Register a receive handler for a device. This handler will then be 6125 * called from __netif_receive_skb. A negative errno code is returned 6126 * on a failure. 6127 * 6128 * The caller must hold the rtnl_mutex. 6129 * 6130 * For a general description of rx_handler, see enum rx_handler_result. 6131 */ 6132 int netdev_rx_handler_register(struct net_device *dev, 6133 rx_handler_func_t *rx_handler, 6134 void *rx_handler_data) 6135 { 6136 if (netdev_is_rx_handler_busy(dev)) 6137 return -EBUSY; 6138 6139 if (dev->priv_flags & IFF_NO_RX_HANDLER) 6140 return -EINVAL; 6141 6142 /* Note: rx_handler_data must be set before rx_handler */ 6143 rcu_assign_pointer(dev->rx_handler_data, rx_handler_data); 6144 rcu_assign_pointer(dev->rx_handler, rx_handler); 6145 6146 return 0; 6147 } 6148 EXPORT_SYMBOL_GPL(netdev_rx_handler_register); 6149 6150 /** 6151 * netdev_rx_handler_unregister - unregister receive handler 6152 * @dev: device to unregister a handler from 6153 * 6154 * Unregister a receive handler from a device. 6155 * 6156 * The caller must hold the rtnl_mutex. 6157 */ 6158 void netdev_rx_handler_unregister(struct net_device *dev) 6159 { 6160 6161 ASSERT_RTNL(); 6162 RCU_INIT_POINTER(dev->rx_handler, NULL); 6163 /* a reader seeing a non NULL rx_handler in a rcu_read_lock() 6164 * section has a guarantee to see a non NULL rx_handler_data 6165 * as well. 6166 */ 6167 synchronize_net(); 6168 RCU_INIT_POINTER(dev->rx_handler_data, NULL); 6169 } 6170 EXPORT_SYMBOL_GPL(netdev_rx_handler_unregister); 6171 6172 /* 6173 * Limit the use of PFMEMALLOC reserves to those protocols that implement 6174 * the special handling of PFMEMALLOC skbs. 6175 */ 6176 static bool skb_pfmemalloc_protocol(struct sk_buff *skb) 6177 { 6178 switch (skb->protocol) { 6179 case htons(ETH_P_ARP): 6180 case htons(ETH_P_IP): 6181 case htons(ETH_P_IPV6): 6182 case htons(ETH_P_8021Q): 6183 case htons(ETH_P_8021AD): 6184 return true; 6185 default: 6186 return false; 6187 } 6188 } 6189 6190 static inline int nf_ingress(struct sk_buff *skb, struct packet_type **pt_prev, 6191 int *ret, struct net_device *orig_dev) 6192 { 6193 if (nf_hook_ingress_active(skb)) { 6194 int ingress_retval; 6195 6196 if (unlikely(*pt_prev)) { 6197 *ret = deliver_skb(skb, *pt_prev, orig_dev); 6198 *pt_prev = NULL; 6199 } 6200 6201 rcu_read_lock(); 6202 ingress_retval = nf_hook_ingress(skb); 6203 rcu_read_unlock(); 6204 return ingress_retval; 6205 } 6206 return 0; 6207 } 6208 6209 static int __netif_receive_skb_core(struct sk_buff **pskb, bool pfmemalloc, 6210 struct packet_type **ppt_prev) 6211 { 6212 enum skb_drop_reason drop_reason = SKB_DROP_REASON_UNHANDLED_PROTO; 6213 struct packet_type *ptype, *pt_prev; 6214 rx_handler_func_t *rx_handler; 6215 struct sk_buff *skb = *pskb; 6216 struct net_device *orig_dev; 6217 bool deliver_exact = false; 6218 int ret = NET_RX_DROP; 6219 __be16 type; 6220 6221 net_timestamp_check(!READ_ONCE(net_hotdata.tstamp_prequeue), skb); 6222 6223 trace_netif_receive_skb(skb); 6224 6225 orig_dev = skb->dev; 6226 if (unlikely(skb_headroom(skb) > SKB_MAX_RX_HEADROOM)) { 6227 drop_reason = SKB_DROP_REASON_SKB_RX_HEADROOM; 6228 goto drop; 6229 } 6230 6231 skb_reset_network_header(skb); 6232 #if !defined(CONFIG_DEBUG_NET) 6233 /* We plan to no longer reset the transport header here. 6234 * Give some time to fuzzers and dev build to catch bugs 6235 * in network stacks. 6236 */ 6237 if (!skb_transport_header_was_set(skb)) 6238 skb_reset_transport_header(skb); 6239 #endif 6240 skb_reset_mac_len(skb); 6241 6242 pt_prev = NULL; 6243 6244 another_round: 6245 skb->skb_iif = skb->dev->ifindex; 6246 6247 __this_cpu_inc(softnet_data.processed); 6248 6249 if (static_branch_unlikely(&generic_xdp_needed_key)) { 6250 int ret2; 6251 6252 migrate_disable(); 6253 ret2 = do_xdp_generic(rcu_dereference(skb->dev->xdp_prog), 6254 &skb); 6255 migrate_enable(); 6256 6257 if (ret2 != XDP_PASS) { 6258 ret = NET_RX_DROP; 6259 goto out; 6260 } 6261 } 6262 6263 if (eth_type_vlan(skb->protocol)) { 6264 skb = skb_vlan_untag(skb); 6265 if (unlikely(!skb)) 6266 goto out; 6267 } 6268 6269 if (skb_skip_tc_classify(skb)) 6270 goto skip_classify; 6271 6272 if (pfmemalloc) 6273 goto skip_taps; 6274 6275 list_for_each_entry_rcu(ptype, &dev_net_rcu(skb->dev)->ptype_all, 6276 list) { 6277 if (unlikely(pt_prev)) 6278 ret = deliver_skb(skb, pt_prev, orig_dev); 6279 pt_prev = ptype; 6280 } 6281 6282 list_for_each_entry_rcu(ptype, &skb->dev->ptype_all, list) { 6283 if (unlikely(pt_prev)) 6284 ret = deliver_skb(skb, pt_prev, orig_dev); 6285 pt_prev = ptype; 6286 } 6287 6288 skip_taps: 6289 #ifdef CONFIG_NET_INGRESS 6290 if (static_branch_unlikely(&ingress_needed_key)) { 6291 bool another = false; 6292 6293 nf_skip_egress(skb, true); 6294 skb = sch_handle_ingress(skb, &pt_prev, &ret, orig_dev, 6295 &another); 6296 if (another) 6297 goto another_round; 6298 if (!skb) 6299 goto out; 6300 6301 nf_skip_egress(skb, false); 6302 if (nf_ingress(skb, &pt_prev, &ret, orig_dev) < 0) 6303 goto out; 6304 } 6305 #endif 6306 skb_reset_redirect(skb); 6307 skip_classify: 6308 if (pfmemalloc && !skb_pfmemalloc_protocol(skb)) { 6309 drop_reason = SKB_DROP_REASON_PFMEMALLOC; 6310 goto drop; 6311 } 6312 6313 if (skb_vlan_tag_present(skb)) { 6314 if (unlikely(pt_prev)) { 6315 ret = deliver_skb(skb, pt_prev, orig_dev); 6316 pt_prev = NULL; 6317 } 6318 if (vlan_do_receive(&skb)) 6319 goto another_round; 6320 else if (unlikely(!skb)) 6321 goto out; 6322 } 6323 6324 rx_handler = rcu_dereference(skb->dev->rx_handler); 6325 if (rx_handler) { 6326 if (unlikely(pt_prev)) { 6327 ret = deliver_skb(skb, pt_prev, orig_dev); 6328 pt_prev = NULL; 6329 } 6330 switch (rx_handler(&skb)) { 6331 case RX_HANDLER_CONSUMED: 6332 ret = NET_RX_SUCCESS; 6333 goto out; 6334 case RX_HANDLER_ANOTHER: 6335 goto another_round; 6336 case RX_HANDLER_EXACT: 6337 deliver_exact = true; 6338 break; 6339 case RX_HANDLER_PASS: 6340 break; 6341 default: 6342 BUG(); 6343 } 6344 } 6345 6346 if (unlikely(skb_vlan_tag_present(skb)) && !netdev_uses_dsa(skb->dev)) { 6347 unsigned int priority_tag_depth = 1; 6348 6349 check_vlan_id: 6350 if (skb_vlan_tag_get_id(skb)) { 6351 /* Vlan id is non 0 and vlan_do_receive() above couldn't 6352 * find vlan device. 6353 */ 6354 skb->pkt_type = PACKET_OTHERHOST; 6355 } else if (eth_type_vlan(skb->protocol)) { 6356 /* Outer header is 802.1P with vlan 0, inner header is 6357 * 802.1Q or 802.1AD and vlan_do_receive() above could 6358 * not find vlan dev for vlan id 0. 6359 */ 6360 if (unlikely(priority_tag_depth >= VLAN_MAX_DEPTH)) { 6361 drop_reason = SKB_DROP_REASON_RECURSION_LIMIT; 6362 goto drop; 6363 } 6364 6365 priority_tag_depth++; 6366 __vlan_hwaccel_clear_tag(skb); 6367 skb = skb_vlan_untag(skb); 6368 if (unlikely(!skb)) 6369 goto out; 6370 if (vlan_do_receive(&skb)) 6371 /* After stripping off 802.1P header with vlan 0 6372 * vlan dev is found for inner header. 6373 */ 6374 goto another_round; 6375 else if (unlikely(!skb)) 6376 goto out; 6377 else 6378 /* We have stripped outer 802.1P vlan 0 header. 6379 * But could not find vlan dev. 6380 * check again for vlan id to set OTHERHOST. 6381 */ 6382 goto check_vlan_id; 6383 } 6384 /* Note: we might in the future use prio bits 6385 * and set skb->priority like in vlan_do_receive() 6386 * For the time being, just ignore Priority Code Point 6387 */ 6388 __vlan_hwaccel_clear_tag(skb); 6389 } 6390 6391 type = skb->protocol; 6392 6393 /* deliver only exact match when indicated */ 6394 if (likely(!deliver_exact)) { 6395 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type, 6396 &ptype_base[ntohs(type) & 6397 PTYPE_HASH_MASK]); 6398 6399 /* orig_dev and skb->dev could belong to different netns; 6400 * Even in such case we need to traverse only the list 6401 * coming from skb->dev, as the ptype owner (packet socket) 6402 * will use dev_net(skb->dev) to do namespace filtering. 6403 */ 6404 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type, 6405 &dev_net_rcu(skb->dev)->ptype_specific); 6406 } 6407 6408 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type, 6409 &orig_dev->ptype_specific); 6410 6411 if (unlikely(skb->dev != orig_dev)) { 6412 deliver_ptype_list_skb(skb, &pt_prev, orig_dev, type, 6413 &skb->dev->ptype_specific); 6414 } 6415 6416 if (pt_prev) { 6417 *ppt_prev = pt_prev; 6418 } else { 6419 drop: 6420 if (!deliver_exact) 6421 dev_core_stats_rx_dropped_inc(skb->dev); 6422 else 6423 dev_core_stats_rx_nohandler_inc(skb->dev); 6424 6425 kfree_skb_reason(skb, drop_reason); 6426 /* Jamal, now you will not able to escape explaining 6427 * me how you were going to use this. :-) 6428 */ 6429 ret = NET_RX_DROP; 6430 } 6431 6432 out: 6433 /* The invariant here is that if *ppt_prev is not NULL 6434 * then skb should also be non-NULL. 6435 * 6436 * Apparently *ppt_prev assignment above holds this invariant due to 6437 * skb dereferencing near it. 6438 */ 6439 *pskb = skb; 6440 return ret; 6441 } 6442 6443 static int __netif_receive_skb_one_core(struct sk_buff *skb, bool pfmemalloc) 6444 { 6445 struct net_device *orig_dev = skb->dev; 6446 struct packet_type *pt_prev = NULL; 6447 int ret; 6448 6449 ret = __netif_receive_skb_core(&skb, pfmemalloc, &pt_prev); 6450 if (pt_prev) 6451 ret = INDIRECT_CALL_INET(pt_prev->func, ipv6_rcv, ip_rcv, skb, 6452 skb->dev, pt_prev, orig_dev); 6453 return ret; 6454 } 6455 6456 /** 6457 * netif_receive_skb_core - special purpose version of netif_receive_skb 6458 * @skb: buffer to process 6459 * 6460 * More direct receive version of netif_receive_skb(). It should 6461 * only be used by callers that have a need to skip RPS and Generic XDP. 6462 * Caller must also take care of handling if ``(page_is_)pfmemalloc``. 6463 * 6464 * This function may only be called from softirq context and interrupts 6465 * should be enabled. 6466 * 6467 * Return values (usually ignored): 6468 * NET_RX_SUCCESS: no congestion 6469 * NET_RX_DROP: packet was dropped 6470 */ 6471 int netif_receive_skb_core(struct sk_buff *skb) 6472 { 6473 int ret; 6474 6475 rcu_read_lock(); 6476 ret = __netif_receive_skb_one_core(skb, false); 6477 rcu_read_unlock(); 6478 6479 return ret; 6480 } 6481 EXPORT_SYMBOL(netif_receive_skb_core); 6482 6483 static inline void __netif_receive_skb_list_ptype(struct list_head *head, 6484 struct packet_type *pt_prev, 6485 struct net_device *orig_dev) 6486 { 6487 struct sk_buff *skb, *next; 6488 6489 if (!pt_prev) 6490 return; 6491 if (list_empty(head)) 6492 return; 6493 if (pt_prev->list_func != NULL) 6494 INDIRECT_CALL_INET(pt_prev->list_func, ipv6_list_rcv, 6495 ip_list_rcv, head, pt_prev, orig_dev); 6496 else 6497 list_for_each_entry_safe(skb, next, head, list) { 6498 skb_list_del_init(skb); 6499 pt_prev->func(skb, skb->dev, pt_prev, orig_dev); 6500 } 6501 } 6502 6503 static void __netif_receive_skb_list_core(struct list_head *head, bool pfmemalloc) 6504 { 6505 /* Fast-path assumptions: 6506 * - There is no RX handler. 6507 * - Only one packet_type matches. 6508 * If either of these fails, we will end up doing some per-packet 6509 * processing in-line, then handling the 'last ptype' for the whole 6510 * sublist. This can't cause out-of-order delivery to any single ptype, 6511 * because the 'last ptype' must be constant across the sublist, and all 6512 * other ptypes are handled per-packet. 6513 */ 6514 /* Current (common) ptype of sublist */ 6515 struct packet_type *pt_curr = NULL; 6516 /* Current (common) orig_dev of sublist */ 6517 struct net_device *od_curr = NULL; 6518 struct sk_buff *skb, *next; 6519 LIST_HEAD(sublist); 6520 6521 list_for_each_entry_safe(skb, next, head, list) { 6522 struct net_device *orig_dev = skb->dev; 6523 struct packet_type *pt_prev = NULL; 6524 6525 skb_list_del_init(skb); 6526 __netif_receive_skb_core(&skb, pfmemalloc, &pt_prev); 6527 if (!pt_prev) 6528 continue; 6529 if (pt_curr != pt_prev || od_curr != orig_dev) { 6530 /* dispatch old sublist */ 6531 __netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr); 6532 /* start new sublist */ 6533 INIT_LIST_HEAD(&sublist); 6534 pt_curr = pt_prev; 6535 od_curr = orig_dev; 6536 } 6537 list_add_tail(&skb->list, &sublist); 6538 } 6539 6540 /* dispatch final sublist */ 6541 __netif_receive_skb_list_ptype(&sublist, pt_curr, od_curr); 6542 } 6543 6544 static int __netif_receive_skb(struct sk_buff *skb) 6545 { 6546 int ret; 6547 6548 if (sk_memalloc_socks() && skb_pfmemalloc(skb)) { 6549 unsigned int noreclaim_flag; 6550 6551 /* 6552 * PFMEMALLOC skbs are special, they should 6553 * - be delivered to SOCK_MEMALLOC sockets only 6554 * - stay away from userspace 6555 * - have bounded memory usage 6556 * 6557 * Use PF_MEMALLOC as this saves us from propagating the allocation 6558 * context down to all allocation sites. 6559 */ 6560 noreclaim_flag = memalloc_noreclaim_save(); 6561 ret = __netif_receive_skb_one_core(skb, true); 6562 memalloc_noreclaim_restore(noreclaim_flag); 6563 } else 6564 ret = __netif_receive_skb_one_core(skb, false); 6565 6566 return ret; 6567 } 6568 6569 static void __netif_receive_skb_list(struct list_head *head) 6570 { 6571 unsigned long noreclaim_flag = 0; 6572 struct sk_buff *skb, *next; 6573 bool pfmemalloc = false; /* Is current sublist PF_MEMALLOC? */ 6574 6575 list_for_each_entry_safe(skb, next, head, list) { 6576 if ((sk_memalloc_socks() && skb_pfmemalloc(skb)) != pfmemalloc) { 6577 struct list_head sublist; 6578 6579 /* Handle the previous sublist */ 6580 list_cut_before(&sublist, head, &skb->list); 6581 if (!list_empty(&sublist)) 6582 __netif_receive_skb_list_core(&sublist, pfmemalloc); 6583 pfmemalloc = !pfmemalloc; 6584 /* See comments in __netif_receive_skb */ 6585 if (pfmemalloc) 6586 noreclaim_flag = memalloc_noreclaim_save(); 6587 else 6588 memalloc_noreclaim_restore(noreclaim_flag); 6589 } 6590 } 6591 /* Handle the remaining sublist */ 6592 if (!list_empty(head)) 6593 __netif_receive_skb_list_core(head, pfmemalloc); 6594 /* Restore pflags */ 6595 if (pfmemalloc) 6596 memalloc_noreclaim_restore(noreclaim_flag); 6597 } 6598 6599 static int generic_xdp_install(struct net_device *dev, struct netdev_bpf *xdp) 6600 { 6601 struct bpf_prog *old = rtnl_dereference(dev->xdp_prog); 6602 struct bpf_prog *new = xdp->prog; 6603 int ret = 0; 6604 6605 switch (xdp->command) { 6606 case XDP_SETUP_PROG: 6607 rcu_assign_pointer(dev->xdp_prog, new); 6608 if (old) 6609 bpf_prog_put(old); 6610 6611 if (old && !new) { 6612 static_branch_dec(&generic_xdp_needed_key); 6613 } else if (new && !old) { 6614 static_branch_inc(&generic_xdp_needed_key); 6615 netif_disable_lro(dev); 6616 dev_disable_gro_hw(dev); 6617 } 6618 break; 6619 6620 default: 6621 ret = -EINVAL; 6622 break; 6623 } 6624 6625 return ret; 6626 } 6627 6628 static int netif_receive_skb_internal(struct sk_buff *skb) 6629 { 6630 int ret; 6631 6632 net_timestamp_check(READ_ONCE(net_hotdata.tstamp_prequeue), skb); 6633 6634 if (skb_defer_rx_timestamp(skb)) 6635 return NET_RX_SUCCESS; 6636 6637 rcu_read_lock(); 6638 #ifdef CONFIG_RPS 6639 if (static_branch_unlikely(&rps_needed)) { 6640 struct rps_dev_flow voidflow, *rflow = &voidflow; 6641 int cpu = get_rps_cpu(skb->dev, skb, &rflow); 6642 6643 if (cpu >= 0) { 6644 ret = enqueue_to_backlog(skb, cpu, &rflow->last_qtail); 6645 rcu_read_unlock(); 6646 return ret; 6647 } 6648 } 6649 #endif 6650 ret = __netif_receive_skb(skb); 6651 rcu_read_unlock(); 6652 return ret; 6653 } 6654 6655 void netif_receive_skb_list_internal(struct list_head *head) 6656 { 6657 struct sk_buff *skb, *next; 6658 LIST_HEAD(sublist); 6659 6660 list_for_each_entry_safe(skb, next, head, list) { 6661 net_timestamp_check(READ_ONCE(net_hotdata.tstamp_prequeue), 6662 skb); 6663 skb_list_del_init(skb); 6664 if (!skb_defer_rx_timestamp(skb)) 6665 list_add_tail(&skb->list, &sublist); 6666 } 6667 list_splice_init(&sublist, head); 6668 6669 rcu_read_lock(); 6670 #ifdef CONFIG_RPS 6671 if (static_branch_unlikely(&rps_needed)) { 6672 list_for_each_entry_safe(skb, next, head, list) { 6673 struct rps_dev_flow voidflow, *rflow = &voidflow; 6674 int cpu = get_rps_cpu(skb->dev, skb, &rflow); 6675 6676 if (cpu >= 0) { 6677 /* Will be handled, remove from list */ 6678 skb_list_del_init(skb); 6679 enqueue_to_backlog(skb, cpu, &rflow->last_qtail); 6680 } 6681 } 6682 } 6683 #endif 6684 __netif_receive_skb_list(head); 6685 rcu_read_unlock(); 6686 } 6687 6688 /** 6689 * netif_receive_skb - process receive buffer from network 6690 * @skb: buffer to process 6691 * 6692 * netif_receive_skb() is the main receive data processing function. 6693 * It always succeeds. The buffer may be dropped during processing 6694 * for congestion control or by the protocol layers. 6695 * 6696 * This function may only be called from softirq context and interrupts 6697 * should be enabled. 6698 * 6699 * Return values (usually ignored): 6700 * NET_RX_SUCCESS: no congestion 6701 * NET_RX_DROP: packet was dropped 6702 */ 6703 int netif_receive_skb(struct sk_buff *skb) 6704 { 6705 int ret; 6706 6707 trace_netif_receive_skb_entry(skb); 6708 6709 ret = netif_receive_skb_internal(skb); 6710 trace_netif_receive_skb_exit(ret); 6711 6712 return ret; 6713 } 6714 EXPORT_SYMBOL(netif_receive_skb); 6715 6716 /** 6717 * netif_receive_skb_list - process many receive buffers from network 6718 * @head: list of skbs to process. 6719 * 6720 * Since return value of netif_receive_skb() is normally ignored, and 6721 * wouldn't be meaningful for a list, this function returns void. 6722 * 6723 * This function may only be called from softirq context and interrupts 6724 * should be enabled. 6725 */ 6726 void netif_receive_skb_list(struct list_head *head) 6727 { 6728 struct sk_buff *skb; 6729 6730 if (list_empty(head)) 6731 return; 6732 if (trace_netif_receive_skb_list_entry_enabled()) { 6733 list_for_each_entry(skb, head, list) 6734 trace_netif_receive_skb_list_entry(skb); 6735 } 6736 netif_receive_skb_list_internal(head); 6737 trace_netif_receive_skb_list_exit(0); 6738 } 6739 EXPORT_SYMBOL(netif_receive_skb_list); 6740 6741 /* Network device is going away, flush any packets still pending */ 6742 static void flush_backlog(struct work_struct *work) 6743 { 6744 struct sk_buff *skb, *tmp; 6745 struct sk_buff_head list; 6746 struct softnet_data *sd; 6747 6748 __skb_queue_head_init(&list); 6749 local_bh_disable(); 6750 sd = this_cpu_ptr(&softnet_data); 6751 6752 backlog_lock_irq_disable(sd); 6753 skb_queue_walk_safe(&sd->input_pkt_queue, skb, tmp) { 6754 if (READ_ONCE(skb->dev->reg_state) == NETREG_UNREGISTERING) { 6755 __skb_unlink(skb, &sd->input_pkt_queue); 6756 __skb_queue_tail(&list, skb); 6757 rps_input_queue_head_incr(sd); 6758 } 6759 } 6760 backlog_unlock_irq_enable(sd); 6761 6762 local_lock_nested_bh(&softnet_data.process_queue_bh_lock); 6763 skb_queue_walk_safe(&sd->process_queue, skb, tmp) { 6764 if (READ_ONCE(skb->dev->reg_state) == NETREG_UNREGISTERING) { 6765 __skb_unlink(skb, &sd->process_queue); 6766 __skb_queue_tail(&list, skb); 6767 rps_input_queue_head_incr(sd); 6768 } 6769 } 6770 local_unlock_nested_bh(&softnet_data.process_queue_bh_lock); 6771 local_bh_enable(); 6772 6773 __skb_queue_purge_reason(&list, SKB_DROP_REASON_DEV_READY); 6774 } 6775 6776 static bool flush_required(int cpu) 6777 { 6778 #if IS_ENABLED(CONFIG_RPS) 6779 struct softnet_data *sd = &per_cpu(softnet_data, cpu); 6780 bool do_flush; 6781 6782 backlog_lock_irq_disable(sd); 6783 6784 /* as insertion into process_queue happens with the rps lock held, 6785 * process_queue access may race only with dequeue 6786 */ 6787 do_flush = !skb_queue_empty(&sd->input_pkt_queue) || 6788 !skb_queue_empty_lockless(&sd->process_queue); 6789 backlog_unlock_irq_enable(sd); 6790 6791 return do_flush; 6792 #endif 6793 /* without RPS we can't safely check input_pkt_queue: during a 6794 * concurrent remote skb_queue_splice() we can detect as empty both 6795 * input_pkt_queue and process_queue even if the latter could end-up 6796 * containing a lot of packets. 6797 */ 6798 return true; 6799 } 6800 6801 struct flush_backlogs { 6802 cpumask_t flush_cpus; 6803 struct work_struct w[]; 6804 }; 6805 6806 static struct flush_backlogs *flush_backlogs_alloc(void) 6807 { 6808 return kmalloc_flex(struct flush_backlogs, w, nr_cpu_ids); 6809 } 6810 6811 static struct flush_backlogs *flush_backlogs_fallback; 6812 static DEFINE_MUTEX(flush_backlogs_mutex); 6813 6814 static void flush_all_backlogs(void) 6815 { 6816 struct flush_backlogs *ptr = flush_backlogs_alloc(); 6817 unsigned int cpu; 6818 6819 if (!ptr) { 6820 mutex_lock(&flush_backlogs_mutex); 6821 ptr = flush_backlogs_fallback; 6822 } 6823 cpumask_clear(&ptr->flush_cpus); 6824 6825 cpus_read_lock(); 6826 6827 for_each_online_cpu(cpu) { 6828 if (flush_required(cpu)) { 6829 INIT_WORK(&ptr->w[cpu], flush_backlog); 6830 queue_work_on(cpu, system_highpri_wq, &ptr->w[cpu]); 6831 __cpumask_set_cpu(cpu, &ptr->flush_cpus); 6832 } 6833 } 6834 6835 /* we can have in flight packet[s] on the cpus we are not flushing, 6836 * synchronize_net() in unregister_netdevice_many() will take care of 6837 * them. 6838 */ 6839 for_each_cpu(cpu, &ptr->flush_cpus) 6840 flush_work(&ptr->w[cpu]); 6841 6842 cpus_read_unlock(); 6843 6844 if (ptr != flush_backlogs_fallback) 6845 kfree(ptr); 6846 else 6847 mutex_unlock(&flush_backlogs_mutex); 6848 } 6849 6850 static void net_rps_send_ipi(struct softnet_data *remsd) 6851 { 6852 #ifdef CONFIG_RPS 6853 while (remsd) { 6854 struct softnet_data *next = remsd->rps_ipi_next; 6855 6856 if (cpu_online(remsd->cpu)) 6857 smp_call_function_single_async(remsd->cpu, &remsd->csd); 6858 remsd = next; 6859 } 6860 #endif 6861 } 6862 6863 /* 6864 * net_rps_action_and_irq_enable sends any pending IPI's for rps. 6865 * Note: called with local irq disabled, but exits with local irq enabled. 6866 */ 6867 static void net_rps_action_and_irq_enable(struct softnet_data *sd) 6868 { 6869 #ifdef CONFIG_RPS 6870 struct softnet_data *remsd = sd->rps_ipi_list; 6871 6872 if (!use_backlog_threads() && remsd) { 6873 sd->rps_ipi_list = NULL; 6874 6875 local_irq_enable(); 6876 6877 /* Send pending IPI's to kick RPS processing on remote cpus. */ 6878 net_rps_send_ipi(remsd); 6879 } else 6880 #endif 6881 local_irq_enable(); 6882 } 6883 6884 static bool sd_has_rps_ipi_waiting(struct softnet_data *sd) 6885 { 6886 #ifdef CONFIG_RPS 6887 return !use_backlog_threads() && sd->rps_ipi_list; 6888 #else 6889 return false; 6890 #endif 6891 } 6892 6893 static int process_backlog(struct napi_struct *napi, int quota) 6894 { 6895 struct softnet_data *sd = container_of(napi, struct softnet_data, backlog); 6896 bool again = true; 6897 int work = 0; 6898 6899 /* Check if we have pending ipi, its better to send them now, 6900 * not waiting net_rx_action() end. 6901 */ 6902 if (sd_has_rps_ipi_waiting(sd)) { 6903 local_irq_disable(); 6904 net_rps_action_and_irq_enable(sd); 6905 } 6906 6907 napi->weight = READ_ONCE(net_hotdata.dev_rx_weight); 6908 while (again) { 6909 struct sk_buff *skb; 6910 6911 local_lock_nested_bh(&softnet_data.process_queue_bh_lock); 6912 while ((skb = __skb_dequeue(&sd->process_queue))) { 6913 local_unlock_nested_bh(&softnet_data.process_queue_bh_lock); 6914 rcu_read_lock(); 6915 __netif_receive_skb(skb); 6916 rcu_read_unlock(); 6917 if (++work >= quota) { 6918 rps_input_queue_head_add(sd, work); 6919 return work; 6920 } 6921 6922 local_lock_nested_bh(&softnet_data.process_queue_bh_lock); 6923 } 6924 local_unlock_nested_bh(&softnet_data.process_queue_bh_lock); 6925 6926 backlog_lock_irq_disable(sd); 6927 if (skb_queue_empty(&sd->input_pkt_queue)) { 6928 /* 6929 * Inline a custom version of __napi_complete(). 6930 * only current cpu owns and manipulates this napi, 6931 * and NAPI_STATE_SCHED is the only possible flag set 6932 * on backlog. 6933 * We can use a plain write instead of clear_bit(), 6934 * and we dont need an smp_mb() memory barrier. 6935 */ 6936 napi->state &= NAPIF_STATE_THREADED; 6937 again = false; 6938 } else { 6939 local_lock_nested_bh(&softnet_data.process_queue_bh_lock); 6940 skb_queue_splice_tail_init(&sd->input_pkt_queue, 6941 &sd->process_queue); 6942 local_unlock_nested_bh(&softnet_data.process_queue_bh_lock); 6943 } 6944 backlog_unlock_irq_enable(sd); 6945 } 6946 6947 if (work) 6948 rps_input_queue_head_add(sd, work); 6949 return work; 6950 } 6951 6952 /** 6953 * __napi_schedule - schedule for receive 6954 * @n: entry to schedule 6955 * 6956 * The entry's receive function will be scheduled to run. 6957 * Consider using __napi_schedule_irqoff() if hard irqs are masked. 6958 */ 6959 void __napi_schedule(struct napi_struct *n) 6960 { 6961 unsigned long flags; 6962 6963 local_irq_save(flags); 6964 ____napi_schedule(this_cpu_ptr(&softnet_data), n); 6965 local_irq_restore(flags); 6966 } 6967 EXPORT_SYMBOL(__napi_schedule); 6968 6969 /** 6970 * napi_schedule_prep - check if napi can be scheduled 6971 * @n: napi context 6972 * 6973 * Test if NAPI routine is already running, and if not mark 6974 * it as running. This is used as a condition variable to 6975 * insure only one NAPI poll instance runs. We also make 6976 * sure there is no pending NAPI disable. 6977 */ 6978 bool napi_schedule_prep(struct napi_struct *n) 6979 { 6980 unsigned long new, val = READ_ONCE(n->state); 6981 6982 do { 6983 if (unlikely(val & NAPIF_STATE_DISABLE)) 6984 return false; 6985 new = val | NAPIF_STATE_SCHED; 6986 6987 /* Sets STATE_MISSED bit if STATE_SCHED was already set 6988 * This was suggested by Alexander Duyck, as compiler 6989 * emits better code than : 6990 * if (val & NAPIF_STATE_SCHED) 6991 * new |= NAPIF_STATE_MISSED; 6992 */ 6993 new |= (val & NAPIF_STATE_SCHED) / NAPIF_STATE_SCHED * 6994 NAPIF_STATE_MISSED; 6995 } while (!try_cmpxchg(&n->state, &val, new)); 6996 6997 return !(val & NAPIF_STATE_SCHED); 6998 } 6999 EXPORT_SYMBOL(napi_schedule_prep); 7000 7001 /** 7002 * __napi_schedule_irqoff - schedule for receive 7003 * @n: entry to schedule 7004 * 7005 * Variant of __napi_schedule() assuming hard irqs are masked. 7006 * 7007 * On PREEMPT_RT enabled kernels this maps to __napi_schedule() 7008 * because the interrupt disabled assumption might not be true 7009 * due to force-threaded interrupts and spinlock substitution. 7010 */ 7011 void __napi_schedule_irqoff(struct napi_struct *n) 7012 { 7013 if (!IS_ENABLED(CONFIG_PREEMPT_RT)) 7014 ____napi_schedule(this_cpu_ptr(&softnet_data), n); 7015 else 7016 __napi_schedule(n); 7017 } 7018 EXPORT_SYMBOL(__napi_schedule_irqoff); 7019 7020 bool napi_complete_done(struct napi_struct *n, int work_done) 7021 { 7022 unsigned long flags, val, new, timeout = 0; 7023 bool ret = true; 7024 7025 /* 7026 * 1) Don't let napi dequeue from the cpu poll list 7027 * just in case its running on a different cpu. 7028 * 2) If we are busy polling, do nothing here, we have 7029 * the guarantee we will be called later. 7030 */ 7031 if (unlikely(n->state & (NAPIF_STATE_NPSVC | 7032 NAPIF_STATE_IN_BUSY_POLL))) 7033 return false; 7034 7035 if (work_done) { 7036 if (n->gro.bitmask) 7037 timeout = napi_get_gro_flush_timeout(n); 7038 n->defer_hard_irqs_count = napi_get_defer_hard_irqs(n); 7039 } 7040 if (n->defer_hard_irqs_count > 0) { 7041 n->defer_hard_irqs_count--; 7042 timeout = napi_get_gro_flush_timeout(n); 7043 if (timeout) 7044 ret = false; 7045 } 7046 7047 /* 7048 * When the NAPI instance uses a timeout and keeps postponing 7049 * it, we need to bound somehow the time packets are kept in 7050 * the GRO layer. 7051 */ 7052 gro_flush_normal(&n->gro, !!timeout); 7053 7054 if (unlikely(!list_empty(&n->poll_list))) { 7055 /* If n->poll_list is not empty, we need to mask irqs */ 7056 local_irq_save(flags); 7057 list_del_init(&n->poll_list); 7058 local_irq_restore(flags); 7059 } 7060 WRITE_ONCE(n->list_owner, -1); 7061 7062 val = READ_ONCE(n->state); 7063 do { 7064 WARN_ON_ONCE(!(val & NAPIF_STATE_SCHED)); 7065 7066 new = val & ~(NAPIF_STATE_MISSED | NAPIF_STATE_SCHED | 7067 NAPIF_STATE_SCHED_THREADED | 7068 NAPIF_STATE_PREFER_BUSY_POLL); 7069 7070 /* If STATE_MISSED was set, leave STATE_SCHED set, 7071 * because we will call napi->poll() one more time. 7072 * This C code was suggested by Alexander Duyck to help gcc. 7073 */ 7074 new |= (val & NAPIF_STATE_MISSED) / NAPIF_STATE_MISSED * 7075 NAPIF_STATE_SCHED; 7076 } while (!try_cmpxchg(&n->state, &val, new)); 7077 7078 if (unlikely(val & NAPIF_STATE_MISSED)) { 7079 __napi_schedule(n); 7080 return false; 7081 } 7082 7083 if (timeout) 7084 hrtimer_start(&n->timer, ns_to_ktime(timeout), 7085 HRTIMER_MODE_REL_PINNED); 7086 return ret; 7087 } 7088 EXPORT_SYMBOL(napi_complete_done); 7089 7090 static void __skb_defer_free_flush(struct skb_defer_node *sdn, int budget) 7091 { 7092 struct llist_node *free_list; 7093 struct sk_buff *skb, *next; 7094 long n = 0; 7095 7096 if (llist_empty(&sdn->defer_list)) 7097 return; 7098 free_list = llist_del_all(&sdn->defer_list); 7099 7100 llist_for_each_entry_safe(skb, next, free_list, ll_node) { 7101 prefetch(next); 7102 napi_consume_skb(skb, budget); 7103 n++; 7104 } 7105 /* Subtract exactly what was drained. Never store an absolute value: 7106 * mixing one with the relative updates in skb_attempt_defer_free() 7107 * lets the counter go negative, and it is read as unsigned there. 7108 */ 7109 atomic_long_sub(n, &sdn->defer_count); 7110 } 7111 7112 void skb_defer_node_flush(struct skb_defer_node *sdn) 7113 { 7114 __skb_defer_free_flush(sdn, 0); 7115 } 7116 7117 static void skb_defer_free_flush(void) 7118 { 7119 struct skb_defer_node *sdn; 7120 int node; 7121 7122 for_each_online_node(node) { 7123 sdn = this_cpu_ptr(net_hotdata.skb_defer_nodes) + node; 7124 __skb_defer_free_flush(sdn, 1); 7125 } 7126 } 7127 7128 #if defined(CONFIG_NET_RX_BUSY_POLL) 7129 7130 enum { 7131 NAPI_F_PREFER_BUSY_POLL = 1, 7132 NAPI_F_END_ON_RESCHED = 2, 7133 }; 7134 7135 static void busy_poll_stop(struct napi_struct *napi, void *have_poll_lock, 7136 unsigned flags, u16 budget) 7137 { 7138 struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx; 7139 unsigned long timeout = 0; 7140 int rc; 7141 7142 /* Busy polling means there is a high chance device driver hard irq 7143 * could not grab NAPI_STATE_SCHED, and that NAPI_STATE_MISSED was 7144 * set in napi_schedule_prep(). 7145 * Since we either call napi->poll() once more or start the timer, 7146 * we can safely clear NAPI_STATE_MISSED. 7147 * 7148 * Note: x86 could use a single "lock and ..." instruction 7149 * to perform these two clear_bit() 7150 */ 7151 clear_bit(NAPI_STATE_MISSED, &napi->state); 7152 clear_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state); 7153 7154 local_bh_disable(); 7155 bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx); 7156 7157 if (flags & NAPI_F_PREFER_BUSY_POLL) { 7158 napi->defer_hard_irqs_count = napi_get_defer_hard_irqs(napi); 7159 if (napi->defer_hard_irqs_count) 7160 timeout = napi_get_gro_flush_timeout(napi); 7161 } 7162 if (timeout) { 7163 netpoll_poll_unlock(have_poll_lock); 7164 7165 /* Drain aged GRO packets before clearing SCHED since the NAPI 7166 * won't run again until after the timer fires. When HZ < 1000, 7167 * GRO age comparison is too coarse, so flush everything. 7168 */ 7169 gro_flush_normal(&napi->gro, HZ >= 1000); 7170 7171 clear_bit(NAPI_STATE_SCHED, &napi->state); 7172 hrtimer_start(&napi->timer, ns_to_ktime(timeout), 7173 HRTIMER_MODE_REL_PINNED); 7174 } else { 7175 /* Use driver poll to re-enable device interrupts. */ 7176 rc = napi->poll(napi, budget); 7177 /* Unless rc == budget we no longer own the NAPI instance, 7178 * IRQ may fire on another CPU, poll this NAPI, and enter GRO. 7179 */ 7180 trace_napi_poll(napi, rc, budget); 7181 netpoll_poll_unlock(have_poll_lock); 7182 if (rc == budget) { 7183 gro_normal_list(&napi->gro); 7184 __napi_schedule(napi); 7185 } 7186 } 7187 7188 bpf_net_ctx_clear(bpf_net_ctx); 7189 local_bh_enable(); 7190 } 7191 7192 static void __napi_busy_loop(unsigned int napi_id, 7193 bool (*loop_end)(void *, unsigned long), 7194 void *loop_end_arg, unsigned flags, u16 budget) 7195 { 7196 unsigned long start_time = loop_end ? busy_loop_current_time() : 0; 7197 int (*napi_poll)(struct napi_struct *napi, int budget); 7198 struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx; 7199 void *have_poll_lock = NULL; 7200 struct napi_struct *napi; 7201 7202 WARN_ON_ONCE(!rcu_read_lock_held()); 7203 7204 restart: 7205 napi_poll = NULL; 7206 7207 napi = napi_by_id(napi_id); 7208 if (!napi) 7209 return; 7210 7211 if (!IS_ENABLED(CONFIG_PREEMPT_RT)) 7212 preempt_disable(); 7213 for (;;) { 7214 int work = 0; 7215 7216 local_bh_disable(); 7217 bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx); 7218 if (!napi_poll) { 7219 unsigned long val = READ_ONCE(napi->state); 7220 7221 /* If multiple threads are competing for this napi, 7222 * we avoid dirtying napi->state as much as we can. 7223 */ 7224 if (val & (NAPIF_STATE_DISABLE | NAPIF_STATE_SCHED | 7225 NAPIF_STATE_IN_BUSY_POLL)) { 7226 if (flags & NAPI_F_PREFER_BUSY_POLL) 7227 set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state); 7228 goto count; 7229 } 7230 if (cmpxchg(&napi->state, val, 7231 val | NAPIF_STATE_IN_BUSY_POLL | 7232 NAPIF_STATE_SCHED) != val) { 7233 if (flags & NAPI_F_PREFER_BUSY_POLL) 7234 set_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state); 7235 goto count; 7236 } 7237 have_poll_lock = netpoll_poll_lock(napi); 7238 napi_poll = napi->poll; 7239 } 7240 work = napi_poll(napi, budget); 7241 trace_napi_poll(napi, work, budget); 7242 gro_normal_list(&napi->gro); 7243 count: 7244 if (work > 0) 7245 __NET_ADD_STATS(dev_net(napi->dev), 7246 LINUX_MIB_BUSYPOLLRXPACKETS, work); 7247 skb_defer_free_flush(); 7248 bpf_net_ctx_clear(bpf_net_ctx); 7249 local_bh_enable(); 7250 7251 if (!loop_end || loop_end(loop_end_arg, start_time)) 7252 break; 7253 7254 if (unlikely(need_resched())) { 7255 if (flags & NAPI_F_END_ON_RESCHED) 7256 break; 7257 if (napi_poll) 7258 busy_poll_stop(napi, have_poll_lock, flags, budget); 7259 if (!IS_ENABLED(CONFIG_PREEMPT_RT)) 7260 preempt_enable(); 7261 rcu_read_unlock(); 7262 cond_resched(); 7263 rcu_read_lock(); 7264 if (loop_end(loop_end_arg, start_time)) 7265 return; 7266 goto restart; 7267 } 7268 cpu_relax(); 7269 } 7270 if (napi_poll) 7271 busy_poll_stop(napi, have_poll_lock, flags, budget); 7272 if (!IS_ENABLED(CONFIG_PREEMPT_RT)) 7273 preempt_enable(); 7274 } 7275 7276 void napi_busy_loop_rcu(unsigned int napi_id, 7277 bool (*loop_end)(void *, unsigned long), 7278 void *loop_end_arg, bool prefer_busy_poll, u16 budget) 7279 { 7280 unsigned flags = NAPI_F_END_ON_RESCHED; 7281 7282 if (prefer_busy_poll) 7283 flags |= NAPI_F_PREFER_BUSY_POLL; 7284 7285 __napi_busy_loop(napi_id, loop_end, loop_end_arg, flags, budget); 7286 } 7287 7288 void napi_busy_loop(unsigned int napi_id, 7289 bool (*loop_end)(void *, unsigned long), 7290 void *loop_end_arg, bool prefer_busy_poll, u16 budget) 7291 { 7292 unsigned flags = prefer_busy_poll ? NAPI_F_PREFER_BUSY_POLL : 0; 7293 7294 rcu_read_lock(); 7295 __napi_busy_loop(napi_id, loop_end, loop_end_arg, flags, budget); 7296 rcu_read_unlock(); 7297 } 7298 EXPORT_SYMBOL(napi_busy_loop); 7299 7300 void napi_suspend_irqs(unsigned int napi_id) 7301 { 7302 struct napi_struct *napi; 7303 7304 rcu_read_lock(); 7305 napi = napi_by_id(napi_id); 7306 if (napi) { 7307 unsigned long timeout = napi_get_irq_suspend_timeout(napi); 7308 7309 if (timeout) 7310 hrtimer_start(&napi->timer, ns_to_ktime(timeout), 7311 HRTIMER_MODE_REL_PINNED); 7312 } 7313 rcu_read_unlock(); 7314 } 7315 7316 void napi_resume_irqs(unsigned int napi_id) 7317 { 7318 struct napi_struct *napi; 7319 7320 rcu_read_lock(); 7321 napi = napi_by_id(napi_id); 7322 if (napi) { 7323 /* If irq_suspend_timeout is set to 0 between the call to 7324 * napi_suspend_irqs and now, the original value still 7325 * determines the safety timeout as intended and napi_watchdog 7326 * will resume irq processing. 7327 */ 7328 if (napi_get_irq_suspend_timeout(napi)) { 7329 local_bh_disable(); 7330 napi_schedule(napi); 7331 local_bh_enable(); 7332 } 7333 } 7334 rcu_read_unlock(); 7335 } 7336 7337 #endif /* CONFIG_NET_RX_BUSY_POLL */ 7338 7339 static void __napi_hash_add_with_id(struct napi_struct *napi, 7340 unsigned int napi_id) 7341 { 7342 napi->gro.cached_napi_id = napi_id; 7343 7344 WRITE_ONCE(napi->napi_id, napi_id); 7345 hlist_add_head_rcu(&napi->napi_hash_node, 7346 &napi_hash[napi->napi_id % HASH_SIZE(napi_hash)]); 7347 } 7348 7349 static void napi_hash_add_with_id(struct napi_struct *napi, 7350 unsigned int napi_id) 7351 { 7352 unsigned long flags; 7353 7354 spin_lock_irqsave(&napi_hash_lock, flags); 7355 WARN_ON_ONCE(napi_by_id(napi_id)); 7356 __napi_hash_add_with_id(napi, napi_id); 7357 spin_unlock_irqrestore(&napi_hash_lock, flags); 7358 } 7359 7360 static void napi_hash_add(struct napi_struct *napi) 7361 { 7362 unsigned long flags; 7363 7364 if (test_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state)) 7365 return; 7366 7367 spin_lock_irqsave(&napi_hash_lock, flags); 7368 7369 /* 0..NR_CPUS range is reserved for sender_cpu use */ 7370 do { 7371 if (unlikely(!napi_id_valid(++napi_gen_id))) 7372 napi_gen_id = MIN_NAPI_ID; 7373 } while (napi_by_id(napi_gen_id)); 7374 7375 __napi_hash_add_with_id(napi, napi_gen_id); 7376 7377 spin_unlock_irqrestore(&napi_hash_lock, flags); 7378 } 7379 7380 /* Warning : caller is responsible to make sure rcu grace period 7381 * is respected before freeing memory containing @napi 7382 */ 7383 static void napi_hash_del(struct napi_struct *napi) 7384 { 7385 unsigned long flags; 7386 7387 spin_lock_irqsave(&napi_hash_lock, flags); 7388 7389 hlist_del_init_rcu(&napi->napi_hash_node); 7390 7391 spin_unlock_irqrestore(&napi_hash_lock, flags); 7392 } 7393 7394 static enum hrtimer_restart napi_watchdog(struct hrtimer *timer) 7395 { 7396 struct napi_struct *napi; 7397 7398 napi = container_of(timer, struct napi_struct, timer); 7399 7400 /* Note : we use a relaxed variant of napi_schedule_prep() not setting 7401 * NAPI_STATE_MISSED, since we do not react to a device IRQ. 7402 */ 7403 if (!napi_disable_pending(napi) && 7404 !test_and_set_bit(NAPI_STATE_SCHED, &napi->state)) { 7405 clear_bit(NAPI_STATE_PREFER_BUSY_POLL, &napi->state); 7406 __napi_schedule_irqoff(napi); 7407 } 7408 7409 return HRTIMER_NORESTART; 7410 } 7411 7412 static void napi_stop_kthread(struct napi_struct *napi) 7413 { 7414 unsigned long val, new; 7415 7416 /* Instance is disabled, we can immediately stop the kthread */ 7417 if (test_bit(NAPI_STATE_DISABLED, &napi->state)) { 7418 clear_bit(NAPI_STATE_THREADED, &napi->state); 7419 clear_bit(NAPI_STATE_THREADED_BUSY_POLL, &napi->state); 7420 goto stop; 7421 } 7422 7423 /* Wait until the napi STATE_THREADED is unset. */ 7424 while (true) { 7425 val = READ_ONCE(napi->state); 7426 7427 /* If napi kthread own this napi or the napi is idle, 7428 * STATE_THREADED can be unset here. 7429 */ 7430 if ((val & NAPIF_STATE_SCHED_THREADED) || 7431 !(val & NAPIF_STATE_SCHED)) { 7432 new = val & (~(NAPIF_STATE_THREADED | 7433 NAPIF_STATE_THREADED_BUSY_POLL)); 7434 } else { 7435 msleep(20); 7436 continue; 7437 } 7438 7439 if (try_cmpxchg(&napi->state, &val, new)) 7440 break; 7441 } 7442 7443 /* Once STATE_THREADED is unset, wait for SCHED_THREADED to be unset by 7444 * the kthread. 7445 */ 7446 while (true) { 7447 if (!test_bit(NAPI_STATE_SCHED_THREADED, &napi->state)) 7448 break; 7449 7450 msleep(20); 7451 } 7452 7453 stop: 7454 kthread_stop(napi->thread); 7455 napi->thread = NULL; 7456 } 7457 7458 static void napi_set_threaded_state(struct napi_struct *napi, 7459 enum netdev_napi_threaded threaded_mode) 7460 { 7461 bool threaded = threaded_mode != NETDEV_NAPI_THREADED_DISABLED; 7462 bool busy_poll = threaded_mode == NETDEV_NAPI_THREADED_BUSY_POLL; 7463 7464 assign_bit(NAPI_STATE_THREADED, &napi->state, threaded); 7465 assign_bit(NAPI_STATE_THREADED_BUSY_POLL, &napi->state, busy_poll); 7466 } 7467 7468 int napi_set_threaded(struct napi_struct *napi, 7469 enum netdev_napi_threaded threaded) 7470 { 7471 if (threaded) { 7472 if (!napi->thread) { 7473 int err = napi_kthread_create(napi); 7474 7475 if (err) 7476 return err; 7477 } 7478 } 7479 7480 if (napi->config) 7481 napi->config->threaded = threaded; 7482 7483 /* Setting/unsetting threaded mode on a napi might not immediately 7484 * take effect, if the current napi instance is actively being 7485 * polled. In this case, the switch between threaded mode and 7486 * softirq mode will happen in the next round of napi_schedule(). 7487 * This should not cause hiccups/stalls to the live traffic. 7488 */ 7489 if (!threaded && napi->thread) { 7490 napi_stop_kthread(napi); 7491 } else { 7492 /* Make sure kthread is created before THREADED bit is set. */ 7493 smp_mb__before_atomic(); 7494 napi_set_threaded_state(napi, threaded); 7495 } 7496 7497 return 0; 7498 } 7499 7500 int netif_set_threaded(struct net_device *dev, 7501 enum netdev_napi_threaded threaded) 7502 { 7503 struct napi_struct *napi; 7504 int i, err = 0; 7505 7506 netdev_assert_locked_or_invisible(dev); 7507 7508 if (threaded) { 7509 list_for_each_entry(napi, &dev->napi_list, dev_list) { 7510 if (!napi->thread) { 7511 err = napi_kthread_create(napi); 7512 if (err) { 7513 threaded = NETDEV_NAPI_THREADED_DISABLED; 7514 break; 7515 } 7516 } 7517 } 7518 } 7519 7520 WRITE_ONCE(dev->threaded, threaded); 7521 7522 /* The error should not occur as the kthreads are already created. */ 7523 list_for_each_entry(napi, &dev->napi_list, dev_list) 7524 WARN_ON_ONCE(napi_set_threaded(napi, threaded)); 7525 7526 /* Override the config for all NAPIs even if currently not listed */ 7527 for (i = 0; i < dev->num_napi_configs; i++) 7528 dev->napi_config[i].threaded = threaded; 7529 7530 return err; 7531 } 7532 7533 /** 7534 * netif_threaded_enable() - enable threaded NAPIs 7535 * @dev: net_device instance 7536 * 7537 * Enable threaded mode for the NAPI instances of the device. This may be useful 7538 * for devices where multiple NAPI instances get scheduled by a single 7539 * interrupt. Threaded NAPI allows moving the NAPI processing to cores other 7540 * than the core where IRQ is mapped. 7541 * 7542 * This function should be called before @dev is registered. 7543 */ 7544 void netif_threaded_enable(struct net_device *dev) 7545 { 7546 WARN_ON_ONCE(netif_set_threaded(dev, NETDEV_NAPI_THREADED_ENABLED)); 7547 } 7548 EXPORT_SYMBOL(netif_threaded_enable); 7549 7550 /** 7551 * netif_queue_set_napi - Associate queue with the napi 7552 * @dev: device to which NAPI and queue belong 7553 * @queue_index: Index of queue 7554 * @type: queue type as RX or TX 7555 * @napi: NAPI context, pass NULL to clear previously set NAPI 7556 * 7557 * Set queue with its corresponding napi context. This should be done after 7558 * registering the NAPI handler for the queue-vector and the queues have been 7559 * mapped to the corresponding interrupt vector. 7560 */ 7561 void netif_queue_set_napi(struct net_device *dev, unsigned int queue_index, 7562 enum netdev_queue_type type, struct napi_struct *napi) 7563 { 7564 struct netdev_rx_queue *rxq; 7565 struct netdev_queue *txq; 7566 7567 if (WARN_ON_ONCE(napi && !napi->dev)) 7568 return; 7569 netdev_assert_locked_ops_compat_or_invisible(dev); 7570 7571 switch (type) { 7572 case NETDEV_QUEUE_TYPE_RX: 7573 rxq = __netif_get_rx_queue(dev, queue_index); 7574 rxq->napi = napi; 7575 return; 7576 case NETDEV_QUEUE_TYPE_TX: 7577 txq = netdev_get_tx_queue(dev, queue_index); 7578 txq->napi = napi; 7579 return; 7580 default: 7581 return; 7582 } 7583 } 7584 EXPORT_SYMBOL(netif_queue_set_napi); 7585 7586 static void 7587 netif_napi_irq_notify(struct irq_affinity_notify *notify, 7588 const cpumask_t *mask) 7589 { 7590 struct napi_struct *napi = 7591 container_of(notify, struct napi_struct, notify); 7592 #ifdef CONFIG_RFS_ACCEL 7593 struct cpu_rmap *rmap = napi->dev->rx_cpu_rmap; 7594 int err; 7595 #endif 7596 7597 if (napi->config && napi->dev->irq_affinity_auto) 7598 cpumask_copy(&napi->config->affinity_mask, mask); 7599 7600 #ifdef CONFIG_RFS_ACCEL 7601 if (napi->dev->rx_cpu_rmap_auto) { 7602 err = cpu_rmap_update(rmap, napi->napi_rmap_idx, mask); 7603 if (err) 7604 netdev_warn(napi->dev, "RMAP update failed (%d)\n", 7605 err); 7606 } 7607 #endif 7608 } 7609 7610 #ifdef CONFIG_RFS_ACCEL 7611 static void netif_napi_affinity_release(struct kref *ref) 7612 { 7613 struct napi_struct *napi = 7614 container_of(ref, struct napi_struct, notify.kref); 7615 struct cpu_rmap *rmap = napi->dev->rx_cpu_rmap; 7616 7617 netdev_assert_locked(napi->dev); 7618 WARN_ON(test_and_clear_bit(NAPI_STATE_HAS_NOTIFIER, 7619 &napi->state)); 7620 7621 if (!napi->dev->rx_cpu_rmap_auto) 7622 return; 7623 rmap->obj[napi->napi_rmap_idx] = NULL; 7624 napi->napi_rmap_idx = -1; 7625 cpu_rmap_put(rmap); 7626 } 7627 7628 int netif_enable_cpu_rmap(struct net_device *dev, unsigned int num_irqs) 7629 { 7630 if (dev->rx_cpu_rmap_auto) 7631 return 0; 7632 7633 dev->rx_cpu_rmap = alloc_irq_cpu_rmap(num_irqs); 7634 if (!dev->rx_cpu_rmap) 7635 return -ENOMEM; 7636 7637 dev->rx_cpu_rmap_auto = true; 7638 return 0; 7639 } 7640 EXPORT_SYMBOL(netif_enable_cpu_rmap); 7641 7642 static void netif_del_cpu_rmap(struct net_device *dev) 7643 { 7644 struct cpu_rmap *rmap = dev->rx_cpu_rmap; 7645 7646 if (!dev->rx_cpu_rmap_auto) 7647 return; 7648 7649 /* Free the rmap */ 7650 cpu_rmap_put(rmap); 7651 dev->rx_cpu_rmap = NULL; 7652 dev->rx_cpu_rmap_auto = false; 7653 } 7654 7655 #else 7656 static void netif_napi_affinity_release(struct kref *ref) 7657 { 7658 } 7659 7660 int netif_enable_cpu_rmap(struct net_device *dev, unsigned int num_irqs) 7661 { 7662 return 0; 7663 } 7664 EXPORT_SYMBOL(netif_enable_cpu_rmap); 7665 7666 static void netif_del_cpu_rmap(struct net_device *dev) 7667 { 7668 } 7669 #endif 7670 7671 void netif_set_affinity_auto(struct net_device *dev) 7672 { 7673 unsigned int i, maxqs, numa; 7674 7675 maxqs = max(dev->num_tx_queues, dev->num_rx_queues); 7676 numa = dev_to_node(&dev->dev); 7677 7678 for (i = 0; i < maxqs; i++) 7679 cpumask_set_cpu(cpumask_local_spread(i, numa), 7680 &dev->napi_config[i].affinity_mask); 7681 7682 dev->irq_affinity_auto = true; 7683 } 7684 EXPORT_SYMBOL(netif_set_affinity_auto); 7685 7686 void netif_napi_set_irq_locked(struct napi_struct *napi, int irq) 7687 { 7688 int rc; 7689 7690 netdev_assert_locked_or_invisible(napi->dev); 7691 7692 if (napi->irq == irq) 7693 return; 7694 7695 /* Remove existing resources */ 7696 if (test_and_clear_bit(NAPI_STATE_HAS_NOTIFIER, &napi->state)) 7697 irq_set_affinity_notifier(napi->irq, NULL); 7698 7699 napi->irq = irq; 7700 if (irq < 0 || 7701 (!napi->dev->rx_cpu_rmap_auto && !napi->dev->irq_affinity_auto)) 7702 return; 7703 7704 /* Abort for buggy drivers */ 7705 if (napi->dev->irq_affinity_auto && WARN_ON_ONCE(!napi->config)) 7706 return; 7707 7708 #ifdef CONFIG_RFS_ACCEL 7709 if (napi->dev->rx_cpu_rmap_auto) { 7710 rc = cpu_rmap_add(napi->dev->rx_cpu_rmap, napi); 7711 if (rc < 0) 7712 return; 7713 7714 cpu_rmap_get(napi->dev->rx_cpu_rmap); 7715 napi->napi_rmap_idx = rc; 7716 } 7717 #endif 7718 7719 /* Use core IRQ notifier */ 7720 napi->notify.notify = netif_napi_irq_notify; 7721 napi->notify.release = netif_napi_affinity_release; 7722 rc = irq_set_affinity_notifier(irq, &napi->notify); 7723 if (rc) { 7724 netdev_warn(napi->dev, "Unable to set IRQ notifier (%d)\n", 7725 rc); 7726 goto put_rmap; 7727 } 7728 7729 set_bit(NAPI_STATE_HAS_NOTIFIER, &napi->state); 7730 return; 7731 7732 put_rmap: 7733 #ifdef CONFIG_RFS_ACCEL 7734 if (napi->dev->rx_cpu_rmap_auto) { 7735 napi->dev->rx_cpu_rmap->obj[napi->napi_rmap_idx] = NULL; 7736 cpu_rmap_put(napi->dev->rx_cpu_rmap); 7737 napi->napi_rmap_idx = -1; 7738 } 7739 #endif 7740 napi->notify.notify = NULL; 7741 napi->notify.release = NULL; 7742 } 7743 EXPORT_SYMBOL(netif_napi_set_irq_locked); 7744 7745 static void napi_restore_config(struct napi_struct *n) 7746 { 7747 n->defer_hard_irqs = n->config->defer_hard_irqs; 7748 n->gro_flush_timeout = n->config->gro_flush_timeout; 7749 n->irq_suspend_timeout = n->config->irq_suspend_timeout; 7750 7751 if (n->dev->irq_affinity_auto && 7752 test_bit(NAPI_STATE_HAS_NOTIFIER, &n->state)) 7753 irq_set_affinity(n->irq, &n->config->affinity_mask); 7754 7755 /* a NAPI ID might be stored in the config, if so use it. if not, use 7756 * napi_hash_add to generate one for us. 7757 */ 7758 if (n->config->napi_id) { 7759 napi_hash_add_with_id(n, n->config->napi_id); 7760 } else { 7761 napi_hash_add(n); 7762 n->config->napi_id = n->napi_id; 7763 } 7764 7765 WARN_ON_ONCE(napi_set_threaded(n, n->config->threaded)); 7766 } 7767 7768 static void napi_save_config(struct napi_struct *n) 7769 { 7770 n->config->defer_hard_irqs = n->defer_hard_irqs; 7771 n->config->gro_flush_timeout = n->gro_flush_timeout; 7772 n->config->irq_suspend_timeout = n->irq_suspend_timeout; 7773 napi_hash_del(n); 7774 } 7775 7776 /* Netlink wants the NAPI list to be sorted by ID, if adding a NAPI which will 7777 * inherit an existing ID try to insert it at the right position. 7778 */ 7779 static void 7780 netif_napi_dev_list_add(struct net_device *dev, struct napi_struct *napi) 7781 { 7782 unsigned int new_id, pos_id; 7783 struct list_head *higher; 7784 struct napi_struct *pos; 7785 7786 new_id = UINT_MAX; 7787 if (napi->config && napi->config->napi_id) 7788 new_id = napi->config->napi_id; 7789 7790 higher = &dev->napi_list; 7791 list_for_each_entry(pos, &dev->napi_list, dev_list) { 7792 if (napi_id_valid(pos->napi_id)) 7793 pos_id = pos->napi_id; 7794 else if (pos->config) 7795 pos_id = pos->config->napi_id; 7796 else 7797 pos_id = UINT_MAX; 7798 7799 if (pos_id <= new_id) 7800 break; 7801 higher = &pos->dev_list; 7802 } 7803 list_add_rcu(&napi->dev_list, higher); /* adds after higher */ 7804 } 7805 7806 /* Double check that napi_get_frags() allocates skbs with 7807 * skb->head being backed by slab, not a page fragment. 7808 * This is to make sure bug fixed in 3226b158e67c 7809 * ("net: avoid 32 x truesize under-estimation for tiny skbs") 7810 * does not accidentally come back. 7811 */ 7812 static void napi_get_frags_check(struct napi_struct *napi) 7813 { 7814 struct sk_buff *skb; 7815 7816 local_bh_disable(); 7817 skb = napi_get_frags(napi); 7818 WARN_ON_ONCE(skb && skb->head_frag); 7819 napi_free_frags(napi); 7820 local_bh_enable(); 7821 } 7822 7823 void netif_napi_add_weight_locked(struct net_device *dev, 7824 struct napi_struct *napi, 7825 int (*poll)(struct napi_struct *, int), 7826 int weight) 7827 { 7828 netdev_assert_locked(dev); 7829 if (WARN_ON(test_and_set_bit(NAPI_STATE_LISTED, &napi->state))) 7830 return; 7831 7832 INIT_LIST_HEAD(&napi->poll_list); 7833 INIT_HLIST_NODE(&napi->napi_hash_node); 7834 hrtimer_setup(&napi->timer, napi_watchdog, CLOCK_MONOTONIC, HRTIMER_MODE_REL_PINNED); 7835 gro_init(&napi->gro); 7836 napi->skb = NULL; 7837 napi->poll = poll; 7838 if (weight > NAPI_POLL_WEIGHT) 7839 netdev_err_once(dev, "%s() called with weight %d\n", __func__, 7840 weight); 7841 napi->weight = weight; 7842 napi->dev = dev; 7843 #ifdef CONFIG_NETPOLL 7844 napi->poll_owner = -1; 7845 #endif 7846 napi->list_owner = -1; 7847 set_bit(NAPI_STATE_SCHED, &napi->state); 7848 set_bit(NAPI_STATE_NPSVC, &napi->state); 7849 set_bit(NAPI_STATE_DISABLED, &napi->state); 7850 netif_napi_dev_list_add(dev, napi); 7851 7852 /* default settings from sysfs are applied to all NAPIs. any per-NAPI 7853 * configuration will be loaded in napi_enable 7854 */ 7855 napi_set_defer_hard_irqs(napi, READ_ONCE(dev->napi_defer_hard_irqs)); 7856 napi_set_gro_flush_timeout(napi, READ_ONCE(dev->gro_flush_timeout)); 7857 7858 napi_get_frags_check(napi); 7859 /* Create kthread for this napi if dev->threaded is set. 7860 * Clear dev->threaded if kthread creation failed so that 7861 * threaded mode will not be enabled in napi_enable(). 7862 */ 7863 if (napi_get_threaded_config(dev, napi)) 7864 if (napi_kthread_create(napi)) 7865 dev->threaded = NETDEV_NAPI_THREADED_DISABLED; 7866 netif_napi_set_irq_locked(napi, -1); 7867 } 7868 EXPORT_SYMBOL(netif_napi_add_weight_locked); 7869 7870 void napi_disable_locked(struct napi_struct *n) 7871 { 7872 unsigned long val, new; 7873 7874 might_sleep(); 7875 netdev_assert_locked(n->dev); 7876 7877 set_bit(NAPI_STATE_DISABLE, &n->state); 7878 7879 val = READ_ONCE(n->state); 7880 do { 7881 while (val & (NAPIF_STATE_SCHED | NAPIF_STATE_NPSVC)) { 7882 usleep_range(20, 200); 7883 val = READ_ONCE(n->state); 7884 } 7885 7886 new = val | NAPIF_STATE_SCHED | NAPIF_STATE_NPSVC | 7887 NAPIF_STATE_DISABLED; 7888 new &= ~(NAPIF_STATE_THREADED | 7889 NAPIF_STATE_THREADED_BUSY_POLL | 7890 NAPIF_STATE_PREFER_BUSY_POLL); 7891 } while (!try_cmpxchg(&n->state, &val, new)); 7892 7893 hrtimer_cancel(&n->timer); 7894 7895 if (n->config) 7896 napi_save_config(n); 7897 else 7898 napi_hash_del(n); 7899 7900 clear_bit(NAPI_STATE_DISABLE, &n->state); 7901 } 7902 EXPORT_SYMBOL(napi_disable_locked); 7903 7904 /** 7905 * napi_disable() - prevent NAPI from scheduling 7906 * @n: NAPI context 7907 * 7908 * Stop NAPI from being scheduled on this context. 7909 * Waits till any outstanding processing completes. 7910 * Takes netdev_lock() for associated net_device. 7911 */ 7912 void napi_disable(struct napi_struct *n) 7913 { 7914 netdev_lock(n->dev); 7915 napi_disable_locked(n); 7916 netdev_unlock(n->dev); 7917 } 7918 EXPORT_SYMBOL(napi_disable); 7919 7920 void napi_enable_locked(struct napi_struct *n) 7921 { 7922 unsigned long new, val = READ_ONCE(n->state); 7923 7924 if (n->config) 7925 napi_restore_config(n); 7926 else 7927 napi_hash_add(n); 7928 7929 do { 7930 BUG_ON(!test_bit(NAPI_STATE_SCHED, &val)); 7931 7932 new = val & ~(NAPIF_STATE_SCHED | NAPIF_STATE_NPSVC | 7933 NAPIF_STATE_DISABLED); 7934 if (n->dev->threaded && n->thread) 7935 new |= NAPIF_STATE_THREADED; 7936 } while (!try_cmpxchg(&n->state, &val, new)); 7937 } 7938 EXPORT_SYMBOL(napi_enable_locked); 7939 7940 /** 7941 * napi_enable() - enable NAPI scheduling 7942 * @n: NAPI context 7943 * 7944 * Enable scheduling of a NAPI instance. 7945 * Must be paired with napi_disable(). 7946 * Takes netdev_lock() for associated net_device. 7947 */ 7948 void napi_enable(struct napi_struct *n) 7949 { 7950 netdev_lock(n->dev); 7951 napi_enable_locked(n); 7952 netdev_unlock(n->dev); 7953 } 7954 EXPORT_SYMBOL(napi_enable); 7955 7956 /* Must be called in process context */ 7957 void __netif_napi_del_locked(struct napi_struct *napi) 7958 { 7959 netdev_assert_locked(napi->dev); 7960 7961 if (!test_and_clear_bit(NAPI_STATE_LISTED, &napi->state)) 7962 return; 7963 7964 /* Make sure NAPI is disabled (or was never enabled). */ 7965 WARN_ON(!test_bit(NAPI_STATE_SCHED, &napi->state)); 7966 7967 if (test_and_clear_bit(NAPI_STATE_HAS_NOTIFIER, &napi->state)) 7968 irq_set_affinity_notifier(napi->irq, NULL); 7969 7970 if (napi->config) { 7971 napi->index = -1; 7972 napi->config = NULL; 7973 } 7974 7975 list_del_rcu(&napi->dev_list); 7976 napi_free_frags(napi); 7977 7978 gro_cleanup(&napi->gro); 7979 7980 if (napi->thread) { 7981 kthread_stop(napi->thread); 7982 napi->thread = NULL; 7983 } 7984 } 7985 EXPORT_SYMBOL(__netif_napi_del_locked); 7986 7987 static int __napi_poll(struct napi_struct *n, bool *repoll) 7988 { 7989 int work, weight; 7990 7991 weight = n->weight; 7992 7993 /* This NAPI_STATE_SCHED test is for avoiding a race 7994 * with netpoll's poll_napi(). Only the entity which 7995 * obtains the lock and sees NAPI_STATE_SCHED set will 7996 * actually make the ->poll() call. Therefore we avoid 7997 * accidentally calling ->poll() when NAPI is not scheduled. 7998 */ 7999 work = 0; 8000 if (napi_is_scheduled(n)) { 8001 work = n->poll(n, weight); 8002 trace_napi_poll(n, work, weight); 8003 8004 xdp_do_check_flushed(n); 8005 } 8006 8007 if (unlikely(work > weight)) 8008 netdev_err_once(n->dev, "NAPI poll function %pS returned %d, exceeding its budget of %d.\n", 8009 n->poll, work, weight); 8010 8011 if (likely(work < weight)) 8012 return work; 8013 8014 /* Drivers must not modify the NAPI state if they 8015 * consume the entire weight. In such cases this code 8016 * still "owns" the NAPI instance and therefore can 8017 * move the instance around on the list at-will. 8018 */ 8019 if (unlikely(napi_disable_pending(n))) { 8020 napi_complete(n); 8021 return work; 8022 } 8023 8024 /* The NAPI context has more processing work, but busy-polling 8025 * is preferred. Exit early. 8026 */ 8027 if (napi_prefer_busy_poll(n)) { 8028 if (napi_complete_done(n, work)) { 8029 /* If timeout is not set, we need to make sure 8030 * that the NAPI is re-scheduled. 8031 */ 8032 napi_schedule(n); 8033 } 8034 return work; 8035 } 8036 8037 /* Flush too old packets. If HZ < 1000, flush all packets */ 8038 gro_flush_normal(&n->gro, HZ >= 1000); 8039 8040 /* Some drivers may have called napi_schedule 8041 * prior to exhausting their budget. 8042 */ 8043 if (unlikely(!list_empty(&n->poll_list))) { 8044 pr_warn_once("%s: Budget exhausted after napi rescheduled\n", 8045 n->dev ? n->dev->name : "backlog"); 8046 return work; 8047 } 8048 8049 *repoll = true; 8050 8051 return work; 8052 } 8053 8054 static int napi_poll(struct napi_struct *n, struct list_head *repoll) 8055 { 8056 bool do_repoll = false; 8057 void *have; 8058 int work; 8059 8060 list_del_init(&n->poll_list); 8061 8062 have = netpoll_poll_lock(n); 8063 8064 work = __napi_poll(n, &do_repoll); 8065 8066 if (do_repoll) { 8067 #if defined(CONFIG_DEBUG_NET) 8068 if (unlikely(!napi_is_scheduled(n))) 8069 pr_crit("repoll requested for device %s %ps but napi is not scheduled.\n", 8070 n->dev->name, n->poll); 8071 #endif 8072 list_add_tail(&n->poll_list, repoll); 8073 } 8074 netpoll_poll_unlock(have); 8075 8076 return work; 8077 } 8078 8079 static int napi_thread_wait(struct napi_struct *napi) 8080 { 8081 set_current_state(TASK_INTERRUPTIBLE); 8082 8083 while (!kthread_should_stop()) { 8084 /* Testing SCHED_THREADED bit here to make sure the current 8085 * kthread owns this napi and could poll on this napi. 8086 * Testing SCHED bit is not enough because SCHED bit might be 8087 * set by some other busy poll thread or by napi_disable(). 8088 */ 8089 if (test_bit(NAPI_STATE_SCHED_THREADED, &napi->state)) { 8090 WARN_ON(!list_empty(&napi->poll_list)); 8091 __set_current_state(TASK_RUNNING); 8092 return 0; 8093 } 8094 8095 schedule(); 8096 set_current_state(TASK_INTERRUPTIBLE); 8097 } 8098 __set_current_state(TASK_RUNNING); 8099 8100 return -1; 8101 } 8102 8103 static void napi_threaded_poll_loop(struct napi_struct *napi, 8104 unsigned long *busy_poll_last_qs) 8105 { 8106 unsigned long last_qs = busy_poll_last_qs ? *busy_poll_last_qs : jiffies; 8107 struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx; 8108 struct softnet_data *sd; 8109 8110 for (;;) { 8111 bool repoll = false; 8112 void *have; 8113 8114 local_bh_disable(); 8115 bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx); 8116 8117 sd = this_cpu_ptr(&softnet_data); 8118 sd->in_napi_threaded_poll = true; 8119 8120 have = netpoll_poll_lock(napi); 8121 __napi_poll(napi, &repoll); 8122 netpoll_poll_unlock(have); 8123 8124 sd->in_napi_threaded_poll = false; 8125 barrier(); 8126 8127 if (sd_has_rps_ipi_waiting(sd)) { 8128 local_irq_disable(); 8129 net_rps_action_and_irq_enable(sd); 8130 } 8131 skb_defer_free_flush(); 8132 bpf_net_ctx_clear(bpf_net_ctx); 8133 8134 /* When busy poll is enabled, the old packets are not flushed in 8135 * napi_complete_done. So flush them here. 8136 */ 8137 if (busy_poll_last_qs) 8138 gro_flush_normal(&napi->gro, HZ >= 1000); 8139 local_bh_enable(); 8140 8141 /* Call cond_resched here to avoid watchdog warnings. */ 8142 if (repoll || busy_poll_last_qs) { 8143 rcu_softirq_qs_periodic(last_qs); 8144 cond_resched(); 8145 } 8146 8147 if (!repoll) 8148 break; 8149 } 8150 8151 if (busy_poll_last_qs) 8152 *busy_poll_last_qs = last_qs; 8153 } 8154 8155 static int napi_threaded_poll(void *data) 8156 { 8157 struct napi_struct *napi = data; 8158 unsigned long last_qs = jiffies; 8159 bool want_busy_poll; 8160 bool in_busy_poll; 8161 unsigned long val; 8162 8163 while (!napi_thread_wait(napi)) { 8164 val = READ_ONCE(napi->state); 8165 8166 want_busy_poll = val & NAPIF_STATE_THREADED_BUSY_POLL; 8167 in_busy_poll = val & NAPIF_STATE_IN_BUSY_POLL; 8168 8169 if (unlikely(val & NAPIF_STATE_DISABLE)) 8170 want_busy_poll = false; 8171 8172 if (want_busy_poll != in_busy_poll) 8173 assign_bit(NAPI_STATE_IN_BUSY_POLL, &napi->state, 8174 want_busy_poll); 8175 8176 napi_threaded_poll_loop(napi, want_busy_poll ? &last_qs : NULL); 8177 } 8178 8179 return 0; 8180 } 8181 8182 static __latent_entropy void net_rx_action(void) 8183 { 8184 struct softnet_data *sd = this_cpu_ptr(&softnet_data); 8185 unsigned long time_limit = jiffies + 8186 usecs_to_jiffies(READ_ONCE(net_hotdata.netdev_budget_usecs)); 8187 struct bpf_net_context __bpf_net_ctx, *bpf_net_ctx; 8188 int budget = READ_ONCE(net_hotdata.netdev_budget); 8189 LIST_HEAD(list); 8190 LIST_HEAD(repoll); 8191 8192 bpf_net_ctx = bpf_net_ctx_set(&__bpf_net_ctx); 8193 start: 8194 sd->in_net_rx_action = true; 8195 local_irq_disable(); 8196 list_splice_init(&sd->poll_list, &list); 8197 local_irq_enable(); 8198 8199 for (;;) { 8200 struct napi_struct *n; 8201 8202 skb_defer_free_flush(); 8203 8204 if (list_empty(&list)) { 8205 if (list_empty(&repoll)) { 8206 sd->in_net_rx_action = false; 8207 barrier(); 8208 /* We need to check if ____napi_schedule() 8209 * had refilled poll_list while 8210 * sd->in_net_rx_action was true. 8211 */ 8212 if (!list_empty(&sd->poll_list)) 8213 goto start; 8214 if (!sd_has_rps_ipi_waiting(sd)) 8215 goto end; 8216 } 8217 break; 8218 } 8219 8220 n = list_first_entry(&list, struct napi_struct, poll_list); 8221 budget -= napi_poll(n, &repoll); 8222 8223 /* If softirq window is exhausted then punt. 8224 * Allow this to run for 2 jiffies since which will allow 8225 * an average latency of 1.5/HZ. 8226 */ 8227 if (unlikely(budget <= 0 || 8228 time_after_eq(jiffies, time_limit))) { 8229 /* Pairs with READ_ONCE() in softnet_seq_show() */ 8230 WRITE_ONCE(sd->time_squeeze, sd->time_squeeze + 1); 8231 break; 8232 } 8233 } 8234 8235 local_irq_disable(); 8236 8237 list_splice_tail_init(&sd->poll_list, &list); 8238 list_splice_tail(&repoll, &list); 8239 list_splice(&list, &sd->poll_list); 8240 if (!list_empty(&sd->poll_list)) 8241 __raise_softirq_irqoff(NET_RX_SOFTIRQ); 8242 else 8243 sd->in_net_rx_action = false; 8244 8245 net_rps_action_and_irq_enable(sd); 8246 end: 8247 bpf_net_ctx_clear(bpf_net_ctx); 8248 } 8249 8250 struct netdev_adjacent { 8251 struct net_device *dev; 8252 netdevice_tracker dev_tracker; 8253 8254 /* upper master flag, there can only be one master device per list */ 8255 bool master; 8256 8257 /* lookup ignore flag */ 8258 bool ignore; 8259 8260 /* counter for the number of times this device was added to us */ 8261 u16 ref_nr; 8262 8263 /* private field for the users */ 8264 void *private; 8265 8266 struct list_head list; 8267 struct rcu_head rcu; 8268 }; 8269 8270 static struct netdev_adjacent *__netdev_find_adj(struct net_device *adj_dev, 8271 struct list_head *adj_list) 8272 { 8273 struct netdev_adjacent *adj; 8274 8275 list_for_each_entry(adj, adj_list, list) { 8276 if (adj->dev == adj_dev) 8277 return adj; 8278 } 8279 return NULL; 8280 } 8281 8282 static int ____netdev_has_upper_dev(struct net_device *upper_dev, 8283 struct netdev_nested_priv *priv) 8284 { 8285 struct net_device *dev = (struct net_device *)priv->data; 8286 8287 return upper_dev == dev; 8288 } 8289 8290 /** 8291 * netdev_has_upper_dev - Check if device is linked to an upper device 8292 * @dev: device 8293 * @upper_dev: upper device to check 8294 * 8295 * Find out if a device is linked to specified upper device and return true 8296 * in case it is. Note that this checks only immediate upper device, 8297 * not through a complete stack of devices. The caller must hold the RTNL lock. 8298 */ 8299 bool netdev_has_upper_dev(struct net_device *dev, 8300 struct net_device *upper_dev) 8301 { 8302 struct netdev_nested_priv priv = { 8303 .data = (void *)upper_dev, 8304 }; 8305 8306 ASSERT_RTNL(); 8307 8308 return netdev_walk_all_upper_dev_rcu(dev, ____netdev_has_upper_dev, 8309 &priv); 8310 } 8311 EXPORT_SYMBOL(netdev_has_upper_dev); 8312 8313 /** 8314 * netdev_has_upper_dev_all_rcu - Check if device is linked to an upper device 8315 * @dev: device 8316 * @upper_dev: upper device to check 8317 * 8318 * Find out if a device is linked to specified upper device and return true 8319 * in case it is. Note that this checks the entire upper device chain. 8320 * The caller must hold rcu lock. 8321 */ 8322 8323 bool netdev_has_upper_dev_all_rcu(struct net_device *dev, 8324 struct net_device *upper_dev) 8325 { 8326 struct netdev_nested_priv priv = { 8327 .data = (void *)upper_dev, 8328 }; 8329 8330 return !!netdev_walk_all_upper_dev_rcu(dev, ____netdev_has_upper_dev, 8331 &priv); 8332 } 8333 EXPORT_SYMBOL(netdev_has_upper_dev_all_rcu); 8334 8335 /** 8336 * netdev_has_any_upper_dev - Check if device is linked to some device 8337 * @dev: device 8338 * 8339 * Find out if a device is linked to an upper device and return true in case 8340 * it is. The caller must hold the RTNL lock. 8341 */ 8342 bool netdev_has_any_upper_dev(struct net_device *dev) 8343 { 8344 ASSERT_RTNL(); 8345 8346 return !list_empty(&dev->adj_list.upper); 8347 } 8348 EXPORT_SYMBOL(netdev_has_any_upper_dev); 8349 8350 /** 8351 * netdev_master_upper_dev_get - Get master upper device 8352 * @dev: device 8353 * 8354 * Find a master upper device and return pointer to it or NULL in case 8355 * it's not there. The caller must hold the RTNL lock. 8356 */ 8357 struct net_device *netdev_master_upper_dev_get(struct net_device *dev) 8358 { 8359 struct netdev_adjacent *upper; 8360 8361 ASSERT_RTNL(); 8362 8363 if (list_empty(&dev->adj_list.upper)) 8364 return NULL; 8365 8366 upper = list_first_entry(&dev->adj_list.upper, 8367 struct netdev_adjacent, list); 8368 if (likely(upper->master)) 8369 return upper->dev; 8370 return NULL; 8371 } 8372 EXPORT_SYMBOL(netdev_master_upper_dev_get); 8373 8374 static struct net_device *__netdev_master_upper_dev_get(struct net_device *dev) 8375 { 8376 struct netdev_adjacent *upper; 8377 8378 ASSERT_RTNL(); 8379 8380 if (list_empty(&dev->adj_list.upper)) 8381 return NULL; 8382 8383 upper = list_first_entry(&dev->adj_list.upper, 8384 struct netdev_adjacent, list); 8385 if (likely(upper->master) && !upper->ignore) 8386 return upper->dev; 8387 return NULL; 8388 } 8389 8390 /** 8391 * netdev_has_any_lower_dev - Check if device is linked to some device 8392 * @dev: device 8393 * 8394 * Find out if a device is linked to a lower device and return true in case 8395 * it is. The caller must hold the RTNL lock. 8396 */ 8397 static bool netdev_has_any_lower_dev(struct net_device *dev) 8398 { 8399 ASSERT_RTNL(); 8400 8401 return !list_empty(&dev->adj_list.lower); 8402 } 8403 8404 void *netdev_adjacent_get_private(struct list_head *adj_list) 8405 { 8406 struct netdev_adjacent *adj; 8407 8408 adj = list_entry(adj_list, struct netdev_adjacent, list); 8409 8410 return adj->private; 8411 } 8412 EXPORT_SYMBOL(netdev_adjacent_get_private); 8413 8414 /** 8415 * netdev_upper_get_next_dev_rcu - Get the next dev from upper list 8416 * @dev: device 8417 * @iter: list_head ** of the current position 8418 * 8419 * Gets the next device from the dev's upper list, starting from iter 8420 * position. The caller must hold RCU read lock. 8421 */ 8422 struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev, 8423 struct list_head **iter) 8424 { 8425 struct netdev_adjacent *upper; 8426 8427 WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held() && 8428 !lockdep_rtnl_is_held()); 8429 8430 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); 8431 8432 if (&upper->list == &dev->adj_list.upper) 8433 return NULL; 8434 8435 *iter = &upper->list; 8436 8437 return upper->dev; 8438 } 8439 EXPORT_SYMBOL(netdev_upper_get_next_dev_rcu); 8440 8441 static struct net_device *__netdev_next_upper_dev(struct net_device *dev, 8442 struct list_head **iter, 8443 bool *ignore) 8444 { 8445 struct netdev_adjacent *upper; 8446 8447 upper = list_entry((*iter)->next, struct netdev_adjacent, list); 8448 8449 if (&upper->list == &dev->adj_list.upper) 8450 return NULL; 8451 8452 *iter = &upper->list; 8453 *ignore = upper->ignore; 8454 8455 return upper->dev; 8456 } 8457 8458 static struct net_device *netdev_next_upper_dev_rcu(struct net_device *dev, 8459 struct list_head **iter) 8460 { 8461 struct netdev_adjacent *upper; 8462 8463 WARN_ON_ONCE(!rcu_read_lock_held() && !lockdep_rtnl_is_held()); 8464 8465 upper = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); 8466 8467 if (&upper->list == &dev->adj_list.upper) 8468 return NULL; 8469 8470 *iter = &upper->list; 8471 8472 return upper->dev; 8473 } 8474 8475 static int __netdev_walk_all_upper_dev(struct net_device *dev, 8476 int (*fn)(struct net_device *dev, 8477 struct netdev_nested_priv *priv), 8478 struct netdev_nested_priv *priv) 8479 { 8480 struct net_device *udev, *next, *now, *dev_stack[MAX_NEST_DEV + 1]; 8481 struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1]; 8482 int ret, cur = 0; 8483 bool ignore; 8484 8485 now = dev; 8486 iter = &dev->adj_list.upper; 8487 8488 while (1) { 8489 if (now != dev) { 8490 ret = fn(now, priv); 8491 if (ret) 8492 return ret; 8493 } 8494 8495 next = NULL; 8496 while (1) { 8497 udev = __netdev_next_upper_dev(now, &iter, &ignore); 8498 if (!udev) 8499 break; 8500 if (ignore) 8501 continue; 8502 8503 next = udev; 8504 niter = &udev->adj_list.upper; 8505 dev_stack[cur] = now; 8506 iter_stack[cur++] = iter; 8507 break; 8508 } 8509 8510 if (!next) { 8511 if (!cur) 8512 return 0; 8513 next = dev_stack[--cur]; 8514 niter = iter_stack[cur]; 8515 } 8516 8517 now = next; 8518 iter = niter; 8519 } 8520 8521 return 0; 8522 } 8523 8524 int netdev_walk_all_upper_dev_rcu(struct net_device *dev, 8525 int (*fn)(struct net_device *dev, 8526 struct netdev_nested_priv *priv), 8527 struct netdev_nested_priv *priv) 8528 { 8529 struct net_device *udev, *next, *now, *dev_stack[MAX_NEST_DEV + 1]; 8530 struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1]; 8531 int ret, cur = 0; 8532 8533 now = dev; 8534 iter = &dev->adj_list.upper; 8535 8536 while (1) { 8537 if (now != dev) { 8538 ret = fn(now, priv); 8539 if (ret) 8540 return ret; 8541 } 8542 8543 next = NULL; 8544 while (1) { 8545 udev = netdev_next_upper_dev_rcu(now, &iter); 8546 if (!udev) 8547 break; 8548 8549 next = udev; 8550 niter = &udev->adj_list.upper; 8551 dev_stack[cur] = now; 8552 iter_stack[cur++] = iter; 8553 break; 8554 } 8555 8556 if (!next) { 8557 if (!cur) 8558 return 0; 8559 next = dev_stack[--cur]; 8560 niter = iter_stack[cur]; 8561 } 8562 8563 now = next; 8564 iter = niter; 8565 } 8566 8567 return 0; 8568 } 8569 EXPORT_SYMBOL_GPL(netdev_walk_all_upper_dev_rcu); 8570 8571 static bool __netdev_has_upper_dev(struct net_device *dev, 8572 struct net_device *upper_dev) 8573 { 8574 struct netdev_nested_priv priv = { 8575 .flags = 0, 8576 .data = (void *)upper_dev, 8577 }; 8578 8579 ASSERT_RTNL(); 8580 8581 return __netdev_walk_all_upper_dev(dev, ____netdev_has_upper_dev, 8582 &priv); 8583 } 8584 8585 /** 8586 * netdev_lower_get_next_private - Get the next ->private from the 8587 * lower neighbour list 8588 * @dev: device 8589 * @iter: list_head ** of the current position 8590 * 8591 * Gets the next netdev_adjacent->private from the dev's lower neighbour 8592 * list, starting from iter position. The caller must hold either hold the 8593 * RTNL lock or its own locking that guarantees that the neighbour lower 8594 * list will remain unchanged. 8595 */ 8596 void *netdev_lower_get_next_private(struct net_device *dev, 8597 struct list_head **iter) 8598 { 8599 struct netdev_adjacent *lower; 8600 8601 lower = list_entry(*iter, struct netdev_adjacent, list); 8602 8603 if (&lower->list == &dev->adj_list.lower) 8604 return NULL; 8605 8606 *iter = lower->list.next; 8607 8608 return lower->private; 8609 } 8610 EXPORT_SYMBOL(netdev_lower_get_next_private); 8611 8612 /** 8613 * netdev_lower_get_next_private_rcu - Get the next ->private from the 8614 * lower neighbour list, RCU 8615 * variant 8616 * @dev: device 8617 * @iter: list_head ** of the current position 8618 * 8619 * Gets the next netdev_adjacent->private from the dev's lower neighbour 8620 * list, starting from iter position. The caller must hold RCU read lock. 8621 */ 8622 void *netdev_lower_get_next_private_rcu(struct net_device *dev, 8623 struct list_head **iter) 8624 { 8625 struct netdev_adjacent *lower; 8626 8627 WARN_ON_ONCE(!rcu_read_lock_held() && !rcu_read_lock_bh_held()); 8628 8629 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); 8630 8631 if (&lower->list == &dev->adj_list.lower) 8632 return NULL; 8633 8634 *iter = &lower->list; 8635 8636 return lower->private; 8637 } 8638 EXPORT_SYMBOL(netdev_lower_get_next_private_rcu); 8639 8640 /** 8641 * netdev_lower_get_next - Get the next device from the lower neighbour 8642 * list 8643 * @dev: device 8644 * @iter: list_head ** of the current position 8645 * 8646 * Gets the next netdev_adjacent from the dev's lower neighbour 8647 * list, starting from iter position. The caller must hold RTNL lock or 8648 * its own locking that guarantees that the neighbour lower 8649 * list will remain unchanged. 8650 */ 8651 void *netdev_lower_get_next(struct net_device *dev, struct list_head **iter) 8652 { 8653 struct netdev_adjacent *lower; 8654 8655 lower = list_entry(*iter, struct netdev_adjacent, list); 8656 8657 if (&lower->list == &dev->adj_list.lower) 8658 return NULL; 8659 8660 *iter = lower->list.next; 8661 8662 return lower->dev; 8663 } 8664 EXPORT_SYMBOL(netdev_lower_get_next); 8665 8666 static struct net_device *netdev_next_lower_dev(struct net_device *dev, 8667 struct list_head **iter) 8668 { 8669 struct netdev_adjacent *lower; 8670 8671 lower = list_entry((*iter)->next, struct netdev_adjacent, list); 8672 8673 if (&lower->list == &dev->adj_list.lower) 8674 return NULL; 8675 8676 *iter = &lower->list; 8677 8678 return lower->dev; 8679 } 8680 8681 static struct net_device *__netdev_next_lower_dev(struct net_device *dev, 8682 struct list_head **iter, 8683 bool *ignore) 8684 { 8685 struct netdev_adjacent *lower; 8686 8687 lower = list_entry((*iter)->next, struct netdev_adjacent, list); 8688 8689 if (&lower->list == &dev->adj_list.lower) 8690 return NULL; 8691 8692 *iter = &lower->list; 8693 *ignore = lower->ignore; 8694 8695 return lower->dev; 8696 } 8697 8698 int netdev_walk_all_lower_dev(struct net_device *dev, 8699 int (*fn)(struct net_device *dev, 8700 struct netdev_nested_priv *priv), 8701 struct netdev_nested_priv *priv) 8702 { 8703 struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1]; 8704 struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1]; 8705 int ret, cur = 0; 8706 8707 now = dev; 8708 iter = &dev->adj_list.lower; 8709 8710 while (1) { 8711 if (now != dev) { 8712 ret = fn(now, priv); 8713 if (ret) 8714 return ret; 8715 } 8716 8717 next = NULL; 8718 while (1) { 8719 ldev = netdev_next_lower_dev(now, &iter); 8720 if (!ldev) 8721 break; 8722 8723 next = ldev; 8724 niter = &ldev->adj_list.lower; 8725 dev_stack[cur] = now; 8726 iter_stack[cur++] = iter; 8727 break; 8728 } 8729 8730 if (!next) { 8731 if (!cur) 8732 return 0; 8733 next = dev_stack[--cur]; 8734 niter = iter_stack[cur]; 8735 } 8736 8737 now = next; 8738 iter = niter; 8739 } 8740 8741 return 0; 8742 } 8743 EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev); 8744 8745 static int __netdev_walk_all_lower_dev(struct net_device *dev, 8746 int (*fn)(struct net_device *dev, 8747 struct netdev_nested_priv *priv), 8748 struct netdev_nested_priv *priv) 8749 { 8750 struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1]; 8751 struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1]; 8752 int ret, cur = 0; 8753 bool ignore; 8754 8755 now = dev; 8756 iter = &dev->adj_list.lower; 8757 8758 while (1) { 8759 if (now != dev) { 8760 ret = fn(now, priv); 8761 if (ret) 8762 return ret; 8763 } 8764 8765 next = NULL; 8766 while (1) { 8767 ldev = __netdev_next_lower_dev(now, &iter, &ignore); 8768 if (!ldev) 8769 break; 8770 if (ignore) 8771 continue; 8772 8773 next = ldev; 8774 niter = &ldev->adj_list.lower; 8775 dev_stack[cur] = now; 8776 iter_stack[cur++] = iter; 8777 break; 8778 } 8779 8780 if (!next) { 8781 if (!cur) 8782 return 0; 8783 next = dev_stack[--cur]; 8784 niter = iter_stack[cur]; 8785 } 8786 8787 now = next; 8788 iter = niter; 8789 } 8790 8791 return 0; 8792 } 8793 8794 struct net_device *netdev_next_lower_dev_rcu(struct net_device *dev, 8795 struct list_head **iter) 8796 { 8797 struct netdev_adjacent *lower; 8798 8799 lower = list_entry_rcu((*iter)->next, struct netdev_adjacent, list); 8800 if (&lower->list == &dev->adj_list.lower) 8801 return NULL; 8802 8803 *iter = &lower->list; 8804 8805 return lower->dev; 8806 } 8807 EXPORT_SYMBOL(netdev_next_lower_dev_rcu); 8808 8809 static u8 __netdev_upper_depth(struct net_device *dev) 8810 { 8811 struct net_device *udev; 8812 struct list_head *iter; 8813 u8 max_depth = 0; 8814 bool ignore; 8815 8816 for (iter = &dev->adj_list.upper, 8817 udev = __netdev_next_upper_dev(dev, &iter, &ignore); 8818 udev; 8819 udev = __netdev_next_upper_dev(dev, &iter, &ignore)) { 8820 if (ignore) 8821 continue; 8822 if (max_depth < udev->upper_level) 8823 max_depth = udev->upper_level; 8824 } 8825 8826 return max_depth; 8827 } 8828 8829 static u8 __netdev_lower_depth(struct net_device *dev) 8830 { 8831 struct net_device *ldev; 8832 struct list_head *iter; 8833 u8 max_depth = 0; 8834 bool ignore; 8835 8836 for (iter = &dev->adj_list.lower, 8837 ldev = __netdev_next_lower_dev(dev, &iter, &ignore); 8838 ldev; 8839 ldev = __netdev_next_lower_dev(dev, &iter, &ignore)) { 8840 if (ignore) 8841 continue; 8842 if (max_depth < ldev->lower_level) 8843 max_depth = ldev->lower_level; 8844 } 8845 8846 return max_depth; 8847 } 8848 8849 static int __netdev_update_upper_level(struct net_device *dev, 8850 struct netdev_nested_priv *__unused) 8851 { 8852 dev->upper_level = __netdev_upper_depth(dev) + 1; 8853 return 0; 8854 } 8855 8856 #ifdef CONFIG_LOCKDEP 8857 static LIST_HEAD(net_unlink_list); 8858 8859 static void net_unlink_todo(struct net_device *dev) 8860 { 8861 if (list_empty(&dev->unlink_list)) 8862 list_add_tail(&dev->unlink_list, &net_unlink_list); 8863 } 8864 #endif 8865 8866 static int __netdev_update_lower_level(struct net_device *dev, 8867 struct netdev_nested_priv *priv) 8868 { 8869 dev->lower_level = __netdev_lower_depth(dev) + 1; 8870 8871 #ifdef CONFIG_LOCKDEP 8872 if (!priv) 8873 return 0; 8874 8875 if (priv->flags & NESTED_SYNC_IMM) 8876 dev->nested_level = dev->lower_level - 1; 8877 if (priv->flags & NESTED_SYNC_TODO) 8878 net_unlink_todo(dev); 8879 #endif 8880 return 0; 8881 } 8882 8883 int netdev_walk_all_lower_dev_rcu(struct net_device *dev, 8884 int (*fn)(struct net_device *dev, 8885 struct netdev_nested_priv *priv), 8886 struct netdev_nested_priv *priv) 8887 { 8888 struct net_device *ldev, *next, *now, *dev_stack[MAX_NEST_DEV + 1]; 8889 struct list_head *niter, *iter, *iter_stack[MAX_NEST_DEV + 1]; 8890 int ret, cur = 0; 8891 8892 now = dev; 8893 iter = &dev->adj_list.lower; 8894 8895 while (1) { 8896 if (now != dev) { 8897 ret = fn(now, priv); 8898 if (ret) 8899 return ret; 8900 } 8901 8902 next = NULL; 8903 while (1) { 8904 ldev = netdev_next_lower_dev_rcu(now, &iter); 8905 if (!ldev) 8906 break; 8907 8908 next = ldev; 8909 niter = &ldev->adj_list.lower; 8910 dev_stack[cur] = now; 8911 iter_stack[cur++] = iter; 8912 break; 8913 } 8914 8915 if (!next) { 8916 if (!cur) 8917 return 0; 8918 next = dev_stack[--cur]; 8919 niter = iter_stack[cur]; 8920 } 8921 8922 now = next; 8923 iter = niter; 8924 } 8925 8926 return 0; 8927 } 8928 EXPORT_SYMBOL_GPL(netdev_walk_all_lower_dev_rcu); 8929 8930 /** 8931 * netdev_lower_get_first_private_rcu - Get the first ->private from the 8932 * lower neighbour list, RCU 8933 * variant 8934 * @dev: device 8935 * 8936 * Gets the first netdev_adjacent->private from the dev's lower neighbour 8937 * list. The caller must hold RCU read lock. 8938 */ 8939 void *netdev_lower_get_first_private_rcu(struct net_device *dev) 8940 { 8941 struct netdev_adjacent *lower; 8942 8943 lower = list_first_or_null_rcu(&dev->adj_list.lower, 8944 struct netdev_adjacent, list); 8945 if (lower) 8946 return lower->private; 8947 return NULL; 8948 } 8949 EXPORT_SYMBOL(netdev_lower_get_first_private_rcu); 8950 8951 /** 8952 * netdev_master_upper_dev_get_rcu - Get master upper device 8953 * @dev: device 8954 * 8955 * Find a master upper device and return pointer to it or NULL in case 8956 * it's not there. The caller must hold the RCU read lock. 8957 */ 8958 struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev) 8959 { 8960 struct netdev_adjacent *upper; 8961 8962 upper = list_first_or_null_rcu(&dev->adj_list.upper, 8963 struct netdev_adjacent, list); 8964 if (upper && likely(upper->master)) 8965 return upper->dev; 8966 return NULL; 8967 } 8968 EXPORT_SYMBOL(netdev_master_upper_dev_get_rcu); 8969 8970 static int netdev_adjacent_sysfs_add(struct net_device *dev, 8971 struct net_device *adj_dev, 8972 struct list_head *dev_list) 8973 { 8974 char linkname[IFNAMSIZ+7]; 8975 8976 sprintf(linkname, dev_list == &dev->adj_list.upper ? 8977 "upper_%s" : "lower_%s", adj_dev->name); 8978 return sysfs_create_link(&(dev->dev.kobj), &(adj_dev->dev.kobj), 8979 linkname); 8980 } 8981 static void netdev_adjacent_sysfs_del(struct net_device *dev, 8982 char *name, 8983 struct list_head *dev_list) 8984 { 8985 char linkname[IFNAMSIZ+7]; 8986 8987 sprintf(linkname, dev_list == &dev->adj_list.upper ? 8988 "upper_%s" : "lower_%s", name); 8989 sysfs_remove_link(&(dev->dev.kobj), linkname); 8990 } 8991 8992 static inline bool netdev_adjacent_is_neigh_list(struct net_device *dev, 8993 struct net_device *adj_dev, 8994 struct list_head *dev_list) 8995 { 8996 return (dev_list == &dev->adj_list.upper || 8997 dev_list == &dev->adj_list.lower) && 8998 net_eq(dev_net(dev), dev_net(adj_dev)); 8999 } 9000 9001 static int __netdev_adjacent_dev_insert(struct net_device *dev, 9002 struct net_device *adj_dev, 9003 struct list_head *dev_list, 9004 void *private, bool master) 9005 { 9006 struct netdev_adjacent *adj; 9007 int ret; 9008 9009 adj = __netdev_find_adj(adj_dev, dev_list); 9010 9011 if (adj) { 9012 adj->ref_nr += 1; 9013 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d\n", 9014 dev->name, adj_dev->name, adj->ref_nr); 9015 9016 return 0; 9017 } 9018 9019 adj = kmalloc_obj(*adj); 9020 if (!adj) 9021 return -ENOMEM; 9022 9023 adj->dev = adj_dev; 9024 adj->master = master; 9025 adj->ref_nr = 1; 9026 adj->private = private; 9027 adj->ignore = false; 9028 netdev_hold(adj_dev, &adj->dev_tracker, GFP_KERNEL); 9029 9030 pr_debug("Insert adjacency: dev %s adj_dev %s adj->ref_nr %d; dev_hold on %s\n", 9031 dev->name, adj_dev->name, adj->ref_nr, adj_dev->name); 9032 9033 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) { 9034 ret = netdev_adjacent_sysfs_add(dev, adj_dev, dev_list); 9035 if (ret) 9036 goto free_adj; 9037 } 9038 9039 /* Ensure that master link is always the first item in list. */ 9040 if (master) { 9041 ret = sysfs_create_link(&(dev->dev.kobj), 9042 &(adj_dev->dev.kobj), "master"); 9043 if (ret) 9044 goto remove_symlinks; 9045 9046 list_add_rcu(&adj->list, dev_list); 9047 } else { 9048 list_add_tail_rcu(&adj->list, dev_list); 9049 } 9050 9051 return 0; 9052 9053 remove_symlinks: 9054 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) 9055 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list); 9056 free_adj: 9057 netdev_put(adj_dev, &adj->dev_tracker); 9058 kfree(adj); 9059 9060 return ret; 9061 } 9062 9063 static void __netdev_adjacent_dev_remove(struct net_device *dev, 9064 struct net_device *adj_dev, 9065 u16 ref_nr, 9066 struct list_head *dev_list) 9067 { 9068 struct netdev_adjacent *adj; 9069 9070 pr_debug("Remove adjacency: dev %s adj_dev %s ref_nr %d\n", 9071 dev->name, adj_dev->name, ref_nr); 9072 9073 adj = __netdev_find_adj(adj_dev, dev_list); 9074 9075 if (!adj) { 9076 pr_err("Adjacency does not exist for device %s from %s\n", 9077 dev->name, adj_dev->name); 9078 WARN_ON(1); 9079 return; 9080 } 9081 9082 if (adj->ref_nr > ref_nr) { 9083 pr_debug("adjacency: %s to %s ref_nr - %d = %d\n", 9084 dev->name, adj_dev->name, ref_nr, 9085 adj->ref_nr - ref_nr); 9086 adj->ref_nr -= ref_nr; 9087 return; 9088 } 9089 9090 if (adj->master) 9091 sysfs_remove_link(&(dev->dev.kobj), "master"); 9092 9093 if (netdev_adjacent_is_neigh_list(dev, adj_dev, dev_list)) 9094 netdev_adjacent_sysfs_del(dev, adj_dev->name, dev_list); 9095 9096 list_del_rcu(&adj->list); 9097 pr_debug("adjacency: dev_put for %s, because link removed from %s to %s\n", 9098 adj_dev->name, dev->name, adj_dev->name); 9099 netdev_put(adj_dev, &adj->dev_tracker); 9100 kfree_rcu(adj, rcu); 9101 } 9102 9103 static int __netdev_adjacent_dev_link_lists(struct net_device *dev, 9104 struct net_device *upper_dev, 9105 struct list_head *up_list, 9106 struct list_head *down_list, 9107 void *private, bool master) 9108 { 9109 int ret; 9110 9111 ret = __netdev_adjacent_dev_insert(dev, upper_dev, up_list, 9112 private, master); 9113 if (ret) 9114 return ret; 9115 9116 ret = __netdev_adjacent_dev_insert(upper_dev, dev, down_list, 9117 private, false); 9118 if (ret) { 9119 __netdev_adjacent_dev_remove(dev, upper_dev, 1, up_list); 9120 return ret; 9121 } 9122 9123 return 0; 9124 } 9125 9126 static void __netdev_adjacent_dev_unlink_lists(struct net_device *dev, 9127 struct net_device *upper_dev, 9128 u16 ref_nr, 9129 struct list_head *up_list, 9130 struct list_head *down_list) 9131 { 9132 __netdev_adjacent_dev_remove(dev, upper_dev, ref_nr, up_list); 9133 __netdev_adjacent_dev_remove(upper_dev, dev, ref_nr, down_list); 9134 } 9135 9136 static int __netdev_adjacent_dev_link_neighbour(struct net_device *dev, 9137 struct net_device *upper_dev, 9138 void *private, bool master) 9139 { 9140 return __netdev_adjacent_dev_link_lists(dev, upper_dev, 9141 &dev->adj_list.upper, 9142 &upper_dev->adj_list.lower, 9143 private, master); 9144 } 9145 9146 static void __netdev_adjacent_dev_unlink_neighbour(struct net_device *dev, 9147 struct net_device *upper_dev) 9148 { 9149 __netdev_adjacent_dev_unlink_lists(dev, upper_dev, 1, 9150 &dev->adj_list.upper, 9151 &upper_dev->adj_list.lower); 9152 } 9153 9154 static int __netdev_upper_dev_link(struct net_device *dev, 9155 struct net_device *upper_dev, bool master, 9156 void *upper_priv, void *upper_info, 9157 struct netdev_nested_priv *priv, 9158 struct netlink_ext_ack *extack) 9159 { 9160 struct netdev_notifier_changeupper_info changeupper_info = { 9161 .info = { 9162 .dev = dev, 9163 .extack = extack, 9164 }, 9165 .upper_dev = upper_dev, 9166 .master = master, 9167 .linking = true, 9168 .upper_info = upper_info, 9169 }; 9170 struct net_device *master_dev; 9171 int ret = 0; 9172 9173 ASSERT_RTNL(); 9174 9175 if (dev == upper_dev) 9176 return -EBUSY; 9177 9178 /* To prevent loops, check if dev is not upper device to upper_dev. */ 9179 if (__netdev_has_upper_dev(upper_dev, dev)) 9180 return -EBUSY; 9181 9182 if ((dev->lower_level + upper_dev->upper_level) > MAX_NEST_DEV) 9183 return -EMLINK; 9184 9185 if (!master) { 9186 if (__netdev_has_upper_dev(dev, upper_dev)) 9187 return -EEXIST; 9188 } else { 9189 master_dev = __netdev_master_upper_dev_get(dev); 9190 if (master_dev) 9191 return master_dev == upper_dev ? -EEXIST : -EBUSY; 9192 } 9193 9194 ret = call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER, 9195 &changeupper_info.info); 9196 ret = notifier_to_errno(ret); 9197 if (ret) 9198 return ret; 9199 9200 ret = __netdev_adjacent_dev_link_neighbour(dev, upper_dev, upper_priv, 9201 master); 9202 if (ret) 9203 return ret; 9204 9205 ret = call_netdevice_notifiers_info(NETDEV_CHANGEUPPER, 9206 &changeupper_info.info); 9207 ret = notifier_to_errno(ret); 9208 if (ret) 9209 goto rollback; 9210 9211 __netdev_update_upper_level(dev, NULL); 9212 __netdev_walk_all_lower_dev(dev, __netdev_update_upper_level, NULL); 9213 9214 __netdev_update_lower_level(upper_dev, priv); 9215 __netdev_walk_all_upper_dev(upper_dev, __netdev_update_lower_level, 9216 priv); 9217 9218 return 0; 9219 9220 rollback: 9221 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev); 9222 9223 return ret; 9224 } 9225 9226 /** 9227 * netdev_upper_dev_link - Add a link to the upper device 9228 * @dev: device 9229 * @upper_dev: new upper device 9230 * @extack: netlink extended ack 9231 * 9232 * Adds a link to device which is upper to this one. The caller must hold 9233 * the RTNL lock. On a failure a negative errno code is returned. 9234 * On success the reference counts are adjusted and the function 9235 * returns zero. 9236 */ 9237 int netdev_upper_dev_link(struct net_device *dev, 9238 struct net_device *upper_dev, 9239 struct netlink_ext_ack *extack) 9240 { 9241 struct netdev_nested_priv priv = { 9242 .flags = NESTED_SYNC_IMM | NESTED_SYNC_TODO, 9243 .data = NULL, 9244 }; 9245 9246 return __netdev_upper_dev_link(dev, upper_dev, false, 9247 NULL, NULL, &priv, extack); 9248 } 9249 EXPORT_SYMBOL(netdev_upper_dev_link); 9250 9251 /** 9252 * netdev_master_upper_dev_link - Add a master link to the upper device 9253 * @dev: device 9254 * @upper_dev: new upper device 9255 * @upper_priv: upper device private 9256 * @upper_info: upper info to be passed down via notifier 9257 * @extack: netlink extended ack 9258 * 9259 * Adds a link to device which is upper to this one. In this case, only 9260 * one master upper device can be linked, although other non-master devices 9261 * might be linked as well. The caller must hold the RTNL lock. 9262 * On a failure a negative errno code is returned. On success the reference 9263 * counts are adjusted and the function returns zero. 9264 */ 9265 int netdev_master_upper_dev_link(struct net_device *dev, 9266 struct net_device *upper_dev, 9267 void *upper_priv, void *upper_info, 9268 struct netlink_ext_ack *extack) 9269 { 9270 struct netdev_nested_priv priv = { 9271 .flags = NESTED_SYNC_IMM | NESTED_SYNC_TODO, 9272 .data = NULL, 9273 }; 9274 9275 return __netdev_upper_dev_link(dev, upper_dev, true, 9276 upper_priv, upper_info, &priv, extack); 9277 } 9278 EXPORT_SYMBOL(netdev_master_upper_dev_link); 9279 9280 static void __netdev_upper_dev_unlink(struct net_device *dev, 9281 struct net_device *upper_dev, 9282 struct netdev_nested_priv *priv) 9283 { 9284 struct netdev_notifier_changeupper_info changeupper_info = { 9285 .info = { 9286 .dev = dev, 9287 }, 9288 .upper_dev = upper_dev, 9289 .linking = false, 9290 }; 9291 9292 ASSERT_RTNL(); 9293 9294 changeupper_info.master = netdev_master_upper_dev_get(dev) == upper_dev; 9295 9296 call_netdevice_notifiers_info(NETDEV_PRECHANGEUPPER, 9297 &changeupper_info.info); 9298 9299 __netdev_adjacent_dev_unlink_neighbour(dev, upper_dev); 9300 9301 call_netdevice_notifiers_info(NETDEV_CHANGEUPPER, 9302 &changeupper_info.info); 9303 9304 __netdev_update_upper_level(dev, NULL); 9305 __netdev_walk_all_lower_dev(dev, __netdev_update_upper_level, NULL); 9306 9307 __netdev_update_lower_level(upper_dev, priv); 9308 __netdev_walk_all_upper_dev(upper_dev, __netdev_update_lower_level, 9309 priv); 9310 } 9311 9312 /** 9313 * netdev_upper_dev_unlink - Removes a link to upper device 9314 * @dev: device 9315 * @upper_dev: new upper device 9316 * 9317 * Removes a link to device which is upper to this one. The caller must hold 9318 * the RTNL lock. 9319 */ 9320 void netdev_upper_dev_unlink(struct net_device *dev, 9321 struct net_device *upper_dev) 9322 { 9323 struct netdev_nested_priv priv = { 9324 .flags = NESTED_SYNC_TODO, 9325 .data = NULL, 9326 }; 9327 9328 __netdev_upper_dev_unlink(dev, upper_dev, &priv); 9329 } 9330 EXPORT_SYMBOL(netdev_upper_dev_unlink); 9331 9332 static void __netdev_adjacent_dev_set(struct net_device *upper_dev, 9333 struct net_device *lower_dev, 9334 bool val) 9335 { 9336 struct netdev_adjacent *adj; 9337 9338 adj = __netdev_find_adj(lower_dev, &upper_dev->adj_list.lower); 9339 if (adj) 9340 adj->ignore = val; 9341 9342 adj = __netdev_find_adj(upper_dev, &lower_dev->adj_list.upper); 9343 if (adj) 9344 adj->ignore = val; 9345 } 9346 9347 static void netdev_adjacent_dev_disable(struct net_device *upper_dev, 9348 struct net_device *lower_dev) 9349 { 9350 __netdev_adjacent_dev_set(upper_dev, lower_dev, true); 9351 } 9352 9353 static void netdev_adjacent_dev_enable(struct net_device *upper_dev, 9354 struct net_device *lower_dev) 9355 { 9356 __netdev_adjacent_dev_set(upper_dev, lower_dev, false); 9357 } 9358 9359 int netdev_adjacent_change_prepare(struct net_device *old_dev, 9360 struct net_device *new_dev, 9361 struct net_device *dev, 9362 struct netlink_ext_ack *extack) 9363 { 9364 struct netdev_nested_priv priv = { 9365 .flags = 0, 9366 .data = NULL, 9367 }; 9368 int err; 9369 9370 if (!new_dev) 9371 return 0; 9372 9373 if (old_dev && new_dev != old_dev) 9374 netdev_adjacent_dev_disable(dev, old_dev); 9375 err = __netdev_upper_dev_link(new_dev, dev, false, NULL, NULL, &priv, 9376 extack); 9377 if (err) { 9378 if (old_dev && new_dev != old_dev) 9379 netdev_adjacent_dev_enable(dev, old_dev); 9380 return err; 9381 } 9382 9383 return 0; 9384 } 9385 EXPORT_SYMBOL(netdev_adjacent_change_prepare); 9386 9387 void netdev_adjacent_change_commit(struct net_device *old_dev, 9388 struct net_device *new_dev, 9389 struct net_device *dev) 9390 { 9391 struct netdev_nested_priv priv = { 9392 .flags = NESTED_SYNC_IMM | NESTED_SYNC_TODO, 9393 .data = NULL, 9394 }; 9395 9396 if (!new_dev || !old_dev) 9397 return; 9398 9399 if (new_dev == old_dev) 9400 return; 9401 9402 netdev_adjacent_dev_enable(dev, old_dev); 9403 __netdev_upper_dev_unlink(old_dev, dev, &priv); 9404 } 9405 EXPORT_SYMBOL(netdev_adjacent_change_commit); 9406 9407 void netdev_adjacent_change_abort(struct net_device *old_dev, 9408 struct net_device *new_dev, 9409 struct net_device *dev) 9410 { 9411 struct netdev_nested_priv priv = { 9412 .flags = 0, 9413 .data = NULL, 9414 }; 9415 9416 if (!new_dev) 9417 return; 9418 9419 if (old_dev && new_dev != old_dev) 9420 netdev_adjacent_dev_enable(dev, old_dev); 9421 9422 __netdev_upper_dev_unlink(new_dev, dev, &priv); 9423 } 9424 EXPORT_SYMBOL(netdev_adjacent_change_abort); 9425 9426 /** 9427 * netdev_bonding_info_change - Dispatch event about slave change 9428 * @dev: device 9429 * @bonding_info: info to dispatch 9430 * 9431 * Send NETDEV_BONDING_INFO to netdev notifiers with info. 9432 * The caller must hold the RTNL lock. 9433 */ 9434 void netdev_bonding_info_change(struct net_device *dev, 9435 struct netdev_bonding_info *bonding_info) 9436 { 9437 struct netdev_notifier_bonding_info info = { 9438 .info.dev = dev, 9439 }; 9440 9441 memcpy(&info.bonding_info, bonding_info, 9442 sizeof(struct netdev_bonding_info)); 9443 call_netdevice_notifiers_info(NETDEV_BONDING_INFO, 9444 &info.info); 9445 } 9446 EXPORT_SYMBOL(netdev_bonding_info_change); 9447 9448 static int netdev_offload_xstats_enable_l3(struct net_device *dev, 9449 struct netlink_ext_ack *extack) 9450 { 9451 struct netdev_notifier_offload_xstats_info info = { 9452 .info.dev = dev, 9453 .info.extack = extack, 9454 .type = NETDEV_OFFLOAD_XSTATS_TYPE_L3, 9455 }; 9456 int err; 9457 int rc; 9458 9459 dev->offload_xstats_l3 = kzalloc_obj(*dev->offload_xstats_l3); 9460 if (!dev->offload_xstats_l3) 9461 return -ENOMEM; 9462 9463 rc = call_netdevice_notifiers_info_robust(NETDEV_OFFLOAD_XSTATS_ENABLE, 9464 NETDEV_OFFLOAD_XSTATS_DISABLE, 9465 &info.info); 9466 err = notifier_to_errno(rc); 9467 if (err) 9468 goto free_stats; 9469 9470 return 0; 9471 9472 free_stats: 9473 kfree(dev->offload_xstats_l3); 9474 dev->offload_xstats_l3 = NULL; 9475 return err; 9476 } 9477 9478 int netdev_offload_xstats_enable(struct net_device *dev, 9479 enum netdev_offload_xstats_type type, 9480 struct netlink_ext_ack *extack) 9481 { 9482 ASSERT_RTNL(); 9483 9484 if (netdev_offload_xstats_enabled(dev, type)) 9485 return -EALREADY; 9486 9487 switch (type) { 9488 case NETDEV_OFFLOAD_XSTATS_TYPE_L3: 9489 return netdev_offload_xstats_enable_l3(dev, extack); 9490 } 9491 9492 WARN_ON(1); 9493 return -EINVAL; 9494 } 9495 EXPORT_SYMBOL(netdev_offload_xstats_enable); 9496 9497 static void netdev_offload_xstats_disable_l3(struct net_device *dev) 9498 { 9499 struct netdev_notifier_offload_xstats_info info = { 9500 .info.dev = dev, 9501 .type = NETDEV_OFFLOAD_XSTATS_TYPE_L3, 9502 }; 9503 9504 call_netdevice_notifiers_info(NETDEV_OFFLOAD_XSTATS_DISABLE, 9505 &info.info); 9506 kfree(dev->offload_xstats_l3); 9507 dev->offload_xstats_l3 = NULL; 9508 } 9509 9510 int netdev_offload_xstats_disable(struct net_device *dev, 9511 enum netdev_offload_xstats_type type) 9512 { 9513 ASSERT_RTNL(); 9514 9515 if (!netdev_offload_xstats_enabled(dev, type)) 9516 return -EALREADY; 9517 9518 switch (type) { 9519 case NETDEV_OFFLOAD_XSTATS_TYPE_L3: 9520 netdev_offload_xstats_disable_l3(dev); 9521 return 0; 9522 } 9523 9524 WARN_ON(1); 9525 return -EINVAL; 9526 } 9527 EXPORT_SYMBOL(netdev_offload_xstats_disable); 9528 9529 static void netdev_offload_xstats_disable_all(struct net_device *dev) 9530 { 9531 netdev_offload_xstats_disable(dev, NETDEV_OFFLOAD_XSTATS_TYPE_L3); 9532 } 9533 9534 static struct rtnl_hw_stats64 * 9535 netdev_offload_xstats_get_ptr(const struct net_device *dev, 9536 enum netdev_offload_xstats_type type) 9537 { 9538 switch (type) { 9539 case NETDEV_OFFLOAD_XSTATS_TYPE_L3: 9540 return dev->offload_xstats_l3; 9541 } 9542 9543 WARN_ON(1); 9544 return NULL; 9545 } 9546 9547 bool netdev_offload_xstats_enabled(const struct net_device *dev, 9548 enum netdev_offload_xstats_type type) 9549 { 9550 ASSERT_RTNL(); 9551 9552 return netdev_offload_xstats_get_ptr(dev, type); 9553 } 9554 EXPORT_SYMBOL(netdev_offload_xstats_enabled); 9555 9556 struct netdev_notifier_offload_xstats_ru { 9557 bool used; 9558 }; 9559 9560 struct netdev_notifier_offload_xstats_rd { 9561 struct rtnl_hw_stats64 stats; 9562 bool used; 9563 }; 9564 9565 static void netdev_hw_stats64_add(struct rtnl_hw_stats64 *dest, 9566 const struct rtnl_hw_stats64 *src) 9567 { 9568 dest->rx_packets += src->rx_packets; 9569 dest->tx_packets += src->tx_packets; 9570 dest->rx_bytes += src->rx_bytes; 9571 dest->tx_bytes += src->tx_bytes; 9572 dest->rx_errors += src->rx_errors; 9573 dest->tx_errors += src->tx_errors; 9574 dest->rx_dropped += src->rx_dropped; 9575 dest->tx_dropped += src->tx_dropped; 9576 dest->multicast += src->multicast; 9577 } 9578 9579 static int netdev_offload_xstats_get_used(struct net_device *dev, 9580 enum netdev_offload_xstats_type type, 9581 bool *p_used, 9582 struct netlink_ext_ack *extack) 9583 { 9584 struct netdev_notifier_offload_xstats_ru report_used = {}; 9585 struct netdev_notifier_offload_xstats_info info = { 9586 .info.dev = dev, 9587 .info.extack = extack, 9588 .type = type, 9589 .report_used = &report_used, 9590 }; 9591 int rc; 9592 9593 WARN_ON(!netdev_offload_xstats_enabled(dev, type)); 9594 rc = call_netdevice_notifiers_info(NETDEV_OFFLOAD_XSTATS_REPORT_USED, 9595 &info.info); 9596 *p_used = report_used.used; 9597 return notifier_to_errno(rc); 9598 } 9599 9600 static int netdev_offload_xstats_get_stats(struct net_device *dev, 9601 enum netdev_offload_xstats_type type, 9602 struct rtnl_hw_stats64 *p_stats, 9603 bool *p_used, 9604 struct netlink_ext_ack *extack) 9605 { 9606 struct netdev_notifier_offload_xstats_rd report_delta = {}; 9607 struct netdev_notifier_offload_xstats_info info = { 9608 .info.dev = dev, 9609 .info.extack = extack, 9610 .type = type, 9611 .report_delta = &report_delta, 9612 }; 9613 struct rtnl_hw_stats64 *stats; 9614 int rc; 9615 9616 stats = netdev_offload_xstats_get_ptr(dev, type); 9617 if (WARN_ON(!stats)) 9618 return -EINVAL; 9619 9620 rc = call_netdevice_notifiers_info(NETDEV_OFFLOAD_XSTATS_REPORT_DELTA, 9621 &info.info); 9622 9623 /* Cache whatever we got, even if there was an error, otherwise the 9624 * successful stats retrievals would get lost. 9625 */ 9626 netdev_hw_stats64_add(stats, &report_delta.stats); 9627 9628 if (p_stats) 9629 *p_stats = *stats; 9630 *p_used = report_delta.used; 9631 9632 return notifier_to_errno(rc); 9633 } 9634 9635 int netdev_offload_xstats_get(struct net_device *dev, 9636 enum netdev_offload_xstats_type type, 9637 struct rtnl_hw_stats64 *p_stats, bool *p_used, 9638 struct netlink_ext_ack *extack) 9639 { 9640 ASSERT_RTNL(); 9641 9642 if (p_stats) 9643 return netdev_offload_xstats_get_stats(dev, type, p_stats, 9644 p_used, extack); 9645 else 9646 return netdev_offload_xstats_get_used(dev, type, p_used, 9647 extack); 9648 } 9649 EXPORT_SYMBOL(netdev_offload_xstats_get); 9650 9651 void 9652 netdev_offload_xstats_report_delta(struct netdev_notifier_offload_xstats_rd *report_delta, 9653 const struct rtnl_hw_stats64 *stats) 9654 { 9655 report_delta->used = true; 9656 netdev_hw_stats64_add(&report_delta->stats, stats); 9657 } 9658 EXPORT_SYMBOL(netdev_offload_xstats_report_delta); 9659 9660 void 9661 netdev_offload_xstats_report_used(struct netdev_notifier_offload_xstats_ru *report_used) 9662 { 9663 report_used->used = true; 9664 } 9665 EXPORT_SYMBOL(netdev_offload_xstats_report_used); 9666 9667 void netdev_offload_xstats_push_delta(struct net_device *dev, 9668 enum netdev_offload_xstats_type type, 9669 const struct rtnl_hw_stats64 *p_stats) 9670 { 9671 struct rtnl_hw_stats64 *stats; 9672 9673 ASSERT_RTNL(); 9674 9675 stats = netdev_offload_xstats_get_ptr(dev, type); 9676 if (WARN_ON(!stats)) 9677 return; 9678 9679 netdev_hw_stats64_add(stats, p_stats); 9680 } 9681 EXPORT_SYMBOL(netdev_offload_xstats_push_delta); 9682 9683 /** 9684 * netdev_get_xmit_slave - Get the xmit slave of master device 9685 * @dev: device 9686 * @skb: The packet 9687 * @all_slaves: assume all the slaves are active 9688 * 9689 * The reference counters are not incremented so the caller must be 9690 * careful with locks. The caller must hold RCU lock. 9691 * %NULL is returned if no slave is found. 9692 */ 9693 9694 struct net_device *netdev_get_xmit_slave(struct net_device *dev, 9695 struct sk_buff *skb, 9696 bool all_slaves) 9697 { 9698 const struct net_device_ops *ops = dev->netdev_ops; 9699 9700 if (!ops->ndo_get_xmit_slave) 9701 return NULL; 9702 return ops->ndo_get_xmit_slave(dev, skb, all_slaves); 9703 } 9704 EXPORT_SYMBOL(netdev_get_xmit_slave); 9705 9706 static struct net_device *netdev_sk_get_lower_dev(struct net_device *dev, 9707 struct sock *sk) 9708 { 9709 const struct net_device_ops *ops = dev->netdev_ops; 9710 9711 if (!ops->ndo_sk_get_lower_dev) 9712 return NULL; 9713 return ops->ndo_sk_get_lower_dev(dev, sk); 9714 } 9715 9716 /** 9717 * netdev_sk_get_lowest_dev - Get the lowest device in chain given device and socket 9718 * @dev: device 9719 * @sk: the socket 9720 * 9721 * %NULL is returned if no lower device is found. 9722 */ 9723 9724 struct net_device *netdev_sk_get_lowest_dev(struct net_device *dev, 9725 struct sock *sk) 9726 { 9727 struct net_device *lower; 9728 9729 lower = netdev_sk_get_lower_dev(dev, sk); 9730 while (lower) { 9731 dev = lower; 9732 lower = netdev_sk_get_lower_dev(dev, sk); 9733 } 9734 9735 return dev; 9736 } 9737 EXPORT_SYMBOL(netdev_sk_get_lowest_dev); 9738 9739 static void netdev_adjacent_add_links(struct net_device *dev) 9740 { 9741 struct netdev_adjacent *iter; 9742 9743 struct net *net = dev_net(dev); 9744 9745 list_for_each_entry(iter, &dev->adj_list.upper, list) { 9746 if (!net_eq(net, dev_net(iter->dev))) 9747 continue; 9748 netdev_adjacent_sysfs_add(iter->dev, dev, 9749 &iter->dev->adj_list.lower); 9750 netdev_adjacent_sysfs_add(dev, iter->dev, 9751 &dev->adj_list.upper); 9752 } 9753 9754 list_for_each_entry(iter, &dev->adj_list.lower, list) { 9755 if (!net_eq(net, dev_net(iter->dev))) 9756 continue; 9757 netdev_adjacent_sysfs_add(iter->dev, dev, 9758 &iter->dev->adj_list.upper); 9759 netdev_adjacent_sysfs_add(dev, iter->dev, 9760 &dev->adj_list.lower); 9761 } 9762 } 9763 9764 static void netdev_adjacent_del_links(struct net_device *dev) 9765 { 9766 struct netdev_adjacent *iter; 9767 9768 struct net *net = dev_net(dev); 9769 9770 list_for_each_entry(iter, &dev->adj_list.upper, list) { 9771 if (!net_eq(net, dev_net(iter->dev))) 9772 continue; 9773 netdev_adjacent_sysfs_del(iter->dev, dev->name, 9774 &iter->dev->adj_list.lower); 9775 netdev_adjacent_sysfs_del(dev, iter->dev->name, 9776 &dev->adj_list.upper); 9777 } 9778 9779 list_for_each_entry(iter, &dev->adj_list.lower, list) { 9780 if (!net_eq(net, dev_net(iter->dev))) 9781 continue; 9782 netdev_adjacent_sysfs_del(iter->dev, dev->name, 9783 &iter->dev->adj_list.upper); 9784 netdev_adjacent_sysfs_del(dev, iter->dev->name, 9785 &dev->adj_list.lower); 9786 } 9787 } 9788 9789 void netdev_adjacent_rename_links(struct net_device *dev, char *oldname) 9790 { 9791 struct netdev_adjacent *iter; 9792 9793 struct net *net = dev_net(dev); 9794 9795 list_for_each_entry(iter, &dev->adj_list.upper, list) { 9796 if (!net_eq(net, dev_net(iter->dev))) 9797 continue; 9798 netdev_adjacent_sysfs_del(iter->dev, oldname, 9799 &iter->dev->adj_list.lower); 9800 netdev_adjacent_sysfs_add(iter->dev, dev, 9801 &iter->dev->adj_list.lower); 9802 } 9803 9804 list_for_each_entry(iter, &dev->adj_list.lower, list) { 9805 if (!net_eq(net, dev_net(iter->dev))) 9806 continue; 9807 netdev_adjacent_sysfs_del(iter->dev, oldname, 9808 &iter->dev->adj_list.upper); 9809 netdev_adjacent_sysfs_add(iter->dev, dev, 9810 &iter->dev->adj_list.upper); 9811 } 9812 } 9813 9814 void *netdev_lower_dev_get_private(struct net_device *dev, 9815 struct net_device *lower_dev) 9816 { 9817 struct netdev_adjacent *lower; 9818 9819 if (!lower_dev) 9820 return NULL; 9821 lower = __netdev_find_adj(lower_dev, &dev->adj_list.lower); 9822 if (!lower) 9823 return NULL; 9824 9825 return lower->private; 9826 } 9827 EXPORT_SYMBOL(netdev_lower_dev_get_private); 9828 9829 9830 /** 9831 * netdev_lower_state_changed - Dispatch event about lower device state change 9832 * @lower_dev: device 9833 * @lower_state_info: state to dispatch 9834 * 9835 * Send NETDEV_CHANGELOWERSTATE to netdev notifiers with info. 9836 * The caller must hold the RTNL lock. 9837 */ 9838 void netdev_lower_state_changed(struct net_device *lower_dev, 9839 void *lower_state_info) 9840 { 9841 struct netdev_notifier_changelowerstate_info changelowerstate_info = { 9842 .info.dev = lower_dev, 9843 }; 9844 9845 ASSERT_RTNL(); 9846 changelowerstate_info.lower_state_info = lower_state_info; 9847 call_netdevice_notifiers_info(NETDEV_CHANGELOWERSTATE, 9848 &changelowerstate_info.info); 9849 } 9850 EXPORT_SYMBOL(netdev_lower_state_changed); 9851 9852 static void dev_change_rx_flags(struct net_device *dev, int flags) 9853 { 9854 const struct net_device_ops *ops = dev->netdev_ops; 9855 9856 if (ops->ndo_change_rx_flags) 9857 ops->ndo_change_rx_flags(dev, flags); 9858 } 9859 9860 int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify) 9861 { 9862 unsigned int old_flags = dev->flags; 9863 unsigned int promiscuity, flags; 9864 kuid_t uid; 9865 kgid_t gid; 9866 9867 netdev_assert_locked_ops_compat(dev); 9868 9869 promiscuity = dev->promiscuity + inc; 9870 if (promiscuity == 0) { 9871 /* 9872 * Avoid overflow. 9873 * If inc causes overflow, untouch promisc and return error. 9874 */ 9875 if (unlikely(inc > 0)) { 9876 netdev_warn(dev, "promiscuity touches roof, set promiscuity failed. promiscuity feature of device might be broken.\n"); 9877 return -EOVERFLOW; 9878 } 9879 flags = old_flags & ~IFF_PROMISC; 9880 } else { 9881 flags = old_flags | IFF_PROMISC; 9882 } 9883 WRITE_ONCE(dev->promiscuity, promiscuity); 9884 if (flags != old_flags) { 9885 WRITE_ONCE(dev->flags, flags); 9886 netdev_info(dev, "%s promiscuous mode\n", 9887 dev->flags & IFF_PROMISC ? "entered" : "left"); 9888 if (audit_enabled) { 9889 current_uid_gid(&uid, &gid); 9890 audit_log(audit_context(), GFP_ATOMIC, 9891 AUDIT_ANOM_PROMISCUOUS, 9892 "dev=%s prom=%d old_prom=%d auid=%u uid=%u gid=%u ses=%u", 9893 dev->name, (dev->flags & IFF_PROMISC), 9894 (old_flags & IFF_PROMISC), 9895 from_kuid(&init_user_ns, audit_get_loginuid(current)), 9896 from_kuid(&init_user_ns, uid), 9897 from_kgid(&init_user_ns, gid), 9898 audit_get_sessionid(current)); 9899 } 9900 9901 dev_change_rx_flags(dev, IFF_PROMISC); 9902 } 9903 if (notify) 9904 __dev_notify_flags(dev, old_flags, IFF_PROMISC, 0, NULL); 9905 return 0; 9906 } 9907 9908 int netif_set_promiscuity(struct net_device *dev, int inc) 9909 { 9910 unsigned int old_flags = dev->flags; 9911 int err; 9912 9913 err = __dev_set_promiscuity(dev, inc, true); 9914 if (err < 0) 9915 return err; 9916 if (dev->flags != old_flags) 9917 dev_set_rx_mode(dev); 9918 return err; 9919 } 9920 9921 int netif_set_allmulti(struct net_device *dev, int inc, bool notify) 9922 { 9923 unsigned int old_flags = dev->flags, old_gflags = dev->gflags; 9924 unsigned int allmulti, flags; 9925 9926 netdev_assert_locked_ops_compat(dev); 9927 9928 allmulti = dev->allmulti + inc; 9929 if (allmulti == 0) { 9930 /* 9931 * Avoid overflow. 9932 * If inc causes overflow, untouch allmulti and return error. 9933 */ 9934 if (unlikely(inc > 0)) { 9935 netdev_warn(dev, "allmulti touches roof, set allmulti failed. allmulti feature of device might be broken.\n"); 9936 return -EOVERFLOW; 9937 } 9938 flags = old_flags & ~IFF_ALLMULTI; 9939 } else { 9940 flags = old_flags | IFF_ALLMULTI; 9941 } 9942 WRITE_ONCE(dev->allmulti, allmulti); 9943 if (flags != old_flags) { 9944 WRITE_ONCE(dev->flags, flags); 9945 netdev_info(dev, "%s allmulticast mode\n", 9946 dev->flags & IFF_ALLMULTI ? "entered" : "left"); 9947 dev_change_rx_flags(dev, IFF_ALLMULTI); 9948 dev_set_rx_mode(dev); 9949 if (notify) 9950 __dev_notify_flags(dev, old_flags, 9951 dev->gflags ^ old_gflags, 0, NULL); 9952 } 9953 return 0; 9954 } 9955 9956 9957 /** 9958 * netif_get_flags() - get flags reported to userspace 9959 * @dev: device 9960 * 9961 * Get the combination of flag bits exported through APIs to userspace. 9962 */ 9963 unsigned int netif_get_flags(const struct net_device *dev) 9964 { 9965 unsigned int flags; 9966 9967 flags = (READ_ONCE(dev->flags) & ~(IFF_PROMISC | 9968 IFF_ALLMULTI | 9969 IFF_RUNNING | 9970 IFF_LOWER_UP | 9971 IFF_DORMANT)) | 9972 (READ_ONCE(dev->gflags) & (IFF_PROMISC | 9973 IFF_ALLMULTI)); 9974 9975 if (netif_running(dev)) { 9976 if (netif_oper_up(dev)) 9977 flags |= IFF_RUNNING; 9978 if (netif_carrier_ok(dev)) 9979 flags |= IFF_LOWER_UP; 9980 if (netif_dormant(dev)) 9981 flags |= IFF_DORMANT; 9982 } 9983 9984 return flags; 9985 } 9986 EXPORT_SYMBOL(netif_get_flags); 9987 9988 int __dev_change_flags(struct net_device *dev, unsigned int flags, 9989 struct netlink_ext_ack *extack) 9990 { 9991 unsigned int old_flags = dev->flags; 9992 int ret; 9993 9994 netdev_assert_locked_ops_compat(dev); 9995 9996 /* 9997 * Set the flags on our device. 9998 */ 9999 10000 dev->flags = (flags & (IFF_DEBUG | IFF_NOTRAILERS | IFF_NOARP | 10001 IFF_DYNAMIC | IFF_MULTICAST | IFF_PORTSEL | 10002 IFF_AUTOMEDIA)) | 10003 (dev->flags & (IFF_UP | IFF_VOLATILE | IFF_PROMISC | 10004 IFF_ALLMULTI)); 10005 10006 /* 10007 * Load in the correct multicast list now the flags have changed. 10008 */ 10009 10010 if ((old_flags ^ flags) & IFF_MULTICAST) 10011 dev_change_rx_flags(dev, IFF_MULTICAST); 10012 10013 dev_set_rx_mode(dev); 10014 10015 /* 10016 * Have we downed the interface. We handle IFF_UP ourselves 10017 * according to user attempts to set it, rather than blindly 10018 * setting it. 10019 */ 10020 10021 ret = 0; 10022 if ((old_flags ^ flags) & IFF_UP) { 10023 if (old_flags & IFF_UP) 10024 __dev_close(dev); 10025 else 10026 ret = __dev_open(dev, extack); 10027 } 10028 10029 if ((flags ^ dev->gflags) & IFF_PROMISC) { 10030 int inc = (flags & IFF_PROMISC) ? 1 : -1; 10031 old_flags = dev->flags; 10032 10033 dev->gflags ^= IFF_PROMISC; 10034 10035 if (__dev_set_promiscuity(dev, inc, false) >= 0) 10036 if (dev->flags != old_flags) 10037 dev_set_rx_mode(dev); 10038 } 10039 10040 /* NOTE: order of synchronization of IFF_PROMISC and IFF_ALLMULTI 10041 * is important. Some (broken) drivers set IFF_PROMISC, when 10042 * IFF_ALLMULTI is requested not asking us and not reporting. 10043 */ 10044 if ((flags ^ dev->gflags) & IFF_ALLMULTI) { 10045 int inc = (flags & IFF_ALLMULTI) ? 1 : -1; 10046 10047 dev->gflags ^= IFF_ALLMULTI; 10048 netif_set_allmulti(dev, inc, false); 10049 } 10050 10051 return ret; 10052 } 10053 10054 void __dev_notify_flags(struct net_device *dev, unsigned int old_flags, 10055 unsigned int gchanges, u32 portid, 10056 const struct nlmsghdr *nlh) 10057 { 10058 unsigned int changes = dev->flags ^ old_flags; 10059 10060 netdev_assert_locked_ops_compat(dev); 10061 10062 if (gchanges) 10063 rtmsg_ifinfo(RTM_NEWLINK, dev, gchanges, GFP_ATOMIC, portid, nlh); 10064 10065 if (changes & IFF_UP) { 10066 if (dev->flags & IFF_UP) 10067 call_netdevice_notifiers(NETDEV_UP, dev); 10068 else 10069 call_netdevice_notifiers(NETDEV_DOWN, dev); 10070 } 10071 10072 if (dev->flags & IFF_UP && 10073 (changes & ~(IFF_UP | IFF_PROMISC | IFF_ALLMULTI | IFF_VOLATILE))) { 10074 struct netdev_notifier_change_info change_info = { 10075 .info = { 10076 .dev = dev, 10077 }, 10078 .flags_changed = changes, 10079 }; 10080 10081 call_netdevice_notifiers_info(NETDEV_CHANGE, &change_info.info); 10082 } 10083 } 10084 10085 int netif_change_flags(struct net_device *dev, unsigned int flags, 10086 struct netlink_ext_ack *extack) 10087 { 10088 int ret; 10089 unsigned int changes, old_flags = dev->flags, old_gflags = dev->gflags; 10090 10091 ret = __dev_change_flags(dev, flags, extack); 10092 if (ret < 0) 10093 return ret; 10094 10095 changes = (old_flags ^ dev->flags) | (old_gflags ^ dev->gflags); 10096 __dev_notify_flags(dev, old_flags, changes, 0, NULL); 10097 return ret; 10098 } 10099 EXPORT_SYMBOL(netif_change_flags); 10100 10101 int __netif_set_mtu(struct net_device *dev, int new_mtu) 10102 { 10103 const struct net_device_ops *ops = dev->netdev_ops; 10104 10105 if (ops->ndo_change_mtu) 10106 return ops->ndo_change_mtu(dev, new_mtu); 10107 10108 /* Pairs with all the lockless reads of dev->mtu in the stack */ 10109 WRITE_ONCE(dev->mtu, new_mtu); 10110 return 0; 10111 } 10112 EXPORT_SYMBOL_NS_GPL(__netif_set_mtu, "NETDEV_INTERNAL"); 10113 10114 int dev_validate_mtu(struct net_device *dev, int new_mtu, 10115 struct netlink_ext_ack *extack) 10116 { 10117 /* MTU must be positive, and in range */ 10118 if (new_mtu < 0 || new_mtu < dev->min_mtu) { 10119 NL_SET_ERR_MSG(extack, "mtu less than device minimum"); 10120 return -EINVAL; 10121 } 10122 10123 if (dev->max_mtu > 0 && new_mtu > dev->max_mtu) { 10124 NL_SET_ERR_MSG(extack, "mtu greater than device maximum"); 10125 return -EINVAL; 10126 } 10127 return 0; 10128 } 10129 10130 /** 10131 * netif_set_mtu_ext() - Change maximum transfer unit 10132 * @dev: device 10133 * @new_mtu: new transfer unit 10134 * @extack: netlink extended ack 10135 * 10136 * Change the maximum transfer size of the network device. 10137 * 10138 * Return: 0 on success, -errno on failure. 10139 */ 10140 int netif_set_mtu_ext(struct net_device *dev, int new_mtu, 10141 struct netlink_ext_ack *extack) 10142 { 10143 int err, orig_mtu; 10144 10145 netdev_assert_locked_ops_compat(dev); 10146 10147 if (new_mtu == dev->mtu) 10148 return 0; 10149 10150 err = dev_validate_mtu(dev, new_mtu, extack); 10151 if (err) 10152 return err; 10153 10154 if (!netif_device_present(dev)) 10155 return -ENODEV; 10156 10157 err = call_netdevice_notifiers(NETDEV_PRECHANGEMTU, dev); 10158 err = notifier_to_errno(err); 10159 if (err) 10160 return err; 10161 10162 orig_mtu = dev->mtu; 10163 err = __netif_set_mtu(dev, new_mtu); 10164 10165 if (!err) { 10166 err = call_netdevice_notifiers_mtu(NETDEV_CHANGEMTU, dev, 10167 orig_mtu); 10168 err = notifier_to_errno(err); 10169 if (err) { 10170 /* setting mtu back and notifying everyone again, 10171 * so that they have a chance to revert changes. 10172 */ 10173 __netif_set_mtu(dev, orig_mtu); 10174 call_netdevice_notifiers_mtu(NETDEV_CHANGEMTU, dev, 10175 new_mtu); 10176 } 10177 } 10178 return err; 10179 } 10180 10181 int netif_set_mtu(struct net_device *dev, int new_mtu) 10182 { 10183 struct netlink_ext_ack extack; 10184 int err; 10185 10186 memset(&extack, 0, sizeof(extack)); 10187 err = netif_set_mtu_ext(dev, new_mtu, &extack); 10188 if (err && extack._msg) 10189 net_err_ratelimited("%s: %s\n", dev->name, extack._msg); 10190 return err; 10191 } 10192 EXPORT_SYMBOL(netif_set_mtu); 10193 10194 int netif_change_tx_queue_len(struct net_device *dev, unsigned long new_len) 10195 { 10196 unsigned int orig_len = dev->tx_queue_len; 10197 int res; 10198 10199 if (new_len > S16_MAX) 10200 return -ERANGE; 10201 10202 if (new_len != orig_len) { 10203 WRITE_ONCE(dev->tx_queue_len, new_len); 10204 res = call_netdevice_notifiers(NETDEV_CHANGE_TX_QUEUE_LEN, dev); 10205 res = notifier_to_errno(res); 10206 if (res) 10207 goto err_rollback; 10208 res = dev_qdisc_change_tx_queue_len(dev); 10209 if (res) 10210 goto err_rollback; 10211 } 10212 10213 return 0; 10214 10215 err_rollback: 10216 netdev_err(dev, "refused to change device tx_queue_len\n"); 10217 WRITE_ONCE(dev->tx_queue_len, orig_len); 10218 return res; 10219 } 10220 10221 void netif_set_group(struct net_device *dev, int new_group) 10222 { 10223 dev->group = new_group; 10224 } 10225 10226 /** 10227 * netif_pre_changeaddr_notify() - Call NETDEV_PRE_CHANGEADDR. 10228 * @dev: device 10229 * @addr: new address 10230 * @extack: netlink extended ack 10231 * 10232 * Return: 0 on success, -errno on failure. 10233 */ 10234 int netif_pre_changeaddr_notify(struct net_device *dev, const char *addr, 10235 struct netlink_ext_ack *extack) 10236 { 10237 struct netdev_notifier_pre_changeaddr_info info = { 10238 .info.dev = dev, 10239 .info.extack = extack, 10240 .dev_addr = addr, 10241 }; 10242 int rc; 10243 10244 rc = call_netdevice_notifiers_info(NETDEV_PRE_CHANGEADDR, &info.info); 10245 return notifier_to_errno(rc); 10246 } 10247 EXPORT_SYMBOL_NS_GPL(netif_pre_changeaddr_notify, "NETDEV_INTERNAL"); 10248 10249 int netif_set_mac_address(struct net_device *dev, struct sockaddr_storage *ss, 10250 struct netlink_ext_ack *extack) 10251 { 10252 const struct net_device_ops *ops = dev->netdev_ops; 10253 int err; 10254 10255 if (!ops->ndo_set_mac_address) 10256 return -EOPNOTSUPP; 10257 if (ss->ss_family != dev->type) 10258 return -EINVAL; 10259 if (!netif_device_present(dev)) 10260 return -ENODEV; 10261 err = netif_pre_changeaddr_notify(dev, ss->__data, extack); 10262 if (err) 10263 return err; 10264 if (memcmp(dev->dev_addr, ss->__data, dev->addr_len)) { 10265 err = ops->ndo_set_mac_address(dev, ss); 10266 if (err) 10267 return err; 10268 } 10269 dev->addr_assign_type = NET_ADDR_SET; 10270 call_netdevice_notifiers(NETDEV_CHANGEADDR, dev); 10271 add_device_randomness(dev->dev_addr, dev->addr_len); 10272 return 0; 10273 } 10274 10275 DECLARE_RWSEM(dev_addr_sem); 10276 10277 /* "sa" is a true struct sockaddr with limited "sa_data" member. */ 10278 int netif_get_mac_address(struct sockaddr *sa, struct net *net, char *dev_name) 10279 { 10280 size_t size = sizeof(sa->sa_data); 10281 struct net_device *dev; 10282 int ret = 0; 10283 10284 down_read(&dev_addr_sem); 10285 rcu_read_lock(); 10286 10287 dev = dev_get_by_name_rcu(net, dev_name); 10288 if (!dev) { 10289 ret = -ENODEV; 10290 goto unlock; 10291 } 10292 if (!dev->addr_len) 10293 memset(sa->sa_data, 0, size); 10294 else 10295 memcpy(sa->sa_data, dev->dev_addr, 10296 min_t(size_t, size, dev->addr_len)); 10297 sa->sa_family = dev->type; 10298 10299 unlock: 10300 rcu_read_unlock(); 10301 up_read(&dev_addr_sem); 10302 return ret; 10303 } 10304 EXPORT_SYMBOL_NS_GPL(netif_get_mac_address, "NETDEV_INTERNAL"); 10305 10306 int netif_change_carrier(struct net_device *dev, bool new_carrier) 10307 { 10308 const struct net_device_ops *ops = dev->netdev_ops; 10309 10310 if (!ops->ndo_change_carrier) 10311 return -EOPNOTSUPP; 10312 if (!netif_device_present(dev)) 10313 return -ENODEV; 10314 return ops->ndo_change_carrier(dev, new_carrier); 10315 } 10316 10317 /** 10318 * dev_get_phys_port_id - Get device physical port ID 10319 * @dev: device 10320 * @ppid: port ID 10321 * 10322 * Get device physical port ID 10323 */ 10324 int dev_get_phys_port_id(struct net_device *dev, 10325 struct netdev_phys_item_id *ppid) 10326 { 10327 const struct net_device_ops *ops = dev->netdev_ops; 10328 10329 if (!ops->ndo_get_phys_port_id) 10330 return -EOPNOTSUPP; 10331 return ops->ndo_get_phys_port_id(dev, ppid); 10332 } 10333 10334 /** 10335 * dev_get_phys_port_name - Get device physical port name 10336 * @dev: device 10337 * @name: port name 10338 * @len: limit of bytes to copy to name 10339 * 10340 * Get device physical port name 10341 */ 10342 int dev_get_phys_port_name(struct net_device *dev, 10343 char *name, size_t len) 10344 { 10345 const struct net_device_ops *ops = dev->netdev_ops; 10346 int err; 10347 10348 if (ops->ndo_get_phys_port_name) { 10349 err = ops->ndo_get_phys_port_name(dev, name, len); 10350 if (err != -EOPNOTSUPP) 10351 return err; 10352 } 10353 return devlink_compat_phys_port_name_get(dev, name, len); 10354 } 10355 10356 /** 10357 * netif_get_port_parent_id() - Get the device's port parent identifier 10358 * @dev: network device 10359 * @ppid: pointer to a storage for the port's parent identifier 10360 * @recurse: allow/disallow recursion to lower devices 10361 * 10362 * Get the devices's port parent identifier. 10363 * 10364 * Return: 0 on success, -errno on failure. 10365 */ 10366 int netif_get_port_parent_id(struct net_device *dev, 10367 struct netdev_phys_item_id *ppid, bool recurse) 10368 { 10369 const struct net_device_ops *ops = dev->netdev_ops; 10370 struct netdev_phys_item_id first = { }; 10371 struct net_device *lower_dev; 10372 struct list_head *iter; 10373 int err; 10374 10375 if (ops->ndo_get_port_parent_id) { 10376 err = ops->ndo_get_port_parent_id(dev, ppid); 10377 if (err != -EOPNOTSUPP) 10378 return err; 10379 } 10380 10381 err = devlink_compat_switch_id_get(dev, ppid); 10382 if (!recurse || err != -EOPNOTSUPP) 10383 return err; 10384 10385 netdev_for_each_lower_dev(dev, lower_dev, iter) { 10386 err = netif_get_port_parent_id(lower_dev, ppid, true); 10387 if (err) 10388 break; 10389 if (!first.id_len) 10390 first = *ppid; 10391 else if (memcmp(&first, ppid, sizeof(*ppid))) 10392 return -EOPNOTSUPP; 10393 } 10394 10395 return err; 10396 } 10397 EXPORT_SYMBOL(netif_get_port_parent_id); 10398 10399 /** 10400 * netdev_port_same_parent_id - Indicate if two network devices have 10401 * the same port parent identifier 10402 * @a: first network device 10403 * @b: second network device 10404 */ 10405 bool netdev_port_same_parent_id(struct net_device *a, struct net_device *b) 10406 { 10407 struct netdev_phys_item_id a_id = { }; 10408 struct netdev_phys_item_id b_id = { }; 10409 10410 if (netif_get_port_parent_id(a, &a_id, true) || 10411 netif_get_port_parent_id(b, &b_id, true)) 10412 return false; 10413 10414 return netdev_phys_item_id_same(&a_id, &b_id); 10415 } 10416 EXPORT_SYMBOL(netdev_port_same_parent_id); 10417 10418 static struct net_device *dev_get_iflink_dev(struct net_device *dev) 10419 { 10420 struct net *net; 10421 10422 ASSERT_RTNL(); 10423 10424 if (!dev->netdev_ops->ndo_get_iflink || !dev->rtnl_link_ops || 10425 !dev->rtnl_link_ops->get_link_net) 10426 return dev; 10427 10428 net = dev->rtnl_link_ops->get_link_net(dev); 10429 return __dev_get_by_index(net, dev_get_iflink(dev)); 10430 } 10431 10432 int netif_change_proto_down(struct net_device *dev, bool proto_down) 10433 { 10434 struct net_device *iflink_dev; 10435 10436 if (!dev->change_proto_down) 10437 return -EOPNOTSUPP; 10438 if (!netif_device_present(dev)) 10439 return -ENODEV; 10440 iflink_dev = dev_get_iflink_dev(dev); 10441 if (!iflink_dev) 10442 return -ENODEV; 10443 WRITE_ONCE(dev->proto_down, proto_down); 10444 if (proto_down) 10445 netif_carrier_off(dev); 10446 else if (dev == iflink_dev || netif_carrier_ok(iflink_dev)) 10447 netif_carrier_on(dev); 10448 return 0; 10449 } 10450 10451 /** 10452 * netdev_change_proto_down_reason_locked - proto down reason 10453 * 10454 * @dev: device 10455 * @mask: proto down mask 10456 * @value: proto down value 10457 */ 10458 void netdev_change_proto_down_reason_locked(struct net_device *dev, 10459 unsigned long mask, u32 value) 10460 { 10461 u32 proto_down_reason; 10462 int b; 10463 10464 if (!mask) { 10465 proto_down_reason = value; 10466 } else { 10467 proto_down_reason = dev->proto_down_reason; 10468 for_each_set_bit(b, &mask, 32) { 10469 if (value & (1 << b)) 10470 proto_down_reason |= BIT(b); 10471 else 10472 proto_down_reason &= ~BIT(b); 10473 } 10474 } 10475 WRITE_ONCE(dev->proto_down_reason, proto_down_reason); 10476 } 10477 10478 struct bpf_xdp_link { 10479 struct bpf_link link; 10480 struct net_device *dev; /* protected by rtnl_lock, no refcnt held */ 10481 int flags; 10482 }; 10483 10484 static enum bpf_xdp_mode dev_xdp_mode(struct net_device *dev, u32 flags) 10485 { 10486 if (flags & XDP_FLAGS_HW_MODE) 10487 return XDP_MODE_HW; 10488 if (flags & XDP_FLAGS_DRV_MODE) 10489 return XDP_MODE_DRV; 10490 if (flags & XDP_FLAGS_SKB_MODE) 10491 return XDP_MODE_SKB; 10492 return dev->netdev_ops->ndo_bpf ? XDP_MODE_DRV : XDP_MODE_SKB; 10493 } 10494 10495 static bpf_op_t dev_xdp_bpf_op(struct net_device *dev, enum bpf_xdp_mode mode) 10496 { 10497 switch (mode) { 10498 case XDP_MODE_SKB: 10499 return generic_xdp_install; 10500 case XDP_MODE_DRV: 10501 case XDP_MODE_HW: 10502 return dev->netdev_ops->ndo_bpf; 10503 default: 10504 return NULL; 10505 } 10506 } 10507 10508 static struct bpf_xdp_link *dev_xdp_link(struct net_device *dev, 10509 enum bpf_xdp_mode mode) 10510 { 10511 return dev->xdp_state[mode].link; 10512 } 10513 10514 static struct bpf_prog *dev_xdp_prog(struct net_device *dev, 10515 enum bpf_xdp_mode mode) 10516 { 10517 struct bpf_xdp_link *link = dev_xdp_link(dev, mode); 10518 10519 if (link) 10520 return link->link.prog; 10521 return dev->xdp_state[mode].prog; 10522 } 10523 10524 u8 dev_xdp_prog_count(struct net_device *dev) 10525 { 10526 u8 count = 0; 10527 int i; 10528 10529 for (i = 0; i < __MAX_XDP_MODE; i++) 10530 if (dev->xdp_state[i].prog || dev->xdp_state[i].link) 10531 count++; 10532 return count; 10533 } 10534 EXPORT_SYMBOL_GPL(dev_xdp_prog_count); 10535 10536 u8 dev_xdp_sb_prog_count(struct net_device *dev) 10537 { 10538 struct bpf_prog *prog; 10539 u8 count = 0; 10540 int i; 10541 10542 for (i = 0; i < __MAX_XDP_MODE; i++) { 10543 prog = dev_xdp_prog(dev, i); 10544 if (prog && !prog->aux->xdp_has_frags) 10545 count++; 10546 } 10547 return count; 10548 } 10549 10550 int netif_xdp_propagate(struct net_device *dev, struct netdev_bpf *bpf) 10551 { 10552 if (!dev->netdev_ops->ndo_bpf) 10553 return -EOPNOTSUPP; 10554 10555 if (dev->cfg->hds_config == ETHTOOL_TCP_DATA_SPLIT_ENABLED && 10556 bpf->command == XDP_SETUP_PROG && 10557 bpf->prog && !bpf->prog->aux->xdp_has_frags) { 10558 NL_SET_ERR_MSG(bpf->extack, 10559 "unable to propagate XDP to device using tcp-data-split"); 10560 return -EBUSY; 10561 } 10562 10563 if (dev_get_min_mp_channel_count(dev)) { 10564 NL_SET_ERR_MSG(bpf->extack, "unable to propagate XDP to device using memory provider"); 10565 return -EBUSY; 10566 } 10567 10568 return dev->netdev_ops->ndo_bpf(dev, bpf); 10569 } 10570 EXPORT_SYMBOL_GPL(netif_xdp_propagate); 10571 10572 u32 dev_xdp_prog_id(struct net_device *dev, enum bpf_xdp_mode mode) 10573 { 10574 struct bpf_prog *prog = dev_xdp_prog(dev, mode); 10575 10576 return prog ? prog->aux->id : 0; 10577 } 10578 10579 static void dev_xdp_set_link(struct net_device *dev, enum bpf_xdp_mode mode, 10580 struct bpf_xdp_link *link) 10581 { 10582 dev->xdp_state[mode].link = link; 10583 dev->xdp_state[mode].prog = NULL; 10584 } 10585 10586 static void dev_xdp_set_prog(struct net_device *dev, enum bpf_xdp_mode mode, 10587 struct bpf_prog *prog) 10588 { 10589 dev->xdp_state[mode].link = NULL; 10590 dev->xdp_state[mode].prog = prog; 10591 } 10592 10593 static int dev_xdp_install(struct net_device *dev, enum bpf_xdp_mode mode, 10594 bpf_op_t bpf_op, struct netlink_ext_ack *extack, 10595 u32 flags, struct bpf_prog *prog) 10596 { 10597 struct netdev_bpf xdp; 10598 int err; 10599 10600 netdev_assert_locked_ops_compat(dev); 10601 10602 if (prog) { 10603 enum bpf_xdp_mode other_mode = mode == XDP_MODE_SKB 10604 ? XDP_MODE_DRV : XDP_MODE_SKB; 10605 bool offload = mode == XDP_MODE_HW; 10606 10607 if (!offload && dev_xdp_prog(dev, other_mode)) { 10608 NL_SET_ERR_MSG(extack, "Native and generic XDP can't be active at the same time"); 10609 return -EEXIST; 10610 } 10611 if (!offload && bpf_prog_is_offloaded(prog->aux)) { 10612 NL_SET_ERR_MSG(extack, "Using offloaded program without HW_MODE flag is not supported"); 10613 return -EINVAL; 10614 } 10615 if (bpf_prog_is_dev_bound(prog->aux) && !bpf_offload_dev_match(prog, dev)) { 10616 NL_SET_ERR_MSG(extack, "Program bound to different device"); 10617 return -EINVAL; 10618 } 10619 if (bpf_prog_is_dev_bound(prog->aux) && mode == XDP_MODE_SKB) { 10620 NL_SET_ERR_MSG(extack, "Can't attach device-bound programs in generic mode"); 10621 return -EINVAL; 10622 } 10623 if (prog->expected_attach_type == BPF_XDP_DEVMAP) { 10624 NL_SET_ERR_MSG(extack, "BPF_XDP_DEVMAP programs can not be attached to a device"); 10625 return -EINVAL; 10626 } 10627 if (prog->expected_attach_type == BPF_XDP_CPUMAP) { 10628 NL_SET_ERR_MSG(extack, "BPF_XDP_CPUMAP programs can not be attached to a device"); 10629 return -EINVAL; 10630 } 10631 } 10632 10633 if (dev->cfg->hds_config == ETHTOOL_TCP_DATA_SPLIT_ENABLED && 10634 prog && !prog->aux->xdp_has_frags) { 10635 NL_SET_ERR_MSG(extack, "unable to install XDP to device using tcp-data-split"); 10636 return -EBUSY; 10637 } 10638 10639 if (dev_get_min_mp_channel_count(dev)) { 10640 NL_SET_ERR_MSG(extack, "unable to install XDP to device using memory provider"); 10641 return -EBUSY; 10642 } 10643 10644 memset(&xdp, 0, sizeof(xdp)); 10645 xdp.command = mode == XDP_MODE_HW ? XDP_SETUP_PROG_HW : XDP_SETUP_PROG; 10646 xdp.extack = extack; 10647 xdp.flags = flags; 10648 xdp.prog = prog; 10649 10650 /* Drivers assume refcnt is already incremented (i.e, prog pointer is 10651 * "moved" into driver), so they don't increment it on their own, but 10652 * they do decrement refcnt when program is detached or replaced. 10653 * Given net_device also owns link/prog, we need to bump refcnt here 10654 * to prevent drivers from underflowing it. 10655 */ 10656 if (prog) 10657 bpf_prog_inc(prog); 10658 err = bpf_op(dev, &xdp); 10659 if (err) { 10660 if (prog) 10661 bpf_prog_put(prog); 10662 return err; 10663 } 10664 10665 if (mode != XDP_MODE_HW) 10666 bpf_prog_change_xdp(dev_xdp_prog(dev, mode), prog); 10667 10668 return 0; 10669 } 10670 10671 static void dev_xdp_uninstall(struct net_device *dev) 10672 { 10673 struct bpf_xdp_link *link; 10674 struct bpf_prog *prog; 10675 enum bpf_xdp_mode mode; 10676 bpf_op_t bpf_op; 10677 10678 ASSERT_RTNL(); 10679 10680 for (mode = XDP_MODE_SKB; mode < __MAX_XDP_MODE; mode++) { 10681 prog = dev_xdp_prog(dev, mode); 10682 if (!prog) 10683 continue; 10684 10685 bpf_op = dev_xdp_bpf_op(dev, mode); 10686 if (!bpf_op) 10687 continue; 10688 10689 WARN_ON(dev_xdp_install(dev, mode, bpf_op, NULL, 0, NULL)); 10690 10691 /* auto-detach link from net device */ 10692 link = dev_xdp_link(dev, mode); 10693 if (link) 10694 link->dev = NULL; 10695 else 10696 bpf_prog_put(prog); 10697 10698 dev_xdp_set_link(dev, mode, NULL); 10699 } 10700 } 10701 10702 static int dev_xdp_attach(struct net_device *dev, struct netlink_ext_ack *extack, 10703 struct bpf_xdp_link *link, struct bpf_prog *new_prog, 10704 struct bpf_prog *old_prog, u32 flags) 10705 { 10706 unsigned int num_modes = hweight32(flags & XDP_FLAGS_MODES); 10707 struct bpf_prog *cur_prog; 10708 struct net_device *upper; 10709 struct list_head *iter; 10710 enum bpf_xdp_mode mode; 10711 bpf_op_t bpf_op; 10712 int err; 10713 10714 ASSERT_RTNL(); 10715 10716 /* either link or prog attachment, never both */ 10717 if (link && (new_prog || old_prog)) 10718 return -EINVAL; 10719 /* link supports only XDP mode flags */ 10720 if (link && (flags & ~XDP_FLAGS_MODES)) { 10721 NL_SET_ERR_MSG(extack, "Invalid XDP flags for BPF link attachment"); 10722 return -EINVAL; 10723 } 10724 /* just one XDP mode bit should be set, zero defaults to drv/skb mode */ 10725 if (num_modes > 1) { 10726 NL_SET_ERR_MSG(extack, "Only one XDP mode flag can be set"); 10727 return -EINVAL; 10728 } 10729 /* avoid ambiguity if offload + drv/skb mode progs are both loaded */ 10730 if (!num_modes && dev_xdp_prog_count(dev) > 1) { 10731 NL_SET_ERR_MSG(extack, 10732 "More than one program loaded, unset mode is ambiguous"); 10733 return -EINVAL; 10734 } 10735 /* old_prog != NULL implies XDP_FLAGS_REPLACE is set */ 10736 if (old_prog && !(flags & XDP_FLAGS_REPLACE)) { 10737 NL_SET_ERR_MSG(extack, "XDP_FLAGS_REPLACE is not specified"); 10738 return -EINVAL; 10739 } 10740 10741 mode = dev_xdp_mode(dev, flags); 10742 /* can't replace attached link */ 10743 if (dev_xdp_link(dev, mode)) { 10744 NL_SET_ERR_MSG(extack, "Can't replace active BPF XDP link"); 10745 return -EBUSY; 10746 } 10747 10748 /* don't allow if an upper device already has a program */ 10749 netdev_for_each_upper_dev_rcu(dev, upper, iter) { 10750 if (dev_xdp_prog_count(upper) > 0) { 10751 NL_SET_ERR_MSG(extack, "Cannot attach when an upper device already has a program"); 10752 return -EEXIST; 10753 } 10754 } 10755 10756 cur_prog = dev_xdp_prog(dev, mode); 10757 /* can't replace attached prog with link */ 10758 if (link && cur_prog) { 10759 NL_SET_ERR_MSG(extack, "Can't replace active XDP program with BPF link"); 10760 return -EBUSY; 10761 } 10762 if ((flags & XDP_FLAGS_REPLACE) && cur_prog != old_prog) { 10763 NL_SET_ERR_MSG(extack, "Active program does not match expected"); 10764 return -EEXIST; 10765 } 10766 10767 /* put effective new program into new_prog */ 10768 if (link) 10769 new_prog = link->link.prog; 10770 10771 if (new_prog) { 10772 if ((flags & XDP_FLAGS_UPDATE_IF_NOEXIST) && cur_prog) { 10773 NL_SET_ERR_MSG(extack, "XDP program already attached"); 10774 return -EBUSY; 10775 } 10776 } 10777 10778 /* don't call drivers if the effective program didn't change */ 10779 if (new_prog != cur_prog) { 10780 bpf_op = dev_xdp_bpf_op(dev, mode); 10781 if (!bpf_op) { 10782 NL_SET_ERR_MSG(extack, "Underlying driver does not support XDP in native mode"); 10783 return -EOPNOTSUPP; 10784 } 10785 10786 err = dev_xdp_install(dev, mode, bpf_op, extack, flags, new_prog); 10787 if (err) 10788 return err; 10789 } 10790 10791 if (link) 10792 dev_xdp_set_link(dev, mode, link); 10793 else 10794 dev_xdp_set_prog(dev, mode, new_prog); 10795 if (cur_prog) 10796 bpf_prog_put(cur_prog); 10797 10798 return 0; 10799 } 10800 10801 static int dev_xdp_attach_link(struct net_device *dev, 10802 struct netlink_ext_ack *extack, 10803 struct bpf_xdp_link *link) 10804 { 10805 return dev_xdp_attach(dev, extack, link, NULL, NULL, link->flags); 10806 } 10807 10808 static int dev_xdp_detach_link(struct net_device *dev, 10809 struct netlink_ext_ack *extack, 10810 struct bpf_xdp_link *link) 10811 { 10812 enum bpf_xdp_mode mode; 10813 bpf_op_t bpf_op; 10814 10815 ASSERT_RTNL(); 10816 10817 mode = dev_xdp_mode(dev, link->flags); 10818 if (dev_xdp_link(dev, mode) != link) 10819 return -EINVAL; 10820 10821 bpf_op = dev_xdp_bpf_op(dev, mode); 10822 WARN_ON(dev_xdp_install(dev, mode, bpf_op, NULL, 0, NULL)); 10823 dev_xdp_set_link(dev, mode, NULL); 10824 return 0; 10825 } 10826 10827 static void bpf_xdp_link_release(struct bpf_link *link) 10828 { 10829 struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link); 10830 10831 rtnl_lock(); 10832 10833 /* if racing with net_device's tear down, xdp_link->dev might be 10834 * already NULL, in which case link was already auto-detached 10835 */ 10836 if (xdp_link->dev) { 10837 netdev_lock_ops(xdp_link->dev); 10838 WARN_ON(dev_xdp_detach_link(xdp_link->dev, NULL, xdp_link)); 10839 netdev_unlock_ops(xdp_link->dev); 10840 xdp_link->dev = NULL; 10841 } 10842 10843 rtnl_unlock(); 10844 } 10845 10846 static int bpf_xdp_link_detach(struct bpf_link *link) 10847 { 10848 bpf_xdp_link_release(link); 10849 return 0; 10850 } 10851 10852 static void bpf_xdp_link_dealloc(struct bpf_link *link) 10853 { 10854 struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link); 10855 10856 kfree(xdp_link); 10857 } 10858 10859 static void bpf_xdp_link_show_fdinfo(const struct bpf_link *link, 10860 struct seq_file *seq) 10861 { 10862 struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link); 10863 u32 ifindex = 0; 10864 10865 rtnl_lock(); 10866 if (xdp_link->dev) 10867 ifindex = xdp_link->dev->ifindex; 10868 rtnl_unlock(); 10869 10870 seq_printf(seq, "ifindex:\t%u\n", ifindex); 10871 } 10872 10873 static int bpf_xdp_link_fill_link_info(const struct bpf_link *link, 10874 struct bpf_link_info *info) 10875 { 10876 struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link); 10877 u32 ifindex = 0; 10878 10879 rtnl_lock(); 10880 if (xdp_link->dev) 10881 ifindex = xdp_link->dev->ifindex; 10882 rtnl_unlock(); 10883 10884 info->xdp.ifindex = ifindex; 10885 return 0; 10886 } 10887 10888 static int bpf_xdp_link_update(struct bpf_link *link, struct bpf_prog *new_prog, 10889 struct bpf_prog *old_prog) 10890 { 10891 struct bpf_xdp_link *xdp_link = container_of(link, struct bpf_xdp_link, link); 10892 enum bpf_xdp_mode mode; 10893 bpf_op_t bpf_op; 10894 int err = 0; 10895 10896 rtnl_lock(); 10897 10898 /* link might have been auto-released already, so fail */ 10899 if (!xdp_link->dev) { 10900 err = -ENOLINK; 10901 goto out_unlock; 10902 } 10903 10904 if (old_prog && link->prog != old_prog) { 10905 err = -EPERM; 10906 goto out_unlock; 10907 } 10908 old_prog = link->prog; 10909 if (old_prog->type != new_prog->type || 10910 old_prog->expected_attach_type != new_prog->expected_attach_type) { 10911 err = -EINVAL; 10912 goto out_unlock; 10913 } 10914 10915 if (old_prog == new_prog) { 10916 /* no-op, don't disturb drivers */ 10917 bpf_prog_put(new_prog); 10918 goto out_unlock; 10919 } 10920 10921 netdev_lock_ops(xdp_link->dev); 10922 mode = dev_xdp_mode(xdp_link->dev, xdp_link->flags); 10923 bpf_op = dev_xdp_bpf_op(xdp_link->dev, mode); 10924 err = dev_xdp_install(xdp_link->dev, mode, bpf_op, NULL, 10925 xdp_link->flags, new_prog); 10926 if (!err) 10927 old_prog = xchg(&link->prog, new_prog); 10928 netdev_unlock_ops(xdp_link->dev); 10929 if (err) 10930 goto out_unlock; 10931 10932 bpf_prog_put(old_prog); 10933 10934 out_unlock: 10935 rtnl_unlock(); 10936 return err; 10937 } 10938 10939 static const struct bpf_link_ops bpf_xdp_link_lops = { 10940 .release = bpf_xdp_link_release, 10941 .dealloc = bpf_xdp_link_dealloc, 10942 .detach = bpf_xdp_link_detach, 10943 .show_fdinfo = bpf_xdp_link_show_fdinfo, 10944 .fill_link_info = bpf_xdp_link_fill_link_info, 10945 .update_prog = bpf_xdp_link_update, 10946 }; 10947 10948 int bpf_xdp_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) 10949 { 10950 struct net *net = current->nsproxy->net_ns; 10951 struct bpf_link_primer link_primer; 10952 struct netlink_ext_ack extack = {}; 10953 struct bpf_xdp_link *link; 10954 struct net_device *dev; 10955 int err, fd; 10956 10957 rtnl_lock(); 10958 dev = dev_get_by_index(net, attr->link_create.target_ifindex); 10959 if (!dev) { 10960 rtnl_unlock(); 10961 return -EINVAL; 10962 } 10963 10964 link = kzalloc_obj(*link, GFP_USER); 10965 if (!link) { 10966 err = -ENOMEM; 10967 goto unlock; 10968 } 10969 10970 bpf_link_init(&link->link, BPF_LINK_TYPE_XDP, &bpf_xdp_link_lops, prog, 10971 attr->link_create.attach_type); 10972 link->dev = dev; 10973 link->flags = attr->link_create.flags; 10974 10975 err = bpf_link_prime(&link->link, &link_primer); 10976 if (err) { 10977 kfree(link); 10978 goto unlock; 10979 } 10980 10981 netdev_lock_ops(dev); 10982 err = dev_xdp_attach_link(dev, &extack, link); 10983 netdev_unlock_ops(dev); 10984 rtnl_unlock(); 10985 10986 if (err) { 10987 link->dev = NULL; 10988 bpf_link_cleanup(&link_primer); 10989 trace_bpf_xdp_link_attach_failed(extack._msg); 10990 goto out_put_dev; 10991 } 10992 10993 fd = bpf_link_settle(&link_primer); 10994 /* link itself doesn't hold dev's refcnt to not complicate shutdown */ 10995 dev_put(dev); 10996 return fd; 10997 10998 unlock: 10999 rtnl_unlock(); 11000 11001 out_put_dev: 11002 dev_put(dev); 11003 return err; 11004 } 11005 11006 /** 11007 * dev_change_xdp_fd - set or clear a bpf program for a device rx path 11008 * @dev: device 11009 * @extack: netlink extended ack 11010 * @fd: new program fd or negative value to clear 11011 * @expected_fd: old program fd that userspace expects to replace or clear 11012 * @flags: xdp-related flags 11013 * 11014 * Set or clear a bpf program for a device 11015 */ 11016 int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack, 11017 int fd, int expected_fd, u32 flags) 11018 { 11019 enum bpf_xdp_mode mode = dev_xdp_mode(dev, flags); 11020 struct bpf_prog *new_prog = NULL, *old_prog = NULL; 11021 int err; 11022 11023 ASSERT_RTNL(); 11024 11025 if (fd >= 0) { 11026 new_prog = bpf_prog_get_type_dev(fd, BPF_PROG_TYPE_XDP, 11027 mode != XDP_MODE_SKB); 11028 if (IS_ERR(new_prog)) 11029 return PTR_ERR(new_prog); 11030 } 11031 11032 if (expected_fd >= 0) { 11033 old_prog = bpf_prog_get_type_dev(expected_fd, BPF_PROG_TYPE_XDP, 11034 mode != XDP_MODE_SKB); 11035 if (IS_ERR(old_prog)) { 11036 err = PTR_ERR(old_prog); 11037 old_prog = NULL; 11038 goto err_out; 11039 } 11040 } 11041 11042 err = dev_xdp_attach(dev, extack, NULL, new_prog, old_prog, flags); 11043 11044 err_out: 11045 if (err && new_prog) 11046 bpf_prog_put(new_prog); 11047 if (old_prog) 11048 bpf_prog_put(old_prog); 11049 return err; 11050 } 11051 11052 u32 dev_get_min_mp_channel_count(const struct net_device *dev) 11053 { 11054 int i; 11055 11056 netdev_assert_locked_ops_compat(dev); 11057 11058 for (i = dev->real_num_rx_queues - 1; i >= 0; i--) 11059 if (dev->_rx[i].mp_params.mp_priv) 11060 /* The channel count is the idx plus 1. */ 11061 return i + 1; 11062 11063 return 0; 11064 } 11065 11066 /** 11067 * dev_index_reserve() - allocate an ifindex in a namespace 11068 * @net: the applicable net namespace 11069 * @ifindex: requested ifindex, pass %0 to get one allocated 11070 * 11071 * Allocate a ifindex for a new device. Caller must either use the ifindex 11072 * to store the device (via list_netdevice()) or call dev_index_release() 11073 * to give the index up. 11074 * 11075 * Return: a suitable unique value for a new device interface number or -errno. 11076 */ 11077 static int dev_index_reserve(struct net *net, u32 ifindex) 11078 { 11079 int err; 11080 11081 if (ifindex > INT_MAX) { 11082 DEBUG_NET_WARN_ON_ONCE(1); 11083 return -EINVAL; 11084 } 11085 11086 if (!ifindex) 11087 err = xa_alloc_cyclic(&net->dev_by_index, &ifindex, NULL, 11088 xa_limit_31b, &net->ifindex, GFP_KERNEL); 11089 else 11090 err = xa_insert(&net->dev_by_index, ifindex, NULL, GFP_KERNEL); 11091 if (err < 0) 11092 return err; 11093 11094 return ifindex; 11095 } 11096 11097 static void dev_index_release(struct net *net, int ifindex) 11098 { 11099 /* Expect only unused indexes, unlist_netdevice() removes the used */ 11100 WARN_ON(xa_erase(&net->dev_by_index, ifindex)); 11101 } 11102 11103 static bool from_cleanup_net(void) 11104 { 11105 #ifdef CONFIG_NET_NS 11106 return current == READ_ONCE(cleanup_net_task); 11107 #else 11108 return false; 11109 #endif 11110 } 11111 11112 /* Delayed registration/unregisteration */ 11113 LIST_HEAD(net_todo_list); 11114 DECLARE_WAIT_QUEUE_HEAD(netdev_unregistering_wq); 11115 atomic_t dev_unreg_count = ATOMIC_INIT(0); 11116 11117 static void net_set_todo(struct net_device *dev) 11118 { 11119 list_add_tail(&dev->todo_list, &net_todo_list); 11120 atomic_inc(&dev_net(dev)->dev_unreg_pending); 11121 } 11122 11123 /* Paired with net_set_todo(). @net must not be used after this call. */ 11124 static void net_unreg_done(struct net *net) 11125 { 11126 if (atomic_dec_and_test(&net->dev_unreg_pending)) 11127 wake_up(&netdev_unregistering_wq); 11128 } 11129 11130 static netdev_features_t netdev_sync_upper_features(struct net_device *lower, 11131 struct net_device *upper, netdev_features_t features) 11132 { 11133 netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES; 11134 netdev_features_t feature; 11135 int feature_bit; 11136 11137 for_each_netdev_feature(upper_disables, feature_bit) { 11138 feature = __NETIF_F_BIT(feature_bit); 11139 if (!(upper->wanted_features & feature) 11140 && (features & feature)) { 11141 netdev_dbg(lower, "Dropping feature %pNF, upper dev %s has it off.\n", 11142 &feature, upper->name); 11143 features &= ~feature; 11144 } 11145 } 11146 11147 return features; 11148 } 11149 11150 static void netdev_sync_lower_features(struct net_device *upper, 11151 struct net_device *lower, netdev_features_t features) 11152 { 11153 netdev_features_t upper_disables = NETIF_F_UPPER_DISABLES; 11154 netdev_features_t feature; 11155 int feature_bit; 11156 11157 for_each_netdev_feature(upper_disables, feature_bit) { 11158 feature = __NETIF_F_BIT(feature_bit); 11159 if (!(features & feature) && (lower->features & feature)) { 11160 netdev_dbg(upper, "Disabling feature %pNF on lower dev %s.\n", 11161 &feature, lower->name); 11162 netdev_lock_ops(lower); 11163 lower->wanted_features &= ~feature; 11164 __netdev_update_features(lower); 11165 11166 if (unlikely(lower->features & feature)) 11167 netdev_WARN(upper, "failed to disable %pNF on %s!\n", 11168 &feature, lower->name); 11169 else 11170 netdev_features_change(lower); 11171 netdev_unlock_ops(lower); 11172 } 11173 } 11174 } 11175 11176 static bool netdev_has_ip_or_hw_csum(netdev_features_t features) 11177 { 11178 netdev_features_t ip_csum_mask = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM; 11179 bool ip_csum = (features & ip_csum_mask) == ip_csum_mask; 11180 bool hw_csum = features & NETIF_F_HW_CSUM; 11181 11182 return ip_csum || hw_csum; 11183 } 11184 11185 static netdev_features_t netdev_fix_features(struct net_device *dev, 11186 netdev_features_t features) 11187 { 11188 /* Fix illegal checksum combinations */ 11189 if ((features & NETIF_F_HW_CSUM) && 11190 (features & (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM))) { 11191 netdev_warn(dev, "mixed HW and IP checksum settings.\n"); 11192 features &= ~(NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM); 11193 } 11194 11195 /* TSO requires that SG is present as well. */ 11196 if ((features & NETIF_F_ALL_TSO) && !(features & NETIF_F_SG)) { 11197 netdev_dbg(dev, "Dropping TSO features since no SG feature.\n"); 11198 features &= ~NETIF_F_ALL_TSO; 11199 } 11200 11201 if ((features & NETIF_F_TSO) && !(features & NETIF_F_HW_CSUM) && 11202 !(features & NETIF_F_IP_CSUM)) { 11203 netdev_dbg(dev, "Dropping TSO features since no CSUM feature.\n"); 11204 features &= ~NETIF_F_TSO; 11205 features &= ~NETIF_F_TSO_ECN; 11206 } 11207 11208 if ((features & NETIF_F_TSO6) && !(features & NETIF_F_HW_CSUM) && 11209 !(features & NETIF_F_IPV6_CSUM)) { 11210 netdev_dbg(dev, "Dropping TSO6 features since no CSUM feature.\n"); 11211 features &= ~NETIF_F_TSO6; 11212 } 11213 11214 /* TSO with IPv4 ID mangling requires IPv4 TSO be enabled */ 11215 if ((features & NETIF_F_TSO_MANGLEID) && !(features & NETIF_F_TSO)) 11216 features &= ~NETIF_F_TSO_MANGLEID; 11217 11218 /* TSO ECN requires that TSO is present as well. */ 11219 if ((features & NETIF_F_ALL_TSO) == NETIF_F_TSO_ECN) 11220 features &= ~NETIF_F_TSO_ECN; 11221 11222 /* Software GSO depends on SG. */ 11223 if ((features & NETIF_F_GSO) && !(features & NETIF_F_SG)) { 11224 netdev_dbg(dev, "Dropping NETIF_F_GSO since no SG feature.\n"); 11225 features &= ~NETIF_F_GSO; 11226 } 11227 11228 /* GSO partial features require GSO partial be set */ 11229 if ((features & dev->gso_partial_features) && 11230 !(features & NETIF_F_GSO_PARTIAL)) { 11231 netdev_dbg(dev, 11232 "Dropping partially supported GSO features since no GSO partial.\n"); 11233 features &= ~dev->gso_partial_features; 11234 } 11235 11236 if (!(features & NETIF_F_RXCSUM)) { 11237 /* NETIF_F_GRO_HW implies doing RXCSUM since every packet 11238 * successfully merged by hardware must also have the 11239 * checksum verified by hardware. If the user does not 11240 * want to enable RXCSUM, logically, we should disable GRO_HW. 11241 */ 11242 if (features & NETIF_F_GRO_HW) { 11243 netdev_dbg(dev, "Dropping NETIF_F_GRO_HW since no RXCSUM feature.\n"); 11244 features &= ~NETIF_F_GRO_HW; 11245 } 11246 } 11247 11248 /* LRO/HW-GRO features cannot be combined with RX-FCS */ 11249 if (features & NETIF_F_RXFCS) { 11250 if (features & NETIF_F_LRO) { 11251 netdev_dbg(dev, "Dropping LRO feature since RX-FCS is requested.\n"); 11252 features &= ~NETIF_F_LRO; 11253 } 11254 11255 if (features & NETIF_F_GRO_HW) { 11256 netdev_dbg(dev, "Dropping HW-GRO feature since RX-FCS is requested.\n"); 11257 features &= ~NETIF_F_GRO_HW; 11258 } 11259 } 11260 11261 if ((features & NETIF_F_GRO_HW) && (features & NETIF_F_LRO)) { 11262 netdev_dbg(dev, "Dropping LRO feature since HW-GRO is requested.\n"); 11263 features &= ~NETIF_F_LRO; 11264 } 11265 11266 if ((features & NETIF_F_HW_TLS_TX) && !netdev_has_ip_or_hw_csum(features)) { 11267 netdev_dbg(dev, "Dropping TLS TX HW offload feature since no CSUM feature.\n"); 11268 features &= ~NETIF_F_HW_TLS_TX; 11269 } 11270 11271 if ((features & NETIF_F_HW_TLS_RX) && !(features & NETIF_F_RXCSUM)) { 11272 netdev_dbg(dev, "Dropping TLS RX HW offload feature since no RXCSUM feature.\n"); 11273 features &= ~NETIF_F_HW_TLS_RX; 11274 } 11275 11276 if ((features & NETIF_F_GSO_UDP_L4) && !netdev_has_ip_or_hw_csum(features)) { 11277 netdev_dbg(dev, "Dropping USO feature since no CSUM feature.\n"); 11278 features &= ~NETIF_F_GSO_UDP_L4; 11279 } 11280 11281 return features; 11282 } 11283 11284 int __netdev_update_features(struct net_device *dev) 11285 { 11286 struct net_device *upper, *lower; 11287 netdev_features_t features; 11288 struct list_head *iter; 11289 int err = -1; 11290 11291 ASSERT_RTNL(); 11292 netdev_assert_locked_ops_compat(dev); 11293 11294 features = netdev_get_wanted_features(dev); 11295 11296 if (dev->netdev_ops->ndo_fix_features) 11297 features = dev->netdev_ops->ndo_fix_features(dev, features); 11298 11299 /* driver might be less strict about feature dependencies */ 11300 features = netdev_fix_features(dev, features); 11301 11302 /* some features can't be enabled if they're off on an upper device */ 11303 netdev_for_each_upper_dev_rcu(dev, upper, iter) 11304 features = netdev_sync_upper_features(dev, upper, features); 11305 11306 if (dev->features == features) 11307 goto sync_lower; 11308 11309 netdev_dbg(dev, "Features changed: %pNF -> %pNF\n", 11310 &dev->features, &features); 11311 11312 if (dev->netdev_ops->ndo_set_features) 11313 err = dev->netdev_ops->ndo_set_features(dev, features); 11314 else 11315 err = 0; 11316 11317 if (unlikely(err < 0)) { 11318 netdev_err(dev, 11319 "set_features() failed (%d); wanted %pNF, left %pNF\n", 11320 err, &features, &dev->features); 11321 /* return non-0 since some features might have changed and 11322 * it's better to fire a spurious notification than miss it 11323 */ 11324 return -1; 11325 } 11326 11327 sync_lower: 11328 /* some features must be disabled on lower devices when disabled 11329 * on an upper device (think: bonding master or bridge) 11330 */ 11331 netdev_for_each_lower_dev(dev, lower, iter) 11332 netdev_sync_lower_features(dev, lower, features); 11333 11334 if (!err) { 11335 netdev_features_t diff = features ^ dev->features; 11336 11337 if (diff & NETIF_F_RX_UDP_TUNNEL_PORT) { 11338 /* udp_tunnel_{get,drop}_rx_info both need 11339 * NETIF_F_RX_UDP_TUNNEL_PORT enabled on the 11340 * device, or they won't do anything. 11341 * Thus we need to update dev->features 11342 * *before* calling udp_tunnel_get_rx_info, 11343 * but *after* calling udp_tunnel_drop_rx_info. 11344 */ 11345 udp_tunnel_nic_lock(dev); 11346 if (features & NETIF_F_RX_UDP_TUNNEL_PORT) { 11347 dev->features = features; 11348 udp_tunnel_get_rx_info(dev); 11349 } else { 11350 udp_tunnel_drop_rx_info(dev); 11351 } 11352 udp_tunnel_nic_unlock(dev); 11353 } 11354 11355 if (diff & NETIF_F_HW_VLAN_CTAG_FILTER) { 11356 if (features & NETIF_F_HW_VLAN_CTAG_FILTER) { 11357 dev->features = features; 11358 err |= vlan_get_rx_ctag_filter_info(dev); 11359 } else { 11360 vlan_drop_rx_ctag_filter_info(dev); 11361 } 11362 } 11363 11364 if (diff & NETIF_F_HW_VLAN_STAG_FILTER) { 11365 if (features & NETIF_F_HW_VLAN_STAG_FILTER) { 11366 dev->features = features; 11367 err |= vlan_get_rx_stag_filter_info(dev); 11368 } else { 11369 vlan_drop_rx_stag_filter_info(dev); 11370 } 11371 } 11372 11373 dev->features = features; 11374 } 11375 11376 return err < 0 ? 0 : 1; 11377 } 11378 11379 /** 11380 * netdev_update_features - recalculate device features 11381 * @dev: the device to check 11382 * 11383 * Recalculate dev->features set and send notifications if it 11384 * has changed. Should be called after driver or hardware dependent 11385 * conditions might have changed that influence the features. 11386 */ 11387 void netdev_update_features(struct net_device *dev) 11388 { 11389 if (__netdev_update_features(dev)) 11390 netdev_features_change(dev); 11391 } 11392 EXPORT_SYMBOL(netdev_update_features); 11393 11394 /** 11395 * netdev_change_features - recalculate device features 11396 * @dev: the device to check 11397 * 11398 * Recalculate dev->features set and send notifications even 11399 * if they have not changed. Should be called instead of 11400 * netdev_update_features() if also dev->vlan_features might 11401 * have changed to allow the changes to be propagated to stacked 11402 * VLAN devices. 11403 */ 11404 void netdev_change_features(struct net_device *dev) 11405 { 11406 __netdev_update_features(dev); 11407 netdev_features_change(dev); 11408 } 11409 EXPORT_SYMBOL(netdev_change_features); 11410 11411 /** 11412 * netif_stacked_transfer_operstate - transfer operstate 11413 * @rootdev: the root or lower level device to transfer state from 11414 * @dev: the device to transfer operstate to 11415 * 11416 * Transfer operational state from root to device. This is normally 11417 * called when a stacking relationship exists between the root 11418 * device and the device(a leaf device). 11419 */ 11420 void netif_stacked_transfer_operstate(const struct net_device *rootdev, 11421 struct net_device *dev) 11422 { 11423 if (rootdev->operstate == IF_OPER_DORMANT) 11424 netif_dormant_on(dev); 11425 else 11426 netif_dormant_off(dev); 11427 11428 if (rootdev->operstate == IF_OPER_TESTING) 11429 netif_testing_on(dev); 11430 else 11431 netif_testing_off(dev); 11432 11433 if (netif_carrier_ok(rootdev)) 11434 netif_carrier_on(dev); 11435 else 11436 netif_carrier_off(dev); 11437 } 11438 EXPORT_SYMBOL(netif_stacked_transfer_operstate); 11439 11440 static int netif_alloc_rx_queues(struct net_device *dev) 11441 { 11442 unsigned int i, count = dev->num_rx_queues; 11443 struct netdev_rx_queue *rx; 11444 size_t sz = count * sizeof(*rx); 11445 int err = 0; 11446 11447 BUG_ON(count < 1); 11448 11449 rx = kvzalloc(sz, GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL); 11450 if (!rx) 11451 return -ENOMEM; 11452 11453 dev->_rx = rx; 11454 11455 for (i = 0; i < count; i++) { 11456 rx[i].dev = dev; 11457 11458 /* XDP RX-queue setup */ 11459 err = xdp_rxq_info_reg(&rx[i].xdp_rxq, dev, i, 0); 11460 if (err < 0) 11461 goto err_rxq_info; 11462 } 11463 return 0; 11464 11465 err_rxq_info: 11466 /* Rollback successful reg's and free other resources */ 11467 while (i--) 11468 xdp_rxq_info_unreg(&rx[i].xdp_rxq); 11469 kvfree(dev->_rx); 11470 dev->_rx = NULL; 11471 return err; 11472 } 11473 11474 static void netif_free_rx_queues(struct net_device *dev) 11475 { 11476 unsigned int i, count = dev->num_rx_queues; 11477 11478 /* netif_alloc_rx_queues alloc failed, resources have been unreg'ed */ 11479 if (!dev->_rx) 11480 return; 11481 11482 for (i = 0; i < count; i++) 11483 xdp_rxq_info_unreg(&dev->_rx[i].xdp_rxq); 11484 11485 kvfree(dev->_rx); 11486 } 11487 11488 static void netdev_init_one_queue(struct net_device *dev, 11489 struct netdev_queue *queue, void *_unused) 11490 { 11491 /* Initialize queue lock */ 11492 spin_lock_init(&queue->_xmit_lock); 11493 netdev_set_xmit_lockdep_class(&queue->_xmit_lock, dev->type); 11494 queue->xmit_lock_owner = -1; 11495 netdev_queue_numa_node_write(queue, NUMA_NO_NODE); 11496 queue->dev = dev; 11497 #ifdef CONFIG_BQL 11498 dql_init(&queue->dql, HZ); 11499 #endif 11500 } 11501 11502 static void netif_free_tx_queues(struct net_device *dev) 11503 { 11504 kvfree(dev->_tx); 11505 } 11506 11507 static int netif_alloc_netdev_queues(struct net_device *dev) 11508 { 11509 unsigned int count = dev->num_tx_queues; 11510 struct netdev_queue *tx; 11511 size_t sz = count * sizeof(*tx); 11512 11513 if (count < 1 || count > 0xffff) 11514 return -EINVAL; 11515 11516 tx = kvzalloc(sz, GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL); 11517 if (!tx) 11518 return -ENOMEM; 11519 11520 dev->_tx = tx; 11521 11522 netdev_for_each_tx_queue(dev, netdev_init_one_queue, NULL); 11523 spin_lock_init(&dev->tx_global_lock); 11524 spin_lock_init(&dev->watchdog_lock); 11525 dev->watchdog_ref_held = false; 11526 return 0; 11527 } 11528 11529 void netif_tx_stop_all_queues(struct net_device *dev) 11530 { 11531 unsigned int i; 11532 11533 for (i = 0; i < dev->num_tx_queues; i++) { 11534 struct netdev_queue *txq = netdev_get_tx_queue(dev, i); 11535 11536 netif_tx_stop_queue(txq); 11537 } 11538 } 11539 EXPORT_SYMBOL(netif_tx_stop_all_queues); 11540 11541 static int netdev_do_alloc_pcpu_stats(struct net_device *dev) 11542 { 11543 void __percpu *v; 11544 11545 /* Drivers implementing ndo_get_peer_dev must support tstat 11546 * accounting, so that skb_do_redirect() can bump the dev's 11547 * RX stats upon network namespace switch. 11548 */ 11549 if (dev->netdev_ops->ndo_get_peer_dev && 11550 dev->pcpu_stat_type != NETDEV_PCPU_STAT_TSTATS) 11551 return -EOPNOTSUPP; 11552 11553 switch (dev->pcpu_stat_type) { 11554 case NETDEV_PCPU_STAT_NONE: 11555 return 0; 11556 case NETDEV_PCPU_STAT_LSTATS: 11557 v = dev->lstats = netdev_alloc_pcpu_stats(struct pcpu_lstats); 11558 break; 11559 case NETDEV_PCPU_STAT_TSTATS: 11560 v = dev->tstats = netdev_alloc_pcpu_stats(struct pcpu_sw_netstats); 11561 break; 11562 case NETDEV_PCPU_STAT_DSTATS: 11563 v = dev->dstats = netdev_alloc_pcpu_stats(struct pcpu_dstats); 11564 break; 11565 default: 11566 return -EINVAL; 11567 } 11568 11569 return v ? 0 : -ENOMEM; 11570 } 11571 11572 static void netdev_do_free_pcpu_stats(struct net_device *dev) 11573 { 11574 switch (dev->pcpu_stat_type) { 11575 case NETDEV_PCPU_STAT_NONE: 11576 return; 11577 case NETDEV_PCPU_STAT_LSTATS: 11578 free_percpu(dev->lstats); 11579 break; 11580 case NETDEV_PCPU_STAT_TSTATS: 11581 free_percpu(dev->tstats); 11582 break; 11583 case NETDEV_PCPU_STAT_DSTATS: 11584 free_percpu(dev->dstats); 11585 break; 11586 } 11587 } 11588 11589 static void netdev_free_phy_link_topology(struct net_device *dev) 11590 { 11591 struct phy_link_topology *topo = dev->link_topo; 11592 11593 if (IS_ENABLED(CONFIG_PHYLIB) && topo) { 11594 xa_destroy(&topo->phys); 11595 xa_destroy(&topo->ports); 11596 kfree(topo); 11597 dev->link_topo = NULL; 11598 } 11599 } 11600 11601 static int netdev_check_ops(struct net_device *dev) 11602 { 11603 const struct net_device_ops *ops = dev->netdev_ops; 11604 11605 if (((dev->hw_features | dev->features) & 11606 NETIF_F_HW_VLAN_CTAG_FILTER) && 11607 (!ops->ndo_vlan_rx_add_vid || !ops->ndo_vlan_rx_kill_vid)) { 11608 netdev_WARN(dev, "Buggy VLAN acceleration in driver!\n"); 11609 return -EINVAL; 11610 } 11611 11612 if (!ops->ndo_hwtstamp_get != !ops->ndo_hwtstamp_set) { 11613 netdev_WARN(dev, "driver implements only one hwtstamp NDO\n"); 11614 return -EINVAL; 11615 } 11616 11617 if (netdev_need_ops_lock(dev) && ops->ndo_set_rx_mode && 11618 !ops->ndo_set_rx_mode_async) { 11619 netdev_WARN(dev, "ops-locked drivers should use ndo_set_rx_mode_async\n"); 11620 return -EINVAL; 11621 } 11622 11623 return 0; 11624 } 11625 11626 /** 11627 * register_netdevice() - register a network device 11628 * @dev: device to register 11629 * 11630 * Take a prepared network device structure and make it externally accessible. 11631 * A %NETDEV_REGISTER message is sent to the netdev notifier chain. 11632 * Callers must hold the rtnl lock - you may want register_netdev() 11633 * instead of this. 11634 */ 11635 int register_netdevice(struct net_device *dev) 11636 { 11637 int ret; 11638 struct net *net = dev_net(dev); 11639 11640 BUILD_BUG_ON(sizeof(netdev_features_t) * BITS_PER_BYTE < 11641 NETDEV_FEATURE_COUNT); 11642 BUG_ON(dev_boot_phase); 11643 ASSERT_RTNL(); 11644 11645 might_sleep(); 11646 11647 /* When net_device's are persistent, this will be fatal. */ 11648 BUG_ON(dev->reg_state != NETREG_UNINITIALIZED); 11649 BUG_ON(!net); 11650 11651 ret = ethtool_check_ops(dev->ethtool_ops); 11652 if (ret) 11653 return ret; 11654 11655 /* rss ctx ID 0 is reserved for the default context, start from 1 */ 11656 xa_init_flags(&dev->ethtool->rss_ctx, XA_FLAGS_ALLOC1); 11657 mutex_init(&dev->ethtool->rss_lock); 11658 11659 spin_lock_init(&dev->addr_list_lock); 11660 netdev_set_addr_lockdep_class(dev); 11661 11662 ret = dev_get_valid_name(net, dev, dev->name); 11663 if (ret < 0) 11664 goto out; 11665 11666 ret = -ENOMEM; 11667 dev->name_node = netdev_name_node_head_alloc(dev); 11668 if (!dev->name_node) 11669 goto out; 11670 11671 /* Init, if this function is available */ 11672 if (dev->netdev_ops->ndo_init) { 11673 ret = dev->netdev_ops->ndo_init(dev); 11674 if (ret) { 11675 if (ret > 0) 11676 ret = -EIO; 11677 goto err_free_name; 11678 } 11679 } 11680 11681 ret = netdev_check_ops(dev); 11682 if (ret) 11683 goto err_uninit; 11684 11685 ret = netdev_do_alloc_pcpu_stats(dev); 11686 if (ret) 11687 goto err_uninit; 11688 11689 ret = dev_index_reserve(net, dev->ifindex); 11690 if (ret < 0) 11691 goto err_free_pcpu; 11692 dev->ifindex = ret; 11693 11694 /* Transfer changeable features to wanted_features and enable 11695 * software offloads (GSO and GRO). 11696 */ 11697 dev->hw_features |= (NETIF_F_SOFT_FEATURES | NETIF_F_SOFT_FEATURES_OFF); 11698 dev->features |= NETIF_F_SOFT_FEATURES; 11699 11700 if (dev->udp_tunnel_nic_info) { 11701 dev->features |= NETIF_F_RX_UDP_TUNNEL_PORT; 11702 dev->hw_features |= NETIF_F_RX_UDP_TUNNEL_PORT; 11703 } 11704 11705 dev->wanted_features = dev->features & dev->hw_features; 11706 11707 if (!(dev->flags & IFF_LOOPBACK)) 11708 dev->hw_features |= NETIF_F_NOCACHE_COPY; 11709 11710 /* If IPv4 TCP segmentation offload is supported we should also 11711 * allow the device to enable segmenting the frame with the option 11712 * of ignoring a static IP ID value. This doesn't enable the 11713 * feature itself but allows the user to enable it later. 11714 */ 11715 if (dev->hw_features & NETIF_F_TSO) 11716 dev->hw_features |= NETIF_F_TSO_MANGLEID; 11717 if (dev->vlan_features & NETIF_F_TSO) 11718 dev->vlan_features |= NETIF_F_TSO_MANGLEID; 11719 if (dev->mpls_features & NETIF_F_TSO) 11720 dev->mpls_features |= NETIF_F_TSO_MANGLEID; 11721 if (dev->hw_enc_features & NETIF_F_TSO) 11722 dev->hw_enc_features |= NETIF_F_TSO_MANGLEID; 11723 11724 /* TSO_MANGLEID belongs in mangleid_features by definition */ 11725 dev->mangleid_features |= NETIF_F_TSO_MANGLEID; 11726 11727 /* Make NETIF_F_HIGHDMA inheritable to VLAN devices. 11728 */ 11729 dev->vlan_features |= NETIF_F_HIGHDMA; 11730 11731 /* Make NETIF_F_SG inheritable to tunnel devices. 11732 */ 11733 dev->hw_enc_features |= NETIF_F_SG | NETIF_F_GSO_PARTIAL; 11734 11735 /* Make NETIF_F_SG inheritable to MPLS. 11736 */ 11737 dev->mpls_features |= NETIF_F_SG; 11738 11739 ret = call_netdevice_notifiers(NETDEV_POST_INIT, dev); 11740 ret = notifier_to_errno(ret); 11741 if (ret) 11742 goto err_ifindex_release; 11743 11744 ret = netdev_register_kobject(dev); 11745 11746 netdev_lock(dev); 11747 WRITE_ONCE(dev->reg_state, ret ? NETREG_UNREGISTERED : NETREG_REGISTERED); 11748 netdev_unlock(dev); 11749 11750 if (ret) 11751 goto err_uninit_notify; 11752 11753 netdev_lock_ops(dev); 11754 __netdev_update_features(dev); 11755 netdev_unlock_ops(dev); 11756 11757 /* 11758 * Default initial state at registry is that the 11759 * device is present. 11760 */ 11761 11762 set_bit(__LINK_STATE_PRESENT, &dev->state); 11763 11764 linkwatch_init_dev(dev); 11765 11766 dev_init_scheduler(dev); 11767 11768 netdev_hold(dev, &dev->dev_registered_tracker, GFP_KERNEL); 11769 list_netdevice(dev); 11770 11771 add_device_randomness(dev->dev_addr, dev->addr_len); 11772 11773 /* If the device has permanent device address, driver should 11774 * set dev_addr and also addr_assign_type should be set to 11775 * NET_ADDR_PERM (default value). 11776 */ 11777 if (dev->addr_assign_type == NET_ADDR_PERM) 11778 memcpy(dev->perm_addr, dev->dev_addr, dev->addr_len); 11779 11780 /* Notify protocols, that a new device appeared. */ 11781 netdev_lock_ops(dev); 11782 ret = call_netdevice_notifiers(NETDEV_REGISTER, dev); 11783 netdev_unlock_ops(dev); 11784 ret = notifier_to_errno(ret); 11785 if (ret) { 11786 /* Expect explicit free_netdev() on failure */ 11787 dev->needs_free_netdev = false; 11788 unregister_netdevice_queue(dev, NULL); 11789 goto out; 11790 } 11791 /* 11792 * Prevent userspace races by waiting until the network 11793 * device is fully setup before sending notifications. 11794 */ 11795 netdev_uevent_add(dev); 11796 if (!(dev->rtnl_link_ops && dev->rtnl_link_initializing)) 11797 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL, 0, NULL); 11798 11799 out: 11800 return ret; 11801 11802 err_uninit_notify: 11803 call_netdevice_notifiers(NETDEV_PRE_UNINIT, dev); 11804 err_ifindex_release: 11805 dev_index_release(net, dev->ifindex); 11806 err_free_pcpu: 11807 netdev_do_free_pcpu_stats(dev); 11808 err_uninit: 11809 if (dev->netdev_ops->ndo_uninit) 11810 dev->netdev_ops->ndo_uninit(dev); 11811 if (dev->priv_destructor) 11812 dev->priv_destructor(dev); 11813 err_free_name: 11814 netdev_name_node_free(dev->name_node); 11815 goto out; 11816 } 11817 EXPORT_SYMBOL(register_netdevice); 11818 11819 /* Initialize the core of a dummy net device. 11820 * The setup steps dummy netdevs need which normal netdevs get by going 11821 * through register_netdevice(). 11822 */ 11823 static void init_dummy_netdev(struct net_device *dev) 11824 { 11825 /* make sure we BUG if trying to hit standard 11826 * register/unregister code path 11827 */ 11828 dev->reg_state = NETREG_DUMMY; 11829 11830 /* a dummy interface is started by default */ 11831 set_bit(__LINK_STATE_PRESENT, &dev->state); 11832 set_bit(__LINK_STATE_START, &dev->state); 11833 11834 /* Note : We dont allocate pcpu_refcnt for dummy devices, 11835 * because users of this 'device' dont need to change 11836 * its refcount. 11837 */ 11838 } 11839 11840 /** 11841 * register_netdev - register a network device 11842 * @dev: device to register 11843 * 11844 * Take a completed network device structure and add it to the kernel 11845 * interfaces. A %NETDEV_REGISTER message is sent to the netdev notifier 11846 * chain. 0 is returned on success. A negative errno code is returned 11847 * on a failure to set up the device, or if the name is a duplicate. 11848 * 11849 * This is a wrapper around register_netdevice that takes the rtnl semaphore 11850 * and expands the device name if you passed a format string to 11851 * alloc_netdev. 11852 */ 11853 int register_netdev(struct net_device *dev) 11854 { 11855 struct net *net = dev_net(dev); 11856 int err; 11857 11858 if (rtnl_net_lock_killable(net)) 11859 return -EINTR; 11860 11861 err = register_netdevice(dev); 11862 11863 rtnl_net_unlock(net); 11864 11865 return err; 11866 } 11867 EXPORT_SYMBOL(register_netdev); 11868 11869 int netdev_refcnt_read(const struct net_device *dev) 11870 { 11871 #ifdef CONFIG_PCPU_DEV_REFCNT 11872 int i, refcnt = 0; 11873 11874 for_each_possible_cpu(i) 11875 refcnt += *per_cpu_ptr(dev->pcpu_refcnt, i); 11876 return refcnt; 11877 #else 11878 return refcount_read(&dev->dev_refcnt); 11879 #endif 11880 } 11881 EXPORT_SYMBOL(netdev_refcnt_read); 11882 11883 int netdev_unregister_timeout_secs __read_mostly = 10; 11884 11885 #define WAIT_REFS_MIN_MSECS 1 11886 #define WAIT_REFS_MAX_MSECS 250 11887 /** 11888 * netdev_wait_allrefs_any - wait until all references are gone. 11889 * @list: list of net_devices to wait on 11890 * 11891 * This is called when unregistering network devices. 11892 * 11893 * Any protocol or device that holds a reference should register 11894 * for netdevice notification, and cleanup and put back the 11895 * reference if they receive an UNREGISTER event. 11896 * We can get stuck here if buggy protocols don't correctly 11897 * call dev_put. 11898 */ 11899 static struct net_device *netdev_wait_allrefs_any(struct list_head *list) 11900 { 11901 unsigned long rebroadcast_time, warning_time; 11902 struct net_device *dev; 11903 int wait = 0; 11904 11905 rebroadcast_time = warning_time = jiffies; 11906 11907 list_for_each_entry(dev, list, todo_list) 11908 if (netdev_refcnt_read(dev) == 1) 11909 return dev; 11910 11911 while (true) { 11912 if (time_after(jiffies, rebroadcast_time + 1 * HZ)) { 11913 rtnl_lock(); 11914 11915 /* Rebroadcast unregister notification */ 11916 list_for_each_entry(dev, list, todo_list) { 11917 struct net *net = dev_net(dev); 11918 11919 __rtnl_net_lock(net); 11920 call_netdevice_notifiers(NETDEV_UNREGISTER, dev); 11921 __rtnl_net_unlock(net); 11922 } 11923 11924 __rtnl_unlock(); 11925 rcu_barrier(); 11926 rtnl_lock(); 11927 11928 list_for_each_entry(dev, list, todo_list) 11929 if (test_bit(__LINK_STATE_LINKWATCH_PENDING, 11930 &dev->state)) { 11931 /* We must not have linkwatch events 11932 * pending on unregister. If this 11933 * happens, we simply run the queue 11934 * unscheduled, resulting in a noop 11935 * for this device. 11936 */ 11937 linkwatch_run_queue(); 11938 break; 11939 } 11940 11941 __rtnl_unlock(); 11942 11943 rebroadcast_time = jiffies; 11944 } 11945 11946 rcu_barrier(); 11947 11948 if (!wait) { 11949 wait = WAIT_REFS_MIN_MSECS; 11950 } else { 11951 msleep(wait); 11952 wait = min(wait << 1, WAIT_REFS_MAX_MSECS); 11953 } 11954 11955 list_for_each_entry(dev, list, todo_list) 11956 if (netdev_refcnt_read(dev) == 1) 11957 return dev; 11958 11959 if (time_after(jiffies, warning_time + 11960 READ_ONCE(netdev_unregister_timeout_secs) * HZ)) { 11961 list_for_each_entry(dev, list, todo_list) { 11962 pr_emerg("unregister_netdevice: waiting for %s to become free. Usage count = %d\n", 11963 dev->name, netdev_refcnt_read(dev)); 11964 ref_tracker_dir_print(&dev->refcnt_tracker, 10); 11965 } 11966 11967 warning_time = jiffies; 11968 } 11969 } 11970 } 11971 11972 /* The sequence is: 11973 * 11974 * rtnl_lock(); 11975 * ... 11976 * register_netdevice(x1); 11977 * register_netdevice(x2); 11978 * ... 11979 * unregister_netdevice(y1); 11980 * unregister_netdevice(y2); 11981 * ... 11982 * rtnl_unlock(); 11983 * free_netdev(y1); 11984 * free_netdev(y2); 11985 * 11986 * We are invoked by rtnl_unlock(). 11987 * This allows us to deal with problems: 11988 * 1) We can delete sysfs objects which invoke hotplug 11989 * without deadlocking with linkwatch via keventd. 11990 * 2) Since we run with the RTNL semaphore not held, we can sleep 11991 * safely in order to wait for the netdev refcnt to drop to zero. 11992 * 11993 * We must not return until all unregister events added during 11994 * the interval the lock was held have been completed. 11995 */ 11996 void netdev_run_todo(void) 11997 { 11998 struct net_device *dev, *tmp; 11999 struct list_head list; 12000 int cnt; 12001 #ifdef CONFIG_LOCKDEP 12002 struct list_head unlink_list; 12003 12004 list_replace_init(&net_unlink_list, &unlink_list); 12005 12006 while (!list_empty(&unlink_list)) { 12007 dev = list_first_entry(&unlink_list, struct net_device, 12008 unlink_list); 12009 list_del_init(&dev->unlink_list); 12010 dev->nested_level = dev->lower_level - 1; 12011 } 12012 #endif 12013 12014 /* Snapshot list, allow later requests */ 12015 list_replace_init(&net_todo_list, &list); 12016 12017 __rtnl_unlock(); 12018 12019 /* Wait for rcu callbacks to finish before next phase */ 12020 if (!list_empty(&list)) 12021 rcu_barrier(); 12022 12023 list_for_each_entry_safe(dev, tmp, &list, todo_list) { 12024 if (unlikely(dev->reg_state != NETREG_UNREGISTERING)) { 12025 netdev_WARN(dev, "run_todo but not unregistering\n"); 12026 list_del(&dev->todo_list); 12027 net_unreg_done(dev_net(dev)); 12028 continue; 12029 } 12030 12031 netdev_lock(dev); 12032 WRITE_ONCE(dev->reg_state, NETREG_UNREGISTERED); 12033 netdev_unlock(dev); 12034 linkwatch_sync_dev(dev); 12035 } 12036 12037 if (!list_empty(&list)) { 12038 rt_flush_dev(NULL); 12039 rt6_uncached_list_flush_dev(NULL); 12040 } 12041 12042 cnt = 0; 12043 while (!list_empty(&list)) { 12044 struct net *net; 12045 12046 dev = netdev_wait_allrefs_any(&list); 12047 list_del(&dev->todo_list); 12048 net = dev_net(dev); 12049 12050 /* paranoia */ 12051 BUG_ON(netdev_refcnt_read(dev) != 1); 12052 BUG_ON(!list_empty(&dev->ptype_all)); 12053 BUG_ON(!list_empty(&dev->ptype_specific)); 12054 WARN_ON(rcu_access_pointer(dev->ip_ptr)); 12055 WARN_ON(rcu_access_pointer(dev->ip6_ptr)); 12056 12057 netdev_name_node_alt_flush(dev); 12058 netdev_name_node_free(dev->name_node); 12059 netdev_do_free_pcpu_stats(dev); 12060 if (dev->priv_destructor) 12061 dev->priv_destructor(dev); 12062 if (dev->needs_free_netdev) 12063 free_netdev(dev); 12064 12065 cnt++; 12066 12067 /* Free network device */ 12068 kobject_put(&dev->dev.kobj); 12069 12070 net_unreg_done(net); 12071 } 12072 if (cnt && atomic_sub_and_test(cnt, &dev_unreg_count)) 12073 wake_up(&netdev_unregistering_wq); 12074 } 12075 12076 /* Collate per-cpu network dstats statistics 12077 * 12078 * Read per-cpu network statistics from dev->dstats and populate the related 12079 * fields in @s. 12080 */ 12081 static void dev_fetch_dstats(struct rtnl_link_stats64 *s, 12082 const struct pcpu_dstats __percpu *dstats) 12083 { 12084 int cpu; 12085 12086 for_each_possible_cpu(cpu) { 12087 u64 rx_packets, rx_bytes, rx_drops; 12088 u64 tx_packets, tx_bytes, tx_drops; 12089 const struct pcpu_dstats *stats; 12090 unsigned int start; 12091 12092 stats = per_cpu_ptr(dstats, cpu); 12093 do { 12094 start = u64_stats_fetch_begin(&stats->syncp); 12095 rx_packets = u64_stats_read(&stats->rx_packets); 12096 rx_bytes = u64_stats_read(&stats->rx_bytes); 12097 rx_drops = u64_stats_read(&stats->rx_drops); 12098 tx_packets = u64_stats_read(&stats->tx_packets); 12099 tx_bytes = u64_stats_read(&stats->tx_bytes); 12100 tx_drops = u64_stats_read(&stats->tx_drops); 12101 } while (u64_stats_fetch_retry(&stats->syncp, start)); 12102 12103 s->rx_packets += rx_packets; 12104 s->rx_bytes += rx_bytes; 12105 s->rx_dropped += rx_drops; 12106 s->tx_packets += tx_packets; 12107 s->tx_bytes += tx_bytes; 12108 s->tx_dropped += tx_drops; 12109 } 12110 } 12111 12112 /* ndo_get_stats64 implementation for dtstats-based accounting. 12113 * 12114 * Populate @s from dev->stats and dev->dstats. This is used internally by the 12115 * core for NETDEV_PCPU_STAT_DSTAT-type stats collection. 12116 */ 12117 static void dev_get_dstats64(const struct net_device *dev, 12118 struct rtnl_link_stats64 *s) 12119 { 12120 netdev_stats_to_stats64(s, &dev->stats); 12121 dev_fetch_dstats(s, dev->dstats); 12122 } 12123 12124 /* Convert net_device_stats to rtnl_link_stats64. rtnl_link_stats64 has 12125 * all the same fields in the same order as net_device_stats, with only 12126 * the type differing, but rtnl_link_stats64 may have additional fields 12127 * at the end for newer counters. 12128 */ 12129 void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64, 12130 const struct net_device_stats *netdev_stats) 12131 { 12132 size_t i, n = sizeof(*netdev_stats) / sizeof(atomic_long_t); 12133 const atomic_long_t *src = (atomic_long_t *)netdev_stats; 12134 u64 *dst = (u64 *)stats64; 12135 12136 BUILD_BUG_ON(n > sizeof(*stats64) / sizeof(u64)); 12137 for (i = 0; i < n; i++) 12138 dst[i] = (unsigned long)atomic_long_read(&src[i]); 12139 /* zero out counters that only exist in rtnl_link_stats64 */ 12140 memset((char *)stats64 + n * sizeof(u64), 0, 12141 sizeof(*stats64) - n * sizeof(u64)); 12142 } 12143 EXPORT_SYMBOL(netdev_stats_to_stats64); 12144 12145 static __cold struct net_device_core_stats __percpu *netdev_core_stats_alloc( 12146 struct net_device *dev) 12147 { 12148 struct net_device_core_stats __percpu *p; 12149 12150 p = alloc_percpu_gfp(struct net_device_core_stats, 12151 GFP_ATOMIC | __GFP_NOWARN); 12152 12153 if (p && cmpxchg(&dev->core_stats, NULL, p)) 12154 free_percpu(p); 12155 12156 /* This READ_ONCE() pairs with the cmpxchg() above */ 12157 return READ_ONCE(dev->core_stats); 12158 } 12159 12160 noinline void netdev_core_stats_inc(struct net_device *dev, u32 offset) 12161 { 12162 /* This READ_ONCE() pairs with the write in netdev_core_stats_alloc() */ 12163 struct net_device_core_stats __percpu *p = READ_ONCE(dev->core_stats); 12164 unsigned long __percpu *field; 12165 12166 if (unlikely(!p)) { 12167 p = netdev_core_stats_alloc(dev); 12168 if (!p) 12169 return; 12170 } 12171 12172 field = (unsigned long __percpu *)((void __percpu *)p + offset); 12173 this_cpu_inc(*field); 12174 } 12175 EXPORT_SYMBOL_GPL(netdev_core_stats_inc); 12176 12177 /** 12178 * dev_get_stats - get network device statistics 12179 * @dev: device to get statistics from 12180 * @storage: place to store stats 12181 * 12182 * Get network statistics from device. Return @storage. 12183 * The device driver may provide its own method by setting 12184 * dev->netdev_ops->get_stats64 or dev->netdev_ops->get_stats; 12185 * otherwise the internal statistics structure is used. 12186 */ 12187 struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev, 12188 struct rtnl_link_stats64 *storage) 12189 { 12190 const struct net_device_ops *ops = dev->netdev_ops; 12191 const struct net_device_core_stats __percpu *p; 12192 12193 /* 12194 * IPv{4,6} and udp tunnels share common stat helpers and use 12195 * different stat type (NETDEV_PCPU_STAT_TSTATS vs 12196 * NETDEV_PCPU_STAT_DSTATS). Ensure the accounting is consistent. 12197 */ 12198 BUILD_BUG_ON(offsetof(struct pcpu_sw_netstats, rx_bytes) != 12199 offsetof(struct pcpu_dstats, rx_bytes)); 12200 BUILD_BUG_ON(offsetof(struct pcpu_sw_netstats, rx_packets) != 12201 offsetof(struct pcpu_dstats, rx_packets)); 12202 BUILD_BUG_ON(offsetof(struct pcpu_sw_netstats, tx_bytes) != 12203 offsetof(struct pcpu_dstats, tx_bytes)); 12204 BUILD_BUG_ON(offsetof(struct pcpu_sw_netstats, tx_packets) != 12205 offsetof(struct pcpu_dstats, tx_packets)); 12206 12207 if (ops->ndo_get_stats64) { 12208 memset(storage, 0, sizeof(*storage)); 12209 ops->ndo_get_stats64(dev, storage); 12210 } else if (ops->ndo_get_stats) { 12211 netdev_stats_to_stats64(storage, ops->ndo_get_stats(dev)); 12212 } else if (dev->pcpu_stat_type == NETDEV_PCPU_STAT_TSTATS) { 12213 dev_get_tstats64(dev, storage); 12214 } else if (dev->pcpu_stat_type == NETDEV_PCPU_STAT_DSTATS) { 12215 dev_get_dstats64(dev, storage); 12216 } else { 12217 netdev_stats_to_stats64(storage, &dev->stats); 12218 } 12219 12220 /* This READ_ONCE() pairs with the write in netdev_core_stats_alloc() */ 12221 p = READ_ONCE(dev->core_stats); 12222 if (p) { 12223 const struct net_device_core_stats *core_stats; 12224 int i; 12225 12226 for_each_possible_cpu(i) { 12227 core_stats = per_cpu_ptr(p, i); 12228 storage->rx_dropped += READ_ONCE(core_stats->rx_dropped); 12229 storage->tx_dropped += READ_ONCE(core_stats->tx_dropped); 12230 storage->rx_nohandler += READ_ONCE(core_stats->rx_nohandler); 12231 storage->rx_otherhost_dropped += READ_ONCE(core_stats->rx_otherhost_dropped); 12232 } 12233 } 12234 return storage; 12235 } 12236 EXPORT_SYMBOL(dev_get_stats); 12237 12238 /** 12239 * dev_fetch_sw_netstats - get per-cpu network device statistics 12240 * @s: place to store stats 12241 * @netstats: per-cpu network stats to read from 12242 * 12243 * Read per-cpu network statistics and populate the related fields in @s. 12244 */ 12245 void dev_fetch_sw_netstats(struct rtnl_link_stats64 *s, 12246 const struct pcpu_sw_netstats __percpu *netstats) 12247 { 12248 int cpu; 12249 12250 for_each_possible_cpu(cpu) { 12251 u64 rx_packets, rx_bytes, tx_packets, tx_bytes; 12252 const struct pcpu_sw_netstats *stats; 12253 unsigned int start; 12254 12255 stats = per_cpu_ptr(netstats, cpu); 12256 do { 12257 start = u64_stats_fetch_begin(&stats->syncp); 12258 rx_packets = u64_stats_read(&stats->rx_packets); 12259 rx_bytes = u64_stats_read(&stats->rx_bytes); 12260 tx_packets = u64_stats_read(&stats->tx_packets); 12261 tx_bytes = u64_stats_read(&stats->tx_bytes); 12262 } while (u64_stats_fetch_retry(&stats->syncp, start)); 12263 12264 s->rx_packets += rx_packets; 12265 s->rx_bytes += rx_bytes; 12266 s->tx_packets += tx_packets; 12267 s->tx_bytes += tx_bytes; 12268 } 12269 } 12270 EXPORT_SYMBOL_GPL(dev_fetch_sw_netstats); 12271 12272 /** 12273 * dev_get_tstats64 - ndo_get_stats64 implementation 12274 * @dev: device to get statistics from 12275 * @s: place to store stats 12276 * 12277 * Populate @s from dev->stats and dev->tstats. Can be used as 12278 * ndo_get_stats64() callback. 12279 */ 12280 void dev_get_tstats64(struct net_device *dev, struct rtnl_link_stats64 *s) 12281 { 12282 netdev_stats_to_stats64(s, &dev->stats); 12283 dev_fetch_sw_netstats(s, dev->tstats); 12284 } 12285 EXPORT_SYMBOL_GPL(dev_get_tstats64); 12286 12287 struct netdev_queue *dev_ingress_queue_create(struct net_device *dev) 12288 { 12289 struct netdev_queue *queue = dev_ingress_queue(dev); 12290 12291 #ifdef CONFIG_NET_CLS_ACT 12292 if (queue) 12293 return queue; 12294 queue = kzalloc_obj(*queue); 12295 if (!queue) 12296 return NULL; 12297 netdev_init_one_queue(dev, queue, NULL); 12298 RCU_INIT_POINTER(queue->qdisc, &noop_qdisc); 12299 RCU_INIT_POINTER(queue->qdisc_sleeping, &noop_qdisc); 12300 rcu_assign_pointer(dev->ingress_queue, queue); 12301 #endif 12302 return queue; 12303 } 12304 12305 static const struct ethtool_ops default_ethtool_ops; 12306 12307 void netdev_set_default_ethtool_ops(struct net_device *dev, 12308 const struct ethtool_ops *ops) 12309 { 12310 if (dev->ethtool_ops == &default_ethtool_ops) 12311 dev->ethtool_ops = ops; 12312 } 12313 EXPORT_SYMBOL_GPL(netdev_set_default_ethtool_ops); 12314 12315 /** 12316 * netdev_sw_irq_coalesce_default_on() - enable SW IRQ coalescing by default 12317 * @dev: netdev to enable the IRQ coalescing on 12318 * 12319 * Sets a conservative default for SW IRQ coalescing. Users can use 12320 * sysfs attributes to override the default values. 12321 */ 12322 void netdev_sw_irq_coalesce_default_on(struct net_device *dev) 12323 { 12324 WARN_ON(dev->reg_state == NETREG_REGISTERED); 12325 12326 if (!IS_ENABLED(CONFIG_PREEMPT_RT)) { 12327 netdev_set_gro_flush_timeout(dev, 20000); 12328 netdev_set_defer_hard_irqs(dev, 1); 12329 } 12330 } 12331 EXPORT_SYMBOL_GPL(netdev_sw_irq_coalesce_default_on); 12332 12333 /** 12334 * alloc_netdev_mqs - allocate network device 12335 * @sizeof_priv: size of private data to allocate space for 12336 * @name: device name format string 12337 * @name_assign_type: origin of device name 12338 * @setup: callback to initialize device 12339 * @txqs: the number of TX subqueues to allocate 12340 * @rxqs: the number of RX subqueues to allocate 12341 * 12342 * Allocates a struct net_device with private data area for driver use 12343 * and performs basic initialization. Also allocates subqueue structs 12344 * for each queue on the device. 12345 */ 12346 struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name, 12347 unsigned char name_assign_type, 12348 void (*setup)(struct net_device *), 12349 unsigned int txqs, unsigned int rxqs) 12350 { 12351 struct net_device *dev; 12352 size_t napi_config_sz; 12353 unsigned int maxqs; 12354 12355 BUG_ON(strlen(name) >= sizeof(dev->name)); 12356 12357 if (txqs < 1) { 12358 pr_err("alloc_netdev: Unable to allocate device with zero queues\n"); 12359 return NULL; 12360 } 12361 12362 if (rxqs < 1) { 12363 pr_err("alloc_netdev: Unable to allocate device with zero RX queues\n"); 12364 return NULL; 12365 } 12366 12367 maxqs = max(txqs, rxqs); 12368 12369 dev = kvzalloc_flex(*dev, priv, sizeof_priv, 12370 GFP_KERNEL_ACCOUNT | __GFP_RETRY_MAYFAIL); 12371 if (!dev) 12372 return NULL; 12373 12374 dev->priv_len = sizeof_priv; 12375 12376 ref_tracker_dir_init(&dev->refcnt_tracker, 128, "netdev"); 12377 #ifdef CONFIG_PCPU_DEV_REFCNT 12378 dev->pcpu_refcnt = alloc_percpu(int); 12379 if (!dev->pcpu_refcnt) 12380 goto free_dev; 12381 __dev_hold(dev); 12382 #else 12383 refcount_set(&dev->dev_refcnt, 1); 12384 #endif 12385 12386 if (dev_addr_init(dev)) 12387 goto free_pcpu; 12388 12389 dev_mc_init(dev); 12390 dev_uc_init(dev); 12391 12392 dev_net_set(dev, &init_net); 12393 12394 dev->gso_max_size = GSO_LEGACY_MAX_SIZE; 12395 dev->xdp_zc_max_segs = 1; 12396 dev->gso_max_segs = GSO_MAX_SEGS; 12397 dev->gro_max_size = GRO_LEGACY_MAX_SIZE; 12398 dev->gso_ipv4_max_size = GSO_LEGACY_MAX_SIZE; 12399 dev->gro_ipv4_max_size = GRO_LEGACY_MAX_SIZE; 12400 dev->tso_max_size = TSO_LEGACY_MAX_SIZE; 12401 dev->tso_max_segs = TSO_MAX_SEGS; 12402 dev->upper_level = 1; 12403 dev->lower_level = 1; 12404 #ifdef CONFIG_LOCKDEP 12405 dev->nested_level = 0; 12406 INIT_LIST_HEAD(&dev->unlink_list); 12407 #endif 12408 12409 INIT_LIST_HEAD(&dev->napi_list); 12410 INIT_LIST_HEAD(&dev->unreg_list); 12411 #ifdef CONFIG_DEBUG_NET_SMALL_RTNL 12412 INIT_LIST_HEAD(&dev->unreg_list_net); 12413 #endif 12414 INIT_LIST_HEAD(&dev->close_list); 12415 INIT_LIST_HEAD(&dev->link_watch_list); 12416 INIT_LIST_HEAD(&dev->adj_list.upper); 12417 INIT_LIST_HEAD(&dev->adj_list.lower); 12418 INIT_LIST_HEAD(&dev->ptype_all); 12419 INIT_LIST_HEAD(&dev->ptype_specific); 12420 INIT_LIST_HEAD(&dev->net_notifier_list); 12421 INIT_LIST_HEAD(&dev->work_node); 12422 #ifdef CONFIG_NET_SCHED 12423 hash_init(dev->qdisc_hash); 12424 #endif 12425 12426 mutex_init(&dev->lock); 12427 /* see also netdev_set_instance_lock_class() */ 12428 lock_set_cmp_fn(&dev->lock, netdev_lock_cmp_fn, NULL); 12429 netif_rx_mode_init(dev); 12430 12431 dev->priv_flags = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM; 12432 setup(dev); 12433 12434 if (!dev->tx_queue_len) { 12435 dev->priv_flags |= IFF_NO_QUEUE; 12436 dev->tx_queue_len = DEFAULT_TX_QUEUE_LEN; 12437 } 12438 12439 dev->num_tx_queues = txqs; 12440 dev->real_num_tx_queues = txqs; 12441 if (netif_alloc_netdev_queues(dev)) 12442 goto free_all; 12443 12444 dev->num_rx_queues = rxqs; 12445 dev->real_num_rx_queues = rxqs; 12446 if (netif_alloc_rx_queues(dev)) 12447 goto free_all; 12448 dev->ethtool = kzalloc_obj(*dev->ethtool, GFP_KERNEL_ACCOUNT); 12449 if (!dev->ethtool) 12450 goto free_all; 12451 12452 if (netdev_alloc_config(dev)) 12453 goto free_all; 12454 12455 dev->num_napi_configs = maxqs; 12456 napi_config_sz = array_size(maxqs, sizeof(*dev->napi_config)); 12457 dev->napi_config = kvzalloc(napi_config_sz, GFP_KERNEL_ACCOUNT); 12458 if (!dev->napi_config) 12459 goto free_all; 12460 12461 strscpy(dev->name, name); 12462 dev->name_assign_type = name_assign_type; 12463 dev->group = INIT_NETDEV_GROUP; 12464 if (!dev->ethtool_ops) 12465 dev->ethtool_ops = &default_ethtool_ops; 12466 12467 nf_hook_netdev_init(dev); 12468 12469 return dev; 12470 12471 free_all: 12472 free_netdev(dev); 12473 return NULL; 12474 12475 free_pcpu: 12476 #ifdef CONFIG_PCPU_DEV_REFCNT 12477 free_percpu(dev->pcpu_refcnt); 12478 free_dev: 12479 #endif 12480 ref_tracker_dir_exit(&dev->refcnt_tracker); 12481 kvfree(dev); 12482 return NULL; 12483 } 12484 EXPORT_SYMBOL(alloc_netdev_mqs); 12485 12486 static void netdev_napi_exit(struct net_device *dev) 12487 { 12488 if (!list_empty(&dev->napi_list)) { 12489 struct napi_struct *p, *n; 12490 12491 netdev_lock(dev); 12492 list_for_each_entry_safe(p, n, &dev->napi_list, dev_list) 12493 __netif_napi_del_locked(p); 12494 netdev_unlock(dev); 12495 12496 synchronize_net(); 12497 } 12498 12499 kvfree(dev->napi_config); 12500 } 12501 12502 /** 12503 * free_netdev - free network device 12504 * @dev: device 12505 * 12506 * This function does the last stage of destroying an allocated device 12507 * interface. The reference to the device object is released. If this 12508 * is the last reference then it will be freed.Must be called in process 12509 * context. 12510 */ 12511 void free_netdev(struct net_device *dev) 12512 { 12513 might_sleep(); 12514 12515 /* When called immediately after register_netdevice() failed the unwind 12516 * handling may still be dismantling the device. Handle that case by 12517 * deferring the free. 12518 */ 12519 if (dev->reg_state == NETREG_UNREGISTERING) { 12520 ASSERT_RTNL(); 12521 dev->needs_free_netdev = true; 12522 return; 12523 } 12524 12525 netdev_free_config(dev); 12526 kfree(dev->ethtool); 12527 netif_free_tx_queues(dev); 12528 netif_free_rx_queues(dev); 12529 12530 kfree(rcu_dereference_protected(dev->ingress_queue, 1)); 12531 12532 __hw_addr_flush(&dev->rx_mode_addr_cache); 12533 12534 /* Flush device addresses */ 12535 dev_addr_flush(dev); 12536 12537 netdev_napi_exit(dev); 12538 12539 netif_del_cpu_rmap(dev); 12540 12541 ref_tracker_dir_exit(&dev->refcnt_tracker); 12542 #ifdef CONFIG_PCPU_DEV_REFCNT 12543 free_percpu(dev->pcpu_refcnt); 12544 dev->pcpu_refcnt = NULL; 12545 #endif 12546 free_percpu(dev->core_stats); 12547 dev->core_stats = NULL; 12548 free_percpu(dev->xdp_bulkq); 12549 dev->xdp_bulkq = NULL; 12550 12551 netdev_free_phy_link_topology(dev); 12552 12553 mutex_destroy(&dev->lock); 12554 12555 /* Compatibility with error handling in drivers */ 12556 if (dev->reg_state == NETREG_UNINITIALIZED || 12557 dev->reg_state == NETREG_DUMMY) { 12558 kvfree(dev); 12559 return; 12560 } 12561 12562 BUG_ON(dev->reg_state != NETREG_UNREGISTERED); 12563 WRITE_ONCE(dev->reg_state, NETREG_RELEASED); 12564 12565 /* will free via device release */ 12566 put_device(&dev->dev); 12567 } 12568 EXPORT_SYMBOL(free_netdev); 12569 12570 /** 12571 * alloc_netdev_dummy - Allocate and initialize a dummy net device. 12572 * @sizeof_priv: size of private data to allocate space for 12573 * 12574 * Return: the allocated net_device on success, NULL otherwise 12575 */ 12576 struct net_device *alloc_netdev_dummy(int sizeof_priv) 12577 { 12578 return alloc_netdev(sizeof_priv, "dummy#", NET_NAME_UNKNOWN, 12579 init_dummy_netdev); 12580 } 12581 EXPORT_SYMBOL_GPL(alloc_netdev_dummy); 12582 12583 /** 12584 * synchronize_net - Synchronize with packet receive processing 12585 * 12586 * Wait for packets currently being received to be done. 12587 * Does not block later packets from starting. 12588 */ 12589 void synchronize_net(void) 12590 { 12591 might_sleep(); 12592 if (from_cleanup_net() || rtnl_is_locked()) 12593 synchronize_rcu_expedited(); 12594 else 12595 synchronize_rcu(); 12596 } 12597 EXPORT_SYMBOL(synchronize_net); 12598 12599 static void netdev_rss_contexts_free(struct net_device *dev) 12600 { 12601 struct ethtool_rxfh_context *ctx; 12602 unsigned long context; 12603 12604 mutex_lock(&dev->ethtool->rss_lock); 12605 xa_for_each(&dev->ethtool->rss_ctx, context, ctx) { 12606 xa_erase(&dev->ethtool->rss_ctx, context); 12607 dev->ethtool_ops->remove_rxfh_context(dev, ctx, context, NULL); 12608 kfree(ctx); 12609 } 12610 xa_destroy(&dev->ethtool->rss_ctx); 12611 mutex_unlock(&dev->ethtool->rss_lock); 12612 } 12613 12614 /** 12615 * unregister_netdevice_queue - remove device from the kernel 12616 * @dev: device 12617 * @head: list 12618 * 12619 * This function shuts down a device interface and removes it 12620 * from the kernel tables. 12621 * If head not NULL, device is queued to be unregistered later. 12622 * 12623 * Callers must hold the rtnl semaphore. You may want 12624 * unregister_netdev() instead of this. 12625 */ 12626 12627 void unregister_netdevice_queue(struct net_device *dev, struct list_head *head) 12628 { 12629 ASSERT_RTNL(); 12630 12631 #ifdef CONFIG_DEBUG_NET_SMALL_RTNL 12632 DEBUG_NET_WARN_ON_ONCE(!list_empty(&dev->unreg_list_net)); 12633 #endif 12634 12635 if (head) { 12636 list_move_tail(&dev->unreg_list, head); 12637 } else { 12638 LIST_HEAD(single); 12639 12640 list_add(&dev->unreg_list, &single); 12641 unregister_netdevice_many(&single); 12642 } 12643 } 12644 EXPORT_SYMBOL(unregister_netdevice_queue); 12645 12646 static void dev_memory_provider_uninstall(struct net_device *dev) 12647 { 12648 unsigned int i; 12649 12650 for (i = 0; i < dev->real_num_rx_queues; i++) { 12651 struct netdev_rx_queue *rxq = &dev->_rx[i]; 12652 12653 __netif_mp_uninstall_rxq(rxq, &rxq->mp_params); 12654 } 12655 } 12656 12657 /* devices must be UP and netdev_lock()'d */ 12658 static void netif_close_many_and_unlock(struct list_head *close_head) 12659 { 12660 struct net_device *dev, *tmp; 12661 12662 netif_close_many(close_head, false); 12663 12664 /* ... now unlock them */ 12665 list_for_each_entry_safe(dev, tmp, close_head, close_list) { 12666 netdev_unlock(dev); 12667 list_del_init(&dev->close_list); 12668 } 12669 } 12670 12671 /* Handle one class of ops-locked devices. Since close requires the lock 12672 * we need to be careful about which classes we allow to nest. 12673 */ 12674 static void netdev_lock_ops_close_many(struct list_head *head, 12675 struct list_head *close_head, 12676 bool leasing) 12677 { 12678 struct net_device *dev; 12679 12680 list_for_each_entry(dev, head, unreg_list) { 12681 if (!(dev->flags & IFF_UP) || !netdev_need_ops_lock(dev) || 12682 netdev_can_create_queue(dev, NULL) != leasing) 12683 continue; 12684 list_add_tail(&dev->close_list, close_head); 12685 netdev_lock(dev); 12686 12687 #ifdef CONFIG_LOCKDEP 12688 /* We can only track up to MAX_LOCK_DEPTH locks per task. 12689 * Reserve half the available slots for additional locks 12690 * possibly taken by notifiers and (soft)irqs. 12691 */ 12692 if (lockdep_depth(current) > MAX_LOCK_DEPTH / 2) 12693 netif_close_many_and_unlock(close_head); 12694 #endif 12695 } 12696 } 12697 12698 bool unregister_netdevice_queued(const struct net_device *dev) 12699 { 12700 ASSERT_RTNL(); 12701 return !list_empty(&dev->unreg_list); 12702 } 12703 12704 void unregister_netdevice_many_notify(struct list_head *head, 12705 u32 portid, const struct nlmsghdr *nlh) 12706 { 12707 struct net_device *dev, *tmp; 12708 LIST_HEAD(close_head); 12709 int cnt = 0; 12710 12711 BUG_ON(dev_boot_phase); 12712 ASSERT_RTNL(); 12713 12714 if (list_empty(head)) 12715 return; 12716 12717 list_for_each_entry_safe(dev, tmp, head, unreg_list) { 12718 /* Some devices call without registering 12719 * for initialization unwind. Remove those 12720 * devices and proceed with the remaining. 12721 */ 12722 if (dev->reg_state == NETREG_UNINITIALIZED) { 12723 pr_debug("unregister_netdevice: device %s/%p never was registered\n", 12724 dev->name, dev); 12725 12726 WARN_ON(1); 12727 list_del(&dev->unreg_list); 12728 continue; 12729 } 12730 dev->dismantle = true; 12731 BUG_ON(dev->reg_state != NETREG_REGISTERED); 12732 } 12733 12734 /* If device is running, close it first. Start with ops locked... */ 12735 netdev_lock_ops_close_many(head, &close_head, true); /* queue leasing */ 12736 netdev_lock_ops_close_many(head, &close_head, false); /* the rest */ 12737 netif_close_many_and_unlock(&close_head); 12738 /* ... now go over the rest. */ 12739 list_for_each_entry(dev, head, unreg_list) { 12740 if (!netdev_need_ops_lock(dev)) 12741 list_add_tail(&dev->close_list, &close_head); 12742 } 12743 netif_close_many(&close_head, true); 12744 12745 list_for_each_entry(dev, head, unreg_list) { 12746 /* And unlink it from device chain. */ 12747 unlist_netdevice(dev); 12748 netdev_lock(dev); 12749 WRITE_ONCE(dev->reg_state, NETREG_UNREGISTERING); 12750 netdev_unlock(dev); 12751 } 12752 flush_all_backlogs(); 12753 12754 synchronize_net(); 12755 12756 list_for_each_entry(dev, head, unreg_list) { 12757 struct sk_buff *skb = NULL; 12758 12759 /* Shutdown queueing discipline. */ 12760 netdev_lock_ops(dev); 12761 dev_shutdown(dev); 12762 dev_tcx_uninstall(dev); 12763 dev_xdp_uninstall(dev); 12764 dev_memory_provider_uninstall(dev); 12765 netdev_work_cancel_all(dev); 12766 netdev_unlock_ops(dev); 12767 bpf_dev_bound_netdev_unregister(dev); 12768 12769 netdev_offload_xstats_disable_all(dev); 12770 12771 /* Notify protocols, that we are about to destroy 12772 * this device. They should clean all the things. 12773 */ 12774 call_netdevice_notifiers(NETDEV_UNREGISTER, dev); 12775 12776 if (!(dev->rtnl_link_ops && dev->rtnl_link_initializing)) 12777 skb = rtmsg_ifinfo_build_skb(RTM_DELLINK, dev, ~0U, 0, 12778 GFP_KERNEL, NULL, 0, 12779 portid, nlh); 12780 12781 /* 12782 * Flush the unicast and multicast chains 12783 */ 12784 dev_uc_flush(dev); 12785 dev_mc_flush(dev); 12786 12787 12788 netdev_rss_contexts_free(dev); 12789 12790 call_netdevice_notifiers(NETDEV_PRE_UNINIT, dev); 12791 12792 if (dev->netdev_ops->ndo_uninit) 12793 dev->netdev_ops->ndo_uninit(dev); 12794 12795 mutex_destroy(&dev->ethtool->rss_lock); 12796 12797 net_shaper_flush_netdev(dev); 12798 12799 if (skb) 12800 rtmsg_ifinfo_send(skb, dev, GFP_KERNEL, portid, nlh); 12801 12802 /* Notifier chain MUST detach us all upper devices. */ 12803 WARN_ON(netdev_has_any_upper_dev(dev)); 12804 WARN_ON(netdev_has_any_lower_dev(dev)); 12805 12806 /* Remove entries from kobject tree */ 12807 netdev_unregister_kobject(dev); 12808 #ifdef CONFIG_XPS 12809 /* Remove XPS queueing entries */ 12810 netif_reset_xps_queues_gt(dev, 0); 12811 #endif 12812 } 12813 12814 synchronize_net(); 12815 12816 list_for_each_entry(dev, head, unreg_list) { 12817 #ifdef CONFIG_DEBUG_NET_SMALL_RTNL 12818 struct net *net = dev_net(dev); 12819 12820 /* spin_lock() can be moved outside of the loop 12821 * once the per-netns RTNL conversion completes. 12822 */ 12823 spin_lock(&net->dev_unreg_lock); 12824 list_del(&dev->unreg_list_net); 12825 spin_unlock(&net->dev_unreg_lock); 12826 #endif 12827 netdev_put(dev, &dev->dev_registered_tracker); 12828 net_set_todo(dev); 12829 cnt++; 12830 } 12831 atomic_add(cnt, &dev_unreg_count); 12832 12833 list_del(head); 12834 } 12835 12836 /** 12837 * unregister_netdevice_many - unregister many devices 12838 * @head: list of devices 12839 * 12840 * Note: As most callers use a stack allocated list_head, 12841 * we force a list_del() to make sure stack won't be corrupted later. 12842 */ 12843 void unregister_netdevice_many(struct list_head *head) 12844 { 12845 unregister_netdevice_many_notify(head, 0, NULL); 12846 } 12847 EXPORT_SYMBOL(unregister_netdevice_many); 12848 12849 #ifdef CONFIG_DEBUG_NET_SMALL_RTNL 12850 void unregister_netdevice_queue_net(struct net *net, struct net_device *dev, 12851 struct list_head *head) 12852 { 12853 netdev_lock(dev); 12854 12855 if (net_eq(dev_net(dev), net)) { 12856 netdev_unlock(dev); 12857 unregister_netdevice_queue(dev, head); 12858 return; 12859 } 12860 12861 net = dev_net(dev); 12862 12863 spin_lock(&net->dev_unreg_lock); 12864 12865 DEBUG_NET_WARN_ON_ONCE(!list_empty(&dev->unreg_list)); 12866 DEBUG_NET_WARN_ON_ONCE(!list_empty(&dev->unreg_list_net)); 12867 12868 list_add_tail(&dev->unreg_list_net, &net->dev_unreg_head); 12869 rtnl_net_queue_work(net); 12870 12871 spin_unlock(&net->dev_unreg_lock); 12872 12873 netdev_unlock(dev); 12874 } 12875 EXPORT_SYMBOL(unregister_netdevice_queue_net); 12876 12877 void unregister_netdevice_queue_many_net(struct net *net, struct list_head *head) 12878 { 12879 struct net_device *dev, *tmp; 12880 12881 spin_lock(&net->dev_unreg_lock); 12882 list_for_each_entry_safe(dev, tmp, head, unreg_list) { 12883 /* Once all cross-netns unregister_netdevice_queue() is 12884 * converted to _net() (or for debugging), remove this check. 12885 */ 12886 if (!net_eq(dev_net(dev), net)) 12887 continue; 12888 12889 DEBUG_NET_WARN_ONCE(!net_eq(dev_net(dev), net), 12890 "%s was unregistered from a different netns.\n", 12891 dev->name); 12892 12893 list_del_init(&dev->unreg_list); 12894 list_move_tail(&dev->unreg_list_net, &net->dev_unreg_head); 12895 } 12896 spin_unlock(&net->dev_unreg_lock); 12897 } 12898 12899 static void unregister_netdevice_move_net(struct net *net_old, 12900 struct net *net, 12901 struct net_device *dev) 12902 { 12903 if (net_old > net) { 12904 spin_lock(&net->dev_unreg_lock); 12905 spin_lock_nested(&net_old->dev_unreg_lock, SINGLE_DEPTH_NESTING); 12906 } else { 12907 spin_lock(&net_old->dev_unreg_lock); 12908 spin_lock_nested(&net->dev_unreg_lock, SINGLE_DEPTH_NESTING); 12909 } 12910 12911 if (!list_empty(&dev->unreg_list_net)) { 12912 list_del(&dev->unreg_list_net); 12913 list_add_tail(&dev->unreg_list_net, &net->dev_unreg_head); 12914 } 12915 12916 spin_unlock(&net_old->dev_unreg_lock); 12917 spin_unlock(&net->dev_unreg_lock); 12918 } 12919 12920 void unregister_netdevice_many_net(struct net *net) 12921 { 12922 struct net_device *dev, *tmp; 12923 LIST_HEAD(unreg_head_net); 12924 LIST_HEAD(unreg_head); 12925 12926 spin_lock(&net->dev_unreg_lock); 12927 list_splice_init(&net->dev_unreg_head, &unreg_head_net); 12928 spin_unlock(&net->dev_unreg_lock); 12929 12930 list_for_each_entry_safe(dev, tmp, &unreg_head_net, unreg_list_net) { 12931 list_del_init(&dev->unreg_list_net); 12932 list_add_tail(&dev->unreg_list, &unreg_head); 12933 } 12934 12935 unregister_netdevice_many(&unreg_head); 12936 } 12937 #endif 12938 12939 /** 12940 * unregister_netdev - remove device from the kernel 12941 * @dev: device 12942 * 12943 * This function shuts down a device interface and removes it 12944 * from the kernel tables. 12945 * 12946 * This is just a wrapper for unregister_netdevice that takes 12947 * the rtnl semaphore. In general you want to use this and not 12948 * unregister_netdevice. 12949 */ 12950 void unregister_netdev(struct net_device *dev) 12951 { 12952 rtnl_net_dev_lock(dev); 12953 unregister_netdevice(dev); 12954 rtnl_net_dev_unlock(dev); 12955 } 12956 EXPORT_SYMBOL(unregister_netdev); 12957 12958 int __dev_change_net_namespace(struct net_device *dev, struct net *net, 12959 const char *pat, int new_ifindex, 12960 struct netlink_ext_ack *extack) 12961 { 12962 struct netdev_name_node *name_node, *tmp; 12963 struct net *net_old = dev_net(dev); 12964 char new_name[IFNAMSIZ] = {}; 12965 int err, new_nsid; 12966 12967 ASSERT_RTNL(); 12968 12969 /* Don't allow namespace local devices to be moved. */ 12970 err = -EINVAL; 12971 if (dev->netns_immutable) { 12972 NL_SET_ERR_MSG(extack, "The interface netns is immutable"); 12973 goto out; 12974 } 12975 12976 /* Ensure the device has been registered */ 12977 if (dev->reg_state != NETREG_REGISTERED) { 12978 NL_SET_ERR_MSG(extack, "The interface isn't registered"); 12979 goto out; 12980 } 12981 12982 /* Get out if there is nothing todo */ 12983 err = 0; 12984 if (net_eq(net_old, net)) 12985 goto out; 12986 12987 /* Pick the destination device name, and ensure 12988 * we can use it in the destination network namespace. 12989 */ 12990 err = -EEXIST; 12991 if (netdev_name_in_use(net, dev->name)) { 12992 /* We get here if we can't use the current device name */ 12993 if (!pat) { 12994 NL_SET_ERR_MSG(extack, 12995 "An interface with the same name exists in the target netns"); 12996 goto out; 12997 } 12998 err = dev_prep_valid_name(net, dev, pat, new_name, EEXIST); 12999 if (err < 0) { 13000 NL_SET_ERR_MSG_FMT(extack, 13001 "Unable to use '%s' for the new interface name in the target netns", 13002 pat); 13003 goto out; 13004 } 13005 } 13006 /* Check that none of the altnames conflicts. */ 13007 err = -EEXIST; 13008 netdev_for_each_altname_safe(dev, name_node, tmp) { 13009 if (!netdev_name_in_use(net, name_node->name)) 13010 continue; 13011 13012 if (!check_net(net_old)) { 13013 __netdev_name_node_alt_destroy(name_node); 13014 continue; 13015 } 13016 13017 NL_SET_ERR_MSG_FMT(extack, 13018 "An interface with the altname %s exists in the target netns", 13019 name_node->name); 13020 goto out; 13021 } 13022 13023 /* Check that new_ifindex isn't used yet. */ 13024 if (new_ifindex) { 13025 err = dev_index_reserve(net, new_ifindex); 13026 if (err < 0) { 13027 NL_SET_ERR_MSG_FMT(extack, 13028 "The ifindex %d is not available in the target netns", 13029 new_ifindex); 13030 goto out; 13031 } 13032 } else { 13033 /* If there is an ifindex conflict assign a new one */ 13034 err = dev_index_reserve(net, dev->ifindex); 13035 if (err == -EBUSY) 13036 err = dev_index_reserve(net, 0); 13037 if (err < 0) { 13038 NL_SET_ERR_MSG(extack, 13039 "Unable to allocate a new ifindex in the target netns"); 13040 goto out; 13041 } 13042 new_ifindex = err; 13043 } 13044 13045 /* 13046 * And now a mini version of register_netdevice unregister_netdevice. 13047 */ 13048 13049 netdev_lock_ops(dev); 13050 /* If device is running close it first. */ 13051 netif_close(dev); 13052 /* And unlink it from device chain */ 13053 unlist_netdevice(dev); 13054 13055 if (!netdev_need_ops_lock(dev)) 13056 netdev_lock(dev); 13057 dev->moving_ns = true; 13058 netdev_unlock(dev); 13059 13060 synchronize_net(); 13061 13062 /* Shutdown queueing discipline. */ 13063 netdev_lock_ops(dev); 13064 dev_shutdown(dev); 13065 netdev_unlock_ops(dev); 13066 13067 /* Notify protocols, that we are about to destroy 13068 * this device. They should clean all the things. 13069 * 13070 * Note that dev->reg_state stays at NETREG_REGISTERED. 13071 * This is wanted because this way 8021q and macvlan know 13072 * the device is just moving and can keep their slaves up. 13073 */ 13074 call_netdevice_notifiers(NETDEV_UNREGISTER, dev); 13075 rcu_barrier(); 13076 13077 new_nsid = peernet2id_alloc(dev_net(dev), net, GFP_KERNEL); 13078 13079 rtmsg_ifinfo_newnet(RTM_DELLINK, dev, ~0U, GFP_KERNEL, &new_nsid, 13080 new_ifindex); 13081 13082 /* 13083 * Flush the unicast and multicast chains 13084 */ 13085 dev_uc_flush(dev); 13086 dev_mc_flush(dev); 13087 13088 /* Send a netdev-removed uevent to the old namespace */ 13089 kobject_uevent(&dev->dev.kobj, KOBJ_REMOVE); 13090 netdev_adjacent_del_links(dev); 13091 13092 /* Move per-net netdevice notifiers that are following the netdevice */ 13093 move_netdevice_notifiers_dev_net(dev, net); 13094 13095 /* Actually switch the network namespace */ 13096 netdev_lock(dev); 13097 dev_net_set(dev, net); 13098 netdev_unlock(dev); 13099 dev->ifindex = new_ifindex; 13100 13101 #ifdef CONFIG_DEBUG_NET_SMALL_RTNL 13102 unregister_netdevice_move_net(net_old, net, dev); 13103 #endif 13104 13105 if (new_name[0]) { 13106 /* Rename the netdev to prepared name */ 13107 write_seqlock_bh(&netdev_rename_lock); 13108 strscpy(dev->name, new_name, IFNAMSIZ); 13109 write_sequnlock_bh(&netdev_rename_lock); 13110 } 13111 13112 /* Fixup kobjects */ 13113 dev_set_uevent_suppress(&dev->dev, 1); 13114 err = device_rename(&dev->dev, dev->name); 13115 dev_set_uevent_suppress(&dev->dev, 0); 13116 WARN_ON(err); 13117 13118 /* Send a netdev-add uevent to the new namespace */ 13119 kobject_uevent(&dev->dev.kobj, KOBJ_ADD); 13120 netdev_adjacent_add_links(dev); 13121 13122 /* Adapt owner in case owning user namespace of target network 13123 * namespace is different from the original one. 13124 */ 13125 err = netdev_change_owner(dev, net_old, net); 13126 WARN_ON(err); 13127 13128 netdev_lock(dev); 13129 dev->moving_ns = false; 13130 if (!netdev_need_ops_lock(dev)) 13131 netdev_unlock(dev); 13132 13133 /* Add the device back in the hashes */ 13134 list_netdevice(dev); 13135 /* Notify protocols, that a new device appeared. */ 13136 call_netdevice_notifiers(NETDEV_REGISTER, dev); 13137 netdev_unlock_ops(dev); 13138 13139 /* 13140 * Prevent userspace races by waiting until the network 13141 * device is fully setup before sending notifications. 13142 */ 13143 rtmsg_ifinfo(RTM_NEWLINK, dev, ~0U, GFP_KERNEL, 0, NULL); 13144 13145 synchronize_net(); 13146 err = 0; 13147 out: 13148 return err; 13149 } 13150 13151 static int dev_cpu_dead(unsigned int oldcpu) 13152 { 13153 struct sk_buff **list_skb; 13154 struct sk_buff *skb; 13155 unsigned int cpu; 13156 int node; 13157 struct softnet_data *sd, *oldsd, *remsd = NULL; 13158 13159 local_irq_disable(); 13160 cpu = smp_processor_id(); 13161 sd = &per_cpu(softnet_data, cpu); 13162 oldsd = &per_cpu(softnet_data, oldcpu); 13163 13164 /* Find end of our completion_queue. */ 13165 list_skb = &sd->completion_queue; 13166 while (*list_skb) 13167 list_skb = &(*list_skb)->next; 13168 /* Append completion queue from offline CPU. */ 13169 *list_skb = oldsd->completion_queue; 13170 oldsd->completion_queue = NULL; 13171 13172 /* Append output queue from offline CPU. */ 13173 if (oldsd->output_queue) { 13174 *sd->output_queue_tailp = oldsd->output_queue; 13175 sd->output_queue_tailp = oldsd->output_queue_tailp; 13176 oldsd->output_queue = NULL; 13177 oldsd->output_queue_tailp = &oldsd->output_queue; 13178 } 13179 /* Append NAPI poll list from offline CPU, with one exception : 13180 * process_backlog() must be called by cpu owning percpu backlog. 13181 * We properly handle process_queue & input_pkt_queue later. 13182 */ 13183 while (!list_empty(&oldsd->poll_list)) { 13184 struct napi_struct *napi = list_first_entry(&oldsd->poll_list, 13185 struct napi_struct, 13186 poll_list); 13187 13188 list_del_init(&napi->poll_list); 13189 if (napi->poll == process_backlog) 13190 napi->state &= NAPIF_STATE_THREADED; 13191 else 13192 ____napi_schedule(sd, napi); 13193 } 13194 13195 raise_softirq_irqoff(NET_TX_SOFTIRQ); 13196 local_irq_enable(); 13197 13198 if (!use_backlog_threads()) { 13199 #ifdef CONFIG_RPS 13200 remsd = oldsd->rps_ipi_list; 13201 oldsd->rps_ipi_list = NULL; 13202 #endif 13203 /* send out pending IPI's on offline CPU */ 13204 net_rps_send_ipi(remsd); 13205 } 13206 13207 /* Process offline CPU's input_pkt_queue */ 13208 while ((skb = __skb_dequeue(&oldsd->process_queue))) { 13209 netif_rx(skb); 13210 rps_input_queue_head_incr(oldsd); 13211 } 13212 while ((skb = skb_dequeue(&oldsd->input_pkt_queue))) { 13213 netif_rx(skb); 13214 rps_input_queue_head_incr(oldsd); 13215 } 13216 13217 for_each_node(node) 13218 skb_defer_node_flush(per_cpu_ptr(net_hotdata.skb_defer_nodes, 13219 oldcpu) + node); 13220 node = cpu_to_node(oldcpu); 13221 if (node_possible(node) && 13222 !cpumask_intersects(cpumask_of_node(node), cpu_online_mask)) { 13223 for_each_possible_cpu(cpu) 13224 skb_defer_node_flush(per_cpu_ptr(net_hotdata.skb_defer_nodes, 13225 cpu) + node); 13226 } 13227 13228 return 0; 13229 } 13230 13231 /** 13232 * netdev_increment_features - increment feature set by one 13233 * @all: current feature set 13234 * @one: new feature set 13235 * @mask: mask feature set 13236 * 13237 * Computes a new feature set after adding a device with feature set 13238 * @one to the master device with current feature set @all. Will not 13239 * enable anything that is off in @mask. Returns the new feature set. 13240 */ 13241 netdev_features_t netdev_increment_features(netdev_features_t all, 13242 netdev_features_t one, netdev_features_t mask) 13243 { 13244 if (mask & NETIF_F_HW_CSUM) 13245 mask |= NETIF_F_CSUM_MASK; 13246 mask |= NETIF_F_VLAN_CHALLENGED; 13247 13248 all |= one & (NETIF_F_ONE_FOR_ALL | NETIF_F_CSUM_MASK) & mask; 13249 all &= one | ~NETIF_F_ALL_FOR_ALL; 13250 13251 /* If one device supports hw checksumming, set for all. */ 13252 if (all & NETIF_F_HW_CSUM) 13253 all &= ~(NETIF_F_CSUM_MASK & ~NETIF_F_HW_CSUM); 13254 13255 return all; 13256 } 13257 EXPORT_SYMBOL(netdev_increment_features); 13258 13259 /** 13260 * netdev_compute_master_upper_features - compute feature from lowers 13261 * @dev: the upper device 13262 * @update_header: whether to update upper device's header_len/headroom/tailroom 13263 * 13264 * Recompute the upper device's feature based on all lower devices. 13265 */ 13266 void netdev_compute_master_upper_features(struct net_device *dev, bool update_header) 13267 { 13268 unsigned int dst_release_flag = IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM; 13269 netdev_features_t gso_partial_features = MASTER_UPPER_DEV_GSO_PARTIAL_FEATURES; 13270 netdev_features_t xfrm_features = MASTER_UPPER_DEV_XFRM_FEATURES; 13271 netdev_features_t mpls_features = MASTER_UPPER_DEV_MPLS_FEATURES; 13272 netdev_features_t vlan_features = MASTER_UPPER_DEV_VLAN_FEATURES; 13273 netdev_features_t enc_features = MASTER_UPPER_DEV_ENC_FEATURES; 13274 unsigned short max_header_len = ETH_HLEN; 13275 unsigned int tso_max_size = TSO_MAX_SIZE; 13276 unsigned short max_headroom = 0; 13277 unsigned short max_tailroom = 0; 13278 u16 tso_max_segs = TSO_MAX_SEGS; 13279 struct net_device *lower_dev; 13280 struct list_head *iter; 13281 13282 mpls_features = netdev_base_features(mpls_features); 13283 vlan_features = netdev_base_features(vlan_features); 13284 enc_features = netdev_base_features(enc_features); 13285 13286 netdev_for_each_lower_dev(dev, lower_dev, iter) { 13287 gso_partial_features = netdev_increment_features(gso_partial_features, 13288 lower_dev->gso_partial_features, 13289 MASTER_UPPER_DEV_GSO_PARTIAL_FEATURES); 13290 13291 vlan_features = netdev_increment_features(vlan_features, 13292 lower_dev->vlan_features, 13293 MASTER_UPPER_DEV_VLAN_FEATURES); 13294 13295 enc_features = netdev_increment_features(enc_features, 13296 lower_dev->hw_enc_features, 13297 MASTER_UPPER_DEV_ENC_FEATURES); 13298 13299 if (IS_ENABLED(CONFIG_XFRM_OFFLOAD)) 13300 xfrm_features = netdev_increment_features(xfrm_features, 13301 lower_dev->hw_enc_features, 13302 MASTER_UPPER_DEV_XFRM_FEATURES); 13303 13304 mpls_features = netdev_increment_features(mpls_features, 13305 lower_dev->mpls_features, 13306 MASTER_UPPER_DEV_MPLS_FEATURES); 13307 13308 dst_release_flag &= lower_dev->priv_flags; 13309 13310 if (update_header) { 13311 max_header_len = max(max_header_len, lower_dev->hard_header_len); 13312 max_headroom = max(max_headroom, lower_dev->needed_headroom); 13313 max_tailroom = max(max_tailroom, lower_dev->needed_tailroom); 13314 } 13315 13316 tso_max_size = min(tso_max_size, lower_dev->tso_max_size); 13317 tso_max_segs = min(tso_max_segs, lower_dev->tso_max_segs); 13318 } 13319 13320 dev->gso_partial_features = gso_partial_features; 13321 dev->vlan_features = vlan_features; 13322 dev->hw_enc_features = enc_features | NETIF_F_GSO_ENCAP_ALL | 13323 NETIF_F_HW_VLAN_CTAG_TX | 13324 NETIF_F_HW_VLAN_STAG_TX; 13325 if (IS_ENABLED(CONFIG_XFRM_OFFLOAD)) 13326 dev->hw_enc_features |= xfrm_features; 13327 dev->mpls_features = mpls_features; 13328 13329 dev->priv_flags &= ~IFF_XMIT_DST_RELEASE; 13330 if ((dev->priv_flags & IFF_XMIT_DST_RELEASE_PERM) && 13331 dst_release_flag == (IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM)) 13332 dev->priv_flags |= IFF_XMIT_DST_RELEASE; 13333 13334 if (update_header) { 13335 dev->hard_header_len = max_header_len; 13336 dev->needed_headroom = max_headroom; 13337 dev->needed_tailroom = max_tailroom; 13338 } 13339 13340 netif_set_tso_max_segs(dev, tso_max_segs); 13341 netif_set_tso_max_size(dev, tso_max_size); 13342 13343 netdev_change_features(dev); 13344 } 13345 EXPORT_SYMBOL(netdev_compute_master_upper_features); 13346 13347 static struct hlist_head * __net_init netdev_create_hash(void) 13348 { 13349 int i; 13350 struct hlist_head *hash; 13351 13352 hash = kmalloc_objs(*hash, NETDEV_HASHENTRIES); 13353 if (hash != NULL) 13354 for (i = 0; i < NETDEV_HASHENTRIES; i++) 13355 INIT_HLIST_HEAD(&hash[i]); 13356 13357 return hash; 13358 } 13359 13360 /* Initialize per network namespace state */ 13361 static int __net_init netdev_init(struct net *net) 13362 { 13363 BUILD_BUG_ON(GRO_HASH_BUCKETS > 13364 BITS_PER_BYTE * sizeof_field(struct gro_node, bitmask)); 13365 13366 INIT_LIST_HEAD(&net->dev_base_head); 13367 13368 net->dev_name_head = netdev_create_hash(); 13369 if (net->dev_name_head == NULL) 13370 goto err_name; 13371 13372 net->dev_index_head = netdev_create_hash(); 13373 if (net->dev_index_head == NULL) 13374 goto err_idx; 13375 13376 xa_init_flags(&net->dev_by_index, XA_FLAGS_ALLOC1); 13377 13378 RAW_INIT_NOTIFIER_HEAD(&net->netdev_chain); 13379 13380 return 0; 13381 13382 err_idx: 13383 kfree(net->dev_name_head); 13384 err_name: 13385 return -ENOMEM; 13386 } 13387 13388 /** 13389 * netdev_drivername - network driver for the device 13390 * @dev: network device 13391 * 13392 * Determine network driver for device. 13393 */ 13394 const char *netdev_drivername(const struct net_device *dev) 13395 { 13396 const struct device_driver *driver; 13397 const struct device *parent; 13398 const char *empty = ""; 13399 13400 parent = dev->dev.parent; 13401 if (!parent) 13402 return empty; 13403 13404 driver = parent->driver; 13405 if (driver && driver->name) 13406 return driver->name; 13407 return empty; 13408 } 13409 13410 static void __netdev_printk(const char *level, const struct net_device *dev, 13411 struct va_format *vaf) 13412 { 13413 if (dev && dev->dev.parent) { 13414 dev_printk_emit(level[1] - '0', 13415 dev->dev.parent, 13416 "%s %s %s%s: %pV", 13417 dev_driver_string(dev->dev.parent), 13418 dev_name(dev->dev.parent), 13419 netdev_name(dev), netdev_reg_state(dev), 13420 vaf); 13421 } else if (dev) { 13422 printk("%s%s%s: %pV", 13423 level, netdev_name(dev), netdev_reg_state(dev), vaf); 13424 } else { 13425 printk("%s(NULL net_device): %pV", level, vaf); 13426 } 13427 } 13428 13429 void netdev_printk(const char *level, const struct net_device *dev, 13430 const char *format, ...) 13431 { 13432 struct va_format vaf; 13433 va_list args; 13434 13435 va_start(args, format); 13436 13437 vaf.fmt = format; 13438 vaf.va = &args; 13439 13440 __netdev_printk(level, dev, &vaf); 13441 13442 va_end(args); 13443 } 13444 EXPORT_SYMBOL(netdev_printk); 13445 13446 #define define_netdev_printk_level(func, level) \ 13447 void func(const struct net_device *dev, const char *fmt, ...) \ 13448 { \ 13449 struct va_format vaf; \ 13450 va_list args; \ 13451 \ 13452 va_start(args, fmt); \ 13453 \ 13454 vaf.fmt = fmt; \ 13455 vaf.va = &args; \ 13456 \ 13457 __netdev_printk(level, dev, &vaf); \ 13458 \ 13459 va_end(args); \ 13460 } \ 13461 EXPORT_SYMBOL(func); 13462 13463 define_netdev_printk_level(netdev_emerg, KERN_EMERG); 13464 define_netdev_printk_level(netdev_alert, KERN_ALERT); 13465 define_netdev_printk_level(netdev_crit, KERN_CRIT); 13466 define_netdev_printk_level(netdev_err, KERN_ERR); 13467 define_netdev_printk_level(netdev_warn, KERN_WARNING); 13468 define_netdev_printk_level(netdev_notice, KERN_NOTICE); 13469 define_netdev_printk_level(netdev_info, KERN_INFO); 13470 13471 static void __net_exit netdev_exit(struct net *net) 13472 { 13473 kfree(net->dev_name_head); 13474 kfree(net->dev_index_head); 13475 xa_destroy(&net->dev_by_index); 13476 if (net != &init_net) 13477 WARN_ON_ONCE(!list_empty(&net->dev_base_head)); 13478 } 13479 13480 static struct pernet_operations __net_initdata netdev_net_ops = { 13481 .init = netdev_init, 13482 .exit = netdev_exit, 13483 }; 13484 13485 static void __net_exit default_device_exit_net(struct net *net) 13486 { 13487 struct net_device *dev, *aux; 13488 /* 13489 * Push all migratable network devices back to the 13490 * initial network namespace 13491 */ 13492 13493 for_each_netdev_safe(net, dev, aux) { 13494 int err; 13495 char fb_name[IFNAMSIZ]; 13496 13497 /* Ignore unmoveable devices (i.e. loopback) */ 13498 if (dev->netns_immutable) 13499 continue; 13500 13501 /* Leave virtual devices for the generic cleanup */ 13502 if (dev->rtnl_link_ops && !dev->rtnl_link_ops->netns_refund) 13503 continue; 13504 13505 /* Push remaining network devices to init_net */ 13506 snprintf(fb_name, IFNAMSIZ, "dev%d", dev->ifindex); 13507 if (netdev_name_in_use(&init_net, fb_name)) 13508 snprintf(fb_name, IFNAMSIZ, "dev%%d"); 13509 13510 err = dev_change_net_namespace(dev, &init_net, fb_name); 13511 if (err) { 13512 pr_emerg("%s: failed to move %s to init_net: %d\n", 13513 __func__, dev->name, err); 13514 BUG(); 13515 } 13516 } 13517 } 13518 13519 static bool __net_exit netns_dev_unreg_pending(const struct list_head *net_list) 13520 { 13521 struct net *net; 13522 13523 list_for_each_entry(net, net_list, exit_list) { 13524 if (atomic_read(&net->dev_unreg_pending)) 13525 return true; 13526 } 13527 return false; 13528 } 13529 13530 static void __net_exit default_device_exit_batch(struct list_head *net_list) 13531 { 13532 /* At exit all network devices most be removed from a network 13533 * namespace. Do this in the reverse order of registration. 13534 * Do this across as many network namespaces as possible to 13535 * improve batching efficiency. 13536 */ 13537 struct net_device *dev; 13538 struct net *net; 13539 LIST_HEAD(dev_kill_list); 13540 13541 rtnl_lock(); 13542 13543 __rtnl_net_lock(&init_net); 13544 13545 list_for_each_entry(net, net_list, exit_list) { 13546 __rtnl_net_lock(net); 13547 default_device_exit_net(net); 13548 __rtnl_net_unlock(net); 13549 13550 cond_resched(); 13551 } 13552 13553 __rtnl_net_unlock(&init_net); 13554 13555 list_for_each_entry(net, net_list, exit_list) { 13556 __rtnl_net_lock(net); 13557 13558 for_each_netdev_reverse(net, dev) { 13559 if (dev->rtnl_link_ops && dev->rtnl_link_ops->dellink) 13560 dev->rtnl_link_ops->dellink(dev, &dev_kill_list); 13561 else 13562 unregister_netdevice_queue(dev, &dev_kill_list); 13563 } 13564 13565 unregister_netdevice_queue_many_net(net, &dev_kill_list); 13566 __rtnl_net_unlock(net); 13567 } 13568 unregister_netdevice_many(&dev_kill_list); 13569 rtnl_unlock(); 13570 13571 /* Devices of these netns unregistered from another context 13572 * (e.g. a veth whose peer was deleted) can still be waiting 13573 * in netdev_run_todo() for their last references, while 13574 * NETDEV_UNREGISTER is rebroadcast for them. Wait for them 13575 * before pernet exit handlers tear down the state used by 13576 * netdevice notifiers, and before the netns is freed. 13577 * No new unregistration can start: all devices are unlisted. 13578 */ 13579 wait_event(netdev_unregistering_wq, !netns_dev_unreg_pending(net_list)); 13580 13581 rtnl_net_flush_workqueue(); 13582 } 13583 13584 static struct pernet_operations __net_initdata default_device_ops = { 13585 .exit_batch = default_device_exit_batch, 13586 }; 13587 13588 static void __init net_dev_struct_check(void) 13589 { 13590 /* TX read-mostly hotpath */ 13591 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, priv_flags_fast); 13592 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, netdev_ops); 13593 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, header_ops); 13594 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, _tx); 13595 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, real_num_tx_queues); 13596 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, gso_max_size); 13597 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, gso_ipv4_max_size); 13598 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, gso_max_segs); 13599 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, gso_partial_features); 13600 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, num_tc); 13601 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, mtu); 13602 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, needed_headroom); 13603 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, tc_to_txq); 13604 #ifdef CONFIG_XPS 13605 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, xps_maps); 13606 #endif 13607 #ifdef CONFIG_NETFILTER_EGRESS 13608 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, nf_hooks_egress); 13609 #endif 13610 #ifdef CONFIG_NET_XGRESS 13611 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_tx, tcx_egress); 13612 #endif 13613 CACHELINE_ASSERT_GROUP_SIZE(struct net_device, net_device_read_tx, 160); 13614 13615 /* TXRX read-mostly hotpath */ 13616 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, lstats); 13617 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, state); 13618 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, flags); 13619 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, hard_header_len); 13620 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, features); 13621 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_txrx, ip6_ptr); 13622 CACHELINE_ASSERT_GROUP_SIZE(struct net_device, net_device_read_txrx, 46); 13623 13624 /* RX read-mostly hotpath */ 13625 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, ptype_specific); 13626 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, ifindex); 13627 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, real_num_rx_queues); 13628 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, _rx); 13629 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, gro_max_size); 13630 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, gro_ipv4_max_size); 13631 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, rx_handler); 13632 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, rx_handler_data); 13633 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, nd_net); 13634 #ifdef CONFIG_NETPOLL 13635 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, npinfo); 13636 #endif 13637 #ifdef CONFIG_NET_XGRESS 13638 CACHELINE_ASSERT_GROUP_MEMBER(struct net_device, net_device_read_rx, tcx_ingress); 13639 #endif 13640 CACHELINE_ASSERT_GROUP_SIZE(struct net_device, net_device_read_rx, 92); 13641 } 13642 13643 /* 13644 * Initialize the DEV module. At boot time this walks the device list and 13645 * unhooks any devices that fail to initialise (normally hardware not 13646 * present) and leaves us with a valid list of present and active devices. 13647 * 13648 */ 13649 13650 /* We allocate 256 pages for each CPU if PAGE_SHIFT is 12 */ 13651 #define SYSTEM_PERCPU_PAGE_POOL_SIZE ((1 << 20) / PAGE_SIZE) 13652 13653 static int net_page_pool_create(int cpuid) 13654 { 13655 #if IS_ENABLED(CONFIG_PAGE_POOL) 13656 struct page_pool_params page_pool_params = { 13657 .pool_size = SYSTEM_PERCPU_PAGE_POOL_SIZE, 13658 .flags = PP_FLAG_SYSTEM_POOL, 13659 .nid = cpu_to_mem(cpuid), 13660 }; 13661 struct page_pool *pp_ptr; 13662 int err; 13663 13664 pp_ptr = page_pool_create_percpu(&page_pool_params, cpuid); 13665 if (IS_ERR(pp_ptr)) 13666 return -ENOMEM; 13667 13668 err = xdp_reg_page_pool(pp_ptr); 13669 if (err) { 13670 page_pool_destroy(pp_ptr); 13671 return err; 13672 } 13673 13674 per_cpu(system_page_pool.pool, cpuid) = pp_ptr; 13675 #endif 13676 return 0; 13677 } 13678 13679 static int backlog_napi_should_run(unsigned int cpu) 13680 { 13681 struct softnet_data *sd = per_cpu_ptr(&softnet_data, cpu); 13682 struct napi_struct *napi = &sd->backlog; 13683 13684 return test_bit(NAPI_STATE_SCHED_THREADED, &napi->state); 13685 } 13686 13687 static void run_backlog_napi(unsigned int cpu) 13688 { 13689 struct softnet_data *sd = per_cpu_ptr(&softnet_data, cpu); 13690 13691 napi_threaded_poll_loop(&sd->backlog, NULL); 13692 } 13693 13694 static void backlog_napi_setup(unsigned int cpu) 13695 { 13696 struct softnet_data *sd = per_cpu_ptr(&softnet_data, cpu); 13697 struct napi_struct *napi = &sd->backlog; 13698 13699 napi->thread = this_cpu_read(backlog_napi); 13700 set_bit(NAPI_STATE_THREADED, &napi->state); 13701 } 13702 13703 static struct smp_hotplug_thread backlog_threads = { 13704 .store = &backlog_napi, 13705 .thread_should_run = backlog_napi_should_run, 13706 .thread_fn = run_backlog_napi, 13707 .thread_comm = "backlog_napi/%u", 13708 .setup = backlog_napi_setup, 13709 }; 13710 13711 /* 13712 * This is called single threaded during boot, so no need 13713 * to take the rtnl semaphore. 13714 */ 13715 static int __init net_dev_init(void) 13716 { 13717 int i, rc = -ENOMEM; 13718 13719 BUG_ON(!dev_boot_phase); 13720 13721 net_dev_struct_check(); 13722 13723 if (dev_proc_init()) 13724 goto out; 13725 13726 if (netdev_kobject_init()) 13727 goto out; 13728 13729 for (i = 0; i < PTYPE_HASH_SIZE; i++) 13730 INIT_LIST_HEAD(&ptype_base[i]); 13731 13732 if (register_pernet_subsys(&netdev_net_ops)) 13733 goto out; 13734 13735 /* 13736 * Initialise the packet receive queues. 13737 */ 13738 13739 flush_backlogs_fallback = flush_backlogs_alloc(); 13740 if (!flush_backlogs_fallback) 13741 goto out; 13742 13743 for_each_possible_cpu(i) { 13744 struct softnet_data *sd = &per_cpu(softnet_data, i); 13745 13746 skb_queue_head_init(&sd->input_pkt_queue); 13747 skb_queue_head_init(&sd->process_queue); 13748 #ifdef CONFIG_XFRM_OFFLOAD 13749 skb_queue_head_init(&sd->xfrm_backlog); 13750 #endif 13751 INIT_LIST_HEAD(&sd->poll_list); 13752 sd->output_queue_tailp = &sd->output_queue; 13753 #ifdef CONFIG_RPS 13754 INIT_CSD(&sd->csd, rps_trigger_softirq, sd); 13755 sd->cpu = i; 13756 #endif 13757 INIT_CSD(&sd->defer_csd, trigger_rx_softirq, sd); 13758 13759 gro_init(&sd->backlog.gro); 13760 sd->backlog.poll = process_backlog; 13761 sd->backlog.weight = weight_p; 13762 INIT_LIST_HEAD(&sd->backlog.poll_list); 13763 13764 if (net_page_pool_create(i)) 13765 goto out; 13766 } 13767 net_hotdata.skb_defer_nodes = 13768 __alloc_percpu(sizeof(struct skb_defer_node) * nr_node_ids, 13769 __alignof__(struct skb_defer_node)); 13770 if (!net_hotdata.skb_defer_nodes) 13771 goto out; 13772 if (use_backlog_threads()) 13773 smpboot_register_percpu_thread(&backlog_threads); 13774 13775 dev_boot_phase = 0; 13776 13777 /* The loopback device is special if any other network devices 13778 * is present in a network namespace the loopback device must 13779 * be present. Since we now dynamically allocate and free the 13780 * loopback device ensure this invariant is maintained by 13781 * keeping the loopback device as the first device on the 13782 * list of network devices. Ensuring the loopback devices 13783 * is the first device that appears and the last network device 13784 * that disappears. 13785 */ 13786 if (register_pernet_device(&loopback_net_ops)) 13787 goto out; 13788 13789 if (register_pernet_device(&default_device_ops)) 13790 goto out; 13791 13792 open_softirq(NET_TX_SOFTIRQ, net_tx_action); 13793 open_softirq(NET_RX_SOFTIRQ, net_rx_action); 13794 13795 rc = cpuhp_setup_state_nocalls(CPUHP_NET_DEV_DEAD, "net/dev:dead", 13796 NULL, dev_cpu_dead); 13797 WARN_ON(rc < 0); 13798 rc = 0; 13799 13800 /* avoid static key IPIs to isolated CPUs */ 13801 if (housekeeping_enabled(HK_TYPE_MISC)) 13802 net_enable_timestamp(); 13803 out: 13804 if (rc < 0) { 13805 for_each_possible_cpu(i) { 13806 struct page_pool *pp_ptr; 13807 13808 pp_ptr = per_cpu(system_page_pool.pool, i); 13809 if (!pp_ptr) 13810 continue; 13811 13812 xdp_unreg_page_pool(pp_ptr); 13813 page_pool_destroy(pp_ptr); 13814 per_cpu(system_page_pool.pool, i) = NULL; 13815 } 13816 } 13817 13818 return rc; 13819 } 13820 13821 subsys_initcall(net_dev_init); 13822