1 /* SPDX-License-Identifier: GPL-2.0-or-later */
2 /*
3 * INET An implementation of the TCP/IP protocol suite for the LINUX
4 * operating system. INET is implemented using the BSD Socket
5 * interface as the means of communication with the user level.
6 *
7 * Definitions for the Interfaces handler.
8 *
9 * Version: @(#)dev.h 1.0.10 08/12/93
10 *
11 * Authors: Ross Biro
12 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
13 * Corey Minyard <wf-rch!minyard@relay.EU.net>
14 * Donald J. Becker, <becker@cesdis.gsfc.nasa.gov>
15 * Alan Cox, <alan@lxorguk.ukuu.org.uk>
16 * Bjorn Ekwall. <bj0rn@blox.se>
17 * Pekka Riikonen <priikone@poseidon.pspt.fi>
18 *
19 * Moved to /usr/include/linux for NET3
20 */
21 #ifndef _LINUX_NETDEVICE_H
22 #define _LINUX_NETDEVICE_H
23
24 #include <linux/timer.h>
25 #include <linux/bug.h>
26 #include <linux/delay.h>
27 #include <linux/atomic.h>
28 #include <linux/prefetch.h>
29 #include <asm/cache.h>
30 #include <asm/byteorder.h>
31 #include <asm/local.h>
32
33 #include <linux/percpu.h>
34 #include <linux/rculist.h>
35 #include <linux/workqueue.h>
36 #include <linux/dynamic_queue_limits.h>
37
38 #include <net/net_namespace.h>
39 #ifdef CONFIG_DCB
40 #include <net/dcbnl.h>
41 #endif
42 #include <net/netprio_cgroup.h>
43 #include <linux/netdev_features.h>
44 #include <linux/neighbour.h>
45 #include <linux/netdevice_xmit.h>
46 #include <uapi/linux/netdevice.h>
47 #include <uapi/linux/if_bonding.h>
48 #include <uapi/linux/pkt_cls.h>
49 #include <uapi/linux/netdev.h>
50 #include <linux/hashtable.h>
51 #include <linux/rbtree.h>
52 #include <net/net_trackers.h>
53 #include <net/net_debug.h>
54 #include <net/dropreason-core.h>
55 #include <net/neighbour_tables.h>
56
57 struct netpoll_info;
58 struct device;
59 struct ethtool_ops;
60 struct kernel_hwtstamp_config;
61 struct phy_device;
62 struct dsa_port;
63 struct ip_tunnel_parm_kern;
64 struct macsec_context;
65 struct macsec_ops;
66 struct netdev_config;
67 struct netdev_name_node;
68 struct sd_flow_limit;
69 struct sfp_bus;
70 /* 802.11 specific */
71 struct wireless_dev;
72 /* 802.15.4 specific */
73 struct wpan_dev;
74 struct mpls_dev;
75 /* UDP Tunnel offloads */
76 struct udp_tunnel_info;
77 struct udp_tunnel_nic_info;
78 struct udp_tunnel_nic;
79 struct bpf_prog;
80 struct xdp_buff;
81 struct xdp_frame;
82 struct xdp_metadata_ops;
83 struct xdp_md;
84 struct ethtool_netdev_state;
85 struct phy_link_topology;
86 struct hwtstamp_provider;
87
88 typedef u32 xdp_features_t;
89
90 void synchronize_net(void);
91 void netdev_set_default_ethtool_ops(struct net_device *dev,
92 const struct ethtool_ops *ops);
93 void netdev_sw_irq_coalesce_default_on(struct net_device *dev);
94
95 /* Backlog congestion levels */
96 #define NET_RX_SUCCESS 0 /* keep 'em coming, baby */
97 #define NET_RX_DROP 1 /* packet dropped */
98
99 #define MAX_NEST_DEV 8
100
101 /*
102 * Transmit return codes: transmit return codes originate from three different
103 * namespaces:
104 *
105 * - qdisc return codes
106 * - driver transmit return codes
107 * - errno values
108 *
109 * Drivers are allowed to return any one of those in their hard_start_xmit()
110 * function. Real network devices commonly used with qdiscs should only return
111 * the driver transmit return codes though - when qdiscs are used, the actual
112 * transmission happens asynchronously, so the value is not propagated to
113 * higher layers. Virtual network devices transmit synchronously; in this case
114 * the driver transmit return codes are consumed by dev_queue_xmit(), and all
115 * others are propagated to higher layers.
116 */
117
118 /* qdisc ->enqueue() return codes. */
119 #define NET_XMIT_SUCCESS 0x00
120 #define NET_XMIT_DROP 0x01 /* skb dropped */
121 #define NET_XMIT_CN 0x02 /* congestion notification */
122 #define NET_XMIT_MASK 0x0f /* qdisc flags in net/sch_generic.h */
123
124 /* NET_XMIT_CN is special. It does not guarantee that this packet is lost. It
125 * indicates that the device will soon be dropping packets, or already drops
126 * some packets of the same priority; prompting us to send less aggressively. */
127 #define net_xmit_eval(e) ((e) == NET_XMIT_CN ? 0 : (e))
128 #define net_xmit_errno(e) ((e) != NET_XMIT_CN ? -ENOBUFS : 0)
129
130 /* Driver transmit return codes */
131 #define NETDEV_TX_MASK 0xf0
132
133 enum netdev_tx {
134 __NETDEV_TX_MIN = INT_MIN, /* make sure enum is signed */
135 NETDEV_TX_OK = 0x00, /* driver took care of packet */
136 NETDEV_TX_BUSY = 0x10, /* driver tx path was busy*/
137 };
138 typedef enum netdev_tx netdev_tx_t;
139
140 /*
141 * Current order: NETDEV_TX_MASK > NET_XMIT_MASK >= 0 is significant;
142 * hard_start_xmit() return < NET_XMIT_MASK means skb was consumed.
143 */
dev_xmit_complete(int rc)144 static inline bool dev_xmit_complete(int rc)
145 {
146 /*
147 * Positive cases with an skb consumed by a driver:
148 * - successful transmission (rc == NETDEV_TX_OK)
149 * - error while transmitting (rc < 0)
150 * - error while queueing to a different device (rc & NET_XMIT_MASK)
151 */
152 if (likely(rc < NET_XMIT_MASK))
153 return true;
154
155 return false;
156 }
157
158 /*
159 * Compute the worst-case header length according to the protocols
160 * used.
161 */
162
163 #if defined(CONFIG_HYPERV_NET)
164 # define LL_MAX_HEADER 128
165 #elif defined(CONFIG_WLAN) || IS_ENABLED(CONFIG_AX25)
166 # if defined(CONFIG_MAC80211_MESH)
167 # define LL_MAX_HEADER 128
168 # else
169 # define LL_MAX_HEADER 96
170 # endif
171 #else
172 # define LL_MAX_HEADER 32
173 #endif
174
175 #if !IS_ENABLED(CONFIG_NET_IPIP) && !IS_ENABLED(CONFIG_NET_IPGRE) && \
176 !IS_ENABLED(CONFIG_IPV6_SIT) && !IS_ENABLED(CONFIG_IPV6_TUNNEL)
177 #define MAX_HEADER LL_MAX_HEADER
178 #else
179 #define MAX_HEADER (LL_MAX_HEADER + 48)
180 #endif
181
182 /*
183 * Old network device statistics. Fields are native words
184 * (unsigned long) so they can be read and written atomically.
185 */
186
187 #define NET_DEV_STAT(FIELD) \
188 union { \
189 unsigned long FIELD; \
190 atomic_long_t __##FIELD; \
191 }
192
193 struct net_device_stats {
194 NET_DEV_STAT(rx_packets);
195 NET_DEV_STAT(tx_packets);
196 NET_DEV_STAT(rx_bytes);
197 NET_DEV_STAT(tx_bytes);
198 NET_DEV_STAT(rx_errors);
199 NET_DEV_STAT(tx_errors);
200 NET_DEV_STAT(rx_dropped);
201 NET_DEV_STAT(tx_dropped);
202 NET_DEV_STAT(multicast);
203 NET_DEV_STAT(collisions);
204 NET_DEV_STAT(rx_length_errors);
205 NET_DEV_STAT(rx_over_errors);
206 NET_DEV_STAT(rx_crc_errors);
207 NET_DEV_STAT(rx_frame_errors);
208 NET_DEV_STAT(rx_fifo_errors);
209 NET_DEV_STAT(rx_missed_errors);
210 NET_DEV_STAT(tx_aborted_errors);
211 NET_DEV_STAT(tx_carrier_errors);
212 NET_DEV_STAT(tx_fifo_errors);
213 NET_DEV_STAT(tx_heartbeat_errors);
214 NET_DEV_STAT(tx_window_errors);
215 NET_DEV_STAT(rx_compressed);
216 NET_DEV_STAT(tx_compressed);
217 };
218 #undef NET_DEV_STAT
219
220 /* per-cpu stats, allocated on demand.
221 * Try to fit them in a single cache line, for dev_get_stats() sake.
222 */
223 struct net_device_core_stats {
224 unsigned long rx_dropped;
225 unsigned long tx_dropped;
226 unsigned long rx_nohandler;
227 unsigned long rx_otherhost_dropped;
228 } __aligned(4 * sizeof(unsigned long));
229
230 #include <linux/cache.h>
231 #include <linux/skbuff.h>
232
233 struct neighbour;
234 struct neigh_parms;
235 struct sk_buff;
236
237 struct netdev_hw_addr {
238 struct list_head list;
239 struct rb_node node;
240 unsigned char addr[MAX_ADDR_LEN];
241 unsigned char type;
242 #define NETDEV_HW_ADDR_T_LAN 1
243 #define NETDEV_HW_ADDR_T_SAN 2
244 #define NETDEV_HW_ADDR_T_UNICAST 3
245 #define NETDEV_HW_ADDR_T_MULTICAST 4
246 bool global_use;
247 int sync_cnt;
248 int refcount;
249 int synced;
250 struct rcu_head rcu_head;
251 };
252
253 struct netdev_hw_addr_list {
254 struct list_head list;
255 int count;
256
257 /* Auxiliary tree for faster lookup on addition and deletion */
258 struct rb_root tree;
259 };
260
261 #define netdev_hw_addr_list_count(l) ((l)->count)
262 #define netdev_hw_addr_list_empty(l) (netdev_hw_addr_list_count(l) == 0)
263 #define netdev_hw_addr_list_for_each(ha, l) \
264 list_for_each_entry(ha, &(l)->list, list)
265
266 #define netdev_uc_count(dev) netdev_hw_addr_list_count(&(dev)->uc)
267 #define netdev_uc_empty(dev) netdev_hw_addr_list_empty(&(dev)->uc)
268 #define netdev_for_each_uc_addr(ha, dev) \
269 netdev_hw_addr_list_for_each(ha, &(dev)->uc)
270 #define netdev_for_each_synced_uc_addr(_ha, _dev) \
271 netdev_for_each_uc_addr((_ha), (_dev)) \
272 if ((_ha)->sync_cnt)
273
274 #define netdev_mc_count(dev) netdev_hw_addr_list_count(&(dev)->mc)
275 #define netdev_mc_empty(dev) netdev_hw_addr_list_empty(&(dev)->mc)
276 #define netdev_for_each_mc_addr(ha, dev) \
277 netdev_hw_addr_list_for_each(ha, &(dev)->mc)
278 #define netdev_for_each_synced_mc_addr(_ha, _dev) \
279 netdev_for_each_mc_addr((_ha), (_dev)) \
280 if ((_ha)->sync_cnt)
281
282 struct hh_cache {
283 unsigned int hh_len;
284 seqlock_t hh_lock;
285
286 /* cached hardware header; allow for machine alignment needs. */
287 #define HH_DATA_MOD 16
288 #define HH_DATA_OFF(__len) \
289 (HH_DATA_MOD - (((__len - 1) & (HH_DATA_MOD - 1)) + 1))
290 #define HH_DATA_ALIGN(__len) \
291 (((__len)+(HH_DATA_MOD-1))&~(HH_DATA_MOD - 1))
292 unsigned long hh_data[HH_DATA_ALIGN(LL_MAX_HEADER) / sizeof(long)];
293 };
294
295 /* Reserve HH_DATA_MOD byte-aligned hard_header_len, but at least that much.
296 * Alternative is:
297 * dev->hard_header_len ? (dev->hard_header_len +
298 * (HH_DATA_MOD - 1)) & ~(HH_DATA_MOD - 1) : 0
299 *
300 * We could use other alignment values, but we must maintain the
301 * relationship HH alignment <= LL alignment.
302 */
303 #define LL_RESERVED_SPACE(dev) \
304 ((((dev)->hard_header_len + READ_ONCE((dev)->needed_headroom)) \
305 & ~(HH_DATA_MOD - 1)) + HH_DATA_MOD)
306 #define LL_RESERVED_SPACE_EXTRA(dev,extra) \
307 ((((dev)->hard_header_len + READ_ONCE((dev)->needed_headroom) + (extra)) \
308 & ~(HH_DATA_MOD - 1)) + HH_DATA_MOD)
309
310 struct header_ops {
311 int (*create) (struct sk_buff *skb, struct net_device *dev,
312 unsigned short type, const void *daddr,
313 const void *saddr, unsigned int len);
314 int (*parse)(const struct sk_buff *skb,
315 const struct net_device *dev,
316 unsigned char *haddr);
317 int (*cache)(const struct neighbour *neigh, struct hh_cache *hh, __be16 type);
318 void (*cache_update)(struct hh_cache *hh,
319 const struct net_device *dev,
320 const unsigned char *haddr);
321 bool (*validate)(const char *ll_header, unsigned int len);
322 __be16 (*parse_protocol)(const struct sk_buff *skb);
323 };
324
325 /* These flag bits are private to the generic network queueing
326 * layer; they may not be explicitly referenced by any other
327 * code.
328 */
329
330 enum netdev_state_t {
331 __LINK_STATE_START,
332 __LINK_STATE_PRESENT,
333 __LINK_STATE_NOCARRIER,
334 __LINK_STATE_LINKWATCH_PENDING,
335 __LINK_STATE_DORMANT,
336 __LINK_STATE_TESTING,
337 };
338
339 struct gro_list {
340 struct list_head list;
341 int count;
342 };
343
344 /*
345 * size of gro hash buckets, must be <= the number of bits in
346 * gro_node::bitmask
347 */
348 #define GRO_HASH_BUCKETS 8
349
350 /**
351 * struct gro_node - structure to support Generic Receive Offload
352 * @bitmask: bitmask to indicate used buckets in @hash
353 * @hash: hashtable of pending aggregated skbs, separated by flows
354 * @rx_list: list of pending ``GRO_NORMAL`` skbs
355 * @rx_count: cached current length of @rx_list
356 * @cached_napi_id: napi_struct::napi_id cached for hotpath, 0 for standalone
357 */
358 struct gro_node {
359 unsigned long bitmask;
360 struct gro_list hash[GRO_HASH_BUCKETS];
361 struct list_head rx_list;
362 u32 rx_count;
363 u32 cached_napi_id;
364 };
365
366 /*
367 * Structure for per-NAPI config
368 */
369 struct napi_config {
370 u64 gro_flush_timeout;
371 u64 irq_suspend_timeout;
372 u32 defer_hard_irqs;
373 cpumask_t affinity_mask;
374 u8 threaded;
375 unsigned int napi_id;
376 };
377
378 /*
379 * Structure for NAPI scheduling similar to tasklet but with weighting
380 */
381 struct napi_struct {
382 /* This field should be first or softnet_data.backlog needs tweaks. */
383 unsigned long state;
384 /* The poll_list must only be managed by the entity which
385 * changes the state of the NAPI_STATE_SCHED bit. This means
386 * whoever atomically sets that bit can add this napi_struct
387 * to the per-CPU poll_list, and whoever clears that bit
388 * can remove from the list right before clearing the bit.
389 */
390 struct list_head poll_list;
391
392 int weight;
393 u32 defer_hard_irqs_count;
394 int (*poll)(struct napi_struct *, int);
395 #ifdef CONFIG_NETPOLL
396 /* CPU actively polling if netpoll is configured */
397 int poll_owner;
398 #endif
399 /* CPU on which NAPI has been scheduled for processing */
400 int list_owner;
401 struct net_device *dev;
402 struct sk_buff *skb;
403 struct gro_node gro;
404 struct hrtimer timer;
405 /* all fields past this point are write-protected by netdev_lock */
406 struct task_struct *thread;
407 unsigned long gro_flush_timeout;
408 unsigned long irq_suspend_timeout;
409 u32 defer_hard_irqs;
410 /* control-path-only fields follow */
411 u32 napi_id;
412 struct list_head dev_list;
413 struct hlist_node napi_hash_node;
414 int irq;
415 struct irq_affinity_notify notify;
416 int napi_rmap_idx;
417 int index;
418 struct napi_config *config;
419 };
420
421 enum {
422 NAPI_STATE_SCHED, /* Poll is scheduled */
423 NAPI_STATE_MISSED, /* reschedule a napi */
424 NAPI_STATE_DISABLE, /* Disable pending */
425 NAPI_STATE_NPSVC, /* Netpoll - don't dequeue from poll_list */
426 NAPI_STATE_LISTED, /* NAPI added to system lists */
427 NAPI_STATE_NO_BUSY_POLL, /* Do not add in napi_hash, no busy polling */
428 NAPI_STATE_IN_BUSY_POLL, /* Do not rearm NAPI interrupt */
429 NAPI_STATE_PREFER_BUSY_POLL, /* prefer busy-polling over softirq processing*/
430 NAPI_STATE_THREADED, /* The poll is performed inside its own thread*/
431 NAPI_STATE_SCHED_THREADED, /* Napi is currently scheduled in threaded mode */
432 NAPI_STATE_HAS_NOTIFIER, /* Napi has an IRQ notifier */
433 NAPI_STATE_THREADED_BUSY_POLL, /* The threaded NAPI poller will busy poll */
434 };
435
436 enum {
437 NAPIF_STATE_SCHED = BIT(NAPI_STATE_SCHED),
438 NAPIF_STATE_MISSED = BIT(NAPI_STATE_MISSED),
439 NAPIF_STATE_DISABLE = BIT(NAPI_STATE_DISABLE),
440 NAPIF_STATE_NPSVC = BIT(NAPI_STATE_NPSVC),
441 NAPIF_STATE_LISTED = BIT(NAPI_STATE_LISTED),
442 NAPIF_STATE_NO_BUSY_POLL = BIT(NAPI_STATE_NO_BUSY_POLL),
443 NAPIF_STATE_IN_BUSY_POLL = BIT(NAPI_STATE_IN_BUSY_POLL),
444 NAPIF_STATE_PREFER_BUSY_POLL = BIT(NAPI_STATE_PREFER_BUSY_POLL),
445 NAPIF_STATE_THREADED = BIT(NAPI_STATE_THREADED),
446 NAPIF_STATE_SCHED_THREADED = BIT(NAPI_STATE_SCHED_THREADED),
447 NAPIF_STATE_HAS_NOTIFIER = BIT(NAPI_STATE_HAS_NOTIFIER),
448 NAPIF_STATE_THREADED_BUSY_POLL = BIT(NAPI_STATE_THREADED_BUSY_POLL),
449 };
450
451 enum gro_result {
452 GRO_MERGED,
453 GRO_MERGED_FREE,
454 GRO_HELD,
455 GRO_NORMAL,
456 GRO_CONSUMED,
457 };
458 typedef enum gro_result gro_result_t;
459
460 /*
461 * enum rx_handler_result - Possible return values for rx_handlers.
462 * @RX_HANDLER_CONSUMED: skb was consumed by rx_handler, do not process it
463 * further.
464 * @RX_HANDLER_ANOTHER: Do another round in receive path. This is indicated in
465 * case skb->dev was changed by rx_handler.
466 * @RX_HANDLER_EXACT: Force exact delivery, no wildcard.
467 * @RX_HANDLER_PASS: Do nothing, pass the skb as if no rx_handler was called.
468 *
469 * rx_handlers are functions called from inside __netif_receive_skb(), to do
470 * special processing of the skb, prior to delivery to protocol handlers.
471 *
472 * Currently, a net_device can only have a single rx_handler registered. Trying
473 * to register a second rx_handler will return -EBUSY.
474 *
475 * To register a rx_handler on a net_device, use netdev_rx_handler_register().
476 * To unregister a rx_handler on a net_device, use
477 * netdev_rx_handler_unregister().
478 *
479 * Upon return, rx_handler is expected to tell __netif_receive_skb() what to
480 * do with the skb.
481 *
482 * If the rx_handler consumed the skb in some way, it should return
483 * RX_HANDLER_CONSUMED. This is appropriate when the rx_handler arranged for
484 * the skb to be delivered in some other way.
485 *
486 * If the rx_handler changed skb->dev, to divert the skb to another
487 * net_device, it should return RX_HANDLER_ANOTHER. The rx_handler for the
488 * new device will be called if it exists.
489 *
490 * If the rx_handler decides the skb should be ignored, it should return
491 * RX_HANDLER_EXACT. The skb will only be delivered to protocol handlers that
492 * are registered on exact device (ptype->dev == skb->dev).
493 *
494 * If the rx_handler didn't change skb->dev, but wants the skb to be normally
495 * delivered, it should return RX_HANDLER_PASS.
496 *
497 * A device without a registered rx_handler will behave as if rx_handler
498 * returned RX_HANDLER_PASS.
499 */
500
501 enum rx_handler_result {
502 RX_HANDLER_CONSUMED,
503 RX_HANDLER_ANOTHER,
504 RX_HANDLER_EXACT,
505 RX_HANDLER_PASS,
506 };
507 typedef enum rx_handler_result rx_handler_result_t;
508 typedef rx_handler_result_t rx_handler_func_t(struct sk_buff **pskb);
509
510 void __napi_schedule(struct napi_struct *n);
511 void __napi_schedule_irqoff(struct napi_struct *n);
512
napi_disable_pending(struct napi_struct * n)513 static inline bool napi_disable_pending(struct napi_struct *n)
514 {
515 return test_bit(NAPI_STATE_DISABLE, &n->state);
516 }
517
napi_prefer_busy_poll(struct napi_struct * n)518 static inline bool napi_prefer_busy_poll(struct napi_struct *n)
519 {
520 return test_bit(NAPI_STATE_PREFER_BUSY_POLL, &n->state);
521 }
522
523 /**
524 * napi_is_scheduled - test if NAPI is scheduled
525 * @n: NAPI context
526 *
527 * This check is "best-effort". With no locking implemented,
528 * a NAPI can be scheduled or terminate right after this check
529 * and produce not precise results.
530 *
531 * NAPI_STATE_SCHED is an internal state, napi_is_scheduled
532 * should not be used normally and napi_schedule should be
533 * used instead.
534 *
535 * Use only if the driver really needs to check if a NAPI
536 * is scheduled for example in the context of delayed timer
537 * that can be skipped if a NAPI is already scheduled.
538 *
539 * Return: True if NAPI is scheduled, False otherwise.
540 */
napi_is_scheduled(struct napi_struct * n)541 static inline bool napi_is_scheduled(struct napi_struct *n)
542 {
543 return test_bit(NAPI_STATE_SCHED, &n->state);
544 }
545
546 bool napi_schedule_prep(struct napi_struct *n);
547
548 /**
549 * napi_schedule - schedule NAPI poll
550 * @n: NAPI context
551 *
552 * Schedule NAPI poll routine to be called if it is not already
553 * running.
554 * Return: true if we schedule a NAPI or false if not.
555 * Refer to napi_schedule_prep() for additional reason on why
556 * a NAPI might not be scheduled.
557 */
napi_schedule(struct napi_struct * n)558 static inline bool napi_schedule(struct napi_struct *n)
559 {
560 if (napi_schedule_prep(n)) {
561 __napi_schedule(n);
562 return true;
563 }
564
565 return false;
566 }
567
568 /**
569 * napi_schedule_irqoff - schedule NAPI poll
570 * @n: NAPI context
571 *
572 * Variant of napi_schedule(), assuming hard irqs are masked.
573 */
napi_schedule_irqoff(struct napi_struct * n)574 static inline void napi_schedule_irqoff(struct napi_struct *n)
575 {
576 if (napi_schedule_prep(n))
577 __napi_schedule_irqoff(n);
578 }
579
580 /**
581 * napi_complete_done - NAPI processing complete
582 * @n: NAPI context
583 * @work_done: number of packets processed
584 *
585 * Mark NAPI processing as complete. Should only be called if poll budget
586 * has not been completely consumed.
587 * Prefer over napi_complete().
588 * Return: false if device should avoid rearming interrupts.
589 */
590 bool napi_complete_done(struct napi_struct *n, int work_done);
591
napi_complete(struct napi_struct * n)592 static inline bool napi_complete(struct napi_struct *n)
593 {
594 return napi_complete_done(n, 0);
595 }
596
597 void netif_threaded_enable(struct net_device *dev);
598 int dev_set_threaded(struct net_device *dev,
599 enum netdev_napi_threaded threaded);
600
601 void napi_disable(struct napi_struct *n);
602 void napi_disable_locked(struct napi_struct *n);
603
604 void napi_enable(struct napi_struct *n);
605 void napi_enable_locked(struct napi_struct *n);
606
607 /**
608 * napi_synchronize - wait until NAPI is not running
609 * @n: NAPI context
610 *
611 * Wait until NAPI is done being scheduled on this context.
612 * Waits till any outstanding processing completes but
613 * does not disable future activations.
614 */
napi_synchronize(const struct napi_struct * n)615 static inline void napi_synchronize(const struct napi_struct *n)
616 {
617 if (IS_ENABLED(CONFIG_SMP))
618 while (test_bit(NAPI_STATE_SCHED, &n->state))
619 msleep(1);
620 else
621 barrier();
622 }
623
624 /**
625 * napi_if_scheduled_mark_missed - if napi is running, set the
626 * NAPIF_STATE_MISSED
627 * @n: NAPI context
628 *
629 * If napi is running, set the NAPIF_STATE_MISSED, and return true if
630 * NAPI is scheduled.
631 **/
napi_if_scheduled_mark_missed(struct napi_struct * n)632 static inline bool napi_if_scheduled_mark_missed(struct napi_struct *n)
633 {
634 unsigned long val, new;
635
636 val = READ_ONCE(n->state);
637 do {
638 if (val & NAPIF_STATE_DISABLE)
639 return true;
640
641 if (!(val & NAPIF_STATE_SCHED))
642 return false;
643
644 new = val | NAPIF_STATE_MISSED;
645 } while (!try_cmpxchg(&n->state, &val, new));
646
647 return true;
648 }
649
650 enum netdev_queue_state_t {
651 __QUEUE_STATE_DRV_XOFF,
652 __QUEUE_STATE_STACK_XOFF,
653 __QUEUE_STATE_FROZEN,
654 };
655
656 #define QUEUE_STATE_DRV_XOFF (1 << __QUEUE_STATE_DRV_XOFF)
657 #define QUEUE_STATE_STACK_XOFF (1 << __QUEUE_STATE_STACK_XOFF)
658 #define QUEUE_STATE_FROZEN (1 << __QUEUE_STATE_FROZEN)
659
660 #define QUEUE_STATE_ANY_XOFF (QUEUE_STATE_DRV_XOFF | QUEUE_STATE_STACK_XOFF)
661 #define QUEUE_STATE_ANY_XOFF_OR_FROZEN (QUEUE_STATE_ANY_XOFF | \
662 QUEUE_STATE_FROZEN)
663 #define QUEUE_STATE_DRV_XOFF_OR_FROZEN (QUEUE_STATE_DRV_XOFF | \
664 QUEUE_STATE_FROZEN)
665
666 /*
667 * __QUEUE_STATE_DRV_XOFF is used by drivers to stop the transmit queue. The
668 * netif_tx_* functions below are used to manipulate this flag. The
669 * __QUEUE_STATE_STACK_XOFF flag is used by the stack to stop the transmit
670 * queue independently. The netif_xmit_*stopped functions below are called
671 * to check if the queue has been stopped by the driver or stack (either
672 * of the XOFF bits are set in the state). Drivers should not need to call
673 * netif_xmit*stopped functions, they should only be using netif_tx_*.
674 */
675
676 struct netdev_queue {
677 /*
678 * read-mostly part
679 */
680 struct net_device *dev;
681 netdevice_tracker dev_tracker;
682
683 struct Qdisc __rcu *qdisc;
684 struct Qdisc __rcu *qdisc_sleeping;
685 #ifdef CONFIG_SYSFS
686 struct kobject kobj;
687 const struct attribute_group **groups;
688 #endif
689 unsigned long tx_maxrate;
690 /*
691 * Number of TX timeouts for this queue
692 * (/sys/class/net/DEV/Q/trans_timeout)
693 */
694 atomic_long_t trans_timeout;
695
696 /* Subordinate device that the queue has been assigned to */
697 struct net_device *sb_dev;
698 #ifdef CONFIG_XDP_SOCKETS
699 /* "ops protected", see comment about net_device::lock */
700 struct xsk_buff_pool *pool;
701 #endif
702
703 /*
704 * write-mostly part
705 */
706 #ifdef CONFIG_BQL
707 struct dql dql;
708 #endif
709 spinlock_t _xmit_lock ____cacheline_aligned_in_smp;
710 int xmit_lock_owner;
711 /*
712 * Time (in jiffies) of last Tx
713 */
714 unsigned long trans_start;
715
716 unsigned long state;
717
718 /*
719 * slow- / control-path part
720 */
721 /* NAPI instance for the queue
722 * "ops protected", see comment about net_device::lock
723 */
724 struct napi_struct *napi;
725
726 #if defined(CONFIG_XPS) && defined(CONFIG_NUMA)
727 int numa_node;
728 #endif
729 } ____cacheline_aligned_in_smp;
730
731 extern int sysctl_fb_tunnels_only_for_init_net;
732 extern int sysctl_devconf_inherit_init_net;
733
734 /*
735 * sysctl_fb_tunnels_only_for_init_net == 0 : For all netns
736 * == 1 : For initns only
737 * == 2 : For none.
738 */
net_has_fallback_tunnels(const struct net * net)739 static inline bool net_has_fallback_tunnels(const struct net *net)
740 {
741 #if IS_ENABLED(CONFIG_SYSCTL)
742 int fb_tunnels_only_for_init_net = READ_ONCE(sysctl_fb_tunnels_only_for_init_net);
743
744 return !fb_tunnels_only_for_init_net ||
745 (net_eq(net, &init_net) && fb_tunnels_only_for_init_net == 1);
746 #else
747 return true;
748 #endif
749 }
750
net_inherit_devconf(void)751 static inline int net_inherit_devconf(void)
752 {
753 #if IS_ENABLED(CONFIG_SYSCTL)
754 return READ_ONCE(sysctl_devconf_inherit_init_net);
755 #else
756 return 0;
757 #endif
758 }
759
netdev_queue_numa_node_read(const struct netdev_queue * q)760 static inline int netdev_queue_numa_node_read(const struct netdev_queue *q)
761 {
762 #if defined(CONFIG_XPS) && defined(CONFIG_NUMA)
763 return q->numa_node;
764 #else
765 return NUMA_NO_NODE;
766 #endif
767 }
768
netdev_queue_numa_node_write(struct netdev_queue * q,int node)769 static inline void netdev_queue_numa_node_write(struct netdev_queue *q, int node)
770 {
771 #if defined(CONFIG_XPS) && defined(CONFIG_NUMA)
772 q->numa_node = node;
773 #endif
774 }
775
776 #ifdef CONFIG_RFS_ACCEL
777 bool rps_may_expire_flow(struct net_device *dev, u16 rxq_index, u32 flow_id,
778 u16 filter_id);
779 #endif
780
781 /* XPS map type and offset of the xps map within net_device->xps_maps[]. */
782 enum xps_map_type {
783 XPS_CPUS = 0,
784 XPS_RXQS,
785 XPS_MAPS_MAX,
786 };
787
788 #ifdef CONFIG_XPS
789 /*
790 * This structure holds an XPS map which can be of variable length. The
791 * map is an array of queues.
792 */
793 struct xps_map {
794 unsigned int len;
795 unsigned int alloc_len;
796 struct rcu_head rcu;
797 u16 queues[];
798 };
799 #define XPS_MAP_SIZE(_num) (sizeof(struct xps_map) + ((_num) * sizeof(u16)))
800 #define XPS_MIN_MAP_ALLOC ((L1_CACHE_ALIGN(offsetof(struct xps_map, queues[1])) \
801 - sizeof(struct xps_map)) / sizeof(u16))
802
803 /*
804 * This structure holds all XPS maps for device. Maps are indexed by CPU.
805 *
806 * We keep track of the number of cpus/rxqs used when the struct is allocated,
807 * in nr_ids. This will help not accessing out-of-bound memory.
808 *
809 * We keep track of the number of traffic classes used when the struct is
810 * allocated, in num_tc. This will be used to navigate the maps, to ensure we're
811 * not crossing its upper bound, as the original dev->num_tc can be updated in
812 * the meantime.
813 */
814 struct xps_dev_maps {
815 struct rcu_head rcu;
816 unsigned int nr_ids;
817 s16 num_tc;
818 struct xps_map __rcu *attr_map[]; /* Either CPUs map or RXQs map */
819 };
820
821 #define XPS_CPU_DEV_MAPS_SIZE(_tcs) (sizeof(struct xps_dev_maps) + \
822 (nr_cpu_ids * (_tcs) * sizeof(struct xps_map *)))
823
824 #define XPS_RXQ_DEV_MAPS_SIZE(_tcs, _rxqs) (sizeof(struct xps_dev_maps) +\
825 (_rxqs * (_tcs) * sizeof(struct xps_map *)))
826
827 #endif /* CONFIG_XPS */
828
829 #define TC_MAX_QUEUE 16
830 #define TC_BITMASK 15
831 /* HW offloaded queuing disciplines txq count and offset maps */
832 struct netdev_tc_txq {
833 u16 count;
834 u16 offset;
835 };
836
837 #if defined(CONFIG_FCOE) || defined(CONFIG_FCOE_MODULE)
838 /*
839 * This structure is to hold information about the device
840 * configured to run FCoE protocol stack.
841 */
842 struct netdev_fcoe_hbainfo {
843 char manufacturer[64];
844 char serial_number[64];
845 char hardware_version[64];
846 char driver_version[64];
847 char optionrom_version[64];
848 char firmware_version[64];
849 char model[256];
850 char model_description[256];
851 };
852 #endif
853
854 #define MAX_PHYS_ITEM_ID_LEN 32
855
856 /* This structure holds a unique identifier to identify some
857 * physical item (port for example) used by a netdevice.
858 */
859 struct netdev_phys_item_id {
860 unsigned char id[MAX_PHYS_ITEM_ID_LEN];
861 unsigned char id_len;
862 };
863
netdev_phys_item_id_same(struct netdev_phys_item_id * a,struct netdev_phys_item_id * b)864 static inline bool netdev_phys_item_id_same(struct netdev_phys_item_id *a,
865 struct netdev_phys_item_id *b)
866 {
867 return a->id_len == b->id_len &&
868 memcmp(a->id, b->id, a->id_len) == 0;
869 }
870
871 typedef u16 (*select_queue_fallback_t)(struct net_device *dev,
872 struct sk_buff *skb,
873 struct net_device *sb_dev);
874
875 enum net_device_path_type {
876 DEV_PATH_ETHERNET = 0,
877 DEV_PATH_VLAN,
878 DEV_PATH_BRIDGE,
879 DEV_PATH_PPPOE,
880 DEV_PATH_DSA,
881 DEV_PATH_MTK_WDMA,
882 DEV_PATH_TUN,
883 };
884
885 struct net_device_path {
886 enum net_device_path_type type;
887 const struct net_device *dev;
888 union {
889 struct {
890 u16 id;
891 __be16 proto;
892 u8 h_dest[ETH_ALEN];
893 } encap;
894 struct {
895 union {
896 struct in_addr src_v4;
897 struct in6_addr src_v6;
898 };
899 union {
900 struct in_addr dst_v4;
901 struct in6_addr dst_v6;
902 };
903
904 u8 l3_proto;
905 } tun;
906 struct {
907 enum {
908 DEV_PATH_BR_VLAN_KEEP,
909 DEV_PATH_BR_VLAN_TAG,
910 DEV_PATH_BR_VLAN_UNTAG,
911 DEV_PATH_BR_VLAN_UNTAG_HW,
912 } vlan_mode;
913 u16 vlan_id;
914 __be16 vlan_proto;
915 } bridge;
916 struct {
917 int port;
918 u16 proto;
919 } dsa;
920 struct {
921 u8 wdma_idx;
922 u8 queue;
923 u16 wcid;
924 u8 bss;
925 u8 amsdu;
926 } mtk_wdma;
927 };
928 };
929
930 #define NET_DEVICE_PATH_STACK_MAX 5
931 #define NET_DEVICE_PATH_VLAN_MAX 2
932
933 struct net_device_path_stack {
934 int num_paths;
935 struct net_device_path path[NET_DEVICE_PATH_STACK_MAX];
936 };
937
938 struct net_device_path_ctx {
939 const struct net_device *dev;
940 u8 daddr[ETH_ALEN];
941
942 int num_vlans;
943 struct {
944 u16 id;
945 __be16 proto;
946 } vlan[NET_DEVICE_PATH_VLAN_MAX];
947 };
948
949 enum tc_setup_type {
950 TC_QUERY_CAPS,
951 TC_SETUP_QDISC_MQPRIO,
952 TC_SETUP_CLSU32,
953 TC_SETUP_CLSFLOWER,
954 TC_SETUP_CLSMATCHALL,
955 TC_SETUP_CLSBPF,
956 TC_SETUP_BLOCK,
957 TC_SETUP_QDISC_CBS,
958 TC_SETUP_QDISC_RED,
959 TC_SETUP_QDISC_PRIO,
960 TC_SETUP_QDISC_MQ,
961 TC_SETUP_QDISC_ETF,
962 TC_SETUP_ROOT_QDISC,
963 TC_SETUP_QDISC_GRED,
964 TC_SETUP_QDISC_TAPRIO,
965 TC_SETUP_FT,
966 TC_SETUP_QDISC_ETS,
967 TC_SETUP_QDISC_TBF,
968 TC_SETUP_QDISC_FIFO,
969 TC_SETUP_QDISC_HTB,
970 TC_SETUP_ACT,
971 };
972
973 /* These structures hold the attributes of bpf state that are being passed
974 * to the netdevice through the bpf op.
975 */
976 enum bpf_netdev_command {
977 /* Set or clear a bpf program used in the earliest stages of packet
978 * rx. The prog will have been loaded as BPF_PROG_TYPE_XDP. The callee
979 * is responsible for calling bpf_prog_put on any old progs that are
980 * stored. In case of error, the callee need not release the new prog
981 * reference, but on success it takes ownership and must bpf_prog_put
982 * when it is no longer used.
983 */
984 XDP_SETUP_PROG,
985 XDP_SETUP_PROG_HW,
986 /* BPF program for offload callbacks, invoked at program load time. */
987 BPF_OFFLOAD_MAP_ALLOC,
988 BPF_OFFLOAD_MAP_FREE,
989 XDP_SETUP_XSK_POOL,
990 };
991
992 struct bpf_prog_offload_ops;
993 struct netlink_ext_ack;
994 struct xdp_umem;
995 struct xdp_dev_bulk_queue;
996 struct bpf_xdp_link;
997
998 enum bpf_xdp_mode {
999 XDP_MODE_SKB = 0,
1000 XDP_MODE_DRV = 1,
1001 XDP_MODE_HW = 2,
1002 __MAX_XDP_MODE
1003 };
1004
1005 struct bpf_xdp_entity {
1006 struct bpf_prog *prog;
1007 struct bpf_xdp_link *link;
1008 };
1009
1010 struct netdev_bpf {
1011 enum bpf_netdev_command command;
1012 union {
1013 /* XDP_SETUP_PROG */
1014 struct {
1015 u32 flags;
1016 struct bpf_prog *prog;
1017 struct netlink_ext_ack *extack;
1018 };
1019 /* BPF_OFFLOAD_MAP_ALLOC, BPF_OFFLOAD_MAP_FREE */
1020 struct {
1021 struct bpf_offloaded_map *offmap;
1022 };
1023 /* XDP_SETUP_XSK_POOL */
1024 struct {
1025 struct xsk_buff_pool *pool;
1026 u16 queue_id;
1027 } xsk;
1028 };
1029 };
1030
1031 /* Flags for ndo_xsk_wakeup. */
1032 #define XDP_WAKEUP_RX (1 << 0)
1033 #define XDP_WAKEUP_TX (1 << 1)
1034
1035 #ifdef CONFIG_XFRM_OFFLOAD
1036 struct xfrmdev_ops {
1037 int (*xdo_dev_state_add)(struct net_device *dev,
1038 struct xfrm_state *x,
1039 struct netlink_ext_ack *extack);
1040 void (*xdo_dev_state_delete)(struct net_device *dev,
1041 struct xfrm_state *x);
1042 void (*xdo_dev_state_free)(struct net_device *dev,
1043 struct xfrm_state *x);
1044 bool (*xdo_dev_offload_ok) (struct sk_buff *skb,
1045 struct xfrm_state *x);
1046 void (*xdo_dev_state_advance_esn) (struct xfrm_state *x);
1047 void (*xdo_dev_state_update_stats) (struct xfrm_state *x);
1048 int (*xdo_dev_policy_add) (struct xfrm_policy *x, struct netlink_ext_ack *extack);
1049 void (*xdo_dev_policy_delete) (struct xfrm_policy *x);
1050 void (*xdo_dev_policy_free) (struct xfrm_policy *x);
1051 };
1052 #endif
1053
1054 struct dev_ifalias {
1055 struct rcu_head rcuhead;
1056 char ifalias[];
1057 };
1058
1059 struct devlink;
1060 struct tlsdev_ops;
1061
1062 struct netdev_net_notifier {
1063 struct list_head list;
1064 struct notifier_block *nb;
1065 };
1066
1067 /*
1068 * This structure defines the management hooks for network devices.
1069 * The following hooks can be defined; unless noted otherwise, they are
1070 * optional and can be filled with a null pointer.
1071 *
1072 * int (*ndo_init)(struct net_device *dev);
1073 * This function is called once when a network device is registered.
1074 * The network device can use this for any late stage initialization
1075 * or semantic validation. It can fail with an error code which will
1076 * be propagated back to register_netdev.
1077 *
1078 * void (*ndo_uninit)(struct net_device *dev);
1079 * This function is called when device is unregistered or when registration
1080 * fails. It is not called if init fails.
1081 *
1082 * int (*ndo_open)(struct net_device *dev);
1083 * This function is called when a network device transitions to the up
1084 * state.
1085 *
1086 * int (*ndo_stop)(struct net_device *dev);
1087 * This function is called when a network device transitions to the down
1088 * state.
1089 *
1090 * netdev_tx_t (*ndo_start_xmit)(struct sk_buff *skb,
1091 * struct net_device *dev);
1092 * Called when a packet needs to be transmitted.
1093 * Returns NETDEV_TX_OK. Can return NETDEV_TX_BUSY, but you should stop
1094 * the queue before that can happen; it's for obsolete devices and weird
1095 * corner cases, but the stack really does a non-trivial amount
1096 * of useless work if you return NETDEV_TX_BUSY.
1097 * Required; cannot be NULL.
1098 *
1099 * netdev_features_t (*ndo_features_check)(struct sk_buff *skb,
1100 * struct net_device *dev
1101 * netdev_features_t features);
1102 * Called by core transmit path to determine if device is capable of
1103 * performing offload operations on a given packet. This is to give
1104 * the device an opportunity to implement any restrictions that cannot
1105 * be otherwise expressed by feature flags. The check is called with
1106 * the set of features that the stack has calculated and it returns
1107 * those the driver believes to be appropriate.
1108 *
1109 * u16 (*ndo_select_queue)(struct net_device *dev, struct sk_buff *skb,
1110 * struct net_device *sb_dev);
1111 * Called to decide which queue to use when device supports multiple
1112 * transmit queues.
1113 *
1114 * void (*ndo_change_rx_flags)(struct net_device *dev, int flags);
1115 * This function is called to allow device receiver to make
1116 * changes to configuration when multicast or promiscuous is enabled.
1117 *
1118 * void (*ndo_set_rx_mode)(struct net_device *dev);
1119 * This function is called device changes address list filtering.
1120 * If driver handles unicast address filtering, it should set
1121 * IFF_UNICAST_FLT in its priv_flags.
1122 *
1123 * int (*ndo_set_mac_address)(struct net_device *dev, void *addr);
1124 * This function is called when the Media Access Control address
1125 * needs to be changed. If this interface is not defined, the
1126 * MAC address can not be changed.
1127 *
1128 * int (*ndo_validate_addr)(struct net_device *dev);
1129 * Test if Media Access Control address is valid for the device.
1130 *
1131 * int (*ndo_do_ioctl)(struct net_device *dev, struct ifreq *ifr, int cmd);
1132 * Old-style ioctl entry point. This is used internally by the
1133 * ieee802154 subsystem but is no longer called by the device
1134 * ioctl handler.
1135 *
1136 * int (*ndo_siocbond)(struct net_device *dev, struct ifreq *ifr, int cmd);
1137 * Used by the bonding driver for its device specific ioctls:
1138 * SIOCBONDENSLAVE, SIOCBONDRELEASE, SIOCBONDSETHWADDR, SIOCBONDCHANGEACTIVE,
1139 * SIOCBONDSLAVEINFOQUERY, and SIOCBONDINFOQUERY
1140 *
1141 * * int (*ndo_eth_ioctl)(struct net_device *dev, struct ifreq *ifr, int cmd);
1142 * Called for ethernet specific ioctls: SIOCGMIIPHY, SIOCGMIIREG,
1143 * SIOCSMIIREG, SIOCSHWTSTAMP and SIOCGHWTSTAMP.
1144 *
1145 * int (*ndo_set_config)(struct net_device *dev, struct ifmap *map);
1146 * Used to set network devices bus interface parameters. This interface
1147 * is retained for legacy reasons; new devices should use the bus
1148 * interface (PCI) for low level management.
1149 *
1150 * int (*ndo_change_mtu)(struct net_device *dev, int new_mtu);
1151 * Called when a user wants to change the Maximum Transfer Unit
1152 * of a device.
1153 *
1154 * void (*ndo_tx_timeout)(struct net_device *dev, unsigned int txqueue);
1155 * Callback used when the transmitter has not made any progress
1156 * for dev->watchdog ticks.
1157 *
1158 * void (*ndo_get_stats64)(struct net_device *dev,
1159 * struct rtnl_link_stats64 *storage);
1160 * struct net_device_stats* (*ndo_get_stats)(struct net_device *dev);
1161 * Called when a user wants to get the network device usage
1162 * statistics. Drivers must do one of the following:
1163 * 1. Define @ndo_get_stats64 to fill in a zero-initialised
1164 * rtnl_link_stats64 structure passed by the caller.
1165 * 2. Define @ndo_get_stats to update a net_device_stats structure
1166 * (which should normally be dev->stats) and return a pointer to
1167 * it. The structure may be changed asynchronously only if each
1168 * field is written atomically.
1169 * 3. Update dev->stats asynchronously and atomically, and define
1170 * neither operation.
1171 *
1172 * bool (*ndo_has_offload_stats)(const struct net_device *dev, int attr_id)
1173 * Return true if this device supports offload stats of this attr_id.
1174 *
1175 * int (*ndo_get_offload_stats)(int attr_id, const struct net_device *dev,
1176 * void *attr_data)
1177 * Get statistics for offload operations by attr_id. Write it into the
1178 * attr_data pointer.
1179 *
1180 * int (*ndo_vlan_rx_add_vid)(struct net_device *dev, __be16 proto, u16 vid);
1181 * If device supports VLAN filtering this function is called when a
1182 * VLAN id is registered.
1183 *
1184 * int (*ndo_vlan_rx_kill_vid)(struct net_device *dev, __be16 proto, u16 vid);
1185 * If device supports VLAN filtering this function is called when a
1186 * VLAN id is unregistered.
1187 *
1188 * void (*ndo_poll_controller)(struct net_device *dev);
1189 *
1190 * SR-IOV management functions.
1191 * int (*ndo_set_vf_mac)(struct net_device *dev, int vf, u8* mac);
1192 * int (*ndo_set_vf_vlan)(struct net_device *dev, int vf, u16 vlan,
1193 * u8 qos, __be16 proto);
1194 * int (*ndo_set_vf_rate)(struct net_device *dev, int vf, int min_tx_rate,
1195 * int max_tx_rate);
1196 * int (*ndo_set_vf_spoofchk)(struct net_device *dev, int vf, bool setting);
1197 * int (*ndo_set_vf_trust)(struct net_device *dev, int vf, bool setting);
1198 * int (*ndo_get_vf_config)(struct net_device *dev,
1199 * int vf, struct ifla_vf_info *ivf);
1200 * int (*ndo_set_vf_link_state)(struct net_device *dev, int vf, int link_state);
1201 * int (*ndo_set_vf_port)(struct net_device *dev, int vf,
1202 * struct nlattr *port[]);
1203 *
1204 * Enable or disable the VF ability to query its RSS Redirection Table and
1205 * Hash Key. This is needed since on some devices VF share this information
1206 * with PF and querying it may introduce a theoretical security risk.
1207 * int (*ndo_set_vf_rss_query_en)(struct net_device *dev, int vf, bool setting);
1208 * int (*ndo_get_vf_port)(struct net_device *dev, int vf, struct sk_buff *skb);
1209 * int (*ndo_setup_tc)(struct net_device *dev, enum tc_setup_type type,
1210 * void *type_data);
1211 * Called to setup any 'tc' scheduler, classifier or action on @dev.
1212 * This is always called from the stack with the rtnl lock held and netif
1213 * tx queues stopped. This allows the netdevice to perform queue
1214 * management safely.
1215 *
1216 * Fiber Channel over Ethernet (FCoE) offload functions.
1217 * int (*ndo_fcoe_enable)(struct net_device *dev);
1218 * Called when the FCoE protocol stack wants to start using LLD for FCoE
1219 * so the underlying device can perform whatever needed configuration or
1220 * initialization to support acceleration of FCoE traffic.
1221 *
1222 * int (*ndo_fcoe_disable)(struct net_device *dev);
1223 * Called when the FCoE protocol stack wants to stop using LLD for FCoE
1224 * so the underlying device can perform whatever needed clean-ups to
1225 * stop supporting acceleration of FCoE traffic.
1226 *
1227 * int (*ndo_fcoe_ddp_setup)(struct net_device *dev, u16 xid,
1228 * struct scatterlist *sgl, unsigned int sgc);
1229 * Called when the FCoE Initiator wants to initialize an I/O that
1230 * is a possible candidate for Direct Data Placement (DDP). The LLD can
1231 * perform necessary setup and returns 1 to indicate the device is set up
1232 * successfully to perform DDP on this I/O, otherwise this returns 0.
1233 *
1234 * int (*ndo_fcoe_ddp_done)(struct net_device *dev, u16 xid);
1235 * Called when the FCoE Initiator/Target is done with the DDPed I/O as
1236 * indicated by the FC exchange id 'xid', so the underlying device can
1237 * clean up and reuse resources for later DDP requests.
1238 *
1239 * int (*ndo_fcoe_ddp_target)(struct net_device *dev, u16 xid,
1240 * struct scatterlist *sgl, unsigned int sgc);
1241 * Called when the FCoE Target wants to initialize an I/O that
1242 * is a possible candidate for Direct Data Placement (DDP). The LLD can
1243 * perform necessary setup and returns 1 to indicate the device is set up
1244 * successfully to perform DDP on this I/O, otherwise this returns 0.
1245 *
1246 * int (*ndo_fcoe_get_hbainfo)(struct net_device *dev,
1247 * struct netdev_fcoe_hbainfo *hbainfo);
1248 * Called when the FCoE Protocol stack wants information on the underlying
1249 * device. This information is utilized by the FCoE protocol stack to
1250 * register attributes with Fiber Channel management service as per the
1251 * FC-GS Fabric Device Management Information(FDMI) specification.
1252 *
1253 * int (*ndo_fcoe_get_wwn)(struct net_device *dev, u64 *wwn, int type);
1254 * Called when the underlying device wants to override default World Wide
1255 * Name (WWN) generation mechanism in FCoE protocol stack to pass its own
1256 * World Wide Port Name (WWPN) or World Wide Node Name (WWNN) to the FCoE
1257 * protocol stack to use.
1258 *
1259 * RFS acceleration.
1260 * int (*ndo_rx_flow_steer)(struct net_device *dev, const struct sk_buff *skb,
1261 * u16 rxq_index, u32 flow_id);
1262 * Set hardware filter for RFS. rxq_index is the target queue index;
1263 * flow_id is a flow ID to be passed to rps_may_expire_flow() later.
1264 * Return the filter ID on success, or a negative error code.
1265 *
1266 * Slave management functions (for bridge, bonding, etc).
1267 * int (*ndo_add_slave)(struct net_device *dev, struct net_device *slave_dev);
1268 * Called to make another netdev an underling.
1269 *
1270 * int (*ndo_del_slave)(struct net_device *dev, struct net_device *slave_dev);
1271 * Called to release previously enslaved netdev.
1272 *
1273 * struct net_device *(*ndo_get_xmit_slave)(struct net_device *dev,
1274 * struct sk_buff *skb,
1275 * bool all_slaves);
1276 * Get the xmit slave of master device. If all_slaves is true, function
1277 * assume all the slaves can transmit.
1278 *
1279 * Feature/offload setting functions.
1280 * netdev_features_t (*ndo_fix_features)(struct net_device *dev,
1281 * netdev_features_t features);
1282 * Adjusts the requested feature flags according to device-specific
1283 * constraints, and returns the resulting flags. Must not modify
1284 * the device state.
1285 *
1286 * int (*ndo_set_features)(struct net_device *dev, netdev_features_t features);
1287 * Called to update device configuration to new features. Passed
1288 * feature set might be less than what was returned by ndo_fix_features()).
1289 * Must return >0 or -errno if it changed dev->features itself.
1290 *
1291 * int (*ndo_fdb_add)(struct ndmsg *ndm, struct nlattr *tb[],
1292 * struct net_device *dev,
1293 * const unsigned char *addr, u16 vid, u16 flags,
1294 * bool *notified, struct netlink_ext_ack *extack);
1295 * Adds an FDB entry to dev for addr.
1296 * Callee shall set *notified to true if it sent any appropriate
1297 * notification(s). Otherwise core will send a generic one.
1298 * int (*ndo_fdb_del)(struct ndmsg *ndm, struct nlattr *tb[],
1299 * struct net_device *dev,
1300 * const unsigned char *addr, u16 vid
1301 * bool *notified, struct netlink_ext_ack *extack);
1302 * Deletes the FDB entry from dev corresponding to addr.
1303 * Callee shall set *notified to true if it sent any appropriate
1304 * notification(s). Otherwise core will send a generic one.
1305 * int (*ndo_fdb_del_bulk)(struct nlmsghdr *nlh, struct net_device *dev,
1306 * struct netlink_ext_ack *extack);
1307 * int (*ndo_fdb_dump)(struct sk_buff *skb, struct netlink_callback *cb,
1308 * struct net_device *dev, struct net_device *filter_dev,
1309 * int *idx)
1310 * Used to add FDB entries to dump requests. Implementers should add
1311 * entries to skb and update idx with the number of entries.
1312 *
1313 * int (*ndo_mdb_add)(struct net_device *dev, struct nlattr *tb[],
1314 * u16 nlmsg_flags, struct netlink_ext_ack *extack);
1315 * Adds an MDB entry to dev.
1316 * int (*ndo_mdb_del)(struct net_device *dev, struct nlattr *tb[],
1317 * struct netlink_ext_ack *extack);
1318 * Deletes the MDB entry from dev.
1319 * int (*ndo_mdb_del_bulk)(struct net_device *dev, struct nlattr *tb[],
1320 * struct netlink_ext_ack *extack);
1321 * Bulk deletes MDB entries from dev.
1322 * int (*ndo_mdb_dump)(struct net_device *dev, struct sk_buff *skb,
1323 * struct netlink_callback *cb);
1324 * Dumps MDB entries from dev. The first argument (marker) in the netlink
1325 * callback is used by core rtnetlink code.
1326 *
1327 * int (*ndo_bridge_setlink)(struct net_device *dev, struct nlmsghdr *nlh,
1328 * u16 flags, struct netlink_ext_ack *extack)
1329 * int (*ndo_bridge_getlink)(struct sk_buff *skb, u32 pid, u32 seq,
1330 * struct net_device *dev, u32 filter_mask,
1331 * int nlflags)
1332 * int (*ndo_bridge_dellink)(struct net_device *dev, struct nlmsghdr *nlh,
1333 * u16 flags);
1334 *
1335 * int (*ndo_change_carrier)(struct net_device *dev, bool new_carrier);
1336 * Called to change device carrier. Soft-devices (like dummy, team, etc)
1337 * which do not represent real hardware may define this to allow their
1338 * userspace components to manage their virtual carrier state. Devices
1339 * that determine carrier state from physical hardware properties (eg
1340 * network cables) or protocol-dependent mechanisms (eg
1341 * USB_CDC_NOTIFY_NETWORK_CONNECTION) should NOT implement this function.
1342 *
1343 * int (*ndo_get_phys_port_id)(struct net_device *dev,
1344 * struct netdev_phys_item_id *ppid);
1345 * Called to get ID of physical port of this device. If driver does
1346 * not implement this, it is assumed that the hw is not able to have
1347 * multiple net devices on single physical port.
1348 *
1349 * int (*ndo_get_port_parent_id)(struct net_device *dev,
1350 * struct netdev_phys_item_id *ppid)
1351 * Called to get the parent ID of the physical port of this device.
1352 *
1353 * void* (*ndo_dfwd_add_station)(struct net_device *pdev,
1354 * struct net_device *dev)
1355 * Called by upper layer devices to accelerate switching or other
1356 * station functionality into hardware. 'pdev is the lowerdev
1357 * to use for the offload and 'dev' is the net device that will
1358 * back the offload. Returns a pointer to the private structure
1359 * the upper layer will maintain.
1360 * void (*ndo_dfwd_del_station)(struct net_device *pdev, void *priv)
1361 * Called by upper layer device to delete the station created
1362 * by 'ndo_dfwd_add_station'. 'pdev' is the net device backing
1363 * the station and priv is the structure returned by the add
1364 * operation.
1365 * int (*ndo_set_tx_maxrate)(struct net_device *dev,
1366 * int queue_index, u32 maxrate);
1367 * Called when a user wants to set a max-rate limitation of specific
1368 * TX queue.
1369 * int (*ndo_get_iflink)(const struct net_device *dev);
1370 * Called to get the iflink value of this device.
1371 * int (*ndo_fill_metadata_dst)(struct net_device *dev, struct sk_buff *skb);
1372 * This function is used to get egress tunnel information for given skb.
1373 * This is useful for retrieving outer tunnel header parameters while
1374 * sampling packet.
1375 * void (*ndo_set_rx_headroom)(struct net_device *dev, int needed_headroom);
1376 * This function is used to specify the headroom that the skb must
1377 * consider when allocation skb during packet reception. Setting
1378 * appropriate rx headroom value allows avoiding skb head copy on
1379 * forward. Setting a negative value resets the rx headroom to the
1380 * default value.
1381 * int (*ndo_bpf)(struct net_device *dev, struct netdev_bpf *bpf);
1382 * This function is used to set or query state related to XDP on the
1383 * netdevice and manage BPF offload. See definition of
1384 * enum bpf_netdev_command for details.
1385 * int (*ndo_xdp_xmit)(struct net_device *dev, int n, struct xdp_frame **xdp,
1386 * u32 flags);
1387 * This function is used to submit @n XDP packets for transmit on a
1388 * netdevice. Returns number of frames successfully transmitted, frames
1389 * that got dropped are freed/returned via xdp_return_frame().
1390 * Returns negative number, means general error invoking ndo, meaning
1391 * no frames were xmit'ed and core-caller will free all frames.
1392 * struct net_device *(*ndo_xdp_get_xmit_slave)(struct net_device *dev,
1393 * struct xdp_buff *xdp);
1394 * Get the xmit slave of master device based on the xdp_buff.
1395 * int (*ndo_xsk_wakeup)(struct net_device *dev, u32 queue_id, u32 flags);
1396 * This function is used to wake up the softirq, ksoftirqd or kthread
1397 * responsible for sending and/or receiving packets on a specific
1398 * queue id bound to an AF_XDP socket. The flags field specifies if
1399 * only RX, only Tx, or both should be woken up using the flags
1400 * XDP_WAKEUP_RX and XDP_WAKEUP_TX.
1401 * int (*ndo_tunnel_ctl)(struct net_device *dev, struct ip_tunnel_parm_kern *p,
1402 * int cmd);
1403 * Add, change, delete or get information on an IPv4 tunnel.
1404 * struct net_device *(*ndo_get_peer_dev)(struct net_device *dev);
1405 * If a device is paired with a peer device, return the peer instance.
1406 * The caller must be under RCU read context.
1407 * int (*ndo_fill_forward_path)(struct net_device_path_ctx *ctx, struct net_device_path *path);
1408 * Get the forwarding path to reach the real device from the HW destination address
1409 * ktime_t (*ndo_get_tstamp)(struct net_device *dev,
1410 * const struct skb_shared_hwtstamps *hwtstamps,
1411 * bool cycles);
1412 * Get hardware timestamp based on normal/adjustable time or free running
1413 * cycle counter. This function is required if physical clock supports a
1414 * free running cycle counter.
1415 *
1416 * int (*ndo_hwtstamp_get)(struct net_device *dev,
1417 * struct kernel_hwtstamp_config *kernel_config);
1418 * Get the currently configured hardware timestamping parameters for the
1419 * NIC device.
1420 *
1421 * int (*ndo_hwtstamp_set)(struct net_device *dev,
1422 * struct kernel_hwtstamp_config *kernel_config,
1423 * struct netlink_ext_ack *extack);
1424 * Change the hardware timestamping parameters for NIC device.
1425 */
1426 struct net_device_ops {
1427 int (*ndo_init)(struct net_device *dev);
1428 void (*ndo_uninit)(struct net_device *dev);
1429 int (*ndo_open)(struct net_device *dev);
1430 int (*ndo_stop)(struct net_device *dev);
1431 netdev_tx_t (*ndo_start_xmit)(struct sk_buff *skb,
1432 struct net_device *dev);
1433 netdev_features_t (*ndo_features_check)(struct sk_buff *skb,
1434 struct net_device *dev,
1435 netdev_features_t features);
1436 u16 (*ndo_select_queue)(struct net_device *dev,
1437 struct sk_buff *skb,
1438 struct net_device *sb_dev);
1439 void (*ndo_change_rx_flags)(struct net_device *dev,
1440 int flags);
1441 void (*ndo_set_rx_mode)(struct net_device *dev);
1442 int (*ndo_set_mac_address)(struct net_device *dev,
1443 void *addr);
1444 int (*ndo_validate_addr)(struct net_device *dev);
1445 int (*ndo_do_ioctl)(struct net_device *dev,
1446 struct ifreq *ifr, int cmd);
1447 int (*ndo_eth_ioctl)(struct net_device *dev,
1448 struct ifreq *ifr, int cmd);
1449 int (*ndo_siocbond)(struct net_device *dev,
1450 struct ifreq *ifr, int cmd);
1451 int (*ndo_siocwandev)(struct net_device *dev,
1452 struct if_settings *ifs);
1453 int (*ndo_siocdevprivate)(struct net_device *dev,
1454 struct ifreq *ifr,
1455 void __user *data, int cmd);
1456 int (*ndo_set_config)(struct net_device *dev,
1457 struct ifmap *map);
1458 int (*ndo_change_mtu)(struct net_device *dev,
1459 int new_mtu);
1460 int (*ndo_neigh_setup)(struct net_device *dev,
1461 struct neigh_parms *);
1462 void (*ndo_tx_timeout) (struct net_device *dev,
1463 unsigned int txqueue);
1464
1465 void (*ndo_get_stats64)(struct net_device *dev,
1466 struct rtnl_link_stats64 *storage);
1467 bool (*ndo_has_offload_stats)(const struct net_device *dev, int attr_id);
1468 int (*ndo_get_offload_stats)(int attr_id,
1469 const struct net_device *dev,
1470 void *attr_data);
1471 struct net_device_stats* (*ndo_get_stats)(struct net_device *dev);
1472
1473 int (*ndo_vlan_rx_add_vid)(struct net_device *dev,
1474 __be16 proto, u16 vid);
1475 int (*ndo_vlan_rx_kill_vid)(struct net_device *dev,
1476 __be16 proto, u16 vid);
1477 #ifdef CONFIG_NET_POLL_CONTROLLER
1478 void (*ndo_poll_controller)(struct net_device *dev);
1479 int (*ndo_netpoll_setup)(struct net_device *dev);
1480 void (*ndo_netpoll_cleanup)(struct net_device *dev);
1481 #endif
1482 int (*ndo_set_vf_mac)(struct net_device *dev,
1483 int queue, u8 *mac);
1484 int (*ndo_set_vf_vlan)(struct net_device *dev,
1485 int queue, u16 vlan,
1486 u8 qos, __be16 proto);
1487 int (*ndo_set_vf_rate)(struct net_device *dev,
1488 int vf, int min_tx_rate,
1489 int max_tx_rate);
1490 int (*ndo_set_vf_spoofchk)(struct net_device *dev,
1491 int vf, bool setting);
1492 int (*ndo_set_vf_trust)(struct net_device *dev,
1493 int vf, bool setting);
1494 int (*ndo_get_vf_config)(struct net_device *dev,
1495 int vf,
1496 struct ifla_vf_info *ivf);
1497 int (*ndo_set_vf_link_state)(struct net_device *dev,
1498 int vf, int link_state);
1499 int (*ndo_get_vf_stats)(struct net_device *dev,
1500 int vf,
1501 struct ifla_vf_stats
1502 *vf_stats);
1503 int (*ndo_set_vf_port)(struct net_device *dev,
1504 int vf,
1505 struct nlattr *port[]);
1506 int (*ndo_get_vf_port)(struct net_device *dev,
1507 int vf, struct sk_buff *skb);
1508 int (*ndo_get_vf_guid)(struct net_device *dev,
1509 int vf,
1510 struct ifla_vf_guid *node_guid,
1511 struct ifla_vf_guid *port_guid);
1512 int (*ndo_set_vf_guid)(struct net_device *dev,
1513 int vf, u64 guid,
1514 int guid_type);
1515 int (*ndo_set_vf_rss_query_en)(
1516 struct net_device *dev,
1517 int vf, bool setting);
1518 int (*ndo_setup_tc)(struct net_device *dev,
1519 enum tc_setup_type type,
1520 void *type_data);
1521 #if IS_ENABLED(CONFIG_FCOE)
1522 int (*ndo_fcoe_enable)(struct net_device *dev);
1523 int (*ndo_fcoe_disable)(struct net_device *dev);
1524 int (*ndo_fcoe_ddp_setup)(struct net_device *dev,
1525 u16 xid,
1526 struct scatterlist *sgl,
1527 unsigned int sgc);
1528 int (*ndo_fcoe_ddp_done)(struct net_device *dev,
1529 u16 xid);
1530 int (*ndo_fcoe_ddp_target)(struct net_device *dev,
1531 u16 xid,
1532 struct scatterlist *sgl,
1533 unsigned int sgc);
1534 int (*ndo_fcoe_get_hbainfo)(struct net_device *dev,
1535 struct netdev_fcoe_hbainfo *hbainfo);
1536 #endif
1537
1538 #if IS_ENABLED(CONFIG_LIBFCOE)
1539 #define NETDEV_FCOE_WWNN 0
1540 #define NETDEV_FCOE_WWPN 1
1541 int (*ndo_fcoe_get_wwn)(struct net_device *dev,
1542 u64 *wwn, int type);
1543 #endif
1544
1545 #ifdef CONFIG_RFS_ACCEL
1546 int (*ndo_rx_flow_steer)(struct net_device *dev,
1547 const struct sk_buff *skb,
1548 u16 rxq_index,
1549 u32 flow_id);
1550 #endif
1551 int (*ndo_add_slave)(struct net_device *dev,
1552 struct net_device *slave_dev,
1553 struct netlink_ext_ack *extack);
1554 int (*ndo_del_slave)(struct net_device *dev,
1555 struct net_device *slave_dev);
1556 struct net_device* (*ndo_get_xmit_slave)(struct net_device *dev,
1557 struct sk_buff *skb,
1558 bool all_slaves);
1559 struct net_device* (*ndo_sk_get_lower_dev)(struct net_device *dev,
1560 struct sock *sk);
1561 netdev_features_t (*ndo_fix_features)(struct net_device *dev,
1562 netdev_features_t features);
1563 int (*ndo_set_features)(struct net_device *dev,
1564 netdev_features_t features);
1565 int (*ndo_neigh_construct)(struct net_device *dev,
1566 struct neighbour *n);
1567 void (*ndo_neigh_destroy)(struct net_device *dev,
1568 struct neighbour *n);
1569
1570 int (*ndo_fdb_add)(struct ndmsg *ndm,
1571 struct nlattr *tb[],
1572 struct net_device *dev,
1573 const unsigned char *addr,
1574 u16 vid,
1575 u16 flags,
1576 bool *notified,
1577 struct netlink_ext_ack *extack);
1578 int (*ndo_fdb_del)(struct ndmsg *ndm,
1579 struct nlattr *tb[],
1580 struct net_device *dev,
1581 const unsigned char *addr,
1582 u16 vid,
1583 bool *notified,
1584 struct netlink_ext_ack *extack);
1585 int (*ndo_fdb_del_bulk)(struct nlmsghdr *nlh,
1586 struct net_device *dev,
1587 struct netlink_ext_ack *extack);
1588 int (*ndo_fdb_dump)(struct sk_buff *skb,
1589 struct netlink_callback *cb,
1590 struct net_device *dev,
1591 struct net_device *filter_dev,
1592 int *idx);
1593 int (*ndo_fdb_get)(struct sk_buff *skb,
1594 struct nlattr *tb[],
1595 struct net_device *dev,
1596 const unsigned char *addr,
1597 u16 vid, u32 portid, u32 seq,
1598 struct netlink_ext_ack *extack);
1599 int (*ndo_mdb_add)(struct net_device *dev,
1600 struct nlattr *tb[],
1601 u16 nlmsg_flags,
1602 struct netlink_ext_ack *extack);
1603 int (*ndo_mdb_del)(struct net_device *dev,
1604 struct nlattr *tb[],
1605 struct netlink_ext_ack *extack);
1606 int (*ndo_mdb_del_bulk)(struct net_device *dev,
1607 struct nlattr *tb[],
1608 struct netlink_ext_ack *extack);
1609 int (*ndo_mdb_dump)(struct net_device *dev,
1610 struct sk_buff *skb,
1611 struct netlink_callback *cb);
1612 int (*ndo_mdb_get)(struct net_device *dev,
1613 struct nlattr *tb[], u32 portid,
1614 u32 seq,
1615 struct netlink_ext_ack *extack);
1616 int (*ndo_bridge_setlink)(struct net_device *dev,
1617 struct nlmsghdr *nlh,
1618 u16 flags,
1619 struct netlink_ext_ack *extack);
1620 int (*ndo_bridge_getlink)(struct sk_buff *skb,
1621 u32 pid, u32 seq,
1622 struct net_device *dev,
1623 u32 filter_mask,
1624 int nlflags);
1625 int (*ndo_bridge_dellink)(struct net_device *dev,
1626 struct nlmsghdr *nlh,
1627 u16 flags);
1628 int (*ndo_change_carrier)(struct net_device *dev,
1629 bool new_carrier);
1630 int (*ndo_get_phys_port_id)(struct net_device *dev,
1631 struct netdev_phys_item_id *ppid);
1632 int (*ndo_get_port_parent_id)(struct net_device *dev,
1633 struct netdev_phys_item_id *ppid);
1634 int (*ndo_get_phys_port_name)(struct net_device *dev,
1635 char *name, size_t len);
1636 void* (*ndo_dfwd_add_station)(struct net_device *pdev,
1637 struct net_device *dev);
1638 void (*ndo_dfwd_del_station)(struct net_device *pdev,
1639 void *priv);
1640
1641 int (*ndo_set_tx_maxrate)(struct net_device *dev,
1642 int queue_index,
1643 u32 maxrate);
1644 int (*ndo_get_iflink)(const struct net_device *dev);
1645 int (*ndo_fill_metadata_dst)(struct net_device *dev,
1646 struct sk_buff *skb);
1647 void (*ndo_set_rx_headroom)(struct net_device *dev,
1648 int needed_headroom);
1649 int (*ndo_bpf)(struct net_device *dev,
1650 struct netdev_bpf *bpf);
1651 int (*ndo_xdp_xmit)(struct net_device *dev, int n,
1652 struct xdp_frame **xdp,
1653 u32 flags);
1654 struct net_device * (*ndo_xdp_get_xmit_slave)(struct net_device *dev,
1655 struct xdp_buff *xdp);
1656 int (*ndo_xsk_wakeup)(struct net_device *dev,
1657 u32 queue_id, u32 flags);
1658 int (*ndo_tunnel_ctl)(struct net_device *dev,
1659 struct ip_tunnel_parm_kern *p,
1660 int cmd);
1661 struct net_device * (*ndo_get_peer_dev)(struct net_device *dev);
1662 int (*ndo_fill_forward_path)(struct net_device_path_ctx *ctx,
1663 struct net_device_path *path);
1664 ktime_t (*ndo_get_tstamp)(struct net_device *dev,
1665 const struct skb_shared_hwtstamps *hwtstamps,
1666 bool cycles);
1667 int (*ndo_hwtstamp_get)(struct net_device *dev,
1668 struct kernel_hwtstamp_config *kernel_config);
1669 int (*ndo_hwtstamp_set)(struct net_device *dev,
1670 struct kernel_hwtstamp_config *kernel_config,
1671 struct netlink_ext_ack *extack);
1672
1673 #if IS_ENABLED(CONFIG_NET_SHAPER)
1674 /**
1675 * @net_shaper_ops: Device shaping offload operations
1676 * see include/net/net_shapers.h
1677 */
1678 const struct net_shaper_ops *net_shaper_ops;
1679 #endif
1680 };
1681
1682 /**
1683 * enum netdev_priv_flags - &struct net_device priv_flags
1684 *
1685 * These are the &struct net_device, they are only set internally
1686 * by drivers and used in the kernel. These flags are invisible to
1687 * userspace; this means that the order of these flags can change
1688 * during any kernel release.
1689 *
1690 * You should add bitfield booleans after either net_device::priv_flags
1691 * (hotpath) or ::threaded (slowpath) instead of extending these flags.
1692 *
1693 * @IFF_802_1Q_VLAN: 802.1Q VLAN device
1694 * @IFF_EBRIDGE: Ethernet bridging device
1695 * @IFF_BONDING: bonding master or slave
1696 * @IFF_ISATAP: ISATAP interface (RFC4214)
1697 * @IFF_WAN_HDLC: WAN HDLC device
1698 * @IFF_XMIT_DST_RELEASE: dev_hard_start_xmit() is allowed to
1699 * release skb->dst
1700 * @IFF_DONT_BRIDGE: disallow bridging this ether dev
1701 * @IFF_DISABLE_NETPOLL: disable netpoll at run-time
1702 * @IFF_MACVLAN_PORT: device used as macvlan port
1703 * @IFF_BRIDGE_PORT: device used as bridge port
1704 * @IFF_OVS_DATAPATH: device used as Open vSwitch datapath port
1705 * @IFF_TX_SKB_SHARING: The interface supports sharing skbs on transmit
1706 * @IFF_UNICAST_FLT: Supports unicast filtering
1707 * @IFF_TEAM_PORT: device used as team port
1708 * @IFF_SUPP_NOFCS: device supports sending custom FCS
1709 * @IFF_LIVE_ADDR_CHANGE: device supports hardware address
1710 * change when it's running
1711 * @IFF_MACVLAN: Macvlan device
1712 * @IFF_XMIT_DST_RELEASE_PERM: IFF_XMIT_DST_RELEASE not taking into account
1713 * underlying stacked devices
1714 * @IFF_L3MDEV_MASTER: device is an L3 master device
1715 * @IFF_NO_QUEUE: device can run without qdisc attached
1716 * @IFF_OPENVSWITCH: device is a Open vSwitch master
1717 * @IFF_L3MDEV_SLAVE: device is enslaved to an L3 master device
1718 * @IFF_TEAM: device is a team device
1719 * @IFF_RXFH_CONFIGURED: device has had Rx Flow indirection table configured
1720 * @IFF_PHONY_HEADROOM: the headroom value is controlled by an external
1721 * entity (i.e. the master device for bridged veth)
1722 * @IFF_MACSEC: device is a MACsec device
1723 * @IFF_NO_RX_HANDLER: device doesn't support the rx_handler hook
1724 * @IFF_FAILOVER: device is a failover master device
1725 * @IFF_FAILOVER_SLAVE: device is lower dev of a failover master device
1726 * @IFF_L3MDEV_RX_HANDLER: only invoke the rx handler of L3 master device
1727 * @IFF_NO_ADDRCONF: prevent ipv6 addrconf
1728 * @IFF_TX_SKB_NO_LINEAR: device/driver is capable of xmitting frames with
1729 * skb_headlen(skb) == 0 (data starts from frag0)
1730 */
1731 enum netdev_priv_flags {
1732 IFF_802_1Q_VLAN = 1<<0,
1733 IFF_EBRIDGE = 1<<1,
1734 IFF_BONDING = 1<<2,
1735 IFF_ISATAP = 1<<3,
1736 IFF_WAN_HDLC = 1<<4,
1737 IFF_XMIT_DST_RELEASE = 1<<5,
1738 IFF_DONT_BRIDGE = 1<<6,
1739 IFF_DISABLE_NETPOLL = 1<<7,
1740 IFF_MACVLAN_PORT = 1<<8,
1741 IFF_BRIDGE_PORT = 1<<9,
1742 IFF_OVS_DATAPATH = 1<<10,
1743 IFF_TX_SKB_SHARING = 1<<11,
1744 IFF_UNICAST_FLT = 1<<12,
1745 IFF_TEAM_PORT = 1<<13,
1746 IFF_SUPP_NOFCS = 1<<14,
1747 IFF_LIVE_ADDR_CHANGE = 1<<15,
1748 IFF_MACVLAN = 1<<16,
1749 IFF_XMIT_DST_RELEASE_PERM = 1<<17,
1750 IFF_L3MDEV_MASTER = 1<<18,
1751 IFF_NO_QUEUE = 1<<19,
1752 IFF_OPENVSWITCH = 1<<20,
1753 IFF_L3MDEV_SLAVE = 1<<21,
1754 IFF_TEAM = 1<<22,
1755 IFF_RXFH_CONFIGURED = 1<<23,
1756 IFF_PHONY_HEADROOM = 1<<24,
1757 IFF_MACSEC = 1<<25,
1758 IFF_NO_RX_HANDLER = 1<<26,
1759 IFF_FAILOVER = 1<<27,
1760 IFF_FAILOVER_SLAVE = 1<<28,
1761 IFF_L3MDEV_RX_HANDLER = 1<<29,
1762 IFF_NO_ADDRCONF = BIT_ULL(30),
1763 IFF_TX_SKB_NO_LINEAR = BIT_ULL(31),
1764 };
1765
1766 /* Specifies the type of the struct net_device::ml_priv pointer */
1767 enum netdev_ml_priv_type {
1768 ML_PRIV_NONE,
1769 ML_PRIV_CAN,
1770 };
1771
1772 enum netdev_stat_type {
1773 NETDEV_PCPU_STAT_NONE,
1774 NETDEV_PCPU_STAT_LSTATS, /* struct pcpu_lstats */
1775 NETDEV_PCPU_STAT_TSTATS, /* struct pcpu_sw_netstats */
1776 NETDEV_PCPU_STAT_DSTATS, /* struct pcpu_dstats */
1777 };
1778
1779 enum netdev_reg_state {
1780 NETREG_UNINITIALIZED = 0,
1781 NETREG_REGISTERED, /* completed register_netdevice */
1782 NETREG_UNREGISTERING, /* called unregister_netdevice */
1783 NETREG_UNREGISTERED, /* completed unregister todo */
1784 NETREG_RELEASED, /* called free_netdev */
1785 NETREG_DUMMY, /* dummy device for NAPI poll */
1786 };
1787
1788 /**
1789 * struct net_device - The DEVICE structure.
1790 *
1791 * Actually, this whole structure is a big mistake. It mixes I/O
1792 * data with strictly "high-level" data, and it has to know about
1793 * almost every data structure used in the INET module.
1794 *
1795 * @priv_flags: flags invisible to userspace defined as bits, see
1796 * enum netdev_priv_flags for the definitions
1797 * @lltx: device supports lockless Tx. Deprecated for real HW
1798 * drivers. Mainly used by logical interfaces, such as
1799 * bonding and tunnels
1800 * @netmem_tx: device support netmem_tx.
1801 *
1802 * @name: This is the first field of the "visible" part of this structure
1803 * (i.e. as seen by users in the "Space.c" file). It is the name
1804 * of the interface.
1805 *
1806 * @name_node: Name hashlist node
1807 * @ifalias: SNMP alias
1808 * @mem_end: Shared memory end
1809 * @mem_start: Shared memory start
1810 * @base_addr: Device I/O address
1811 * @irq: Device IRQ number
1812 *
1813 * @state: Generic network queuing layer state, see netdev_state_t
1814 * @dev_list: The global list of network devices
1815 * @napi_list: List entry used for polling NAPI devices
1816 * @unreg_list: List entry when we are unregistering the
1817 * device; see the function unregister_netdev
1818 * @close_list: List entry used when we are closing the device
1819 * @ptype_all: Device-specific packet handlers for all protocols
1820 * @ptype_specific: Device-specific, protocol-specific packet handlers
1821 *
1822 * @adj_list: Directly linked devices, like slaves for bonding
1823 * @features: Currently active device features
1824 * @hw_features: User-changeable features
1825 *
1826 * @wanted_features: User-requested features
1827 * @vlan_features: Mask of features inheritable by VLAN devices
1828 *
1829 * @hw_enc_features: Mask of features inherited by encapsulating devices
1830 * This field indicates what encapsulation
1831 * offloads the hardware is capable of doing,
1832 * and drivers will need to set them appropriately.
1833 *
1834 * @mpls_features: Mask of features inheritable by MPLS
1835 * @gso_partial_features: value(s) from NETIF_F_GSO\*
1836 * @mangleid_features: Mask of features requiring MANGLEID, will be
1837 * disabled together with the latter.
1838 *
1839 * @ifindex: interface index
1840 * @group: The group the device belongs to
1841 *
1842 * @stats: Statistics struct, which was left as a legacy, use
1843 * rtnl_link_stats64 instead
1844 *
1845 * @core_stats: core networking counters,
1846 * do not use this in drivers
1847 * @carrier_up_count: Number of times the carrier has been up
1848 * @carrier_down_count: Number of times the carrier has been down
1849 *
1850 * @wireless_handlers: List of functions to handle Wireless Extensions,
1851 * instead of ioctl,
1852 * see <net/iw_handler.h> for details.
1853 *
1854 * @netdev_ops: Includes several pointers to callbacks,
1855 * if one wants to override the ndo_*() functions
1856 * @xdp_metadata_ops: Includes pointers to XDP metadata callbacks.
1857 * @xsk_tx_metadata_ops: Includes pointers to AF_XDP TX metadata callbacks.
1858 * @ethtool_ops: Management operations
1859 * @l3mdev_ops: Layer 3 master device operations
1860 * @ndisc_ops: Includes callbacks for different IPv6 neighbour
1861 * discovery handling. Necessary for e.g. 6LoWPAN.
1862 * @xfrmdev_ops: Transformation offload operations
1863 * @tlsdev_ops: Transport Layer Security offload operations
1864 * @header_ops: Includes callbacks for creating,parsing,caching,etc
1865 * of Layer 2 headers.
1866 *
1867 * @flags: Interface flags (a la BSD)
1868 * @xdp_features: XDP capability supported by the device
1869 * @gflags: Global flags ( kept as legacy )
1870 * @priv_len: Size of the ->priv flexible array
1871 * @priv: Flexible array containing private data
1872 * @operstate: RFC2863 operstate
1873 * @link_mode: Mapping policy to operstate
1874 * @if_port: Selectable AUI, TP, ...
1875 * @dma: DMA channel
1876 * @mtu: Interface MTU value
1877 * @min_mtu: Interface Minimum MTU value
1878 * @max_mtu: Interface Maximum MTU value
1879 * @type: Interface hardware type
1880 * @hard_header_len: Maximum hardware header length.
1881 * @min_header_len: Minimum hardware header length
1882 *
1883 * @needed_headroom: Extra headroom the hardware may need, but not in all
1884 * cases can this be guaranteed
1885 * @needed_tailroom: Extra tailroom the hardware may need, but not in all
1886 * cases can this be guaranteed. Some cases also use
1887 * LL_MAX_HEADER instead to allocate the skb
1888 *
1889 * interface address info:
1890 *
1891 * @perm_addr: Permanent hw address
1892 * @addr_assign_type: Hw address assignment type
1893 * @addr_len: Hardware address length
1894 * @upper_level: Maximum depth level of upper devices.
1895 * @lower_level: Maximum depth level of lower devices.
1896 * @threaded: napi threaded state.
1897 * @neigh_priv_len: Used in neigh_alloc()
1898 * @dev_id: Used to differentiate devices that share
1899 * the same link layer address
1900 * @dev_port: Used to differentiate devices that share
1901 * the same function
1902 * @addr_list_lock: XXX: need comments on this one
1903 * @name_assign_type: network interface name assignment type
1904 * @uc_promisc: Counter that indicates promiscuous mode
1905 * has been enabled due to the need to listen to
1906 * additional unicast addresses in a device that
1907 * does not implement ndo_set_rx_mode()
1908 * @uc: unicast mac addresses
1909 * @mc: multicast mac addresses
1910 * @dev_addrs: list of device hw addresses
1911 * @queues_kset: Group of all Kobjects in the Tx and RX queues
1912 * @promiscuity: Number of times the NIC is told to work in
1913 * promiscuous mode; if it becomes 0 the NIC will
1914 * exit promiscuous mode
1915 * @allmulti: Counter, enables or disables allmulticast mode
1916 *
1917 * @vlan_info: VLAN info
1918 * @dsa_ptr: dsa specific data
1919 * @tipc_ptr: TIPC specific data
1920 * @atalk_ptr: AppleTalk link
1921 * @ip_ptr: IPv4 specific data
1922 * @ip6_ptr: IPv6 specific data
1923 * @ax25_ptr: AX.25 specific data
1924 * @ieee80211_ptr: IEEE 802.11 specific data, assign before registering
1925 * @ieee802154_ptr: IEEE 802.15.4 low-rate Wireless Personal Area Network
1926 * device struct
1927 * @mpls_ptr: mpls_dev struct pointer
1928 * @mctp_ptr: MCTP specific data
1929 * @psp_dev: PSP crypto device registered for this netdev
1930 *
1931 * @dev_addr: Hw address (before bcast,
1932 * because most packets are unicast)
1933 *
1934 * @_rx: Array of RX queues
1935 * @num_rx_queues: Number of RX queues
1936 * allocated at register_netdev() time
1937 * @real_num_rx_queues: Number of RX queues currently active in device
1938 * @xdp_prog: XDP sockets filter program pointer
1939 *
1940 * @rx_handler: handler for received packets
1941 * @rx_handler_data: XXX: need comments on this one
1942 * @tcx_ingress: BPF & clsact qdisc specific data for ingress processing
1943 * @ingress_queue: XXX: need comments on this one
1944 * @nf_hooks_ingress: netfilter hooks executed for ingress packets
1945 * @broadcast: hw bcast address
1946 *
1947 * @rx_cpu_rmap: CPU reverse-mapping for RX completion interrupts,
1948 * indexed by RX queue number. Assigned by driver.
1949 * This must only be set if the ndo_rx_flow_steer
1950 * operation is defined
1951 * @index_hlist: Device index hash chain
1952 *
1953 * @_tx: Array of TX queues
1954 * @num_tx_queues: Number of TX queues allocated at alloc_netdev_mq() time
1955 * @real_num_tx_queues: Number of TX queues currently active in device
1956 * @qdisc: Root qdisc from userspace point of view
1957 * @tx_queue_len: Max frames per queue allowed
1958 * @tx_global_lock: XXX: need comments on this one
1959 * @xdp_bulkq: XDP device bulk queue
1960 * @xps_maps: all CPUs/RXQs maps for XPS device
1961 *
1962 * @xps_maps: XXX: need comments on this one
1963 * @tcx_egress: BPF & clsact qdisc specific data for egress processing
1964 * @nf_hooks_egress: netfilter hooks executed for egress packets
1965 * @qdisc_hash: qdisc hash table
1966 * @watchdog_timeo: Represents the timeout that is used by
1967 * the watchdog (see dev_watchdog())
1968 * @watchdog_timer: List of timers
1969 *
1970 * @proto_down_reason: reason a netdev interface is held down
1971 * @pcpu_refcnt: Number of references to this device
1972 * @dev_refcnt: Number of references to this device
1973 * @refcnt_tracker: Tracker directory for tracked references to this device
1974 * @todo_list: Delayed register/unregister
1975 * @link_watch_list: XXX: need comments on this one
1976 *
1977 * @reg_state: Register/unregister state machine
1978 * @dismantle: Device is going to be freed
1979 * @needs_free_netdev: Should unregister perform free_netdev?
1980 * @priv_destructor: Called from unregister
1981 * @npinfo: XXX: need comments on this one
1982 * @nd_net: Network namespace this network device is inside
1983 * protected by @lock
1984 *
1985 * @ml_priv: Mid-layer private
1986 * @ml_priv_type: Mid-layer private type
1987 *
1988 * @pcpu_stat_type: Type of device statistics which the core should
1989 * allocate/free: none, lstats, tstats, dstats. none
1990 * means the driver is handling statistics allocation/
1991 * freeing internally.
1992 * @lstats: Loopback statistics: packets, bytes
1993 * @tstats: Tunnel statistics: RX/TX packets, RX/TX bytes
1994 * @dstats: Dummy statistics: RX/TX/drop packets, RX/TX bytes
1995 *
1996 * @garp_port: GARP
1997 * @mrp_port: MRP
1998 *
1999 * @dm_private: Drop monitor private
2000 *
2001 * @dev: Class/net/name entry
2002 * @sysfs_groups: Space for optional device, statistics and wireless
2003 * sysfs groups
2004 *
2005 * @sysfs_rx_queue_group: Space for optional per-rx queue attributes
2006 * @rtnl_link_ops: Rtnl_link_ops
2007 * @stat_ops: Optional ops for queue-aware statistics
2008 * @queue_mgmt_ops: Optional ops for queue management
2009 *
2010 * @gso_max_size: Maximum size of generic segmentation offload
2011 * @tso_max_size: Device (as in HW) limit on the max TSO request size
2012 * @gso_max_segs: Maximum number of segments that can be passed to the
2013 * NIC for GSO
2014 * @tso_max_segs: Device (as in HW) limit on the max TSO segment count
2015 * @gso_ipv4_max_size: Maximum size of generic segmentation offload,
2016 * for IPv4.
2017 *
2018 * @dcbnl_ops: Data Center Bridging netlink ops
2019 * @num_tc: Number of traffic classes in the net device
2020 * @tc_to_txq: XXX: need comments on this one
2021 * @prio_tc_map: XXX: need comments on this one
2022 *
2023 * @fcoe_ddp_xid: Max exchange id for FCoE LRO by ddp
2024 *
2025 * @priomap: XXX: need comments on this one
2026 * @link_topo: Physical link topology tracking attached PHYs
2027 * @phydev: Physical device may attach itself
2028 * for hardware timestamping
2029 * @sfp_bus: attached &struct sfp_bus structure.
2030 *
2031 * @qdisc_tx_busylock: lockdep class annotating Qdisc->busylock spinlock
2032 *
2033 * @proto_down: protocol port state information can be sent to the
2034 * switch driver and used to set the phys state of the
2035 * switch port.
2036 *
2037 * @irq_affinity_auto: driver wants the core to store and re-assign the IRQ
2038 * affinity. Set by netif_enable_irq_affinity(), then
2039 * the driver must create a persistent napi by
2040 * netif_napi_add_config() and finally bind the napi to
2041 * IRQ (via netif_napi_set_irq()).
2042 *
2043 * @rx_cpu_rmap_auto: driver wants the core to manage the ARFS rmap.
2044 * Set by calling netif_enable_cpu_rmap().
2045 *
2046 * @see_all_hwtstamp_requests: device wants to see calls to
2047 * ndo_hwtstamp_set() for all timestamp requests
2048 * regardless of source, even if those aren't
2049 * HWTSTAMP_SOURCE_NETDEV
2050 * @change_proto_down: device supports setting carrier via IFLA_PROTO_DOWN
2051 * @netns_immutable: interface can't change network namespaces
2052 * @fcoe_mtu: device supports maximum FCoE MTU, 2158 bytes
2053 *
2054 * @net_notifier_list: List of per-net netdev notifier block
2055 * that follow this device when it is moved
2056 * to another network namespace.
2057 *
2058 * @macsec_ops: MACsec offloading ops
2059 *
2060 * @udp_tunnel_nic_info: static structure describing the UDP tunnel
2061 * offload capabilities of the device
2062 * @udp_tunnel_nic: UDP tunnel offload state
2063 * @ethtool: ethtool related state
2064 * @xdp_state: stores info on attached XDP BPF programs
2065 *
2066 * @nested_level: Used as a parameter of spin_lock_nested() of
2067 * dev->addr_list_lock.
2068 * @unlink_list: As netif_addr_lock() can be called recursively,
2069 * keep a list of interfaces to be deleted.
2070 * @gro_max_size: Maximum size of aggregated packet in generic
2071 * receive offload (GRO)
2072 * @gro_ipv4_max_size: Maximum size of aggregated packet in generic
2073 * receive offload (GRO), for IPv4.
2074 * @xdp_zc_max_segs: Maximum number of segments supported by AF_XDP
2075 * zero copy driver
2076 *
2077 * @dev_addr_shadow: Copy of @dev_addr to catch direct writes.
2078 * @linkwatch_dev_tracker: refcount tracker used by linkwatch.
2079 * @watchdog_dev_tracker: refcount tracker used by watchdog.
2080 * @dev_registered_tracker: tracker for reference held while
2081 * registered
2082 * @offload_xstats_l3: L3 HW stats for this netdevice.
2083 *
2084 * @devlink_port: Pointer to related devlink port structure.
2085 * Assigned by a driver before netdev registration using
2086 * SET_NETDEV_DEVLINK_PORT macro. This pointer is static
2087 * during the time netdevice is registered.
2088 *
2089 * @dpll_pin: Pointer to the SyncE source pin of a DPLL subsystem,
2090 * where the clock is recovered.
2091 *
2092 * @max_pacing_offload_horizon: max EDT offload horizon in nsec.
2093 * @napi_config: An array of napi_config structures containing per-NAPI
2094 * settings.
2095 * @num_napi_configs: number of allocated NAPI config structs,
2096 * always >= max(num_rx_queues, num_tx_queues).
2097 * @gro_flush_timeout: timeout for GRO layer in NAPI
2098 * @napi_defer_hard_irqs: If not zero, provides a counter that would
2099 * allow to avoid NIC hard IRQ, on busy queues.
2100 *
2101 * @neighbours: List heads pointing to this device's neighbours'
2102 * dev_list, one per address-family.
2103 * @hwprov: Tracks which PTP performs hardware packet time stamping.
2104 *
2105 * FIXME: cleanup struct net_device such that network protocol info
2106 * moves out.
2107 */
2108
2109 struct net_device {
2110 /* Cacheline organization can be found documented in
2111 * Documentation/networking/net_cachelines/net_device.rst.
2112 * Please update the document when adding new fields.
2113 */
2114
2115 /* TX read-mostly hotpath */
2116 __cacheline_group_begin(net_device_read_tx);
2117 struct_group(priv_flags_fast,
2118 unsigned long priv_flags:32;
2119 unsigned long lltx:1;
2120 unsigned long netmem_tx:1;
2121 );
2122 const struct net_device_ops *netdev_ops;
2123 const struct header_ops *header_ops;
2124 struct netdev_queue *_tx;
2125 netdev_features_t gso_partial_features;
2126 unsigned int real_num_tx_queues;
2127 unsigned int gso_max_size;
2128 unsigned int gso_ipv4_max_size;
2129 u16 gso_max_segs;
2130 s16 num_tc;
2131 /* Note : dev->mtu is often read without holding a lock.
2132 * Writers usually hold RTNL.
2133 * It is recommended to use READ_ONCE() to annotate the reads,
2134 * and to use WRITE_ONCE() to annotate the writes.
2135 */
2136 unsigned int mtu;
2137 unsigned short needed_headroom;
2138 struct netdev_tc_txq tc_to_txq[TC_MAX_QUEUE];
2139 #ifdef CONFIG_XPS
2140 struct xps_dev_maps __rcu *xps_maps[XPS_MAPS_MAX];
2141 #endif
2142 #ifdef CONFIG_NETFILTER_EGRESS
2143 struct nf_hook_entries __rcu *nf_hooks_egress;
2144 #endif
2145 #ifdef CONFIG_NET_XGRESS
2146 struct bpf_mprog_entry __rcu *tcx_egress;
2147 #endif
2148 __cacheline_group_end(net_device_read_tx);
2149
2150 /* TXRX read-mostly hotpath */
2151 __cacheline_group_begin(net_device_read_txrx);
2152 union {
2153 struct pcpu_lstats __percpu *lstats;
2154 struct pcpu_sw_netstats __percpu *tstats;
2155 struct pcpu_dstats __percpu *dstats;
2156 };
2157 unsigned long state;
2158 unsigned int flags;
2159 unsigned short hard_header_len;
2160 enum netdev_stat_type pcpu_stat_type:8;
2161 netdev_features_t features;
2162 struct inet6_dev __rcu *ip6_ptr;
2163 __cacheline_group_end(net_device_read_txrx);
2164
2165 /* RX read-mostly hotpath */
2166 __cacheline_group_begin(net_device_read_rx);
2167 struct bpf_prog __rcu *xdp_prog;
2168 struct list_head ptype_specific;
2169 int ifindex;
2170 unsigned int real_num_rx_queues;
2171 struct netdev_rx_queue *_rx;
2172 unsigned int gro_max_size;
2173 unsigned int gro_ipv4_max_size;
2174 rx_handler_func_t __rcu *rx_handler;
2175 void __rcu *rx_handler_data;
2176 possible_net_t nd_net;
2177 #ifdef CONFIG_NETPOLL
2178 struct netpoll_info __rcu *npinfo;
2179 #endif
2180 #ifdef CONFIG_NET_XGRESS
2181 struct bpf_mprog_entry __rcu *tcx_ingress;
2182 #endif
2183 __cacheline_group_end(net_device_read_rx);
2184
2185 char name[IFNAMSIZ];
2186 struct netdev_name_node *name_node;
2187 struct dev_ifalias __rcu *ifalias;
2188 /*
2189 * I/O specific fields
2190 * FIXME: Merge these and struct ifmap into one
2191 */
2192 unsigned long mem_end;
2193 unsigned long mem_start;
2194 unsigned long base_addr;
2195
2196 /*
2197 * Some hardware also needs these fields (state,dev_list,
2198 * napi_list,unreg_list,close_list) but they are not
2199 * part of the usual set specified in Space.c.
2200 */
2201
2202
2203 struct list_head dev_list;
2204 struct list_head napi_list;
2205 struct list_head unreg_list;
2206 struct list_head close_list;
2207 struct list_head ptype_all;
2208
2209 struct {
2210 struct list_head upper;
2211 struct list_head lower;
2212 } adj_list;
2213
2214 /* Read-mostly cache-line for fast-path access */
2215 xdp_features_t xdp_features;
2216 const struct xdp_metadata_ops *xdp_metadata_ops;
2217 const struct xsk_tx_metadata_ops *xsk_tx_metadata_ops;
2218 unsigned short gflags;
2219
2220 unsigned short needed_tailroom;
2221
2222 netdev_features_t hw_features;
2223 netdev_features_t wanted_features;
2224 netdev_features_t vlan_features;
2225 netdev_features_t hw_enc_features;
2226 netdev_features_t mpls_features;
2227 netdev_features_t mangleid_features;
2228
2229 unsigned int min_mtu;
2230 unsigned int max_mtu;
2231 unsigned short type;
2232 unsigned char min_header_len;
2233 unsigned char name_assign_type;
2234
2235 int group;
2236
2237 struct net_device_stats stats; /* not used by modern drivers */
2238
2239 struct net_device_core_stats __percpu *core_stats;
2240
2241 /* Stats to monitor link on/off, flapping */
2242 atomic_t carrier_up_count;
2243 atomic_t carrier_down_count;
2244
2245 #ifdef CONFIG_WIRELESS_EXT
2246 const struct iw_handler_def *wireless_handlers;
2247 #endif
2248 const struct ethtool_ops *ethtool_ops;
2249 #ifdef CONFIG_NET_L3_MASTER_DEV
2250 const struct l3mdev_ops *l3mdev_ops;
2251 #endif
2252 #if IS_ENABLED(CONFIG_IPV6)
2253 const struct ndisc_ops *ndisc_ops;
2254 #endif
2255
2256 #ifdef CONFIG_XFRM_OFFLOAD
2257 const struct xfrmdev_ops *xfrmdev_ops;
2258 #endif
2259
2260 #if IS_ENABLED(CONFIG_TLS_DEVICE)
2261 const struct tlsdev_ops *tlsdev_ops;
2262 #endif
2263
2264 unsigned int operstate;
2265 unsigned char link_mode;
2266
2267 unsigned char if_port;
2268 unsigned char dma;
2269
2270 /* Interface address info. */
2271 unsigned char perm_addr[MAX_ADDR_LEN];
2272 unsigned char addr_assign_type;
2273 unsigned char addr_len;
2274 unsigned char upper_level;
2275 unsigned char lower_level;
2276 u8 threaded;
2277
2278 unsigned short neigh_priv_len;
2279 unsigned short dev_id;
2280 unsigned short dev_port;
2281 int irq;
2282 u32 priv_len;
2283
2284 spinlock_t addr_list_lock;
2285
2286 struct netdev_hw_addr_list uc;
2287 struct netdev_hw_addr_list mc;
2288 struct netdev_hw_addr_list dev_addrs;
2289
2290 #ifdef CONFIG_SYSFS
2291 struct kset *queues_kset;
2292 #endif
2293 #ifdef CONFIG_LOCKDEP
2294 struct list_head unlink_list;
2295 #endif
2296 unsigned int promiscuity;
2297 unsigned int allmulti;
2298 bool uc_promisc;
2299 #ifdef CONFIG_LOCKDEP
2300 unsigned char nested_level;
2301 #endif
2302
2303
2304 /* Protocol-specific pointers */
2305 struct in_device __rcu *ip_ptr;
2306 /** @fib_nh_head: nexthops associated with this netdev */
2307 struct hlist_head fib_nh_head;
2308
2309 #if IS_ENABLED(CONFIG_VLAN_8021Q)
2310 struct vlan_info __rcu *vlan_info;
2311 #endif
2312 #if IS_ENABLED(CONFIG_NET_DSA)
2313 struct dsa_port *dsa_ptr;
2314 #endif
2315 #if IS_ENABLED(CONFIG_TIPC)
2316 struct tipc_bearer __rcu *tipc_ptr;
2317 #endif
2318 #if IS_ENABLED(CONFIG_ATALK)
2319 void *atalk_ptr;
2320 #endif
2321 #if IS_ENABLED(CONFIG_AX25)
2322 struct ax25_dev __rcu *ax25_ptr;
2323 #endif
2324 #if IS_ENABLED(CONFIG_CFG80211)
2325 struct wireless_dev *ieee80211_ptr;
2326 #endif
2327 #if IS_ENABLED(CONFIG_IEEE802154) || IS_ENABLED(CONFIG_6LOWPAN)
2328 struct wpan_dev *ieee802154_ptr;
2329 #endif
2330 #if IS_ENABLED(CONFIG_MPLS_ROUTING)
2331 struct mpls_dev __rcu *mpls_ptr;
2332 #endif
2333 #if IS_ENABLED(CONFIG_MCTP)
2334 struct mctp_dev __rcu *mctp_ptr;
2335 #endif
2336 #if IS_ENABLED(CONFIG_INET_PSP)
2337 struct psp_dev __rcu *psp_dev;
2338 #endif
2339
2340 /*
2341 * Cache lines mostly used on receive path (including eth_type_trans())
2342 */
2343 /* Interface address info used in eth_type_trans() */
2344 const unsigned char *dev_addr;
2345
2346 unsigned int num_rx_queues;
2347 #define GRO_LEGACY_MAX_SIZE 65536u
2348 /* TCP minimal MSS is 8 (TCP_MIN_GSO_SIZE),
2349 * and shinfo->gso_segs is a 16bit field.
2350 */
2351 #define GRO_MAX_SIZE (8 * 65535u)
2352 unsigned int xdp_zc_max_segs;
2353 struct netdev_queue __rcu *ingress_queue;
2354 #ifdef CONFIG_NETFILTER_INGRESS
2355 struct nf_hook_entries __rcu *nf_hooks_ingress;
2356 #endif
2357
2358 unsigned char broadcast[MAX_ADDR_LEN];
2359 #ifdef CONFIG_RFS_ACCEL
2360 struct cpu_rmap *rx_cpu_rmap;
2361 #endif
2362 struct hlist_node index_hlist;
2363
2364 /*
2365 * Cache lines mostly used on transmit path
2366 */
2367 unsigned int num_tx_queues;
2368 struct Qdisc __rcu *qdisc;
2369 unsigned int tx_queue_len;
2370 spinlock_t tx_global_lock;
2371
2372 struct xdp_dev_bulk_queue __percpu *xdp_bulkq;
2373
2374 #ifdef CONFIG_NET_SCHED
2375 DECLARE_HASHTABLE (qdisc_hash, 4);
2376 #endif
2377 /* These may be needed for future network-power-down code. */
2378 struct timer_list watchdog_timer;
2379 int watchdog_timeo;
2380
2381 u32 proto_down_reason;
2382
2383 struct list_head todo_list;
2384
2385 #ifdef CONFIG_PCPU_DEV_REFCNT
2386 int __percpu *pcpu_refcnt;
2387 #else
2388 refcount_t dev_refcnt;
2389 #endif
2390 struct ref_tracker_dir refcnt_tracker;
2391
2392 struct list_head link_watch_list;
2393
2394 u8 reg_state;
2395
2396 bool dismantle;
2397
2398 /** @moving_ns: device is changing netns, protected by @lock */
2399 bool moving_ns;
2400 /** @rtnl_link_initializing: Device being created, suppress events */
2401 bool rtnl_link_initializing;
2402
2403 bool needs_free_netdev;
2404 void (*priv_destructor)(struct net_device *dev);
2405
2406 /* mid-layer private */
2407 void *ml_priv;
2408 enum netdev_ml_priv_type ml_priv_type;
2409
2410 #if IS_ENABLED(CONFIG_GARP)
2411 struct garp_port __rcu *garp_port;
2412 #endif
2413 #if IS_ENABLED(CONFIG_MRP)
2414 struct mrp_port __rcu *mrp_port;
2415 #endif
2416 #if IS_ENABLED(CONFIG_NET_DROP_MONITOR)
2417 struct dm_hw_stat_delta __rcu *dm_private;
2418 #endif
2419 struct device dev;
2420 const struct attribute_group *sysfs_groups[5];
2421 const struct attribute_group *sysfs_rx_queue_group;
2422
2423 const struct rtnl_link_ops *rtnl_link_ops;
2424
2425 const struct netdev_stat_ops *stat_ops;
2426
2427 const struct netdev_queue_mgmt_ops *queue_mgmt_ops;
2428
2429 /* for setting kernel sock attribute on TCP connection setup */
2430 #define GSO_MAX_SEGS 65535u
2431 #define GSO_LEGACY_MAX_SIZE 65536u
2432 /* TCP minimal MSS is 8 (TCP_MIN_GSO_SIZE),
2433 * and shinfo->gso_segs is a 16bit field.
2434 */
2435 #define GSO_MAX_SIZE (8 * GSO_MAX_SEGS)
2436
2437 #define TSO_LEGACY_MAX_SIZE 65536
2438 #define TSO_MAX_SIZE UINT_MAX
2439 unsigned int tso_max_size;
2440 #define TSO_MAX_SEGS U16_MAX
2441 u16 tso_max_segs;
2442
2443 #ifdef CONFIG_DCB
2444 const struct dcbnl_rtnl_ops *dcbnl_ops;
2445 #endif
2446 u8 prio_tc_map[TC_BITMASK + 1];
2447
2448 #if IS_ENABLED(CONFIG_FCOE)
2449 unsigned int fcoe_ddp_xid;
2450 #endif
2451 #if IS_ENABLED(CONFIG_CGROUP_NET_PRIO)
2452 struct netprio_map __rcu *priomap;
2453 #endif
2454 struct phy_link_topology *link_topo;
2455 struct phy_device *phydev;
2456 struct sfp_bus *sfp_bus;
2457 struct lock_class_key *qdisc_tx_busylock;
2458 bool proto_down;
2459 bool irq_affinity_auto;
2460 bool rx_cpu_rmap_auto;
2461
2462 /* priv_flags_slow, ungrouped to save space */
2463 unsigned long see_all_hwtstamp_requests:1;
2464 unsigned long change_proto_down:1;
2465 unsigned long netns_immutable:1;
2466 unsigned long fcoe_mtu:1;
2467
2468 struct list_head net_notifier_list;
2469
2470 #if IS_ENABLED(CONFIG_MACSEC)
2471 /* MACsec management functions */
2472 const struct macsec_ops *macsec_ops;
2473 #endif
2474 const struct udp_tunnel_nic_info *udp_tunnel_nic_info;
2475 struct udp_tunnel_nic *udp_tunnel_nic;
2476
2477 /** @cfg: net_device queue-related configuration */
2478 struct netdev_config *cfg;
2479 /**
2480 * @cfg_pending: same as @cfg but when device is being actively
2481 * reconfigured includes any changes to the configuration
2482 * requested by the user, but which may or may not be rejected.
2483 */
2484 struct netdev_config *cfg_pending;
2485 struct ethtool_netdev_state *ethtool;
2486
2487 /* protected by rtnl_lock */
2488 struct bpf_xdp_entity xdp_state[__MAX_XDP_MODE];
2489
2490 u8 dev_addr_shadow[MAX_ADDR_LEN];
2491 netdevice_tracker linkwatch_dev_tracker;
2492 netdevice_tracker watchdog_dev_tracker;
2493 netdevice_tracker dev_registered_tracker;
2494 struct rtnl_hw_stats64 *offload_xstats_l3;
2495
2496 struct devlink_port *devlink_port;
2497
2498 #if IS_ENABLED(CONFIG_DPLL)
2499 struct dpll_pin __rcu *dpll_pin;
2500 #endif
2501 #if IS_ENABLED(CONFIG_PAGE_POOL)
2502 /** @page_pools: page pools created for this netdevice */
2503 struct hlist_head page_pools;
2504 #endif
2505
2506 /** @irq_moder: dim parameters used if IS_ENABLED(CONFIG_DIMLIB). */
2507 struct dim_irq_moder *irq_moder;
2508
2509 u64 max_pacing_offload_horizon;
2510 struct napi_config *napi_config;
2511 u32 num_napi_configs;
2512 u32 napi_defer_hard_irqs;
2513 unsigned long gro_flush_timeout;
2514
2515 /**
2516 * @up: copy of @state's IFF_UP, but safe to read with just @lock.
2517 * May report false negatives while the device is being opened
2518 * or closed (@lock does not protect .ndo_open, or .ndo_close).
2519 */
2520 bool up;
2521
2522 /**
2523 * @request_ops_lock: request the core to run all @netdev_ops and
2524 * @ethtool_ops under the @lock.
2525 */
2526 bool request_ops_lock;
2527
2528 /**
2529 * @lock: netdev-scope lock, protects a small selection of fields.
2530 * Should always be taken using netdev_lock() / netdev_unlock() helpers.
2531 * Drivers are free to use it for other protection.
2532 *
2533 * For the drivers that implement shaper or queue API, the scope
2534 * of this lock is expanded to cover most ndo/queue/ethtool/sysfs
2535 * operations. Drivers may opt-in to this behavior by setting
2536 * @request_ops_lock.
2537 *
2538 * @lock protection mixes with rtnl_lock in multiple ways, fields are
2539 * either:
2540 *
2541 * - simply protected by the instance @lock;
2542 *
2543 * - double protected - writers hold both locks, readers hold either;
2544 *
2545 * - ops protected - protected by the lock held around the NDOs
2546 * and other callbacks, that is the instance lock on devices for
2547 * which netdev_need_ops_lock() returns true, otherwise by rtnl_lock;
2548 *
2549 * - double ops protected - always protected by rtnl_lock but for
2550 * devices for which netdev_need_ops_lock() returns true - also
2551 * the instance lock.
2552 *
2553 * Simply protects:
2554 * @gro_flush_timeout, @napi_defer_hard_irqs, @napi_list,
2555 * @net_shaper_hierarchy, @reg_state, @threaded
2556 *
2557 * Double protects:
2558 * @up, @moving_ns, @nd_net, @xdp_features
2559 *
2560 * Double ops protects:
2561 * @real_num_rx_queues, @real_num_tx_queues
2562 *
2563 * Also protects some fields in:
2564 * struct napi_struct, struct netdev_queue, struct netdev_rx_queue
2565 *
2566 * Ordering: take after rtnl_lock.
2567 */
2568 struct mutex lock;
2569
2570 #if IS_ENABLED(CONFIG_NET_SHAPER)
2571 /**
2572 * @net_shaper_hierarchy: data tracking the current shaper status
2573 * see include/net/net_shapers.h
2574 */
2575 struct net_shaper_hierarchy *net_shaper_hierarchy;
2576 #endif
2577
2578 struct hlist_head neighbours[NEIGH_NR_TABLES];
2579
2580 struct hwtstamp_provider __rcu *hwprov;
2581
2582 u8 priv[] ____cacheline_aligned
2583 __counted_by(priv_len);
2584 } ____cacheline_aligned;
2585 #define to_net_dev(d) container_of(d, struct net_device, dev)
2586
2587 /*
2588 * Driver should use this to assign devlink port instance to a netdevice
2589 * before it registers the netdevice. Therefore devlink_port is static
2590 * during the netdev lifetime after it is registered.
2591 */
2592 #define SET_NETDEV_DEVLINK_PORT(dev, port) \
2593 ({ \
2594 WARN_ON((dev)->reg_state != NETREG_UNINITIALIZED); \
2595 ((dev)->devlink_port = (port)); \
2596 })
2597
netif_elide_gro(const struct net_device * dev)2598 static inline bool netif_elide_gro(const struct net_device *dev)
2599 {
2600 if (!(dev->features & NETIF_F_GRO) || dev->xdp_prog)
2601 return true;
2602 return false;
2603 }
2604
2605 #define NETDEV_ALIGN 32
2606
2607 static inline
netdev_get_prio_tc_map(const struct net_device * dev,u32 prio)2608 int netdev_get_prio_tc_map(const struct net_device *dev, u32 prio)
2609 {
2610 return dev->prio_tc_map[prio & TC_BITMASK];
2611 }
2612
2613 static inline
netdev_set_prio_tc_map(struct net_device * dev,u8 prio,u8 tc)2614 int netdev_set_prio_tc_map(struct net_device *dev, u8 prio, u8 tc)
2615 {
2616 if (tc >= dev->num_tc)
2617 return -EINVAL;
2618
2619 dev->prio_tc_map[prio & TC_BITMASK] = tc & TC_BITMASK;
2620 return 0;
2621 }
2622
2623 int netdev_txq_to_tc(struct net_device *dev, unsigned int txq);
2624 void netdev_reset_tc(struct net_device *dev);
2625 int netdev_set_tc_queue(struct net_device *dev, u8 tc, u16 count, u16 offset);
2626 int netdev_set_num_tc(struct net_device *dev, u8 num_tc);
2627
2628 static inline
netdev_get_num_tc(struct net_device * dev)2629 int netdev_get_num_tc(struct net_device *dev)
2630 {
2631 return dev->num_tc;
2632 }
2633
net_prefetch(void * p)2634 static inline void net_prefetch(void *p)
2635 {
2636 prefetch(p);
2637 #if L1_CACHE_BYTES < 128
2638 prefetch((u8 *)p + L1_CACHE_BYTES);
2639 #endif
2640 }
2641
net_prefetchw(void * p)2642 static inline void net_prefetchw(void *p)
2643 {
2644 prefetchw(p);
2645 #if L1_CACHE_BYTES < 128
2646 prefetchw((u8 *)p + L1_CACHE_BYTES);
2647 #endif
2648 }
2649
2650 void netdev_unbind_sb_channel(struct net_device *dev,
2651 struct net_device *sb_dev);
2652 int netdev_bind_sb_channel_queue(struct net_device *dev,
2653 struct net_device *sb_dev,
2654 u8 tc, u16 count, u16 offset);
2655 int netdev_set_sb_channel(struct net_device *dev, u16 channel);
netdev_get_sb_channel(struct net_device * dev)2656 static inline int netdev_get_sb_channel(struct net_device *dev)
2657 {
2658 return max_t(int, -dev->num_tc, 0);
2659 }
2660
2661 static inline
netdev_get_tx_queue(const struct net_device * dev,unsigned int index)2662 struct netdev_queue *netdev_get_tx_queue(const struct net_device *dev,
2663 unsigned int index)
2664 {
2665 DEBUG_NET_WARN_ON_ONCE(index >= dev->num_tx_queues);
2666 return &dev->_tx[index];
2667 }
2668
skb_get_tx_queue(const struct net_device * dev,const struct sk_buff * skb)2669 static inline struct netdev_queue *skb_get_tx_queue(const struct net_device *dev,
2670 const struct sk_buff *skb)
2671 {
2672 return netdev_get_tx_queue(dev, skb_get_queue_mapping(skb));
2673 }
2674
netdev_for_each_tx_queue(struct net_device * dev,void (* f)(struct net_device *,struct netdev_queue *,void *),void * arg)2675 static inline void netdev_for_each_tx_queue(struct net_device *dev,
2676 void (*f)(struct net_device *,
2677 struct netdev_queue *,
2678 void *),
2679 void *arg)
2680 {
2681 unsigned int i;
2682
2683 for (i = 0; i < dev->num_tx_queues; i++)
2684 f(dev, &dev->_tx[i], arg);
2685 }
2686
2687 u16 netdev_pick_tx(struct net_device *dev, struct sk_buff *skb,
2688 struct net_device *sb_dev);
2689 struct netdev_queue *netdev_core_pick_tx(struct net_device *dev,
2690 struct sk_buff *skb,
2691 struct net_device *sb_dev);
2692
2693 /* returns the headroom that the master device needs to take in account
2694 * when forwarding to this dev
2695 */
netdev_get_fwd_headroom(struct net_device * dev)2696 static inline unsigned netdev_get_fwd_headroom(struct net_device *dev)
2697 {
2698 return dev->priv_flags & IFF_PHONY_HEADROOM ? 0 : dev->needed_headroom;
2699 }
2700
netdev_set_rx_headroom(struct net_device * dev,int new_hr)2701 static inline void netdev_set_rx_headroom(struct net_device *dev, int new_hr)
2702 {
2703 if (dev->netdev_ops->ndo_set_rx_headroom)
2704 dev->netdev_ops->ndo_set_rx_headroom(dev, new_hr);
2705 }
2706
2707 /* set the device rx headroom to the dev's default */
netdev_reset_rx_headroom(struct net_device * dev)2708 static inline void netdev_reset_rx_headroom(struct net_device *dev)
2709 {
2710 netdev_set_rx_headroom(dev, -1);
2711 }
2712
netdev_get_ml_priv(struct net_device * dev,enum netdev_ml_priv_type type)2713 static inline void *netdev_get_ml_priv(struct net_device *dev,
2714 enum netdev_ml_priv_type type)
2715 {
2716 if (dev->ml_priv_type != type)
2717 return NULL;
2718
2719 return dev->ml_priv;
2720 }
2721
netdev_set_ml_priv(struct net_device * dev,void * ml_priv,enum netdev_ml_priv_type type)2722 static inline void netdev_set_ml_priv(struct net_device *dev,
2723 void *ml_priv,
2724 enum netdev_ml_priv_type type)
2725 {
2726 WARN(dev->ml_priv_type && dev->ml_priv_type != type,
2727 "Overwriting already set ml_priv_type (%u) with different ml_priv_type (%u)!\n",
2728 dev->ml_priv_type, type);
2729 WARN(!dev->ml_priv_type && dev->ml_priv,
2730 "Overwriting already set ml_priv and ml_priv_type is ML_PRIV_NONE!\n");
2731
2732 dev->ml_priv = ml_priv;
2733 dev->ml_priv_type = type;
2734 }
2735
2736 /*
2737 * Net namespace inlines
2738 */
2739 static inline
dev_net(const struct net_device * dev)2740 struct net *dev_net(const struct net_device *dev)
2741 {
2742 return read_pnet(&dev->nd_net);
2743 }
2744
2745 static inline
dev_net_rcu(const struct net_device * dev)2746 struct net *dev_net_rcu(const struct net_device *dev)
2747 {
2748 return read_pnet_rcu(&dev->nd_net);
2749 }
2750
2751 static inline
dev_net_set(struct net_device * dev,struct net * net)2752 void dev_net_set(struct net_device *dev, struct net *net)
2753 {
2754 write_pnet(&dev->nd_net, net);
2755 }
2756
2757 /**
2758 * netdev_priv - access network device private data
2759 * @dev: network device
2760 *
2761 * Get network device private data
2762 */
netdev_priv(const struct net_device * dev)2763 static inline void *netdev_priv(const struct net_device *dev)
2764 {
2765 return (void *)dev->priv;
2766 }
2767
2768 /* Set the sysfs physical device reference for the network logical device
2769 * if set prior to registration will cause a symlink during initialization.
2770 */
2771 #define SET_NETDEV_DEV(net, pdev) ((net)->dev.parent = (pdev))
2772
2773 /* Set the sysfs device type for the network logical device to allow
2774 * fine-grained identification of different network device types. For
2775 * example Ethernet, Wireless LAN, Bluetooth, WiMAX etc.
2776 */
2777 #define SET_NETDEV_DEVTYPE(net, devtype) ((net)->dev.type = (devtype))
2778
2779 void netif_queue_set_napi(struct net_device *dev, unsigned int queue_index,
2780 enum netdev_queue_type type,
2781 struct napi_struct *napi);
2782
netdev_lock(struct net_device * dev)2783 static inline void netdev_lock(struct net_device *dev)
2784 {
2785 mutex_lock(&dev->lock);
2786 }
2787
netdev_unlock(struct net_device * dev)2788 static inline void netdev_unlock(struct net_device *dev)
2789 {
2790 mutex_unlock(&dev->lock);
2791 }
2792 /* Additional netdev_lock()-related helpers are in net/netdev_lock.h */
2793
2794 void netif_napi_set_irq_locked(struct napi_struct *napi, int irq);
2795
netif_napi_set_irq(struct napi_struct * napi,int irq)2796 static inline void netif_napi_set_irq(struct napi_struct *napi, int irq)
2797 {
2798 netdev_lock(napi->dev);
2799 netif_napi_set_irq_locked(napi, irq);
2800 netdev_unlock(napi->dev);
2801 }
2802
2803 /* Default NAPI poll() weight
2804 * Device drivers are strongly advised to not use bigger value
2805 */
2806 #define NAPI_POLL_WEIGHT 64
2807
2808 void netif_napi_add_weight_locked(struct net_device *dev,
2809 struct napi_struct *napi,
2810 int (*poll)(struct napi_struct *, int),
2811 int weight);
2812
2813 static inline void
netif_napi_add_weight(struct net_device * dev,struct napi_struct * napi,int (* poll)(struct napi_struct *,int),int weight)2814 netif_napi_add_weight(struct net_device *dev, struct napi_struct *napi,
2815 int (*poll)(struct napi_struct *, int), int weight)
2816 {
2817 netdev_lock(dev);
2818 netif_napi_add_weight_locked(dev, napi, poll, weight);
2819 netdev_unlock(dev);
2820 }
2821
2822 /**
2823 * netif_napi_add() - initialize a NAPI context
2824 * @dev: network device
2825 * @napi: NAPI context
2826 * @poll: polling function
2827 *
2828 * netif_napi_add() must be used to initialize a NAPI context prior to calling
2829 * *any* of the other NAPI-related functions.
2830 */
2831 static inline void
netif_napi_add(struct net_device * dev,struct napi_struct * napi,int (* poll)(struct napi_struct *,int))2832 netif_napi_add(struct net_device *dev, struct napi_struct *napi,
2833 int (*poll)(struct napi_struct *, int))
2834 {
2835 netif_napi_add_weight(dev, napi, poll, NAPI_POLL_WEIGHT);
2836 }
2837
2838 static inline void
netif_napi_add_locked(struct net_device * dev,struct napi_struct * napi,int (* poll)(struct napi_struct *,int))2839 netif_napi_add_locked(struct net_device *dev, struct napi_struct *napi,
2840 int (*poll)(struct napi_struct *, int))
2841 {
2842 netif_napi_add_weight_locked(dev, napi, poll, NAPI_POLL_WEIGHT);
2843 }
2844
2845 static inline void
netif_napi_add_tx_weight(struct net_device * dev,struct napi_struct * napi,int (* poll)(struct napi_struct *,int),int weight)2846 netif_napi_add_tx_weight(struct net_device *dev,
2847 struct napi_struct *napi,
2848 int (*poll)(struct napi_struct *, int),
2849 int weight)
2850 {
2851 set_bit(NAPI_STATE_NO_BUSY_POLL, &napi->state);
2852 netif_napi_add_weight(dev, napi, poll, weight);
2853 }
2854
2855 static inline void
netif_napi_add_config_locked(struct net_device * dev,struct napi_struct * napi,int (* poll)(struct napi_struct *,int),int index)2856 netif_napi_add_config_locked(struct net_device *dev, struct napi_struct *napi,
2857 int (*poll)(struct napi_struct *, int), int index)
2858 {
2859 napi->index = index;
2860 napi->config = &dev->napi_config[index];
2861 netif_napi_add_weight_locked(dev, napi, poll, NAPI_POLL_WEIGHT);
2862 }
2863
2864 /**
2865 * netif_napi_add_config - initialize a NAPI context with persistent config
2866 * @dev: network device
2867 * @napi: NAPI context
2868 * @poll: polling function
2869 * @index: the NAPI index
2870 */
2871 static inline void
netif_napi_add_config(struct net_device * dev,struct napi_struct * napi,int (* poll)(struct napi_struct *,int),int index)2872 netif_napi_add_config(struct net_device *dev, struct napi_struct *napi,
2873 int (*poll)(struct napi_struct *, int), int index)
2874 {
2875 netdev_lock(dev);
2876 netif_napi_add_config_locked(dev, napi, poll, index);
2877 netdev_unlock(dev);
2878 }
2879
2880 /**
2881 * netif_napi_add_tx() - initialize a NAPI context to be used for Tx only
2882 * @dev: network device
2883 * @napi: NAPI context
2884 * @poll: polling function
2885 *
2886 * This variant of netif_napi_add() should be used from drivers using NAPI
2887 * to exclusively poll a TX queue.
2888 * This will avoid we add it into napi_hash[], thus polluting this hash table.
2889 */
netif_napi_add_tx(struct net_device * dev,struct napi_struct * napi,int (* poll)(struct napi_struct *,int))2890 static inline void netif_napi_add_tx(struct net_device *dev,
2891 struct napi_struct *napi,
2892 int (*poll)(struct napi_struct *, int))
2893 {
2894 netif_napi_add_tx_weight(dev, napi, poll, NAPI_POLL_WEIGHT);
2895 }
2896
2897 void __netif_napi_del_locked(struct napi_struct *napi);
2898
2899 /**
2900 * __netif_napi_del - remove a NAPI context
2901 * @napi: NAPI context
2902 *
2903 * Warning: caller must observe RCU grace period before freeing memory
2904 * containing @napi. Drivers might want to call this helper to combine
2905 * all the needed RCU grace periods into a single one.
2906 */
__netif_napi_del(struct napi_struct * napi)2907 static inline void __netif_napi_del(struct napi_struct *napi)
2908 {
2909 netdev_lock(napi->dev);
2910 __netif_napi_del_locked(napi);
2911 netdev_unlock(napi->dev);
2912 }
2913
netif_napi_del_locked(struct napi_struct * napi)2914 static inline void netif_napi_del_locked(struct napi_struct *napi)
2915 {
2916 __netif_napi_del_locked(napi);
2917 synchronize_net();
2918 }
2919
2920 /**
2921 * netif_napi_del - remove a NAPI context
2922 * @napi: NAPI context
2923 *
2924 * netif_napi_del() removes a NAPI context from the network device NAPI list
2925 */
netif_napi_del(struct napi_struct * napi)2926 static inline void netif_napi_del(struct napi_struct *napi)
2927 {
2928 __netif_napi_del(napi);
2929 synchronize_net();
2930 }
2931
2932 int netif_enable_cpu_rmap(struct net_device *dev, unsigned int num_irqs);
2933 void netif_set_affinity_auto(struct net_device *dev);
2934
2935 struct packet_type {
2936 __be16 type; /* This is really htons(ether_type). */
2937 bool ignore_outgoing;
2938 struct net_device *dev; /* NULL is wildcarded here */
2939 netdevice_tracker dev_tracker;
2940 int (*func) (struct sk_buff *,
2941 struct net_device *,
2942 struct packet_type *,
2943 struct net_device *);
2944 void (*list_func) (struct list_head *,
2945 struct packet_type *,
2946 struct net_device *);
2947 bool (*id_match)(struct packet_type *ptype,
2948 struct sock *sk);
2949 struct net *af_packet_net;
2950 void *af_packet_priv;
2951 struct list_head list;
2952 };
2953
2954 struct offload_callbacks {
2955 struct sk_buff *(*gso_segment)(struct sk_buff *skb,
2956 netdev_features_t features);
2957 struct sk_buff *(*gro_receive)(struct list_head *head,
2958 struct sk_buff *skb);
2959 int (*gro_complete)(struct sk_buff *skb, int nhoff);
2960 };
2961
2962 struct packet_offload {
2963 __be16 type; /* This is really htons(ether_type). */
2964 u16 priority;
2965 struct offload_callbacks callbacks;
2966 struct list_head list;
2967 };
2968
2969 /* often modified stats are per-CPU, other are shared (netdev->stats) */
2970 struct pcpu_sw_netstats {
2971 u64_stats_t rx_packets;
2972 u64_stats_t rx_bytes;
2973 u64_stats_t tx_packets;
2974 u64_stats_t tx_bytes;
2975 struct u64_stats_sync syncp;
2976 } __aligned(4 * sizeof(u64));
2977
2978 struct pcpu_dstats {
2979 u64_stats_t rx_packets;
2980 u64_stats_t rx_bytes;
2981 u64_stats_t tx_packets;
2982 u64_stats_t tx_bytes;
2983 u64_stats_t rx_drops;
2984 u64_stats_t tx_drops;
2985 struct u64_stats_sync syncp;
2986 } __aligned(8 * sizeof(u64));
2987
2988 struct pcpu_lstats {
2989 u64_stats_t packets;
2990 u64_stats_t bytes;
2991 struct u64_stats_sync syncp;
2992 } __aligned(2 * sizeof(u64));
2993
2994 void dev_lstats_read(struct net_device *dev, u64 *packets, u64 *bytes);
2995
dev_sw_netstats_rx_add(struct net_device * dev,unsigned int len)2996 static inline void dev_sw_netstats_rx_add(struct net_device *dev, unsigned int len)
2997 {
2998 struct pcpu_sw_netstats *tstats = this_cpu_ptr(dev->tstats);
2999
3000 u64_stats_update_begin(&tstats->syncp);
3001 u64_stats_add(&tstats->rx_bytes, len);
3002 u64_stats_inc(&tstats->rx_packets);
3003 u64_stats_update_end(&tstats->syncp);
3004 }
3005
dev_sw_netstats_tx_add(struct net_device * dev,unsigned int packets,unsigned int len)3006 static inline void dev_sw_netstats_tx_add(struct net_device *dev,
3007 unsigned int packets,
3008 unsigned int len)
3009 {
3010 struct pcpu_sw_netstats *tstats = this_cpu_ptr(dev->tstats);
3011
3012 u64_stats_update_begin(&tstats->syncp);
3013 u64_stats_add(&tstats->tx_bytes, len);
3014 u64_stats_add(&tstats->tx_packets, packets);
3015 u64_stats_update_end(&tstats->syncp);
3016 }
3017
dev_lstats_add(struct net_device * dev,unsigned int len)3018 static inline void dev_lstats_add(struct net_device *dev, unsigned int len)
3019 {
3020 struct pcpu_lstats *lstats = this_cpu_ptr(dev->lstats);
3021
3022 u64_stats_update_begin(&lstats->syncp);
3023 u64_stats_add(&lstats->bytes, len);
3024 u64_stats_inc(&lstats->packets);
3025 u64_stats_update_end(&lstats->syncp);
3026 }
3027
dev_dstats_rx_add(struct net_device * dev,unsigned int len)3028 static inline void dev_dstats_rx_add(struct net_device *dev,
3029 unsigned int len)
3030 {
3031 struct pcpu_dstats *dstats = this_cpu_ptr(dev->dstats);
3032
3033 u64_stats_update_begin(&dstats->syncp);
3034 u64_stats_inc(&dstats->rx_packets);
3035 u64_stats_add(&dstats->rx_bytes, len);
3036 u64_stats_update_end(&dstats->syncp);
3037 }
3038
dev_dstats_rx_dropped(struct net_device * dev)3039 static inline void dev_dstats_rx_dropped(struct net_device *dev)
3040 {
3041 struct pcpu_dstats *dstats = this_cpu_ptr(dev->dstats);
3042
3043 u64_stats_update_begin(&dstats->syncp);
3044 u64_stats_inc(&dstats->rx_drops);
3045 u64_stats_update_end(&dstats->syncp);
3046 }
3047
dev_dstats_rx_dropped_add(struct net_device * dev,unsigned int packets)3048 static inline void dev_dstats_rx_dropped_add(struct net_device *dev,
3049 unsigned int packets)
3050 {
3051 struct pcpu_dstats *dstats = this_cpu_ptr(dev->dstats);
3052
3053 u64_stats_update_begin(&dstats->syncp);
3054 u64_stats_add(&dstats->rx_drops, packets);
3055 u64_stats_update_end(&dstats->syncp);
3056 }
3057
dev_dstats_tx_add(struct net_device * dev,unsigned int len)3058 static inline void dev_dstats_tx_add(struct net_device *dev,
3059 unsigned int len)
3060 {
3061 struct pcpu_dstats *dstats = this_cpu_ptr(dev->dstats);
3062
3063 u64_stats_update_begin(&dstats->syncp);
3064 u64_stats_inc(&dstats->tx_packets);
3065 u64_stats_add(&dstats->tx_bytes, len);
3066 u64_stats_update_end(&dstats->syncp);
3067 }
3068
dev_dstats_tx_dropped(struct net_device * dev)3069 static inline void dev_dstats_tx_dropped(struct net_device *dev)
3070 {
3071 struct pcpu_dstats *dstats = this_cpu_ptr(dev->dstats);
3072
3073 u64_stats_update_begin(&dstats->syncp);
3074 u64_stats_inc(&dstats->tx_drops);
3075 u64_stats_update_end(&dstats->syncp);
3076 }
3077
3078 #define __netdev_alloc_pcpu_stats(type, gfp) \
3079 ({ \
3080 typeof(type) __percpu *pcpu_stats = alloc_percpu_gfp(type, gfp);\
3081 if (pcpu_stats) { \
3082 int __cpu; \
3083 for_each_possible_cpu(__cpu) { \
3084 typeof(type) *stat; \
3085 stat = per_cpu_ptr(pcpu_stats, __cpu); \
3086 u64_stats_init(&stat->syncp); \
3087 } \
3088 } \
3089 pcpu_stats; \
3090 })
3091
3092 #define netdev_alloc_pcpu_stats(type) \
3093 __netdev_alloc_pcpu_stats(type, GFP_KERNEL)
3094
3095 #define devm_netdev_alloc_pcpu_stats(dev, type) \
3096 ({ \
3097 typeof(type) __percpu *pcpu_stats = devm_alloc_percpu(dev, type);\
3098 if (pcpu_stats) { \
3099 int __cpu; \
3100 for_each_possible_cpu(__cpu) { \
3101 typeof(type) *stat; \
3102 stat = per_cpu_ptr(pcpu_stats, __cpu); \
3103 u64_stats_init(&stat->syncp); \
3104 } \
3105 } \
3106 pcpu_stats; \
3107 })
3108
3109 enum netdev_lag_tx_type {
3110 NETDEV_LAG_TX_TYPE_UNKNOWN,
3111 NETDEV_LAG_TX_TYPE_RANDOM,
3112 NETDEV_LAG_TX_TYPE_BROADCAST,
3113 NETDEV_LAG_TX_TYPE_ROUNDROBIN,
3114 NETDEV_LAG_TX_TYPE_ACTIVEBACKUP,
3115 NETDEV_LAG_TX_TYPE_HASH,
3116 };
3117
3118 enum netdev_lag_hash {
3119 NETDEV_LAG_HASH_NONE,
3120 NETDEV_LAG_HASH_L2,
3121 NETDEV_LAG_HASH_L34,
3122 NETDEV_LAG_HASH_L23,
3123 NETDEV_LAG_HASH_E23,
3124 NETDEV_LAG_HASH_E34,
3125 NETDEV_LAG_HASH_VLAN_SRCMAC,
3126 NETDEV_LAG_HASH_UNKNOWN,
3127 };
3128
3129 struct netdev_lag_upper_info {
3130 enum netdev_lag_tx_type tx_type;
3131 enum netdev_lag_hash hash_type;
3132 };
3133
3134 struct netdev_lag_lower_state_info {
3135 u8 link_up : 1,
3136 tx_enabled : 1;
3137 };
3138
3139 #include <linux/notifier.h>
3140
3141 /* netdevice notifier chain. Please remember to update netdev_cmd_to_name()
3142 * and the rtnetlink notification exclusion list in rtnetlink_event() when
3143 * adding new types.
3144 */
3145 enum netdev_cmd {
3146 NETDEV_UP = 1, /* For now you can't veto a device up/down */
3147 NETDEV_DOWN,
3148 NETDEV_REBOOT, /* Tell a protocol stack a network interface
3149 detected a hardware crash and restarted
3150 - we can use this eg to kick tcp sessions
3151 once done */
3152 NETDEV_CHANGE, /* Notify device state change */
3153 NETDEV_REGISTER,
3154 NETDEV_UNREGISTER,
3155 NETDEV_CHANGEMTU, /* notify after mtu change happened */
3156 NETDEV_CHANGEADDR, /* notify after the address change */
3157 NETDEV_PRE_CHANGEADDR, /* notify before the address change */
3158 NETDEV_GOING_DOWN,
3159 NETDEV_CHANGENAME,
3160 NETDEV_FEAT_CHANGE,
3161 NETDEV_BONDING_FAILOVER,
3162 NETDEV_PRE_UP,
3163 NETDEV_PRE_TYPE_CHANGE,
3164 NETDEV_POST_TYPE_CHANGE,
3165 NETDEV_POST_INIT,
3166 NETDEV_PRE_UNINIT,
3167 NETDEV_RELEASE,
3168 NETDEV_NOTIFY_PEERS,
3169 NETDEV_JOIN,
3170 NETDEV_CHANGEUPPER,
3171 NETDEV_RESEND_IGMP,
3172 NETDEV_PRECHANGEMTU, /* notify before mtu change happened */
3173 NETDEV_CHANGEINFODATA,
3174 NETDEV_BONDING_INFO,
3175 NETDEV_PRECHANGEUPPER,
3176 NETDEV_CHANGELOWERSTATE,
3177 NETDEV_UDP_TUNNEL_PUSH_INFO,
3178 NETDEV_UDP_TUNNEL_DROP_INFO,
3179 NETDEV_CHANGE_TX_QUEUE_LEN,
3180 NETDEV_CVLAN_FILTER_PUSH_INFO,
3181 NETDEV_CVLAN_FILTER_DROP_INFO,
3182 NETDEV_SVLAN_FILTER_PUSH_INFO,
3183 NETDEV_SVLAN_FILTER_DROP_INFO,
3184 NETDEV_OFFLOAD_XSTATS_ENABLE,
3185 NETDEV_OFFLOAD_XSTATS_DISABLE,
3186 NETDEV_OFFLOAD_XSTATS_REPORT_USED,
3187 NETDEV_OFFLOAD_XSTATS_REPORT_DELTA,
3188 NETDEV_XDP_FEAT_CHANGE,
3189 };
3190 const char *netdev_cmd_to_name(enum netdev_cmd cmd);
3191
3192 int register_netdevice_notifier(struct notifier_block *nb);
3193 int unregister_netdevice_notifier(struct notifier_block *nb);
3194 int register_netdevice_notifier_net(struct net *net, struct notifier_block *nb);
3195 int unregister_netdevice_notifier_net(struct net *net,
3196 struct notifier_block *nb);
3197 int register_netdevice_notifier_dev_net(struct net_device *dev,
3198 struct notifier_block *nb,
3199 struct netdev_net_notifier *nn);
3200 int unregister_netdevice_notifier_dev_net(struct net_device *dev,
3201 struct notifier_block *nb,
3202 struct netdev_net_notifier *nn);
3203
3204 struct netdev_notifier_info {
3205 struct net_device *dev;
3206 struct netlink_ext_ack *extack;
3207 };
3208
3209 struct netdev_notifier_info_ext {
3210 struct netdev_notifier_info info; /* must be first */
3211 union {
3212 u32 mtu;
3213 } ext;
3214 };
3215
3216 struct netdev_notifier_change_info {
3217 struct netdev_notifier_info info; /* must be first */
3218 unsigned int flags_changed;
3219 };
3220
3221 struct netdev_notifier_changeupper_info {
3222 struct netdev_notifier_info info; /* must be first */
3223 struct net_device *upper_dev; /* new upper dev */
3224 bool master; /* is upper dev master */
3225 bool linking; /* is the notification for link or unlink */
3226 void *upper_info; /* upper dev info */
3227 };
3228
3229 struct netdev_notifier_changelowerstate_info {
3230 struct netdev_notifier_info info; /* must be first */
3231 void *lower_state_info; /* is lower dev state */
3232 };
3233
3234 struct netdev_notifier_pre_changeaddr_info {
3235 struct netdev_notifier_info info; /* must be first */
3236 const unsigned char *dev_addr;
3237 };
3238
3239 enum netdev_offload_xstats_type {
3240 NETDEV_OFFLOAD_XSTATS_TYPE_L3 = 1,
3241 };
3242
3243 struct netdev_notifier_offload_xstats_info {
3244 struct netdev_notifier_info info; /* must be first */
3245 enum netdev_offload_xstats_type type;
3246
3247 union {
3248 /* NETDEV_OFFLOAD_XSTATS_REPORT_DELTA */
3249 struct netdev_notifier_offload_xstats_rd *report_delta;
3250 /* NETDEV_OFFLOAD_XSTATS_REPORT_USED */
3251 struct netdev_notifier_offload_xstats_ru *report_used;
3252 };
3253 };
3254
3255 int netdev_offload_xstats_enable(struct net_device *dev,
3256 enum netdev_offload_xstats_type type,
3257 struct netlink_ext_ack *extack);
3258 int netdev_offload_xstats_disable(struct net_device *dev,
3259 enum netdev_offload_xstats_type type);
3260 bool netdev_offload_xstats_enabled(const struct net_device *dev,
3261 enum netdev_offload_xstats_type type);
3262 int netdev_offload_xstats_get(struct net_device *dev,
3263 enum netdev_offload_xstats_type type,
3264 struct rtnl_hw_stats64 *stats, bool *used,
3265 struct netlink_ext_ack *extack);
3266 void
3267 netdev_offload_xstats_report_delta(struct netdev_notifier_offload_xstats_rd *rd,
3268 const struct rtnl_hw_stats64 *stats);
3269 void
3270 netdev_offload_xstats_report_used(struct netdev_notifier_offload_xstats_ru *ru);
3271 void netdev_offload_xstats_push_delta(struct net_device *dev,
3272 enum netdev_offload_xstats_type type,
3273 const struct rtnl_hw_stats64 *stats);
3274
netdev_notifier_info_init(struct netdev_notifier_info * info,struct net_device * dev)3275 static inline void netdev_notifier_info_init(struct netdev_notifier_info *info,
3276 struct net_device *dev)
3277 {
3278 info->dev = dev;
3279 info->extack = NULL;
3280 }
3281
3282 static inline struct net_device *
netdev_notifier_info_to_dev(const struct netdev_notifier_info * info)3283 netdev_notifier_info_to_dev(const struct netdev_notifier_info *info)
3284 {
3285 return info->dev;
3286 }
3287
3288 static inline struct netlink_ext_ack *
netdev_notifier_info_to_extack(const struct netdev_notifier_info * info)3289 netdev_notifier_info_to_extack(const struct netdev_notifier_info *info)
3290 {
3291 return info->extack;
3292 }
3293
3294 int call_netdevice_notifiers(unsigned long val, struct net_device *dev);
3295 int call_netdevice_notifiers_info(unsigned long val,
3296 struct netdev_notifier_info *info);
3297
3298 #define for_each_netdev(net, d) \
3299 list_for_each_entry(d, &(net)->dev_base_head, dev_list)
3300 #define for_each_netdev_reverse(net, d) \
3301 list_for_each_entry_reverse(d, &(net)->dev_base_head, dev_list)
3302 #define for_each_netdev_rcu(net, d) \
3303 list_for_each_entry_rcu(d, &(net)->dev_base_head, dev_list)
3304 #define for_each_netdev_safe(net, d, n) \
3305 list_for_each_entry_safe(d, n, &(net)->dev_base_head, dev_list)
3306 #define for_each_netdev_continue(net, d) \
3307 list_for_each_entry_continue(d, &(net)->dev_base_head, dev_list)
3308 #define for_each_netdev_continue_reverse(net, d) \
3309 list_for_each_entry_continue_reverse(d, &(net)->dev_base_head, \
3310 dev_list)
3311 #define for_each_netdev_continue_rcu(net, d) \
3312 list_for_each_entry_continue_rcu(d, &(net)->dev_base_head, dev_list)
3313 #define for_each_netdev_in_bond_rcu(bond, slave) \
3314 for_each_netdev_rcu(dev_net_rcu(bond), slave) \
3315 if (netdev_master_upper_dev_get_rcu(slave) == (bond))
3316 #define net_device_entry(lh) list_entry(lh, struct net_device, dev_list)
3317
3318 #define for_each_netdev_dump(net, d, ifindex) \
3319 for (; (d = xa_find(&(net)->dev_by_index, &ifindex, \
3320 ULONG_MAX, XA_PRESENT)); ifindex++)
3321
next_net_device(struct net_device * dev)3322 static inline struct net_device *next_net_device(struct net_device *dev)
3323 {
3324 struct list_head *lh;
3325 struct net *net;
3326
3327 net = dev_net(dev);
3328 lh = dev->dev_list.next;
3329 return lh == &net->dev_base_head ? NULL : net_device_entry(lh);
3330 }
3331
next_net_device_rcu(struct net_device * dev)3332 static inline struct net_device *next_net_device_rcu(struct net_device *dev)
3333 {
3334 struct list_head *lh;
3335 struct net *net;
3336
3337 net = dev_net(dev);
3338 lh = rcu_dereference(list_next_rcu(&dev->dev_list));
3339 return lh == &net->dev_base_head ? NULL : net_device_entry(lh);
3340 }
3341
first_net_device(struct net * net)3342 static inline struct net_device *first_net_device(struct net *net)
3343 {
3344 return list_empty(&net->dev_base_head) ? NULL :
3345 net_device_entry(net->dev_base_head.next);
3346 }
3347
3348 int netdev_boot_setup_check(struct net_device *dev);
3349 struct net_device *dev_getbyhwaddr(struct net *net, unsigned short type,
3350 const char *hwaddr);
3351 struct net_device *dev_getbyhwaddr_rcu(struct net *net, unsigned short type,
3352 const char *hwaddr);
3353 struct net_device *dev_getfirstbyhwtype(struct net *net, unsigned short type);
3354 void dev_add_pack(struct packet_type *pt);
3355 void dev_remove_pack(struct packet_type *pt);
3356 void __dev_remove_pack(struct packet_type *pt);
3357 void dev_add_offload(struct packet_offload *po);
3358 void dev_remove_offload(struct packet_offload *po);
3359
3360 int dev_get_iflink(const struct net_device *dev);
3361 int dev_fill_metadata_dst(struct net_device *dev, struct sk_buff *skb);
3362 int dev_fill_forward_path(const struct net_device *dev, const u8 *daddr,
3363 struct net_device_path_stack *stack);
3364 struct net_device *dev_get_by_name(struct net *net, const char *name);
3365 struct net_device *dev_get_by_name_rcu(struct net *net, const char *name);
3366 struct net_device *__dev_get_by_name(struct net *net, const char *name);
3367 bool netdev_name_in_use(struct net *net, const char *name);
3368 int dev_alloc_name(struct net_device *dev, const char *name);
3369 int netif_open(struct net_device *dev, struct netlink_ext_ack *extack);
3370 int dev_open(struct net_device *dev, struct netlink_ext_ack *extack);
3371 void netif_close(struct net_device *dev);
3372 void dev_close(struct net_device *dev);
3373 void netif_close_many(struct list_head *head, bool unlink);
3374 void netif_disable_lro(struct net_device *dev);
3375 void dev_disable_lro(struct net_device *dev);
3376 int dev_loopback_xmit(struct net *net, struct sock *sk, struct sk_buff *newskb);
3377 u16 dev_pick_tx_zero(struct net_device *dev, struct sk_buff *skb,
3378 struct net_device *sb_dev);
3379
3380 int __dev_queue_xmit(struct sk_buff *skb, struct net_device *sb_dev);
3381 int __dev_direct_xmit(struct sk_buff *skb, u16 queue_id);
3382
dev_queue_xmit(struct sk_buff * skb)3383 static inline int dev_queue_xmit(struct sk_buff *skb)
3384 {
3385 return __dev_queue_xmit(skb, NULL);
3386 }
3387
dev_queue_xmit_accel(struct sk_buff * skb,struct net_device * sb_dev)3388 static inline int dev_queue_xmit_accel(struct sk_buff *skb,
3389 struct net_device *sb_dev)
3390 {
3391 return __dev_queue_xmit(skb, sb_dev);
3392 }
3393
dev_direct_xmit(struct sk_buff * skb,u16 queue_id)3394 static inline int dev_direct_xmit(struct sk_buff *skb, u16 queue_id)
3395 {
3396 int ret;
3397
3398 ret = __dev_direct_xmit(skb, queue_id);
3399 if (!dev_xmit_complete(ret))
3400 kfree_skb(skb);
3401 return ret;
3402 }
3403
3404 int register_netdevice(struct net_device *dev);
3405 void unregister_netdevice_queue(struct net_device *dev, struct list_head *head);
3406 void unregister_netdevice_many(struct list_head *head);
unregister_netdevice(struct net_device * dev)3407 static inline void unregister_netdevice(struct net_device *dev)
3408 {
3409 unregister_netdevice_queue(dev, NULL);
3410 }
3411
3412 int netdev_refcnt_read(const struct net_device *dev);
3413 void free_netdev(struct net_device *dev);
3414
3415 struct net_device *netdev_get_xmit_slave(struct net_device *dev,
3416 struct sk_buff *skb,
3417 bool all_slaves);
3418 struct net_device *netdev_sk_get_lowest_dev(struct net_device *dev,
3419 struct sock *sk);
3420 struct net_device *dev_get_by_index(struct net *net, int ifindex);
3421 struct net_device *__dev_get_by_index(struct net *net, int ifindex);
3422 struct net_device *netdev_get_by_index(struct net *net, int ifindex,
3423 netdevice_tracker *tracker, gfp_t gfp);
3424 struct net_device *netdev_get_by_index_lock(struct net *net, int ifindex);
3425 struct net_device *netdev_get_by_name(struct net *net, const char *name,
3426 netdevice_tracker *tracker, gfp_t gfp);
3427 struct net_device *netdev_get_by_flags_rcu(struct net *net, netdevice_tracker *tracker,
3428 unsigned short flags, unsigned short mask);
3429 struct net_device *dev_get_by_index_rcu(struct net *net, int ifindex);
3430 void netdev_copy_name(struct net_device *dev, char *name);
3431
dev_hard_header(struct sk_buff * skb,struct net_device * dev,unsigned short type,const void * daddr,const void * saddr,unsigned int len)3432 static inline int dev_hard_header(struct sk_buff *skb, struct net_device *dev,
3433 unsigned short type,
3434 const void *daddr, const void *saddr,
3435 unsigned int len)
3436 {
3437 if (!dev->header_ops || !dev->header_ops->create)
3438 return 0;
3439
3440 return dev->header_ops->create(skb, dev, type, daddr, saddr, len);
3441 }
3442
dev_parse_header(const struct sk_buff * skb,unsigned char * haddr)3443 static inline int dev_parse_header(const struct sk_buff *skb,
3444 unsigned char *haddr)
3445 {
3446 const struct net_device *dev = skb->dev;
3447
3448 if (!dev->header_ops || !dev->header_ops->parse)
3449 return 0;
3450 return dev->header_ops->parse(skb, dev, haddr);
3451 }
3452
dev_parse_header_protocol(const struct sk_buff * skb)3453 static inline __be16 dev_parse_header_protocol(const struct sk_buff *skb)
3454 {
3455 const struct net_device *dev = skb->dev;
3456
3457 if (!dev->header_ops || !dev->header_ops->parse_protocol)
3458 return 0;
3459 return dev->header_ops->parse_protocol(skb);
3460 }
3461
3462 /* ll_header must have at least hard_header_len allocated */
dev_validate_header(const struct net_device * dev,char * ll_header,int len)3463 static inline bool dev_validate_header(const struct net_device *dev,
3464 char *ll_header, int len)
3465 {
3466 if (likely(len >= dev->hard_header_len))
3467 return true;
3468 if (len < dev->min_header_len)
3469 return false;
3470
3471 if (capable(CAP_SYS_RAWIO)) {
3472 memset(ll_header + len, 0, dev->hard_header_len - len);
3473 return true;
3474 }
3475
3476 if (dev->header_ops && dev->header_ops->validate)
3477 return dev->header_ops->validate(ll_header, len);
3478
3479 return false;
3480 }
3481
dev_has_header(const struct net_device * dev)3482 static inline bool dev_has_header(const struct net_device *dev)
3483 {
3484 return dev->header_ops && dev->header_ops->create;
3485 }
3486
3487 struct numa_drop_counters {
3488 atomic_t drops0 ____cacheline_aligned_in_smp;
3489 atomic_t drops1 ____cacheline_aligned_in_smp;
3490 };
3491
numa_drop_read(const struct numa_drop_counters * ndc)3492 static inline int numa_drop_read(const struct numa_drop_counters *ndc)
3493 {
3494 return atomic_read(&ndc->drops0) + atomic_read(&ndc->drops1);
3495 }
3496
numa_drop_add(struct numa_drop_counters * ndc,int val)3497 static inline void numa_drop_add(struct numa_drop_counters *ndc, int val)
3498 {
3499 int n = numa_node_id() % 2;
3500
3501 if (n)
3502 atomic_add(val, &ndc->drops1);
3503 else
3504 atomic_add(val, &ndc->drops0);
3505 }
3506
numa_drop_reset(struct numa_drop_counters * ndc)3507 static inline void numa_drop_reset(struct numa_drop_counters *ndc)
3508 {
3509 atomic_set(&ndc->drops0, 0);
3510 atomic_set(&ndc->drops1, 0);
3511 }
3512
3513 /*
3514 * Incoming packets are placed on per-CPU queues
3515 */
3516 struct softnet_data {
3517 struct list_head poll_list;
3518 struct sk_buff_head process_queue;
3519 local_lock_t process_queue_bh_lock;
3520
3521 /* stats */
3522 unsigned int processed;
3523 unsigned int time_squeeze;
3524 #ifdef CONFIG_RPS
3525 struct softnet_data *rps_ipi_list;
3526 #endif
3527
3528 unsigned int received_rps;
3529 bool in_net_rx_action;
3530 bool in_napi_threaded_poll;
3531
3532 #ifdef CONFIG_NET_FLOW_LIMIT
3533 struct sd_flow_limit __rcu *flow_limit;
3534 #endif
3535 struct Qdisc *output_queue;
3536 struct Qdisc **output_queue_tailp;
3537 struct sk_buff *completion_queue;
3538 #ifdef CONFIG_XFRM_OFFLOAD
3539 struct sk_buff_head xfrm_backlog;
3540 #endif
3541 /* written and read only by owning cpu: */
3542 struct netdev_xmit xmit;
3543 #ifdef CONFIG_RPS
3544 /* input_queue_head should be written by cpu owning this struct,
3545 * and only read by other cpus. Worth using a cache line.
3546 */
3547 unsigned int input_queue_head ____cacheline_aligned_in_smp;
3548
3549 /* Elements below can be accessed between CPUs for RPS/RFS */
3550 call_single_data_t csd ____cacheline_aligned_in_smp;
3551 struct softnet_data *rps_ipi_next;
3552 unsigned int cpu;
3553
3554 /* We force a cacheline alignment from here, to hold together
3555 * input_queue_tail, input_pkt_queue and backlog.state.
3556 * We add holes so that backlog.state is the last field
3557 * of this cache line.
3558 */
3559 long pad[3] ____cacheline_aligned_in_smp;
3560 unsigned int input_queue_tail;
3561 #endif
3562 struct sk_buff_head input_pkt_queue;
3563
3564 struct napi_struct backlog;
3565
3566 struct numa_drop_counters drop_counters;
3567
3568 int defer_ipi_scheduled ____cacheline_aligned_in_smp;
3569 call_single_data_t defer_csd;
3570 };
3571
3572 DECLARE_PER_CPU_ALIGNED(struct softnet_data, softnet_data);
3573
3574 struct page_pool_bh {
3575 struct page_pool *pool;
3576 local_lock_t bh_lock;
3577 };
3578 DECLARE_PER_CPU(struct page_pool_bh, system_page_pool);
3579
3580 #define XMIT_RECURSION_LIMIT 8
3581
3582 #ifndef CONFIG_PREEMPT_RT
dev_recursion_level(void)3583 static inline int dev_recursion_level(void)
3584 {
3585 return this_cpu_read(softnet_data.xmit.recursion);
3586 }
3587
dev_xmit_recursion(void)3588 static inline bool dev_xmit_recursion(void)
3589 {
3590 return unlikely(__this_cpu_read(softnet_data.xmit.recursion) >
3591 XMIT_RECURSION_LIMIT);
3592 }
3593
dev_xmit_recursion_inc(void)3594 static inline void dev_xmit_recursion_inc(void)
3595 {
3596 __this_cpu_inc(softnet_data.xmit.recursion);
3597 }
3598
dev_xmit_recursion_dec(void)3599 static inline void dev_xmit_recursion_dec(void)
3600 {
3601 __this_cpu_dec(softnet_data.xmit.recursion);
3602 }
3603 #else
dev_recursion_level(void)3604 static inline int dev_recursion_level(void)
3605 {
3606 return current->net_xmit.recursion;
3607 }
3608
dev_xmit_recursion(void)3609 static inline bool dev_xmit_recursion(void)
3610 {
3611 return unlikely(current->net_xmit.recursion > XMIT_RECURSION_LIMIT);
3612 }
3613
dev_xmit_recursion_inc(void)3614 static inline void dev_xmit_recursion_inc(void)
3615 {
3616 current->net_xmit.recursion++;
3617 }
3618
dev_xmit_recursion_dec(void)3619 static inline void dev_xmit_recursion_dec(void)
3620 {
3621 current->net_xmit.recursion--;
3622 }
3623 #endif
3624
3625 void __netif_schedule(struct Qdisc *q);
3626 void netif_schedule_queue(struct netdev_queue *txq);
3627
netif_tx_schedule_all(struct net_device * dev)3628 static inline void netif_tx_schedule_all(struct net_device *dev)
3629 {
3630 unsigned int i;
3631
3632 for (i = 0; i < dev->num_tx_queues; i++)
3633 netif_schedule_queue(netdev_get_tx_queue(dev, i));
3634 }
3635
netif_tx_start_queue(struct netdev_queue * dev_queue)3636 static __always_inline void netif_tx_start_queue(struct netdev_queue *dev_queue)
3637 {
3638 clear_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state);
3639 }
3640
3641 /**
3642 * netif_start_queue - allow transmit
3643 * @dev: network device
3644 *
3645 * Allow upper layers to call the device hard_start_xmit routine.
3646 */
netif_start_queue(struct net_device * dev)3647 static inline void netif_start_queue(struct net_device *dev)
3648 {
3649 netif_tx_start_queue(netdev_get_tx_queue(dev, 0));
3650 }
3651
netif_tx_start_all_queues(struct net_device * dev)3652 static inline void netif_tx_start_all_queues(struct net_device *dev)
3653 {
3654 unsigned int i;
3655
3656 for (i = 0; i < dev->num_tx_queues; i++) {
3657 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
3658 netif_tx_start_queue(txq);
3659 }
3660 }
3661
3662 void netif_tx_wake_queue(struct netdev_queue *dev_queue);
3663
3664 /**
3665 * netif_wake_queue - restart transmit
3666 * @dev: network device
3667 *
3668 * Allow upper layers to call the device hard_start_xmit routine.
3669 * Used for flow control when transmit resources are available.
3670 */
netif_wake_queue(struct net_device * dev)3671 static inline void netif_wake_queue(struct net_device *dev)
3672 {
3673 netif_tx_wake_queue(netdev_get_tx_queue(dev, 0));
3674 }
3675
netif_tx_wake_all_queues(struct net_device * dev)3676 static inline void netif_tx_wake_all_queues(struct net_device *dev)
3677 {
3678 unsigned int i;
3679
3680 for (i = 0; i < dev->num_tx_queues; i++) {
3681 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
3682 netif_tx_wake_queue(txq);
3683 }
3684 }
3685
netif_tx_stop_queue(struct netdev_queue * dev_queue)3686 static __always_inline void netif_tx_stop_queue(struct netdev_queue *dev_queue)
3687 {
3688 /* Paired with READ_ONCE() from dev_watchdog() */
3689 WRITE_ONCE(dev_queue->trans_start, jiffies);
3690
3691 /* This barrier is paired with smp_mb() from dev_watchdog() */
3692 smp_mb__before_atomic();
3693
3694 /* Must be an atomic op see netif_txq_try_stop() */
3695 set_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state);
3696 }
3697
3698 /**
3699 * netif_stop_queue - stop transmitted packets
3700 * @dev: network device
3701 *
3702 * Stop upper layers calling the device hard_start_xmit routine.
3703 * Used for flow control when transmit resources are unavailable.
3704 */
netif_stop_queue(struct net_device * dev)3705 static inline void netif_stop_queue(struct net_device *dev)
3706 {
3707 netif_tx_stop_queue(netdev_get_tx_queue(dev, 0));
3708 }
3709
3710 void netif_tx_stop_all_queues(struct net_device *dev);
3711
netif_tx_queue_stopped(const struct netdev_queue * dev_queue)3712 static inline bool netif_tx_queue_stopped(const struct netdev_queue *dev_queue)
3713 {
3714 return test_bit(__QUEUE_STATE_DRV_XOFF, &dev_queue->state);
3715 }
3716
3717 /**
3718 * netif_queue_stopped - test if transmit queue is flowblocked
3719 * @dev: network device
3720 *
3721 * Test if transmit queue on device is currently unable to send.
3722 */
netif_queue_stopped(const struct net_device * dev)3723 static inline bool netif_queue_stopped(const struct net_device *dev)
3724 {
3725 return netif_tx_queue_stopped(netdev_get_tx_queue(dev, 0));
3726 }
3727
netif_xmit_stopped(const struct netdev_queue * dev_queue)3728 static inline bool netif_xmit_stopped(const struct netdev_queue *dev_queue)
3729 {
3730 return dev_queue->state & QUEUE_STATE_ANY_XOFF;
3731 }
3732
3733 static inline bool
netif_xmit_frozen_or_stopped(const struct netdev_queue * dev_queue)3734 netif_xmit_frozen_or_stopped(const struct netdev_queue *dev_queue)
3735 {
3736 return dev_queue->state & QUEUE_STATE_ANY_XOFF_OR_FROZEN;
3737 }
3738
3739 static inline bool
netif_xmit_frozen_or_drv_stopped(const struct netdev_queue * dev_queue)3740 netif_xmit_frozen_or_drv_stopped(const struct netdev_queue *dev_queue)
3741 {
3742 return dev_queue->state & QUEUE_STATE_DRV_XOFF_OR_FROZEN;
3743 }
3744
3745 /**
3746 * netdev_queue_set_dql_min_limit - set dql minimum limit
3747 * @dev_queue: pointer to transmit queue
3748 * @min_limit: dql minimum limit
3749 *
3750 * Forces xmit_more() to return true until the minimum threshold
3751 * defined by @min_limit is reached (or until the tx queue is
3752 * empty). Warning: to be use with care, misuse will impact the
3753 * latency.
3754 */
netdev_queue_set_dql_min_limit(struct netdev_queue * dev_queue,unsigned int min_limit)3755 static inline void netdev_queue_set_dql_min_limit(struct netdev_queue *dev_queue,
3756 unsigned int min_limit)
3757 {
3758 #ifdef CONFIG_BQL
3759 dev_queue->dql.min_limit = min_limit;
3760 #endif
3761 }
3762
netdev_queue_dql_avail(const struct netdev_queue * txq)3763 static inline int netdev_queue_dql_avail(const struct netdev_queue *txq)
3764 {
3765 #ifdef CONFIG_BQL
3766 /* Non-BQL migrated drivers will return 0, too. */
3767 return dql_avail(&txq->dql);
3768 #else
3769 return 0;
3770 #endif
3771 }
3772
3773 /**
3774 * netdev_txq_bql_enqueue_prefetchw - prefetch bql data for write
3775 * @dev_queue: pointer to transmit queue
3776 *
3777 * BQL enabled drivers might use this helper in their ndo_start_xmit(),
3778 * to give appropriate hint to the CPU.
3779 */
netdev_txq_bql_enqueue_prefetchw(struct netdev_queue * dev_queue)3780 static inline void netdev_txq_bql_enqueue_prefetchw(struct netdev_queue *dev_queue)
3781 {
3782 #ifdef CONFIG_BQL
3783 prefetchw(&dev_queue->dql.num_queued);
3784 #endif
3785 }
3786
3787 /**
3788 * netdev_txq_bql_complete_prefetchw - prefetch bql data for write
3789 * @dev_queue: pointer to transmit queue
3790 *
3791 * BQL enabled drivers might use this helper in their TX completion path,
3792 * to give appropriate hint to the CPU.
3793 */
netdev_txq_bql_complete_prefetchw(struct netdev_queue * dev_queue)3794 static inline void netdev_txq_bql_complete_prefetchw(struct netdev_queue *dev_queue)
3795 {
3796 #ifdef CONFIG_BQL
3797 prefetchw(&dev_queue->dql.limit);
3798 #endif
3799 }
3800
3801 /**
3802 * netdev_tx_sent_queue - report the number of bytes queued to a given tx queue
3803 * @dev_queue: network device queue
3804 * @bytes: number of bytes queued to the device queue
3805 *
3806 * Report the number of bytes queued for sending/completion to the network
3807 * device hardware queue. @bytes should be a good approximation and should
3808 * exactly match netdev_completed_queue() @bytes.
3809 * This is typically called once per packet, from ndo_start_xmit().
3810 */
netdev_tx_sent_queue(struct netdev_queue * dev_queue,unsigned int bytes)3811 static inline void netdev_tx_sent_queue(struct netdev_queue *dev_queue,
3812 unsigned int bytes)
3813 {
3814 #ifdef CONFIG_BQL
3815 dql_queued(&dev_queue->dql, bytes);
3816
3817 if (likely(dql_avail(&dev_queue->dql) >= 0))
3818 return;
3819
3820 /* Paired with READ_ONCE() from dev_watchdog() */
3821 WRITE_ONCE(dev_queue->trans_start, jiffies);
3822
3823 /* This barrier is paired with smp_mb() from dev_watchdog() */
3824 smp_mb__before_atomic();
3825
3826 set_bit(__QUEUE_STATE_STACK_XOFF, &dev_queue->state);
3827
3828 /*
3829 * The XOFF flag must be set before checking the dql_avail below,
3830 * because in netdev_tx_completed_queue we update the dql_completed
3831 * before checking the XOFF flag.
3832 */
3833 smp_mb__after_atomic();
3834
3835 /* check again in case another CPU has just made room avail */
3836 if (unlikely(dql_avail(&dev_queue->dql) >= 0))
3837 clear_bit(__QUEUE_STATE_STACK_XOFF, &dev_queue->state);
3838 #endif
3839 }
3840
3841 /* Variant of netdev_tx_sent_queue() for drivers that are aware
3842 * that they should not test BQL status themselves.
3843 * We do want to change __QUEUE_STATE_STACK_XOFF only for the last
3844 * skb of a batch.
3845 * Returns true if the doorbell must be used to kick the NIC.
3846 */
__netdev_tx_sent_queue(struct netdev_queue * dev_queue,unsigned int bytes,bool xmit_more)3847 static inline bool __netdev_tx_sent_queue(struct netdev_queue *dev_queue,
3848 unsigned int bytes,
3849 bool xmit_more)
3850 {
3851 if (xmit_more) {
3852 #ifdef CONFIG_BQL
3853 dql_queued(&dev_queue->dql, bytes);
3854 #endif
3855 return netif_tx_queue_stopped(dev_queue);
3856 }
3857 netdev_tx_sent_queue(dev_queue, bytes);
3858 return true;
3859 }
3860
3861 /**
3862 * netdev_sent_queue - report the number of bytes queued to hardware
3863 * @dev: network device
3864 * @bytes: number of bytes queued to the hardware device queue
3865 *
3866 * Report the number of bytes queued for sending/completion to the network
3867 * device hardware queue#0. @bytes should be a good approximation and should
3868 * exactly match netdev_completed_queue() @bytes.
3869 * This is typically called once per packet, from ndo_start_xmit().
3870 */
netdev_sent_queue(struct net_device * dev,unsigned int bytes)3871 static inline void netdev_sent_queue(struct net_device *dev, unsigned int bytes)
3872 {
3873 netdev_tx_sent_queue(netdev_get_tx_queue(dev, 0), bytes);
3874 }
3875
__netdev_sent_queue(struct net_device * dev,unsigned int bytes,bool xmit_more)3876 static inline bool __netdev_sent_queue(struct net_device *dev,
3877 unsigned int bytes,
3878 bool xmit_more)
3879 {
3880 return __netdev_tx_sent_queue(netdev_get_tx_queue(dev, 0), bytes,
3881 xmit_more);
3882 }
3883
3884 /**
3885 * netdev_tx_completed_queue - report number of packets/bytes at TX completion.
3886 * @dev_queue: network device queue
3887 * @pkts: number of packets (currently ignored)
3888 * @bytes: number of bytes dequeued from the device queue
3889 *
3890 * Must be called at most once per TX completion round (and not per
3891 * individual packet), so that BQL can adjust its limits appropriately.
3892 */
netdev_tx_completed_queue(struct netdev_queue * dev_queue,unsigned int pkts,unsigned int bytes)3893 static inline void netdev_tx_completed_queue(struct netdev_queue *dev_queue,
3894 unsigned int pkts, unsigned int bytes)
3895 {
3896 #ifdef CONFIG_BQL
3897 if (unlikely(!bytes))
3898 return;
3899
3900 dql_completed(&dev_queue->dql, bytes);
3901
3902 /*
3903 * Without the memory barrier there is a small possibility that
3904 * netdev_tx_sent_queue will miss the update and cause the queue to
3905 * be stopped forever
3906 */
3907 smp_mb(); /* NOTE: netdev_txq_completed_mb() assumes this exists */
3908
3909 if (unlikely(dql_avail(&dev_queue->dql) < 0))
3910 return;
3911
3912 if (test_and_clear_bit(__QUEUE_STATE_STACK_XOFF, &dev_queue->state))
3913 netif_schedule_queue(dev_queue);
3914 #endif
3915 }
3916
3917 /**
3918 * netdev_completed_queue - report bytes and packets completed by device
3919 * @dev: network device
3920 * @pkts: actual number of packets sent over the medium
3921 * @bytes: actual number of bytes sent over the medium
3922 *
3923 * Report the number of bytes and packets transmitted by the network device
3924 * hardware queue over the physical medium, @bytes must exactly match the
3925 * @bytes amount passed to netdev_sent_queue()
3926 */
netdev_completed_queue(struct net_device * dev,unsigned int pkts,unsigned int bytes)3927 static inline void netdev_completed_queue(struct net_device *dev,
3928 unsigned int pkts, unsigned int bytes)
3929 {
3930 netdev_tx_completed_queue(netdev_get_tx_queue(dev, 0), pkts, bytes);
3931 }
3932
netdev_tx_reset_queue(struct netdev_queue * q)3933 static inline void netdev_tx_reset_queue(struct netdev_queue *q)
3934 {
3935 #ifdef CONFIG_BQL
3936 clear_bit(__QUEUE_STATE_STACK_XOFF, &q->state);
3937 dql_reset(&q->dql);
3938 #endif
3939 }
3940
3941 /**
3942 * netdev_tx_reset_subqueue - reset the BQL stats and state of a netdev queue
3943 * @dev: network device
3944 * @qid: stack index of the queue to reset
3945 */
netdev_tx_reset_subqueue(const struct net_device * dev,u32 qid)3946 static inline void netdev_tx_reset_subqueue(const struct net_device *dev,
3947 u32 qid)
3948 {
3949 netdev_tx_reset_queue(netdev_get_tx_queue(dev, qid));
3950 }
3951
3952 /**
3953 * netdev_reset_queue - reset the packets and bytes count of a network device
3954 * @dev_queue: network device
3955 *
3956 * Reset the bytes and packet count of a network device and clear the
3957 * software flow control OFF bit for this network device
3958 */
netdev_reset_queue(struct net_device * dev_queue)3959 static inline void netdev_reset_queue(struct net_device *dev_queue)
3960 {
3961 netdev_tx_reset_subqueue(dev_queue, 0);
3962 }
3963
3964 /**
3965 * netdev_cap_txqueue - check if selected tx queue exceeds device queues
3966 * @dev: network device
3967 * @queue_index: given tx queue index
3968 *
3969 * Returns 0 if given tx queue index >= number of device tx queues,
3970 * otherwise returns the originally passed tx queue index.
3971 */
netdev_cap_txqueue(struct net_device * dev,u16 queue_index)3972 static inline u16 netdev_cap_txqueue(struct net_device *dev, u16 queue_index)
3973 {
3974 if (unlikely(queue_index >= dev->real_num_tx_queues)) {
3975 net_warn_ratelimited("%s selects TX queue %d, but real number of TX queues is %d\n",
3976 dev->name, queue_index,
3977 dev->real_num_tx_queues);
3978 return 0;
3979 }
3980
3981 return queue_index;
3982 }
3983
3984 /**
3985 * netif_running - test if up
3986 * @dev: network device
3987 *
3988 * Test if the device has been brought up.
3989 */
netif_running(const struct net_device * dev)3990 static inline bool netif_running(const struct net_device *dev)
3991 {
3992 return test_bit(__LINK_STATE_START, &dev->state);
3993 }
3994
3995 /*
3996 * Routines to manage the subqueues on a device. We only need start,
3997 * stop, and a check if it's stopped. All other device management is
3998 * done at the overall netdevice level.
3999 * Also test the device if we're multiqueue.
4000 */
4001
4002 /**
4003 * netif_start_subqueue - allow sending packets on subqueue
4004 * @dev: network device
4005 * @queue_index: sub queue index
4006 *
4007 * Start individual transmit queue of a device with multiple transmit queues.
4008 */
netif_start_subqueue(struct net_device * dev,u16 queue_index)4009 static inline void netif_start_subqueue(struct net_device *dev, u16 queue_index)
4010 {
4011 struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
4012
4013 netif_tx_start_queue(txq);
4014 }
4015
4016 /**
4017 * netif_stop_subqueue - stop sending packets on subqueue
4018 * @dev: network device
4019 * @queue_index: sub queue index
4020 *
4021 * Stop individual transmit queue of a device with multiple transmit queues.
4022 */
netif_stop_subqueue(struct net_device * dev,u16 queue_index)4023 static inline void netif_stop_subqueue(struct net_device *dev, u16 queue_index)
4024 {
4025 struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
4026 netif_tx_stop_queue(txq);
4027 }
4028
4029 /**
4030 * __netif_subqueue_stopped - test status of subqueue
4031 * @dev: network device
4032 * @queue_index: sub queue index
4033 *
4034 * Check individual transmit queue of a device with multiple transmit queues.
4035 */
__netif_subqueue_stopped(const struct net_device * dev,u16 queue_index)4036 static inline bool __netif_subqueue_stopped(const struct net_device *dev,
4037 u16 queue_index)
4038 {
4039 struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
4040
4041 return netif_tx_queue_stopped(txq);
4042 }
4043
4044 /**
4045 * netif_subqueue_stopped - test status of subqueue
4046 * @dev: network device
4047 * @skb: sub queue buffer pointer
4048 *
4049 * Check individual transmit queue of a device with multiple transmit queues.
4050 */
netif_subqueue_stopped(const struct net_device * dev,struct sk_buff * skb)4051 static inline bool netif_subqueue_stopped(const struct net_device *dev,
4052 struct sk_buff *skb)
4053 {
4054 return __netif_subqueue_stopped(dev, skb_get_queue_mapping(skb));
4055 }
4056
4057 /**
4058 * netif_wake_subqueue - allow sending packets on subqueue
4059 * @dev: network device
4060 * @queue_index: sub queue index
4061 *
4062 * Resume individual transmit queue of a device with multiple transmit queues.
4063 */
netif_wake_subqueue(struct net_device * dev,u16 queue_index)4064 static inline void netif_wake_subqueue(struct net_device *dev, u16 queue_index)
4065 {
4066 struct netdev_queue *txq = netdev_get_tx_queue(dev, queue_index);
4067
4068 netif_tx_wake_queue(txq);
4069 }
4070
4071 #ifdef CONFIG_XPS
4072 int netif_set_xps_queue(struct net_device *dev, const struct cpumask *mask,
4073 u16 index);
4074 int __netif_set_xps_queue(struct net_device *dev, const unsigned long *mask,
4075 u16 index, enum xps_map_type type);
4076
4077 /**
4078 * netif_attr_test_mask - Test a CPU or Rx queue set in a mask
4079 * @j: CPU/Rx queue index
4080 * @mask: bitmask of all cpus/rx queues
4081 * @nr_bits: number of bits in the bitmask
4082 *
4083 * Test if a CPU or Rx queue index is set in a mask of all CPU/Rx queues.
4084 */
netif_attr_test_mask(unsigned long j,const unsigned long * mask,unsigned int nr_bits)4085 static inline bool netif_attr_test_mask(unsigned long j,
4086 const unsigned long *mask,
4087 unsigned int nr_bits)
4088 {
4089 cpu_max_bits_warn(j, nr_bits);
4090 return test_bit(j, mask);
4091 }
4092
4093 /**
4094 * netif_attr_test_online - Test for online CPU/Rx queue
4095 * @j: CPU/Rx queue index
4096 * @online_mask: bitmask for CPUs/Rx queues that are online
4097 * @nr_bits: number of bits in the bitmask
4098 *
4099 * Returns: true if a CPU/Rx queue is online.
4100 */
netif_attr_test_online(unsigned long j,const unsigned long * online_mask,unsigned int nr_bits)4101 static inline bool netif_attr_test_online(unsigned long j,
4102 const unsigned long *online_mask,
4103 unsigned int nr_bits)
4104 {
4105 cpu_max_bits_warn(j, nr_bits);
4106
4107 if (online_mask)
4108 return test_bit(j, online_mask);
4109
4110 return (j < nr_bits);
4111 }
4112
4113 /**
4114 * netif_attrmask_next - get the next CPU/Rx queue in a cpu/Rx queues mask
4115 * @n: CPU/Rx queue index
4116 * @srcp: the cpumask/Rx queue mask pointer
4117 * @nr_bits: number of bits in the bitmask
4118 *
4119 * Returns: next (after n) CPU/Rx queue index in the mask;
4120 * >= nr_bits if no further CPUs/Rx queues set.
4121 */
netif_attrmask_next(int n,const unsigned long * srcp,unsigned int nr_bits)4122 static inline unsigned int netif_attrmask_next(int n, const unsigned long *srcp,
4123 unsigned int nr_bits)
4124 {
4125 /* -1 is a legal arg here. */
4126 if (n != -1)
4127 cpu_max_bits_warn(n, nr_bits);
4128
4129 if (srcp)
4130 return find_next_bit(srcp, nr_bits, n + 1);
4131
4132 return n + 1;
4133 }
4134
4135 /**
4136 * netif_attrmask_next_and - get the next CPU/Rx queue in \*src1p & \*src2p
4137 * @n: CPU/Rx queue index
4138 * @src1p: the first CPUs/Rx queues mask pointer
4139 * @src2p: the second CPUs/Rx queues mask pointer
4140 * @nr_bits: number of bits in the bitmask
4141 *
4142 * Returns: next (after n) CPU/Rx queue index set in both masks;
4143 * >= nr_bits if no further CPUs/Rx queues set in both.
4144 */
netif_attrmask_next_and(int n,const unsigned long * src1p,const unsigned long * src2p,unsigned int nr_bits)4145 static inline int netif_attrmask_next_and(int n, const unsigned long *src1p,
4146 const unsigned long *src2p,
4147 unsigned int nr_bits)
4148 {
4149 /* -1 is a legal arg here. */
4150 if (n != -1)
4151 cpu_max_bits_warn(n, nr_bits);
4152
4153 if (src1p && src2p)
4154 return find_next_and_bit(src1p, src2p, nr_bits, n + 1);
4155 else if (src1p)
4156 return find_next_bit(src1p, nr_bits, n + 1);
4157 else if (src2p)
4158 return find_next_bit(src2p, nr_bits, n + 1);
4159
4160 return n + 1;
4161 }
4162 #else
netif_set_xps_queue(struct net_device * dev,const struct cpumask * mask,u16 index)4163 static inline int netif_set_xps_queue(struct net_device *dev,
4164 const struct cpumask *mask,
4165 u16 index)
4166 {
4167 return 0;
4168 }
4169
__netif_set_xps_queue(struct net_device * dev,const unsigned long * mask,u16 index,enum xps_map_type type)4170 static inline int __netif_set_xps_queue(struct net_device *dev,
4171 const unsigned long *mask,
4172 u16 index, enum xps_map_type type)
4173 {
4174 return 0;
4175 }
4176 #endif
4177
4178 /**
4179 * netif_is_multiqueue - test if device has multiple transmit queues
4180 * @dev: network device
4181 *
4182 * Check if device has multiple transmit queues
4183 */
netif_is_multiqueue(const struct net_device * dev)4184 static inline bool netif_is_multiqueue(const struct net_device *dev)
4185 {
4186 return dev->num_tx_queues > 1;
4187 }
4188
4189 int netif_set_real_num_tx_queues(struct net_device *dev, unsigned int txq);
4190 int netif_set_real_num_rx_queues(struct net_device *dev, unsigned int rxq);
4191 int netif_set_real_num_queues(struct net_device *dev,
4192 unsigned int txq, unsigned int rxq);
4193
4194 int netif_get_num_default_rss_queues(void);
4195
4196 void dev_kfree_skb_irq_reason(struct sk_buff *skb, enum skb_drop_reason reason);
4197 void dev_kfree_skb_any_reason(struct sk_buff *skb, enum skb_drop_reason reason);
4198
4199 /*
4200 * It is not allowed to call kfree_skb() or consume_skb() from hardware
4201 * interrupt context or with hardware interrupts being disabled.
4202 * (in_hardirq() || irqs_disabled())
4203 *
4204 * We provide four helpers that can be used in following contexts :
4205 *
4206 * dev_kfree_skb_irq(skb) when caller drops a packet from irq context,
4207 * replacing kfree_skb(skb)
4208 *
4209 * dev_consume_skb_irq(skb) when caller consumes a packet from irq context.
4210 * Typically used in place of consume_skb(skb) in TX completion path
4211 *
4212 * dev_kfree_skb_any(skb) when caller doesn't know its current irq context,
4213 * replacing kfree_skb(skb)
4214 *
4215 * dev_consume_skb_any(skb) when caller doesn't know its current irq context,
4216 * and consumed a packet. Used in place of consume_skb(skb)
4217 */
dev_kfree_skb_irq(struct sk_buff * skb)4218 static inline void dev_kfree_skb_irq(struct sk_buff *skb)
4219 {
4220 dev_kfree_skb_irq_reason(skb, SKB_DROP_REASON_NOT_SPECIFIED);
4221 }
4222
dev_consume_skb_irq(struct sk_buff * skb)4223 static inline void dev_consume_skb_irq(struct sk_buff *skb)
4224 {
4225 dev_kfree_skb_irq_reason(skb, SKB_CONSUMED);
4226 }
4227
dev_kfree_skb_any(struct sk_buff * skb)4228 static inline void dev_kfree_skb_any(struct sk_buff *skb)
4229 {
4230 dev_kfree_skb_any_reason(skb, SKB_DROP_REASON_NOT_SPECIFIED);
4231 }
4232
dev_consume_skb_any(struct sk_buff * skb)4233 static inline void dev_consume_skb_any(struct sk_buff *skb)
4234 {
4235 dev_kfree_skb_any_reason(skb, SKB_CONSUMED);
4236 }
4237
4238 u32 bpf_prog_run_generic_xdp(struct sk_buff *skb, struct xdp_buff *xdp,
4239 const struct bpf_prog *xdp_prog);
4240 void generic_xdp_tx(struct sk_buff *skb, const struct bpf_prog *xdp_prog);
4241 int do_xdp_generic(const struct bpf_prog *xdp_prog, struct sk_buff **pskb);
4242 int netif_rx(struct sk_buff *skb);
4243 int __netif_rx(struct sk_buff *skb);
4244
4245 int netif_receive_skb(struct sk_buff *skb);
4246 int netif_receive_skb_core(struct sk_buff *skb);
4247 void netif_receive_skb_list_internal(struct list_head *head);
4248 void netif_receive_skb_list(struct list_head *head);
4249 gro_result_t gro_receive_skb(struct gro_node *gro, struct sk_buff *skb);
4250
napi_gro_receive(struct napi_struct * napi,struct sk_buff * skb)4251 static inline gro_result_t napi_gro_receive(struct napi_struct *napi,
4252 struct sk_buff *skb)
4253 {
4254 return gro_receive_skb(&napi->gro, skb);
4255 }
4256
4257 struct sk_buff *napi_get_frags(struct napi_struct *napi);
4258 gro_result_t napi_gro_frags(struct napi_struct *napi);
4259
napi_free_frags(struct napi_struct * napi)4260 static inline void napi_free_frags(struct napi_struct *napi)
4261 {
4262 kfree_skb(napi->skb);
4263 napi->skb = NULL;
4264 }
4265
4266 bool netdev_is_rx_handler_busy(struct net_device *dev);
4267 int netdev_rx_handler_register(struct net_device *dev,
4268 rx_handler_func_t *rx_handler,
4269 void *rx_handler_data);
4270 void netdev_rx_handler_unregister(struct net_device *dev);
4271
4272 bool dev_valid_name(const char *name);
is_socket_ioctl_cmd(unsigned int cmd)4273 static inline bool is_socket_ioctl_cmd(unsigned int cmd)
4274 {
4275 return _IOC_TYPE(cmd) == SOCK_IOC_TYPE;
4276 }
4277 int get_user_ifreq(struct ifreq *ifr, void __user **ifrdata, void __user *arg);
4278 int put_user_ifreq(struct ifreq *ifr, void __user *arg);
4279 int dev_ioctl(struct net *net, unsigned int cmd, struct ifreq *ifr,
4280 void __user *data, bool *need_copyout);
4281 int dev_ifconf(struct net *net, struct ifconf __user *ifc);
4282 int dev_eth_ioctl(struct net_device *dev,
4283 struct ifreq *ifr, unsigned int cmd);
4284 int generic_hwtstamp_get_lower(struct net_device *dev,
4285 struct kernel_hwtstamp_config *kernel_cfg);
4286 int generic_hwtstamp_set_lower(struct net_device *dev,
4287 struct kernel_hwtstamp_config *kernel_cfg,
4288 struct netlink_ext_ack *extack);
4289 int dev_ethtool(struct net *net, struct ifreq *ifr, void __user *userdata);
4290 unsigned int netif_get_flags(const struct net_device *dev);
4291 int __dev_change_flags(struct net_device *dev, unsigned int flags,
4292 struct netlink_ext_ack *extack);
4293 int netif_change_flags(struct net_device *dev, unsigned int flags,
4294 struct netlink_ext_ack *extack);
4295 int dev_change_flags(struct net_device *dev, unsigned int flags,
4296 struct netlink_ext_ack *extack);
4297 int netif_set_alias(struct net_device *dev, const char *alias, size_t len);
4298 int dev_set_alias(struct net_device *, const char *, size_t);
4299 int dev_get_alias(const struct net_device *, char *, size_t);
4300 int __dev_change_net_namespace(struct net_device *dev, struct net *net,
4301 const char *pat, int new_ifindex,
4302 struct netlink_ext_ack *extack);
4303 int dev_change_net_namespace(struct net_device *dev, struct net *net,
4304 const char *pat);
4305 int __netif_set_mtu(struct net_device *dev, int new_mtu);
4306 int netif_set_mtu(struct net_device *dev, int new_mtu);
4307 int dev_set_mtu(struct net_device *, int);
4308 int netif_pre_changeaddr_notify(struct net_device *dev, const char *addr,
4309 struct netlink_ext_ack *extack);
4310 int netif_set_mac_address(struct net_device *dev, struct sockaddr_storage *ss,
4311 struct netlink_ext_ack *extack);
4312 int dev_set_mac_address(struct net_device *dev, struct sockaddr_storage *ss,
4313 struct netlink_ext_ack *extack);
4314 int dev_set_mac_address_user(struct net_device *dev, struct sockaddr_storage *ss,
4315 struct netlink_ext_ack *extack);
4316 int netif_get_mac_address(struct sockaddr *sa, struct net *net, char *dev_name);
4317 int netif_get_port_parent_id(struct net_device *dev,
4318 struct netdev_phys_item_id *ppid, bool recurse);
4319 bool netdev_port_same_parent_id(struct net_device *a, struct net_device *b);
4320
4321 struct sk_buff *validate_xmit_skb_list(struct sk_buff *skb, struct net_device *dev, bool *again);
4322 struct sk_buff *dev_hard_start_xmit(struct sk_buff *skb, struct net_device *dev,
4323 struct netdev_queue *txq, int *ret);
4324
4325 int bpf_xdp_link_attach(const union bpf_attr *attr, struct bpf_prog *prog);
4326 u8 dev_xdp_prog_count(struct net_device *dev);
4327 int netif_xdp_propagate(struct net_device *dev, struct netdev_bpf *bpf);
4328 int dev_xdp_propagate(struct net_device *dev, struct netdev_bpf *bpf);
4329 u8 dev_xdp_sb_prog_count(struct net_device *dev);
4330 u32 dev_xdp_prog_id(struct net_device *dev, enum bpf_xdp_mode mode);
4331
4332 u32 dev_get_min_mp_channel_count(const struct net_device *dev);
4333
4334 int __dev_forward_skb(struct net_device *dev, struct sk_buff *skb);
4335 int dev_forward_skb(struct net_device *dev, struct sk_buff *skb);
4336 int dev_forward_skb_nomtu(struct net_device *dev, struct sk_buff *skb);
4337 bool is_skb_forwardable(const struct net_device *dev,
4338 const struct sk_buff *skb);
4339
__is_skb_forwardable(const struct net_device * dev,const struct sk_buff * skb,const bool check_mtu)4340 static __always_inline bool __is_skb_forwardable(const struct net_device *dev,
4341 const struct sk_buff *skb,
4342 const bool check_mtu)
4343 {
4344 const u32 vlan_hdr_len = 4; /* VLAN_HLEN */
4345 unsigned int len;
4346
4347 if (!(dev->flags & IFF_UP))
4348 return false;
4349
4350 if (!check_mtu)
4351 return true;
4352
4353 len = dev->mtu + dev->hard_header_len + vlan_hdr_len;
4354 if (skb->len <= len)
4355 return true;
4356
4357 /* if TSO is enabled, we don't care about the length as the packet
4358 * could be forwarded without being segmented before
4359 */
4360 if (skb_is_gso(skb))
4361 return true;
4362
4363 return false;
4364 }
4365
4366 void netdev_core_stats_inc(struct net_device *dev, u32 offset);
4367
4368 #define DEV_CORE_STATS_INC(FIELD) \
4369 static inline void dev_core_stats_##FIELD##_inc(struct net_device *dev) \
4370 { \
4371 netdev_core_stats_inc(dev, \
4372 offsetof(struct net_device_core_stats, FIELD)); \
4373 }
4374 DEV_CORE_STATS_INC(rx_dropped)
DEV_CORE_STATS_INC(tx_dropped)4375 DEV_CORE_STATS_INC(tx_dropped)
4376 DEV_CORE_STATS_INC(rx_nohandler)
4377 DEV_CORE_STATS_INC(rx_otherhost_dropped)
4378 #undef DEV_CORE_STATS_INC
4379
4380 static __always_inline int ____dev_forward_skb(struct net_device *dev,
4381 struct sk_buff *skb,
4382 const bool check_mtu)
4383 {
4384 if (skb_orphan_frags(skb, GFP_ATOMIC) ||
4385 unlikely(!__is_skb_forwardable(dev, skb, check_mtu))) {
4386 dev_core_stats_rx_dropped_inc(dev);
4387 kfree_skb(skb);
4388 return NET_RX_DROP;
4389 }
4390
4391 skb_scrub_packet(skb, !net_eq(dev_net(dev), dev_net(skb->dev)));
4392 skb->priority = 0;
4393 return 0;
4394 }
4395
4396 bool dev_nit_active_rcu(const struct net_device *dev);
dev_nit_active(const struct net_device * dev)4397 static inline bool dev_nit_active(const struct net_device *dev)
4398 {
4399 bool ret;
4400
4401 rcu_read_lock();
4402 ret = dev_nit_active_rcu(dev);
4403 rcu_read_unlock();
4404 return ret;
4405 }
4406
4407 void dev_queue_xmit_nit(struct sk_buff *skb, struct net_device *dev);
4408
__dev_put(struct net_device * dev)4409 static inline void __dev_put(struct net_device *dev)
4410 {
4411 if (dev) {
4412 #ifdef CONFIG_PCPU_DEV_REFCNT
4413 this_cpu_dec(*dev->pcpu_refcnt);
4414 #else
4415 refcount_dec(&dev->dev_refcnt);
4416 #endif
4417 }
4418 }
4419
__dev_hold(struct net_device * dev)4420 static inline void __dev_hold(struct net_device *dev)
4421 {
4422 if (dev) {
4423 #ifdef CONFIG_PCPU_DEV_REFCNT
4424 this_cpu_inc(*dev->pcpu_refcnt);
4425 #else
4426 refcount_inc(&dev->dev_refcnt);
4427 #endif
4428 }
4429 }
4430
__netdev_tracker_alloc(struct net_device * dev,netdevice_tracker * tracker,gfp_t gfp)4431 static inline void __netdev_tracker_alloc(struct net_device *dev,
4432 netdevice_tracker *tracker,
4433 gfp_t gfp)
4434 {
4435 #ifdef CONFIG_NET_DEV_REFCNT_TRACKER
4436 ref_tracker_alloc(&dev->refcnt_tracker, tracker, gfp);
4437 #endif
4438 }
4439
4440 /* netdev_tracker_alloc() can upgrade a prior untracked reference
4441 * taken by dev_get_by_name()/dev_get_by_index() to a tracked one.
4442 */
netdev_tracker_alloc(struct net_device * dev,netdevice_tracker * tracker,gfp_t gfp)4443 static inline void netdev_tracker_alloc(struct net_device *dev,
4444 netdevice_tracker *tracker, gfp_t gfp)
4445 {
4446 #ifdef CONFIG_NET_DEV_REFCNT_TRACKER
4447 refcount_dec(&dev->refcnt_tracker.no_tracker);
4448 __netdev_tracker_alloc(dev, tracker, gfp);
4449 #endif
4450 }
4451
netdev_tracker_free(struct net_device * dev,netdevice_tracker * tracker)4452 static inline void netdev_tracker_free(struct net_device *dev,
4453 netdevice_tracker *tracker)
4454 {
4455 #ifdef CONFIG_NET_DEV_REFCNT_TRACKER
4456 ref_tracker_free(&dev->refcnt_tracker, tracker);
4457 #endif
4458 }
4459
netdev_hold(struct net_device * dev,netdevice_tracker * tracker,gfp_t gfp)4460 static inline void netdev_hold(struct net_device *dev,
4461 netdevice_tracker *tracker, gfp_t gfp)
4462 {
4463 if (dev) {
4464 __dev_hold(dev);
4465 __netdev_tracker_alloc(dev, tracker, gfp);
4466 }
4467 }
4468
netdev_put(struct net_device * dev,netdevice_tracker * tracker)4469 static inline void netdev_put(struct net_device *dev,
4470 netdevice_tracker *tracker)
4471 {
4472 if (dev) {
4473 netdev_tracker_free(dev, tracker);
4474 __dev_put(dev);
4475 }
4476 }
4477
4478 /**
4479 * dev_hold - get reference to device
4480 * @dev: network device
4481 *
4482 * Hold reference to device to keep it from being freed.
4483 * Try using netdev_hold() instead.
4484 */
dev_hold(struct net_device * dev)4485 static inline void dev_hold(struct net_device *dev)
4486 {
4487 netdev_hold(dev, NULL, GFP_ATOMIC);
4488 }
4489
4490 /**
4491 * dev_put - release reference to device
4492 * @dev: network device
4493 *
4494 * Release reference to device to allow it to be freed.
4495 * Try using netdev_put() instead.
4496 */
dev_put(struct net_device * dev)4497 static inline void dev_put(struct net_device *dev)
4498 {
4499 netdev_put(dev, NULL);
4500 }
4501
DEFINE_FREE(dev_put,struct net_device *,if (_T)dev_put (_T))4502 DEFINE_FREE(dev_put, struct net_device *, if (_T) dev_put(_T))
4503
4504 static inline void netdev_ref_replace(struct net_device *odev,
4505 struct net_device *ndev,
4506 netdevice_tracker *tracker,
4507 gfp_t gfp)
4508 {
4509 if (odev)
4510 netdev_tracker_free(odev, tracker);
4511
4512 __dev_hold(ndev);
4513 __dev_put(odev);
4514
4515 if (ndev)
4516 __netdev_tracker_alloc(ndev, tracker, gfp);
4517 }
4518
4519 /* Carrier loss detection, dial on demand. The functions netif_carrier_on
4520 * and _off may be called from IRQ context, but it is caller
4521 * who is responsible for serialization of these calls.
4522 *
4523 * The name carrier is inappropriate, these functions should really be
4524 * called netif_lowerlayer_*() because they represent the state of any
4525 * kind of lower layer not just hardware media.
4526 */
4527 void linkwatch_fire_event(struct net_device *dev);
4528
4529 /**
4530 * linkwatch_sync_dev - sync linkwatch for the given device
4531 * @dev: network device to sync linkwatch for
4532 *
4533 * Sync linkwatch for the given device, removing it from the
4534 * pending work list (if queued).
4535 */
4536 void linkwatch_sync_dev(struct net_device *dev);
4537 void __linkwatch_sync_dev(struct net_device *dev);
4538
4539 /**
4540 * netif_carrier_ok - test if carrier present
4541 * @dev: network device
4542 *
4543 * Check if carrier is present on device
4544 */
netif_carrier_ok(const struct net_device * dev)4545 static inline bool netif_carrier_ok(const struct net_device *dev)
4546 {
4547 return !test_bit(__LINK_STATE_NOCARRIER, &dev->state);
4548 }
4549
4550 unsigned long dev_trans_start(struct net_device *dev);
4551
4552 void netdev_watchdog_up(struct net_device *dev);
4553
4554 void netif_carrier_on(struct net_device *dev);
4555 void netif_carrier_off(struct net_device *dev);
4556 void netif_carrier_event(struct net_device *dev);
4557
4558 /**
4559 * netif_dormant_on - mark device as dormant.
4560 * @dev: network device
4561 *
4562 * Mark device as dormant (as per RFC2863).
4563 *
4564 * The dormant state indicates that the relevant interface is not
4565 * actually in a condition to pass packets (i.e., it is not 'up') but is
4566 * in a "pending" state, waiting for some external event. For "on-
4567 * demand" interfaces, this new state identifies the situation where the
4568 * interface is waiting for events to place it in the up state.
4569 */
netif_dormant_on(struct net_device * dev)4570 static inline void netif_dormant_on(struct net_device *dev)
4571 {
4572 if (!test_and_set_bit(__LINK_STATE_DORMANT, &dev->state))
4573 linkwatch_fire_event(dev);
4574 }
4575
4576 /**
4577 * netif_dormant_off - set device as not dormant.
4578 * @dev: network device
4579 *
4580 * Device is not in dormant state.
4581 */
netif_dormant_off(struct net_device * dev)4582 static inline void netif_dormant_off(struct net_device *dev)
4583 {
4584 if (test_and_clear_bit(__LINK_STATE_DORMANT, &dev->state))
4585 linkwatch_fire_event(dev);
4586 }
4587
4588 /**
4589 * netif_dormant - test if device is dormant
4590 * @dev: network device
4591 *
4592 * Check if device is dormant.
4593 */
netif_dormant(const struct net_device * dev)4594 static inline bool netif_dormant(const struct net_device *dev)
4595 {
4596 return test_bit(__LINK_STATE_DORMANT, &dev->state);
4597 }
4598
4599
4600 /**
4601 * netif_testing_on - mark device as under test.
4602 * @dev: network device
4603 *
4604 * Mark device as under test (as per RFC2863).
4605 *
4606 * The testing state indicates that some test(s) must be performed on
4607 * the interface. After completion, of the test, the interface state
4608 * will change to up, dormant, or down, as appropriate.
4609 */
netif_testing_on(struct net_device * dev)4610 static inline void netif_testing_on(struct net_device *dev)
4611 {
4612 if (!test_and_set_bit(__LINK_STATE_TESTING, &dev->state))
4613 linkwatch_fire_event(dev);
4614 }
4615
4616 /**
4617 * netif_testing_off - set device as not under test.
4618 * @dev: network device
4619 *
4620 * Device is not in testing state.
4621 */
netif_testing_off(struct net_device * dev)4622 static inline void netif_testing_off(struct net_device *dev)
4623 {
4624 if (test_and_clear_bit(__LINK_STATE_TESTING, &dev->state))
4625 linkwatch_fire_event(dev);
4626 }
4627
4628 /**
4629 * netif_testing - test if device is under test
4630 * @dev: network device
4631 *
4632 * Check if device is under test
4633 */
netif_testing(const struct net_device * dev)4634 static inline bool netif_testing(const struct net_device *dev)
4635 {
4636 return test_bit(__LINK_STATE_TESTING, &dev->state);
4637 }
4638
4639
4640 /**
4641 * netif_oper_up - test if device is operational
4642 * @dev: network device
4643 *
4644 * Check if carrier is operational
4645 */
netif_oper_up(const struct net_device * dev)4646 static inline bool netif_oper_up(const struct net_device *dev)
4647 {
4648 unsigned int operstate = READ_ONCE(dev->operstate);
4649
4650 return operstate == IF_OPER_UP ||
4651 operstate == IF_OPER_UNKNOWN /* backward compat */;
4652 }
4653
4654 /**
4655 * netif_device_present - is device available or removed
4656 * @dev: network device
4657 *
4658 * Check if device has not been removed from system.
4659 */
netif_device_present(const struct net_device * dev)4660 static inline bool netif_device_present(const struct net_device *dev)
4661 {
4662 return test_bit(__LINK_STATE_PRESENT, &dev->state);
4663 }
4664
4665 void netif_device_detach(struct net_device *dev);
4666
4667 void netif_device_attach(struct net_device *dev);
4668
4669 /*
4670 * Network interface message level settings
4671 */
4672
4673 enum {
4674 NETIF_MSG_DRV_BIT,
4675 NETIF_MSG_PROBE_BIT,
4676 NETIF_MSG_LINK_BIT,
4677 NETIF_MSG_TIMER_BIT,
4678 NETIF_MSG_IFDOWN_BIT,
4679 NETIF_MSG_IFUP_BIT,
4680 NETIF_MSG_RX_ERR_BIT,
4681 NETIF_MSG_TX_ERR_BIT,
4682 NETIF_MSG_TX_QUEUED_BIT,
4683 NETIF_MSG_INTR_BIT,
4684 NETIF_MSG_TX_DONE_BIT,
4685 NETIF_MSG_RX_STATUS_BIT,
4686 NETIF_MSG_PKTDATA_BIT,
4687 NETIF_MSG_HW_BIT,
4688 NETIF_MSG_WOL_BIT,
4689
4690 /* When you add a new bit above, update netif_msg_class_names array
4691 * in net/ethtool/common.c
4692 */
4693 NETIF_MSG_CLASS_COUNT,
4694 };
4695 /* Both ethtool_ops interface and internal driver implementation use u32 */
4696 static_assert(NETIF_MSG_CLASS_COUNT <= 32);
4697
4698 #define __NETIF_MSG_BIT(bit) ((u32)1 << (bit))
4699 #define __NETIF_MSG(name) __NETIF_MSG_BIT(NETIF_MSG_ ## name ## _BIT)
4700
4701 #define NETIF_MSG_DRV __NETIF_MSG(DRV)
4702 #define NETIF_MSG_PROBE __NETIF_MSG(PROBE)
4703 #define NETIF_MSG_LINK __NETIF_MSG(LINK)
4704 #define NETIF_MSG_TIMER __NETIF_MSG(TIMER)
4705 #define NETIF_MSG_IFDOWN __NETIF_MSG(IFDOWN)
4706 #define NETIF_MSG_IFUP __NETIF_MSG(IFUP)
4707 #define NETIF_MSG_RX_ERR __NETIF_MSG(RX_ERR)
4708 #define NETIF_MSG_TX_ERR __NETIF_MSG(TX_ERR)
4709 #define NETIF_MSG_TX_QUEUED __NETIF_MSG(TX_QUEUED)
4710 #define NETIF_MSG_INTR __NETIF_MSG(INTR)
4711 #define NETIF_MSG_TX_DONE __NETIF_MSG(TX_DONE)
4712 #define NETIF_MSG_RX_STATUS __NETIF_MSG(RX_STATUS)
4713 #define NETIF_MSG_PKTDATA __NETIF_MSG(PKTDATA)
4714 #define NETIF_MSG_HW __NETIF_MSG(HW)
4715 #define NETIF_MSG_WOL __NETIF_MSG(WOL)
4716
4717 #define netif_msg_drv(p) ((p)->msg_enable & NETIF_MSG_DRV)
4718 #define netif_msg_probe(p) ((p)->msg_enable & NETIF_MSG_PROBE)
4719 #define netif_msg_link(p) ((p)->msg_enable & NETIF_MSG_LINK)
4720 #define netif_msg_timer(p) ((p)->msg_enable & NETIF_MSG_TIMER)
4721 #define netif_msg_ifdown(p) ((p)->msg_enable & NETIF_MSG_IFDOWN)
4722 #define netif_msg_ifup(p) ((p)->msg_enable & NETIF_MSG_IFUP)
4723 #define netif_msg_rx_err(p) ((p)->msg_enable & NETIF_MSG_RX_ERR)
4724 #define netif_msg_tx_err(p) ((p)->msg_enable & NETIF_MSG_TX_ERR)
4725 #define netif_msg_tx_queued(p) ((p)->msg_enable & NETIF_MSG_TX_QUEUED)
4726 #define netif_msg_intr(p) ((p)->msg_enable & NETIF_MSG_INTR)
4727 #define netif_msg_tx_done(p) ((p)->msg_enable & NETIF_MSG_TX_DONE)
4728 #define netif_msg_rx_status(p) ((p)->msg_enable & NETIF_MSG_RX_STATUS)
4729 #define netif_msg_pktdata(p) ((p)->msg_enable & NETIF_MSG_PKTDATA)
4730 #define netif_msg_hw(p) ((p)->msg_enable & NETIF_MSG_HW)
4731 #define netif_msg_wol(p) ((p)->msg_enable & NETIF_MSG_WOL)
4732
netif_msg_init(int debug_value,int default_msg_enable_bits)4733 static inline u32 netif_msg_init(int debug_value, int default_msg_enable_bits)
4734 {
4735 /* use default */
4736 if (debug_value < 0 || debug_value >= (sizeof(u32) * 8))
4737 return default_msg_enable_bits;
4738 if (debug_value == 0) /* no output */
4739 return 0;
4740 /* set low N bits */
4741 return (1U << debug_value) - 1;
4742 }
4743
__netif_tx_lock(struct netdev_queue * txq,int cpu)4744 static inline void __netif_tx_lock(struct netdev_queue *txq, int cpu)
4745 {
4746 spin_lock(&txq->_xmit_lock);
4747 /* Pairs with READ_ONCE() in netif_tx_owned() */
4748 WRITE_ONCE(txq->xmit_lock_owner, cpu);
4749 }
4750
__netif_tx_acquire(struct netdev_queue * txq)4751 static inline bool __netif_tx_acquire(struct netdev_queue *txq)
4752 {
4753 __acquire(&txq->_xmit_lock);
4754 return true;
4755 }
4756
__netif_tx_release(struct netdev_queue * txq)4757 static inline void __netif_tx_release(struct netdev_queue *txq)
4758 {
4759 __release(&txq->_xmit_lock);
4760 }
4761
__netif_tx_lock_bh(struct netdev_queue * txq)4762 static inline void __netif_tx_lock_bh(struct netdev_queue *txq)
4763 {
4764 spin_lock_bh(&txq->_xmit_lock);
4765 /* Pairs with READ_ONCE() in netif_tx_owned() */
4766 WRITE_ONCE(txq->xmit_lock_owner, smp_processor_id());
4767 }
4768
__netif_tx_trylock(struct netdev_queue * txq)4769 static inline bool __netif_tx_trylock(struct netdev_queue *txq)
4770 {
4771 bool ok = spin_trylock(&txq->_xmit_lock);
4772
4773 if (likely(ok)) {
4774 /* Pairs with READ_ONCE() in netif_tx_owned() */
4775 WRITE_ONCE(txq->xmit_lock_owner, smp_processor_id());
4776 }
4777 return ok;
4778 }
4779
__netif_tx_unlock(struct netdev_queue * txq)4780 static inline void __netif_tx_unlock(struct netdev_queue *txq)
4781 {
4782 /* Pairs with READ_ONCE() in netif_tx_owned() */
4783 WRITE_ONCE(txq->xmit_lock_owner, -1);
4784 spin_unlock(&txq->_xmit_lock);
4785 }
4786
__netif_tx_unlock_bh(struct netdev_queue * txq)4787 static inline void __netif_tx_unlock_bh(struct netdev_queue *txq)
4788 {
4789 /* Pairs with READ_ONCE() in netif_tx_owned() */
4790 WRITE_ONCE(txq->xmit_lock_owner, -1);
4791 spin_unlock_bh(&txq->_xmit_lock);
4792 }
4793
4794 /*
4795 * txq->trans_start can be read locklessly from dev_watchdog()
4796 */
txq_trans_update(const struct net_device * dev,struct netdev_queue * txq)4797 static inline void txq_trans_update(const struct net_device *dev,
4798 struct netdev_queue *txq)
4799 {
4800 if (!dev->lltx)
4801 WRITE_ONCE(txq->trans_start, jiffies);
4802 }
4803
txq_trans_cond_update(struct netdev_queue * txq)4804 static inline void txq_trans_cond_update(struct netdev_queue *txq)
4805 {
4806 unsigned long now = jiffies;
4807
4808 if (READ_ONCE(txq->trans_start) != now)
4809 WRITE_ONCE(txq->trans_start, now);
4810 }
4811
4812 /* legacy drivers only, netdev_start_xmit() sets txq->trans_start */
netif_trans_update(struct net_device * dev)4813 static inline void netif_trans_update(struct net_device *dev)
4814 {
4815 struct netdev_queue *txq = netdev_get_tx_queue(dev, 0);
4816
4817 txq_trans_cond_update(txq);
4818 }
4819
4820 /**
4821 * netif_tx_lock - grab network device transmit lock
4822 * @dev: network device
4823 *
4824 * Get network device transmit lock
4825 */
4826 void netif_tx_lock(struct net_device *dev);
4827
netif_tx_lock_bh(struct net_device * dev)4828 static inline void netif_tx_lock_bh(struct net_device *dev)
4829 {
4830 local_bh_disable();
4831 netif_tx_lock(dev);
4832 }
4833
4834 void netif_tx_unlock(struct net_device *dev);
4835
netif_tx_unlock_bh(struct net_device * dev)4836 static inline void netif_tx_unlock_bh(struct net_device *dev)
4837 {
4838 netif_tx_unlock(dev);
4839 local_bh_enable();
4840 }
4841
4842 #define HARD_TX_LOCK(dev, txq, cpu) { \
4843 if (!(dev)->lltx) { \
4844 __netif_tx_lock(txq, cpu); \
4845 } else { \
4846 __netif_tx_acquire(txq); \
4847 } \
4848 }
4849
4850 #define HARD_TX_TRYLOCK(dev, txq) \
4851 (!(dev)->lltx ? \
4852 __netif_tx_trylock(txq) : \
4853 __netif_tx_acquire(txq))
4854
4855 #define HARD_TX_UNLOCK(dev, txq) { \
4856 if (!(dev)->lltx) { \
4857 __netif_tx_unlock(txq); \
4858 } else { \
4859 __netif_tx_release(txq); \
4860 } \
4861 }
4862
netif_tx_disable(struct net_device * dev)4863 static inline void netif_tx_disable(struct net_device *dev)
4864 {
4865 unsigned int i;
4866 int cpu;
4867
4868 local_bh_disable();
4869 cpu = smp_processor_id();
4870 spin_lock(&dev->tx_global_lock);
4871 for (i = 0; i < dev->num_tx_queues; i++) {
4872 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
4873
4874 __netif_tx_lock(txq, cpu);
4875 netif_tx_stop_queue(txq);
4876 __netif_tx_unlock(txq);
4877 }
4878 spin_unlock(&dev->tx_global_lock);
4879 local_bh_enable();
4880 }
4881
4882 #ifndef CONFIG_PREEMPT_RT
netif_tx_owned(struct netdev_queue * txq,unsigned int cpu)4883 static inline bool netif_tx_owned(struct netdev_queue *txq, unsigned int cpu)
4884 {
4885 /* Other cpus might concurrently change txq->xmit_lock_owner
4886 * to -1 or to their cpu id, but not to our id.
4887 */
4888 return READ_ONCE(txq->xmit_lock_owner) == cpu;
4889 }
4890
4891 #else
netif_tx_owned(struct netdev_queue * txq,unsigned int cpu)4892 static inline bool netif_tx_owned(struct netdev_queue *txq, unsigned int cpu)
4893 {
4894 return rt_mutex_owner(&txq->_xmit_lock.lock) == current;
4895 }
4896
4897 #endif
4898
netif_addr_lock(struct net_device * dev)4899 static inline void netif_addr_lock(struct net_device *dev)
4900 {
4901 unsigned char nest_level = 0;
4902
4903 #ifdef CONFIG_LOCKDEP
4904 nest_level = dev->nested_level;
4905 #endif
4906 spin_lock_nested(&dev->addr_list_lock, nest_level);
4907 }
4908
netif_addr_lock_bh(struct net_device * dev)4909 static inline void netif_addr_lock_bh(struct net_device *dev)
4910 {
4911 unsigned char nest_level = 0;
4912
4913 #ifdef CONFIG_LOCKDEP
4914 nest_level = dev->nested_level;
4915 #endif
4916 local_bh_disable();
4917 spin_lock_nested(&dev->addr_list_lock, nest_level);
4918 }
4919
netif_addr_unlock(struct net_device * dev)4920 static inline void netif_addr_unlock(struct net_device *dev)
4921 {
4922 spin_unlock(&dev->addr_list_lock);
4923 }
4924
netif_addr_unlock_bh(struct net_device * dev)4925 static inline void netif_addr_unlock_bh(struct net_device *dev)
4926 {
4927 spin_unlock_bh(&dev->addr_list_lock);
4928 }
4929
4930 /*
4931 * dev_addrs walker. Should be used only for read access. Call with
4932 * rcu_read_lock held.
4933 */
4934 #define for_each_dev_addr(dev, ha) \
4935 list_for_each_entry_rcu(ha, &dev->dev_addrs.list, list)
4936
4937 /* These functions live elsewhere (drivers/net/net_init.c, but related) */
4938
4939 void ether_setup(struct net_device *dev);
4940
4941 /* Allocate dummy net_device */
4942 struct net_device *alloc_netdev_dummy(int sizeof_priv);
4943
4944 /* Support for loadable net-drivers */
4945 struct net_device *alloc_netdev_mqs(int sizeof_priv, const char *name,
4946 unsigned char name_assign_type,
4947 void (*setup)(struct net_device *),
4948 unsigned int txqs, unsigned int rxqs);
4949 #define alloc_netdev(sizeof_priv, name, name_assign_type, setup) \
4950 alloc_netdev_mqs(sizeof_priv, name, name_assign_type, setup, 1, 1)
4951
4952 #define alloc_netdev_mq(sizeof_priv, name, name_assign_type, setup, count) \
4953 alloc_netdev_mqs(sizeof_priv, name, name_assign_type, setup, count, \
4954 count)
4955
4956 int register_netdev(struct net_device *dev);
4957 void unregister_netdev(struct net_device *dev);
4958
4959 int devm_register_netdev(struct device *dev, struct net_device *ndev);
4960
4961 /* General hardware address lists handling functions */
4962 int __hw_addr_sync(struct netdev_hw_addr_list *to_list,
4963 struct netdev_hw_addr_list *from_list, int addr_len);
4964 int __hw_addr_sync_multiple(struct netdev_hw_addr_list *to_list,
4965 struct netdev_hw_addr_list *from_list,
4966 int addr_len);
4967 void __hw_addr_unsync(struct netdev_hw_addr_list *to_list,
4968 struct netdev_hw_addr_list *from_list, int addr_len);
4969 int __hw_addr_sync_dev(struct netdev_hw_addr_list *list,
4970 struct net_device *dev,
4971 int (*sync)(struct net_device *, const unsigned char *),
4972 int (*unsync)(struct net_device *,
4973 const unsigned char *));
4974 int __hw_addr_ref_sync_dev(struct netdev_hw_addr_list *list,
4975 struct net_device *dev,
4976 int (*sync)(struct net_device *,
4977 const unsigned char *, int),
4978 int (*unsync)(struct net_device *,
4979 const unsigned char *, int));
4980 void __hw_addr_ref_unsync_dev(struct netdev_hw_addr_list *list,
4981 struct net_device *dev,
4982 int (*unsync)(struct net_device *,
4983 const unsigned char *, int));
4984 void __hw_addr_unsync_dev(struct netdev_hw_addr_list *list,
4985 struct net_device *dev,
4986 int (*unsync)(struct net_device *,
4987 const unsigned char *));
4988 void __hw_addr_init(struct netdev_hw_addr_list *list);
4989
4990 /* Functions used for device addresses handling */
4991 void dev_addr_mod(struct net_device *dev, unsigned int offset,
4992 const void *addr, size_t len);
4993
4994 static inline void
__dev_addr_set(struct net_device * dev,const void * addr,size_t len)4995 __dev_addr_set(struct net_device *dev, const void *addr, size_t len)
4996 {
4997 dev_addr_mod(dev, 0, addr, len);
4998 }
4999
dev_addr_set(struct net_device * dev,const u8 * addr)5000 static inline void dev_addr_set(struct net_device *dev, const u8 *addr)
5001 {
5002 __dev_addr_set(dev, addr, dev->addr_len);
5003 }
5004
5005 int dev_addr_add(struct net_device *dev, const unsigned char *addr,
5006 unsigned char addr_type);
5007 int dev_addr_del(struct net_device *dev, const unsigned char *addr,
5008 unsigned char addr_type);
5009
5010 /* Functions used for unicast addresses handling */
5011 int dev_uc_add(struct net_device *dev, const unsigned char *addr);
5012 int dev_uc_add_excl(struct net_device *dev, const unsigned char *addr);
5013 int dev_uc_del(struct net_device *dev, const unsigned char *addr);
5014 int dev_uc_sync(struct net_device *to, struct net_device *from);
5015 int dev_uc_sync_multiple(struct net_device *to, struct net_device *from);
5016 void dev_uc_unsync(struct net_device *to, struct net_device *from);
5017 void dev_uc_flush(struct net_device *dev);
5018 void dev_uc_init(struct net_device *dev);
5019
5020 /**
5021 * __dev_uc_sync - Synchronize device's unicast list
5022 * @dev: device to sync
5023 * @sync: function to call if address should be added
5024 * @unsync: function to call if address should be removed
5025 *
5026 * Add newly added addresses to the interface, and release
5027 * addresses that have been deleted.
5028 */
__dev_uc_sync(struct net_device * dev,int (* sync)(struct net_device *,const unsigned char *),int (* unsync)(struct net_device *,const unsigned char *))5029 static inline int __dev_uc_sync(struct net_device *dev,
5030 int (*sync)(struct net_device *,
5031 const unsigned char *),
5032 int (*unsync)(struct net_device *,
5033 const unsigned char *))
5034 {
5035 return __hw_addr_sync_dev(&dev->uc, dev, sync, unsync);
5036 }
5037
5038 /**
5039 * __dev_uc_unsync - Remove synchronized addresses from device
5040 * @dev: device to sync
5041 * @unsync: function to call if address should be removed
5042 *
5043 * Remove all addresses that were added to the device by dev_uc_sync().
5044 */
__dev_uc_unsync(struct net_device * dev,int (* unsync)(struct net_device *,const unsigned char *))5045 static inline void __dev_uc_unsync(struct net_device *dev,
5046 int (*unsync)(struct net_device *,
5047 const unsigned char *))
5048 {
5049 __hw_addr_unsync_dev(&dev->uc, dev, unsync);
5050 }
5051
5052 /* Functions used for multicast addresses handling */
5053 int dev_mc_add(struct net_device *dev, const unsigned char *addr);
5054 int dev_mc_add_global(struct net_device *dev, const unsigned char *addr);
5055 int dev_mc_add_excl(struct net_device *dev, const unsigned char *addr);
5056 int dev_mc_del(struct net_device *dev, const unsigned char *addr);
5057 int dev_mc_del_global(struct net_device *dev, const unsigned char *addr);
5058 int dev_mc_sync(struct net_device *to, struct net_device *from);
5059 int dev_mc_sync_multiple(struct net_device *to, struct net_device *from);
5060 void dev_mc_unsync(struct net_device *to, struct net_device *from);
5061 void dev_mc_flush(struct net_device *dev);
5062 void dev_mc_init(struct net_device *dev);
5063
5064 /**
5065 * __dev_mc_sync - Synchronize device's multicast list
5066 * @dev: device to sync
5067 * @sync: function to call if address should be added
5068 * @unsync: function to call if address should be removed
5069 *
5070 * Add newly added addresses to the interface, and release
5071 * addresses that have been deleted.
5072 */
__dev_mc_sync(struct net_device * dev,int (* sync)(struct net_device *,const unsigned char *),int (* unsync)(struct net_device *,const unsigned char *))5073 static inline int __dev_mc_sync(struct net_device *dev,
5074 int (*sync)(struct net_device *,
5075 const unsigned char *),
5076 int (*unsync)(struct net_device *,
5077 const unsigned char *))
5078 {
5079 return __hw_addr_sync_dev(&dev->mc, dev, sync, unsync);
5080 }
5081
5082 /**
5083 * __dev_mc_unsync - Remove synchronized addresses from device
5084 * @dev: device to sync
5085 * @unsync: function to call if address should be removed
5086 *
5087 * Remove all addresses that were added to the device by dev_mc_sync().
5088 */
__dev_mc_unsync(struct net_device * dev,int (* unsync)(struct net_device *,const unsigned char *))5089 static inline void __dev_mc_unsync(struct net_device *dev,
5090 int (*unsync)(struct net_device *,
5091 const unsigned char *))
5092 {
5093 __hw_addr_unsync_dev(&dev->mc, dev, unsync);
5094 }
5095
5096 /* Functions used for secondary unicast and multicast support */
5097 void dev_set_rx_mode(struct net_device *dev);
5098 int netif_set_promiscuity(struct net_device *dev, int inc);
5099 int dev_set_promiscuity(struct net_device *dev, int inc);
5100 int netif_set_allmulti(struct net_device *dev, int inc, bool notify);
5101 int dev_set_allmulti(struct net_device *dev, int inc);
5102 void netif_state_change(struct net_device *dev);
5103 void netdev_state_change(struct net_device *dev);
5104 void __netdev_notify_peers(struct net_device *dev);
5105 void netdev_notify_peers(struct net_device *dev);
5106 void netdev_features_change(struct net_device *dev);
5107 /* Load a device via the kmod */
5108 void dev_load(struct net *net, const char *name);
5109 struct rtnl_link_stats64 *dev_get_stats(struct net_device *dev,
5110 struct rtnl_link_stats64 *storage);
5111 void netdev_stats_to_stats64(struct rtnl_link_stats64 *stats64,
5112 const struct net_device_stats *netdev_stats);
5113 void dev_fetch_sw_netstats(struct rtnl_link_stats64 *s,
5114 const struct pcpu_sw_netstats __percpu *netstats);
5115 void dev_get_tstats64(struct net_device *dev, struct rtnl_link_stats64 *s);
5116
5117 enum {
5118 NESTED_SYNC_IMM_BIT,
5119 NESTED_SYNC_TODO_BIT,
5120 };
5121
5122 #define __NESTED_SYNC_BIT(bit) ((u32)1 << (bit))
5123 #define __NESTED_SYNC(name) __NESTED_SYNC_BIT(NESTED_SYNC_ ## name ## _BIT)
5124
5125 #define NESTED_SYNC_IMM __NESTED_SYNC(IMM)
5126 #define NESTED_SYNC_TODO __NESTED_SYNC(TODO)
5127
5128 struct netdev_nested_priv {
5129 unsigned char flags;
5130 void *data;
5131 };
5132
5133 bool netdev_has_upper_dev(struct net_device *dev, struct net_device *upper_dev);
5134 struct net_device *netdev_upper_get_next_dev_rcu(struct net_device *dev,
5135 struct list_head **iter);
5136
5137 /* iterate through upper list, must be called under RCU read lock */
5138 #define netdev_for_each_upper_dev_rcu(dev, updev, iter) \
5139 for (iter = &(dev)->adj_list.upper, \
5140 updev = netdev_upper_get_next_dev_rcu(dev, &(iter)); \
5141 updev; \
5142 updev = netdev_upper_get_next_dev_rcu(dev, &(iter)))
5143
5144 int netdev_walk_all_upper_dev_rcu(struct net_device *dev,
5145 int (*fn)(struct net_device *upper_dev,
5146 struct netdev_nested_priv *priv),
5147 struct netdev_nested_priv *priv);
5148
5149 bool netdev_has_upper_dev_all_rcu(struct net_device *dev,
5150 struct net_device *upper_dev);
5151
5152 bool netdev_has_any_upper_dev(struct net_device *dev);
5153
5154 void *netdev_lower_get_next_private(struct net_device *dev,
5155 struct list_head **iter);
5156 void *netdev_lower_get_next_private_rcu(struct net_device *dev,
5157 struct list_head **iter);
5158
5159 #define netdev_for_each_lower_private(dev, priv, iter) \
5160 for (iter = (dev)->adj_list.lower.next, \
5161 priv = netdev_lower_get_next_private(dev, &(iter)); \
5162 priv; \
5163 priv = netdev_lower_get_next_private(dev, &(iter)))
5164
5165 #define netdev_for_each_lower_private_rcu(dev, priv, iter) \
5166 for (iter = &(dev)->adj_list.lower, \
5167 priv = netdev_lower_get_next_private_rcu(dev, &(iter)); \
5168 priv; \
5169 priv = netdev_lower_get_next_private_rcu(dev, &(iter)))
5170
5171 void *netdev_lower_get_next(struct net_device *dev,
5172 struct list_head **iter);
5173
5174 #define netdev_for_each_lower_dev(dev, ldev, iter) \
5175 for (iter = (dev)->adj_list.lower.next, \
5176 ldev = netdev_lower_get_next(dev, &(iter)); \
5177 ldev; \
5178 ldev = netdev_lower_get_next(dev, &(iter)))
5179
5180 struct net_device *netdev_next_lower_dev_rcu(struct net_device *dev,
5181 struct list_head **iter);
5182 int netdev_walk_all_lower_dev(struct net_device *dev,
5183 int (*fn)(struct net_device *lower_dev,
5184 struct netdev_nested_priv *priv),
5185 struct netdev_nested_priv *priv);
5186 int netdev_walk_all_lower_dev_rcu(struct net_device *dev,
5187 int (*fn)(struct net_device *lower_dev,
5188 struct netdev_nested_priv *priv),
5189 struct netdev_nested_priv *priv);
5190
5191 void *netdev_adjacent_get_private(struct list_head *adj_list);
5192 void *netdev_lower_get_first_private_rcu(struct net_device *dev);
5193 struct net_device *netdev_master_upper_dev_get(struct net_device *dev);
5194 struct net_device *netdev_master_upper_dev_get_rcu(struct net_device *dev);
5195 int netdev_upper_dev_link(struct net_device *dev, struct net_device *upper_dev,
5196 struct netlink_ext_ack *extack);
5197 int netdev_master_upper_dev_link(struct net_device *dev,
5198 struct net_device *upper_dev,
5199 void *upper_priv, void *upper_info,
5200 struct netlink_ext_ack *extack);
5201 void netdev_upper_dev_unlink(struct net_device *dev,
5202 struct net_device *upper_dev);
5203 int netdev_adjacent_change_prepare(struct net_device *old_dev,
5204 struct net_device *new_dev,
5205 struct net_device *dev,
5206 struct netlink_ext_ack *extack);
5207 void netdev_adjacent_change_commit(struct net_device *old_dev,
5208 struct net_device *new_dev,
5209 struct net_device *dev);
5210 void netdev_adjacent_change_abort(struct net_device *old_dev,
5211 struct net_device *new_dev,
5212 struct net_device *dev);
5213 void netdev_adjacent_rename_links(struct net_device *dev, char *oldname);
5214 void *netdev_lower_dev_get_private(struct net_device *dev,
5215 struct net_device *lower_dev);
5216 void netdev_lower_state_changed(struct net_device *lower_dev,
5217 void *lower_state_info);
5218
5219 #define NETDEV_RSS_KEY_LEN 256
5220 extern u8 netdev_rss_key[NETDEV_RSS_KEY_LEN] __read_mostly;
5221 void netdev_rss_key_fill(void *buffer, size_t len);
5222
5223 int skb_checksum_help(struct sk_buff *skb);
5224 int skb_crc32c_csum_help(struct sk_buff *skb);
5225 int skb_csum_hwoffload_help(struct sk_buff *skb,
5226 const netdev_features_t features);
5227
5228 struct netdev_bonding_info {
5229 ifslave slave;
5230 ifbond master;
5231 };
5232
5233 struct netdev_notifier_bonding_info {
5234 struct netdev_notifier_info info; /* must be first */
5235 struct netdev_bonding_info bonding_info;
5236 };
5237
5238 void netdev_bonding_info_change(struct net_device *dev,
5239 struct netdev_bonding_info *bonding_info);
5240
5241 #if IS_ENABLED(CONFIG_ETHTOOL_NETLINK)
5242 void ethtool_notify(struct net_device *dev, unsigned int cmd);
5243 #else
ethtool_notify(struct net_device * dev,unsigned int cmd)5244 static inline void ethtool_notify(struct net_device *dev, unsigned int cmd)
5245 {
5246 }
5247 #endif
5248
5249 __be16 skb_network_protocol(struct sk_buff *skb, int *depth);
5250
can_checksum_protocol(netdev_features_t features,__be16 protocol)5251 static inline bool can_checksum_protocol(netdev_features_t features,
5252 __be16 protocol)
5253 {
5254 if (protocol == htons(ETH_P_FCOE))
5255 return !!(features & NETIF_F_FCOE_CRC);
5256
5257 /* Assume this is an IP checksum (not SCTP CRC) */
5258
5259 if (features & NETIF_F_HW_CSUM) {
5260 /* Can checksum everything */
5261 return true;
5262 }
5263
5264 switch (protocol) {
5265 case htons(ETH_P_IP):
5266 return !!(features & NETIF_F_IP_CSUM);
5267 case htons(ETH_P_IPV6):
5268 return !!(features & NETIF_F_IPV6_CSUM);
5269 default:
5270 return false;
5271 }
5272 }
5273
5274 #ifdef CONFIG_BUG
5275 void netdev_rx_csum_fault(struct net_device *dev, struct sk_buff *skb);
5276 #else
netdev_rx_csum_fault(struct net_device * dev,struct sk_buff * skb)5277 static inline void netdev_rx_csum_fault(struct net_device *dev,
5278 struct sk_buff *skb)
5279 {
5280 }
5281 #endif
5282 /* rx skb timestamps */
5283 void net_enable_timestamp(void);
5284 void net_disable_timestamp(void);
5285
netdev_get_tstamp(struct net_device * dev,const struct skb_shared_hwtstamps * hwtstamps,bool cycles)5286 static inline ktime_t netdev_get_tstamp(struct net_device *dev,
5287 const struct skb_shared_hwtstamps *hwtstamps,
5288 bool cycles)
5289 {
5290 const struct net_device_ops *ops = dev->netdev_ops;
5291
5292 if (ops->ndo_get_tstamp)
5293 return ops->ndo_get_tstamp(dev, hwtstamps, cycles);
5294
5295 return hwtstamps->hwtstamp;
5296 }
5297
5298 #ifndef CONFIG_PREEMPT_RT
netdev_xmit_set_more(bool more)5299 static inline void netdev_xmit_set_more(bool more)
5300 {
5301 __this_cpu_write(softnet_data.xmit.more, more);
5302 }
5303
netdev_xmit_more(void)5304 static inline bool netdev_xmit_more(void)
5305 {
5306 return __this_cpu_read(softnet_data.xmit.more);
5307 }
5308 #else
netdev_xmit_set_more(bool more)5309 static inline void netdev_xmit_set_more(bool more)
5310 {
5311 current->net_xmit.more = more;
5312 }
5313
netdev_xmit_more(void)5314 static inline bool netdev_xmit_more(void)
5315 {
5316 return current->net_xmit.more;
5317 }
5318 #endif
5319
__netdev_start_xmit(const struct net_device_ops * ops,struct sk_buff * skb,struct net_device * dev,bool more)5320 static inline netdev_tx_t __netdev_start_xmit(const struct net_device_ops *ops,
5321 struct sk_buff *skb, struct net_device *dev,
5322 bool more)
5323 {
5324 netdev_xmit_set_more(more);
5325 return ops->ndo_start_xmit(skb, dev);
5326 }
5327
netdev_start_xmit(struct sk_buff * skb,struct net_device * dev,struct netdev_queue * txq,bool more)5328 static inline netdev_tx_t netdev_start_xmit(struct sk_buff *skb, struct net_device *dev,
5329 struct netdev_queue *txq, bool more)
5330 {
5331 const struct net_device_ops *ops = dev->netdev_ops;
5332 netdev_tx_t rc;
5333
5334 rc = __netdev_start_xmit(ops, skb, dev, more);
5335 if (rc == NETDEV_TX_OK)
5336 txq_trans_update(dev, txq);
5337
5338 return rc;
5339 }
5340
5341 int netdev_class_create_file_ns(const struct class_attribute *class_attr,
5342 const void *ns);
5343 void netdev_class_remove_file_ns(const struct class_attribute *class_attr,
5344 const void *ns);
5345
5346 extern const struct kobj_ns_type_operations net_ns_type_operations;
5347
5348 const char *netdev_drivername(const struct net_device *dev);
5349
netdev_intersect_features(netdev_features_t f1,netdev_features_t f2)5350 static inline netdev_features_t netdev_intersect_features(netdev_features_t f1,
5351 netdev_features_t f2)
5352 {
5353 if ((f1 ^ f2) & NETIF_F_HW_CSUM) {
5354 if (f1 & NETIF_F_HW_CSUM)
5355 f1 |= (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
5356 else
5357 f2 |= (NETIF_F_IP_CSUM|NETIF_F_IPV6_CSUM);
5358 }
5359
5360 return f1 & f2;
5361 }
5362
netdev_get_wanted_features(struct net_device * dev)5363 static inline netdev_features_t netdev_get_wanted_features(
5364 struct net_device *dev)
5365 {
5366 return (dev->features & ~dev->hw_features) | dev->wanted_features;
5367 }
5368 netdev_features_t netdev_increment_features(netdev_features_t all,
5369 netdev_features_t one, netdev_features_t mask);
5370
5371 /* Allow TSO being used on stacked device :
5372 * Performing the GSO segmentation before last device
5373 * is a performance improvement.
5374 */
netdev_add_tso_features(netdev_features_t features,netdev_features_t mask)5375 static inline netdev_features_t netdev_add_tso_features(netdev_features_t features,
5376 netdev_features_t mask)
5377 {
5378 return netdev_increment_features(features, NETIF_F_ALL_TSO |
5379 NETIF_F_ALL_FOR_ALL, mask);
5380 }
5381
5382 int __netdev_update_features(struct net_device *dev);
5383 void netdev_update_features(struct net_device *dev);
5384 void netdev_change_features(struct net_device *dev);
5385 void netdev_compute_master_upper_features(struct net_device *dev, bool update_header);
5386
5387 void netif_stacked_transfer_operstate(const struct net_device *rootdev,
5388 struct net_device *dev);
5389
5390 netdev_features_t passthru_features_check(struct sk_buff *skb,
5391 struct net_device *dev,
5392 netdev_features_t features);
5393 netdev_features_t netif_skb_features(struct sk_buff *skb);
5394 void skb_warn_bad_offload(const struct sk_buff *skb);
5395
net_gso_ok(netdev_features_t features,int gso_type)5396 static inline bool net_gso_ok(netdev_features_t features, int gso_type)
5397 {
5398 netdev_features_t feature;
5399
5400 if (gso_type & (SKB_GSO_TCP_FIXEDID | SKB_GSO_TCP_FIXEDID_INNER))
5401 gso_type |= __SKB_GSO_TCP_FIXEDID;
5402
5403 feature = ((netdev_features_t)gso_type << NETIF_F_GSO_SHIFT) & NETIF_F_GSO_MASK;
5404
5405 /* check flags correspondence */
5406 BUILD_BUG_ON(SKB_GSO_TCPV4 != (NETIF_F_TSO >> NETIF_F_GSO_SHIFT));
5407 BUILD_BUG_ON(SKB_GSO_DODGY != (NETIF_F_GSO_ROBUST >> NETIF_F_GSO_SHIFT));
5408 BUILD_BUG_ON(SKB_GSO_TCP_ECN != (NETIF_F_TSO_ECN >> NETIF_F_GSO_SHIFT));
5409 BUILD_BUG_ON(__SKB_GSO_TCP_FIXEDID != (NETIF_F_TSO_MANGLEID >> NETIF_F_GSO_SHIFT));
5410 BUILD_BUG_ON(SKB_GSO_TCPV6 != (NETIF_F_TSO6 >> NETIF_F_GSO_SHIFT));
5411 BUILD_BUG_ON(SKB_GSO_FCOE != (NETIF_F_FSO >> NETIF_F_GSO_SHIFT));
5412 BUILD_BUG_ON(SKB_GSO_GRE != (NETIF_F_GSO_GRE >> NETIF_F_GSO_SHIFT));
5413 BUILD_BUG_ON(SKB_GSO_GRE_CSUM != (NETIF_F_GSO_GRE_CSUM >> NETIF_F_GSO_SHIFT));
5414 BUILD_BUG_ON(SKB_GSO_IPXIP4 != (NETIF_F_GSO_IPXIP4 >> NETIF_F_GSO_SHIFT));
5415 BUILD_BUG_ON(SKB_GSO_IPXIP6 != (NETIF_F_GSO_IPXIP6 >> NETIF_F_GSO_SHIFT));
5416 BUILD_BUG_ON(SKB_GSO_UDP_TUNNEL != (NETIF_F_GSO_UDP_TUNNEL >> NETIF_F_GSO_SHIFT));
5417 BUILD_BUG_ON(SKB_GSO_UDP_TUNNEL_CSUM != (NETIF_F_GSO_UDP_TUNNEL_CSUM >> NETIF_F_GSO_SHIFT));
5418 BUILD_BUG_ON(SKB_GSO_PARTIAL != (NETIF_F_GSO_PARTIAL >> NETIF_F_GSO_SHIFT));
5419 BUILD_BUG_ON(SKB_GSO_TUNNEL_REMCSUM != (NETIF_F_GSO_TUNNEL_REMCSUM >> NETIF_F_GSO_SHIFT));
5420 BUILD_BUG_ON(SKB_GSO_SCTP != (NETIF_F_GSO_SCTP >> NETIF_F_GSO_SHIFT));
5421 BUILD_BUG_ON(SKB_GSO_ESP != (NETIF_F_GSO_ESP >> NETIF_F_GSO_SHIFT));
5422 BUILD_BUG_ON(SKB_GSO_UDP != (NETIF_F_GSO_UDP >> NETIF_F_GSO_SHIFT));
5423 BUILD_BUG_ON(SKB_GSO_UDP_L4 != (NETIF_F_GSO_UDP_L4 >> NETIF_F_GSO_SHIFT));
5424 BUILD_BUG_ON(SKB_GSO_FRAGLIST != (NETIF_F_GSO_FRAGLIST >> NETIF_F_GSO_SHIFT));
5425 BUILD_BUG_ON(SKB_GSO_TCP_ACCECN !=
5426 (NETIF_F_GSO_ACCECN >> NETIF_F_GSO_SHIFT));
5427
5428 return (features & feature) == feature;
5429 }
5430
skb_gso_ok(struct sk_buff * skb,netdev_features_t features)5431 static inline bool skb_gso_ok(struct sk_buff *skb, netdev_features_t features)
5432 {
5433 return net_gso_ok(features, skb_shinfo(skb)->gso_type) &&
5434 (!skb_has_frag_list(skb) || (features & NETIF_F_FRAGLIST));
5435 }
5436
netif_needs_gso(struct sk_buff * skb,netdev_features_t features)5437 static inline bool netif_needs_gso(struct sk_buff *skb,
5438 netdev_features_t features)
5439 {
5440 return skb_is_gso(skb) && (!skb_gso_ok(skb, features) ||
5441 unlikely((skb->ip_summed != CHECKSUM_PARTIAL) &&
5442 (skb->ip_summed != CHECKSUM_UNNECESSARY)));
5443 }
5444
5445 void netif_set_tso_max_size(struct net_device *dev, unsigned int size);
5446 void netif_set_tso_max_segs(struct net_device *dev, unsigned int segs);
5447 void netif_inherit_tso_max(struct net_device *to,
5448 const struct net_device *from);
5449
5450 static inline unsigned int
netif_get_gro_max_size(const struct net_device * dev,const struct sk_buff * skb)5451 netif_get_gro_max_size(const struct net_device *dev, const struct sk_buff *skb)
5452 {
5453 /* pairs with WRITE_ONCE() in netif_set_gro(_ipv4)_max_size() */
5454 return skb->protocol == htons(ETH_P_IPV6) ?
5455 READ_ONCE(dev->gro_max_size) :
5456 READ_ONCE(dev->gro_ipv4_max_size);
5457 }
5458
5459 static inline unsigned int
netif_get_gso_max_size(const struct net_device * dev,const struct sk_buff * skb)5460 netif_get_gso_max_size(const struct net_device *dev, const struct sk_buff *skb)
5461 {
5462 /* pairs with WRITE_ONCE() in netif_set_gso(_ipv4)_max_size() */
5463 return skb->protocol == htons(ETH_P_IPV6) ?
5464 READ_ONCE(dev->gso_max_size) :
5465 READ_ONCE(dev->gso_ipv4_max_size);
5466 }
5467
netif_is_macsec(const struct net_device * dev)5468 static inline bool netif_is_macsec(const struct net_device *dev)
5469 {
5470 return dev->priv_flags & IFF_MACSEC;
5471 }
5472
netif_is_macvlan(const struct net_device * dev)5473 static inline bool netif_is_macvlan(const struct net_device *dev)
5474 {
5475 return dev->priv_flags & IFF_MACVLAN;
5476 }
5477
netif_is_macvlan_port(const struct net_device * dev)5478 static inline bool netif_is_macvlan_port(const struct net_device *dev)
5479 {
5480 return dev->priv_flags & IFF_MACVLAN_PORT;
5481 }
5482
netif_is_bond_master(const struct net_device * dev)5483 static inline bool netif_is_bond_master(const struct net_device *dev)
5484 {
5485 return dev->flags & IFF_MASTER && dev->priv_flags & IFF_BONDING;
5486 }
5487
netif_is_bond_slave(const struct net_device * dev)5488 static inline bool netif_is_bond_slave(const struct net_device *dev)
5489 {
5490 return dev->flags & IFF_SLAVE && dev->priv_flags & IFF_BONDING;
5491 }
5492
netif_supports_nofcs(struct net_device * dev)5493 static inline bool netif_supports_nofcs(struct net_device *dev)
5494 {
5495 return dev->priv_flags & IFF_SUPP_NOFCS;
5496 }
5497
netif_has_l3_rx_handler(const struct net_device * dev)5498 static inline bool netif_has_l3_rx_handler(const struct net_device *dev)
5499 {
5500 return dev->priv_flags & IFF_L3MDEV_RX_HANDLER;
5501 }
5502
netif_is_l3_master(const struct net_device * dev)5503 static inline bool netif_is_l3_master(const struct net_device *dev)
5504 {
5505 return dev->priv_flags & IFF_L3MDEV_MASTER;
5506 }
5507
netif_is_l3_slave(const struct net_device * dev)5508 static inline bool netif_is_l3_slave(const struct net_device *dev)
5509 {
5510 return dev->priv_flags & IFF_L3MDEV_SLAVE;
5511 }
5512
dev_sdif(const struct net_device * dev)5513 static inline int dev_sdif(const struct net_device *dev)
5514 {
5515 #ifdef CONFIG_NET_L3_MASTER_DEV
5516 if (netif_is_l3_slave(dev))
5517 return dev->ifindex;
5518 #endif
5519 return 0;
5520 }
5521
netif_is_bridge_master(const struct net_device * dev)5522 static inline bool netif_is_bridge_master(const struct net_device *dev)
5523 {
5524 return dev->priv_flags & IFF_EBRIDGE;
5525 }
5526
netif_is_bridge_port(const struct net_device * dev)5527 static inline bool netif_is_bridge_port(const struct net_device *dev)
5528 {
5529 return dev->priv_flags & IFF_BRIDGE_PORT;
5530 }
5531
netif_is_ovs_master(const struct net_device * dev)5532 static inline bool netif_is_ovs_master(const struct net_device *dev)
5533 {
5534 return dev->priv_flags & IFF_OPENVSWITCH;
5535 }
5536
netif_is_ovs_port(const struct net_device * dev)5537 static inline bool netif_is_ovs_port(const struct net_device *dev)
5538 {
5539 return dev->priv_flags & IFF_OVS_DATAPATH;
5540 }
5541
netif_is_any_bridge_master(const struct net_device * dev)5542 static inline bool netif_is_any_bridge_master(const struct net_device *dev)
5543 {
5544 return netif_is_bridge_master(dev) || netif_is_ovs_master(dev);
5545 }
5546
netif_is_any_bridge_port(const struct net_device * dev)5547 static inline bool netif_is_any_bridge_port(const struct net_device *dev)
5548 {
5549 return netif_is_bridge_port(dev) || netif_is_ovs_port(dev);
5550 }
5551
netif_is_team_master(const struct net_device * dev)5552 static inline bool netif_is_team_master(const struct net_device *dev)
5553 {
5554 return dev->priv_flags & IFF_TEAM;
5555 }
5556
netif_is_team_port(const struct net_device * dev)5557 static inline bool netif_is_team_port(const struct net_device *dev)
5558 {
5559 return dev->priv_flags & IFF_TEAM_PORT;
5560 }
5561
netif_is_lag_master(const struct net_device * dev)5562 static inline bool netif_is_lag_master(const struct net_device *dev)
5563 {
5564 return netif_is_bond_master(dev) || netif_is_team_master(dev);
5565 }
5566
netif_is_lag_port(const struct net_device * dev)5567 static inline bool netif_is_lag_port(const struct net_device *dev)
5568 {
5569 return netif_is_bond_slave(dev) || netif_is_team_port(dev);
5570 }
5571
netif_is_rxfh_configured(const struct net_device * dev)5572 static inline bool netif_is_rxfh_configured(const struct net_device *dev)
5573 {
5574 return dev->priv_flags & IFF_RXFH_CONFIGURED;
5575 }
5576
netif_is_failover(const struct net_device * dev)5577 static inline bool netif_is_failover(const struct net_device *dev)
5578 {
5579 return dev->priv_flags & IFF_FAILOVER;
5580 }
5581
netif_is_failover_slave(const struct net_device * dev)5582 static inline bool netif_is_failover_slave(const struct net_device *dev)
5583 {
5584 return dev->priv_flags & IFF_FAILOVER_SLAVE;
5585 }
5586
5587 /* This device needs to keep skb dst for qdisc enqueue or ndo_start_xmit() */
netif_keep_dst(struct net_device * dev)5588 static inline void netif_keep_dst(struct net_device *dev)
5589 {
5590 dev->priv_flags &= ~(IFF_XMIT_DST_RELEASE | IFF_XMIT_DST_RELEASE_PERM);
5591 }
5592
5593 /* return true if dev can't cope with mtu frames that need vlan tag insertion */
netif_reduces_vlan_mtu(struct net_device * dev)5594 static inline bool netif_reduces_vlan_mtu(struct net_device *dev)
5595 {
5596 /* TODO: reserve and use an additional IFF bit, if we get more users */
5597 return netif_is_macsec(dev);
5598 }
5599
5600 extern struct pernet_operations __net_initdata loopback_net_ops;
5601
5602 /* Logging, debugging and troubleshooting/diagnostic helpers. */
5603
5604 /* netdev_printk helpers, similar to dev_printk */
5605
netdev_name(const struct net_device * dev)5606 static inline const char *netdev_name(const struct net_device *dev)
5607 {
5608 if (!dev->name[0] || strchr(dev->name, '%'))
5609 return "(unnamed net_device)";
5610 return dev->name;
5611 }
5612
netdev_reg_state(const struct net_device * dev)5613 static inline const char *netdev_reg_state(const struct net_device *dev)
5614 {
5615 u8 reg_state = READ_ONCE(dev->reg_state);
5616
5617 switch (reg_state) {
5618 case NETREG_UNINITIALIZED: return " (uninitialized)";
5619 case NETREG_REGISTERED: return "";
5620 case NETREG_UNREGISTERING: return " (unregistering)";
5621 case NETREG_UNREGISTERED: return " (unregistered)";
5622 case NETREG_RELEASED: return " (released)";
5623 case NETREG_DUMMY: return " (dummy)";
5624 }
5625
5626 WARN_ONCE(1, "%s: unknown reg_state %d\n", dev->name, reg_state);
5627 return " (unknown)";
5628 }
5629
5630 #define MODULE_ALIAS_NETDEV(device) \
5631 MODULE_ALIAS("netdev-" device)
5632
5633 /*
5634 * netdev_WARN() acts like dev_printk(), but with the key difference
5635 * of using a WARN/WARN_ON to get the message out, including the
5636 * file/line information and a backtrace.
5637 */
5638 #define netdev_WARN(dev, format, args...) \
5639 WARN(1, "netdevice: %s%s: " format, netdev_name(dev), \
5640 netdev_reg_state(dev), ##args)
5641
5642 #define netdev_WARN_ONCE(dev, format, args...) \
5643 WARN_ONCE(1, "netdevice: %s%s: " format, netdev_name(dev), \
5644 netdev_reg_state(dev), ##args)
5645
5646 /*
5647 * The list of packet types we will receive (as opposed to discard)
5648 * and the routines to invoke.
5649 *
5650 * Why 16. Because with 16 the only overlap we get on a hash of the
5651 * low nibble of the protocol value is RARP/SNAP/X.25.
5652 *
5653 * 0800 IP
5654 * 0001 802.3
5655 * 0002 AX.25
5656 * 0004 802.2
5657 * 8035 RARP
5658 * 0005 SNAP
5659 * 0805 X.25
5660 * 0806 ARP
5661 * 8137 IPX
5662 * 0009 Localtalk
5663 * 86DD IPv6
5664 */
5665 #define PTYPE_HASH_SIZE (16)
5666 #define PTYPE_HASH_MASK (PTYPE_HASH_SIZE - 1)
5667
5668 extern struct list_head ptype_base[PTYPE_HASH_SIZE] __read_mostly;
5669
5670 extern struct net_device *blackhole_netdev;
5671
5672 /* Note: Avoid these macros in fast path, prefer per-cpu or per-queue counters. */
5673 #define DEV_STATS_INC(DEV, FIELD) atomic_long_inc(&(DEV)->stats.__##FIELD)
5674 #define DEV_STATS_ADD(DEV, FIELD, VAL) \
5675 atomic_long_add((VAL), &(DEV)->stats.__##FIELD)
5676 #define DEV_STATS_READ(DEV, FIELD) atomic_long_read(&(DEV)->stats.__##FIELD)
5677
5678 #endif /* _LINUX_NETDEVICE_H */
5679