1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 #ifndef _NET_CORE_DEV_H 3 #define _NET_CORE_DEV_H 4 5 #include <linux/cleanup.h> 6 #include <linux/types.h> 7 #include <linux/rwsem.h> 8 #include <linux/netdevice.h> 9 #include <net/netdev_lock.h> 10 11 struct net; 12 struct netlink_ext_ack; 13 struct netdev_queue_config; 14 struct cpumask; 15 struct pp_memory_provider_params; 16 17 /* Random bits of netdevice that don't need to be exposed */ 18 #define FLOW_LIMIT_HISTORY (1 << 7) /* must be ^2 and !overflow buckets */ 19 struct sd_flow_limit { 20 struct rcu_head rcu; 21 unsigned int count; 22 u8 log_buckets; 23 unsigned int history_head; 24 u16 history[FLOW_LIMIT_HISTORY]; 25 u8 buckets[]; 26 }; 27 28 extern int netdev_flow_limit_table_len; 29 30 struct napi_struct * 31 netdev_napi_by_id_lock(struct net *net, unsigned int napi_id); 32 struct net_device *dev_get_by_napi_id(unsigned int napi_id); 33 34 struct net_device *netdev_put_lock(struct net_device *dev, struct net *net, 35 netdevice_tracker *tracker); 36 37 static inline struct net_device * 38 __netdev_put_lock(struct net_device *dev, struct net *net) 39 { 40 return netdev_put_lock(dev, net, NULL); 41 } 42 43 struct net_device * 44 netdev_xa_find_lock(struct net *net, struct net_device *dev, 45 unsigned long *index); 46 47 DEFINE_FREE(netdev_unlock, struct net_device *, if (_T) netdev_unlock(_T)); 48 49 #define for_each_netdev_lock_scoped(net, var_name, ifindex) \ 50 for (struct net_device *var_name __free(netdev_unlock) = NULL; \ 51 (var_name = netdev_xa_find_lock(net, var_name, &ifindex)); \ 52 ifindex++) 53 54 struct net_device * 55 netdev_get_by_index_lock_ops_compat(struct net *net, int ifindex); 56 struct net_device * 57 netdev_xa_find_lock_ops_compat(struct net *net, struct net_device *dev, 58 unsigned long *index); 59 60 DEFINE_FREE(netdev_unlock_ops_compat, struct net_device *, 61 if (_T) netdev_unlock_ops_compat(_T)); 62 63 #define for_each_netdev_lock_ops_compat_scoped(net, var_name, ifindex) \ 64 for (struct net_device *var_name __free(netdev_unlock_ops_compat) = NULL; \ 65 (var_name = netdev_xa_find_lock_ops_compat(net, var_name, \ 66 &ifindex)); \ 67 ifindex++) 68 69 #ifdef CONFIG_PROC_FS 70 int __init dev_proc_init(void); 71 #else 72 #define dev_proc_init() 0 73 #endif 74 75 void linkwatch_init_dev(struct net_device *dev); 76 void linkwatch_run_queue(void); 77 78 void dev_addr_flush(struct net_device *dev); 79 int dev_addr_init(struct net_device *dev); 80 void dev_addr_check(struct net_device *dev); 81 void __hw_addr_flush(struct netdev_hw_addr_list *list); 82 83 #if IS_ENABLED(CONFIG_NET_SHAPER) 84 void net_shaper_flush_netdev(struct net_device *dev); 85 void net_shaper_set_real_num_tx_queues(struct net_device *dev, 86 unsigned int txq); 87 #else 88 static inline void net_shaper_flush_netdev(struct net_device *dev) {} 89 static inline void net_shaper_set_real_num_tx_queues(struct net_device *dev, 90 unsigned int txq) {} 91 #endif 92 93 /* sysctls not referred to from outside net/core/ */ 94 extern int netdev_unregister_timeout_secs; 95 extern int weight_p; 96 extern int dev_weight_rx_bias; 97 extern int dev_weight_tx_bias; 98 extern u8 netdev_rss_key[NETDEV_RSS_KEY_LEN] __read_mostly; 99 extern bool netdev_rss_key_initialized; 100 101 extern struct rw_semaphore dev_addr_sem; 102 103 /* rtnl helpers */ 104 extern struct list_head net_todo_list; 105 void netdev_run_todo(void); 106 107 int netdev_alloc_config(struct net_device *dev); 108 void __netdev_free_config(struct netdev_config *cfg); 109 void netdev_free_config(struct net_device *dev); 110 int netdev_reconfig_start(struct net_device *dev); 111 112 int netdev_queue_config_validate(struct net_device *dev, int rxq_idx, 113 struct netdev_queue_config *qcfg, 114 struct netlink_ext_ack *extack); 115 int netdev_queue_config_revalidate(struct net_device *dev, 116 struct netlink_ext_ack *extack); 117 118 bool netif_rxq_has_mp(struct net_device *dev, unsigned int rxq_idx); 119 bool netif_rxq_is_leased(struct net_device *dev, unsigned int rxq_idx); 120 bool netif_is_queue_leasee(const struct net_device *dev); 121 122 void __netif_mp_uninstall_rxq(struct netdev_rx_queue *rxq, 123 const struct pp_memory_provider_params *p); 124 125 void netif_rxq_cleanup_unlease(struct netdev_rx_queue *phys_rxq, 126 struct netdev_rx_queue *virt_rxq); 127 128 /* netdev management, shared between various uAPI entry points */ 129 struct netdev_name_node { 130 struct hlist_node hlist; 131 struct list_head list; 132 struct net_device *dev; 133 const char *name; 134 struct rcu_head rcu; 135 }; 136 137 int netdev_get_name(struct net *net, char *name, int ifindex); 138 int netif_change_name(struct net_device *dev, const char *newname); 139 int dev_change_name(struct net_device *dev, const char *newname); 140 141 #define netdev_for_each_altname(dev, namenode) \ 142 list_for_each_entry((namenode), &(dev)->name_node->list, list) 143 #define netdev_for_each_altname_safe(dev, namenode, next) \ 144 list_for_each_entry_safe((namenode), (next), &(dev)->name_node->list, \ 145 list) 146 147 int netdev_name_node_alt_create(struct net_device *dev, const char *name); 148 int netdev_name_node_alt_destroy(struct net_device *dev, const char *name); 149 150 int dev_validate_mtu(struct net_device *dev, int mtu, 151 struct netlink_ext_ack *extack); 152 int netif_set_mtu_ext(struct net_device *dev, int new_mtu, 153 struct netlink_ext_ack *extack); 154 155 int dev_get_phys_port_id(struct net_device *dev, 156 struct netdev_phys_item_id *ppid); 157 int dev_get_phys_port_name(struct net_device *dev, 158 char *name, size_t len); 159 160 int netif_change_proto_down(struct net_device *dev, bool proto_down); 161 int dev_change_proto_down(struct net_device *dev, bool proto_down); 162 void netdev_change_proto_down_reason_locked(struct net_device *dev, 163 unsigned long mask, u32 value); 164 165 typedef int (*bpf_op_t)(struct net_device *dev, struct netdev_bpf *bpf); 166 int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack, 167 int fd, int expected_fd, u32 flags); 168 169 int netif_change_tx_queue_len(struct net_device *dev, unsigned long new_len); 170 int dev_change_tx_queue_len(struct net_device *dev, unsigned long new_len); 171 void netif_set_group(struct net_device *dev, int new_group); 172 void dev_set_group(struct net_device *dev, int new_group); 173 int netif_change_carrier(struct net_device *dev, bool new_carrier); 174 int dev_change_carrier(struct net_device *dev, bool new_carrier); 175 176 void __dev_set_rx_mode(struct net_device *dev); 177 int __dev_set_promiscuity(struct net_device *dev, int inc, bool notify); 178 void netif_rx_mode_init(struct net_device *dev); 179 void netif_rx_mode_run(struct net_device *dev); 180 void netif_rx_mode_sync(struct net_device *dev); 181 void netif_rx_mode_cancel_retry(struct net_device *dev); 182 183 /* Events for the async netdev work, tracked in netdev->work_core_pending. */ 184 enum netdev_work_core { 185 NETDEV_WORK_RX_MODE = BIT(0), /* run the rx_mode update */ 186 }; 187 188 void __netdev_work_core_sched(struct net_device *dev, unsigned long event); 189 unsigned long 190 __netdev_work_core_cancel(struct net_device *dev, unsigned long mask); 191 void netdev_work_cancel_all(struct net_device *dev); 192 193 void __dev_notify_flags(struct net_device *dev, unsigned int old_flags, 194 unsigned int gchanges, u32 portid, 195 const struct nlmsghdr *nlh); 196 197 void unregister_netdevice_many_notify(struct list_head *head, 198 u32 portid, const struct nlmsghdr *nlh); 199 200 static inline void netif_set_up(struct net_device *dev, bool value) 201 { 202 if (value) 203 dev->flags |= IFF_UP; 204 else 205 dev->flags &= ~IFF_UP; 206 207 if (!netdev_need_ops_lock(dev)) 208 netdev_lock(dev); 209 dev->up = value; 210 if (!netdev_need_ops_lock(dev)) 211 netdev_unlock(dev); 212 } 213 214 static inline void netif_set_gso_max_size(struct net_device *dev, 215 unsigned int size) 216 { 217 /* dev->gso_max_size is read locklessly from sk_setup_caps() */ 218 WRITE_ONCE(dev->gso_max_size, size); 219 if (size <= GSO_LEGACY_MAX_SIZE) 220 WRITE_ONCE(dev->gso_ipv4_max_size, size); 221 } 222 223 static inline void netif_set_gso_max_segs(struct net_device *dev, 224 unsigned int segs) 225 { 226 /* dev->gso_max_segs is read locklessly from sk_setup_caps() */ 227 WRITE_ONCE(dev->gso_max_segs, segs); 228 } 229 230 static inline void netif_set_gro_max_size(struct net_device *dev, 231 unsigned int size) 232 { 233 /* This pairs with the READ_ONCE() in skb_gro_receive() */ 234 WRITE_ONCE(dev->gro_max_size, size); 235 if (size <= GRO_LEGACY_MAX_SIZE) 236 WRITE_ONCE(dev->gro_ipv4_max_size, size); 237 } 238 239 static inline void netif_set_gso_ipv4_max_size(struct net_device *dev, 240 unsigned int size) 241 { 242 /* dev->gso_ipv4_max_size is read locklessly from sk_setup_caps() */ 243 WRITE_ONCE(dev->gso_ipv4_max_size, size); 244 } 245 246 static inline void netif_set_gro_ipv4_max_size(struct net_device *dev, 247 unsigned int size) 248 { 249 /* This pairs with the READ_ONCE() in skb_gro_receive() */ 250 WRITE_ONCE(dev->gro_ipv4_max_size, size); 251 } 252 253 /** 254 * napi_get_defer_hard_irqs - get the NAPI's defer_hard_irqs 255 * @n: napi struct to get the defer_hard_irqs field from 256 * 257 * Return: the per-NAPI value of the defar_hard_irqs field. 258 */ 259 static inline u32 napi_get_defer_hard_irqs(const struct napi_struct *n) 260 { 261 return READ_ONCE(n->defer_hard_irqs); 262 } 263 264 /** 265 * napi_set_defer_hard_irqs - set the defer_hard_irqs for a napi 266 * @n: napi_struct to set the defer_hard_irqs field 267 * @defer: the value the field should be set to 268 */ 269 static inline void napi_set_defer_hard_irqs(struct napi_struct *n, u32 defer) 270 { 271 WRITE_ONCE(n->defer_hard_irqs, defer); 272 } 273 274 /** 275 * netdev_set_defer_hard_irqs - set defer_hard_irqs for all NAPIs of a netdev 276 * @netdev: the net_device for which all NAPIs will have defer_hard_irqs set 277 * @defer: the defer_hard_irqs value to set 278 */ 279 static inline void netdev_set_defer_hard_irqs(struct net_device *netdev, 280 u32 defer) 281 { 282 unsigned int count = max(netdev->num_rx_queues, 283 netdev->num_tx_queues); 284 struct napi_struct *napi; 285 int i; 286 287 WRITE_ONCE(netdev->napi_defer_hard_irqs, defer); 288 list_for_each_entry(napi, &netdev->napi_list, dev_list) 289 napi_set_defer_hard_irqs(napi, defer); 290 291 for (i = 0; i < count; i++) 292 netdev->napi_config[i].defer_hard_irqs = defer; 293 } 294 295 /** 296 * napi_get_gro_flush_timeout - get the gro_flush_timeout 297 * @n: napi struct to get the gro_flush_timeout from 298 * 299 * Return: the per-NAPI value of the gro_flush_timeout field. 300 */ 301 static inline unsigned long 302 napi_get_gro_flush_timeout(const struct napi_struct *n) 303 { 304 return READ_ONCE(n->gro_flush_timeout); 305 } 306 307 /** 308 * napi_set_gro_flush_timeout - set the gro_flush_timeout for a napi 309 * @n: napi struct to set the gro_flush_timeout 310 * @timeout: timeout value to set 311 * 312 * napi_set_gro_flush_timeout sets the per-NAPI gro_flush_timeout 313 */ 314 static inline void napi_set_gro_flush_timeout(struct napi_struct *n, 315 unsigned long timeout) 316 { 317 WRITE_ONCE(n->gro_flush_timeout, timeout); 318 } 319 320 /** 321 * netdev_set_gro_flush_timeout - set gro_flush_timeout of a netdev's NAPIs 322 * @netdev: the net_device for which all NAPIs will have gro_flush_timeout set 323 * @timeout: the timeout value to set 324 */ 325 static inline void netdev_set_gro_flush_timeout(struct net_device *netdev, 326 unsigned long timeout) 327 { 328 unsigned int count = max(netdev->num_rx_queues, 329 netdev->num_tx_queues); 330 struct napi_struct *napi; 331 int i; 332 333 WRITE_ONCE(netdev->gro_flush_timeout, timeout); 334 list_for_each_entry(napi, &netdev->napi_list, dev_list) 335 napi_set_gro_flush_timeout(napi, timeout); 336 337 for (i = 0; i < count; i++) 338 netdev->napi_config[i].gro_flush_timeout = timeout; 339 } 340 341 /** 342 * napi_get_irq_suspend_timeout - get the irq_suspend_timeout 343 * @n: napi struct to get the irq_suspend_timeout from 344 * 345 * Return: the per-NAPI value of the irq_suspend_timeout field. 346 */ 347 static inline unsigned long 348 napi_get_irq_suspend_timeout(const struct napi_struct *n) 349 { 350 return READ_ONCE(n->irq_suspend_timeout); 351 } 352 353 /** 354 * napi_set_irq_suspend_timeout - set the irq_suspend_timeout for a napi 355 * @n: napi struct to set the irq_suspend_timeout 356 * @timeout: timeout value to set 357 * 358 * napi_set_irq_suspend_timeout sets the per-NAPI irq_suspend_timeout 359 */ 360 static inline void napi_set_irq_suspend_timeout(struct napi_struct *n, 361 unsigned long timeout) 362 { 363 WRITE_ONCE(n->irq_suspend_timeout, timeout); 364 } 365 366 static inline enum netdev_napi_threaded napi_get_threaded(struct napi_struct *n) 367 { 368 if (test_bit(NAPI_STATE_THREADED_BUSY_POLL, &n->state)) 369 return NETDEV_NAPI_THREADED_BUSY_POLL; 370 371 if (test_bit(NAPI_STATE_THREADED, &n->state)) 372 return NETDEV_NAPI_THREADED_ENABLED; 373 374 return NETDEV_NAPI_THREADED_DISABLED; 375 } 376 377 static inline enum netdev_napi_threaded 378 napi_get_threaded_config(struct net_device *dev, struct napi_struct *n) 379 { 380 if (n->config) 381 return n->config->threaded; 382 return dev->threaded; 383 } 384 385 int napi_set_threaded(struct napi_struct *n, 386 enum netdev_napi_threaded threaded); 387 388 int netif_set_threaded(struct net_device *dev, 389 enum netdev_napi_threaded threaded); 390 391 int rps_cpumask_housekeeping(struct cpumask *mask); 392 393 #if defined(CONFIG_DEBUG_NET) && defined(CONFIG_BPF_SYSCALL) 394 void xdp_do_check_flushed(struct napi_struct *napi); 395 #else 396 static inline void xdp_do_check_flushed(struct napi_struct *napi) { } 397 #endif 398 399 /* Best effort check that NAPI is not idle (can't be scheduled to run) */ 400 static inline void napi_assert_will_not_race(const struct napi_struct *napi) 401 { 402 /* uninitialized instance, can't race */ 403 if (!napi->poll_list.next) 404 return; 405 406 /* SCHED bit is set on disabled instances */ 407 WARN_ON(!test_bit(NAPI_STATE_SCHED, &napi->state)); 408 WARN_ON(READ_ONCE(napi->list_owner) != -1); 409 } 410 411 struct skb_defer_node; 412 void skb_defer_node_flush(struct skb_defer_node *sdn); 413 void kick_defer_list_purge(unsigned int cpu); 414 415 int dev_set_hwtstamp_phylib(struct net_device *dev, 416 struct kernel_hwtstamp_config *cfg, 417 struct netlink_ext_ack *extack); 418 int dev_get_hwtstamp_phylib(struct net_device *dev, 419 struct kernel_hwtstamp_config *cfg); 420 int net_hwtstamp_validate(const struct kernel_hwtstamp_config *cfg); 421 422 #endif 423