1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 #ifndef _NET_CORE_DEV_H 3 #define _NET_CORE_DEV_H 4 5 #include <linux/cleanup.h> 6 #include <linux/types.h> 7 #include <linux/rwsem.h> 8 #include <linux/netdevice.h> 9 #include <net/netdev_lock.h> 10 11 struct net; 12 struct netlink_ext_ack; 13 struct netdev_queue_config; 14 struct cpumask; 15 16 /* Random bits of netdevice that don't need to be exposed */ 17 #define FLOW_LIMIT_HISTORY (1 << 7) /* must be ^2 and !overflow buckets */ 18 struct sd_flow_limit { 19 struct rcu_head rcu; 20 unsigned int count; 21 u8 log_buckets; 22 unsigned int history_head; 23 u16 history[FLOW_LIMIT_HISTORY]; 24 u8 buckets[]; 25 }; 26 27 extern int netdev_flow_limit_table_len; 28 29 struct napi_struct * 30 netdev_napi_by_id_lock(struct net *net, unsigned int napi_id); 31 struct net_device *dev_get_by_napi_id(unsigned int napi_id); 32 33 struct net_device *__netdev_put_lock(struct net_device *dev, struct net *net); 34 struct net_device * 35 netdev_xa_find_lock(struct net *net, struct net_device *dev, 36 unsigned long *index); 37 38 DEFINE_FREE(netdev_unlock, struct net_device *, if (_T) netdev_unlock(_T)); 39 40 #define for_each_netdev_lock_scoped(net, var_name, ifindex) \ 41 for (struct net_device *var_name __free(netdev_unlock) = NULL; \ 42 (var_name = netdev_xa_find_lock(net, var_name, &ifindex)); \ 43 ifindex++) 44 45 struct net_device * 46 netdev_get_by_index_lock_ops_compat(struct net *net, int ifindex); 47 struct net_device * 48 netdev_xa_find_lock_ops_compat(struct net *net, struct net_device *dev, 49 unsigned long *index); 50 51 DEFINE_FREE(netdev_unlock_ops_compat, struct net_device *, 52 if (_T) netdev_unlock_ops_compat(_T)); 53 54 #define for_each_netdev_lock_ops_compat_scoped(net, var_name, ifindex) \ 55 for (struct net_device *var_name __free(netdev_unlock_ops_compat) = NULL; \ 56 (var_name = netdev_xa_find_lock_ops_compat(net, var_name, \ 57 &ifindex)); \ 58 ifindex++) 59 60 #ifdef CONFIG_PROC_FS 61 int __init dev_proc_init(void); 62 #else 63 #define dev_proc_init() 0 64 #endif 65 66 void linkwatch_init_dev(struct net_device *dev); 67 void linkwatch_run_queue(void); 68 69 void dev_addr_flush(struct net_device *dev); 70 int dev_addr_init(struct net_device *dev); 71 void dev_addr_check(struct net_device *dev); 72 73 #if IS_ENABLED(CONFIG_NET_SHAPER) 74 void net_shaper_flush_netdev(struct net_device *dev); 75 void net_shaper_set_real_num_tx_queues(struct net_device *dev, 76 unsigned int txq); 77 #else 78 static inline void net_shaper_flush_netdev(struct net_device *dev) {} 79 static inline void net_shaper_set_real_num_tx_queues(struct net_device *dev, 80 unsigned int txq) {} 81 #endif 82 83 /* sysctls not referred to from outside net/core/ */ 84 extern int netdev_unregister_timeout_secs; 85 extern int weight_p; 86 extern int dev_weight_rx_bias; 87 extern int dev_weight_tx_bias; 88 89 extern struct rw_semaphore dev_addr_sem; 90 91 /* rtnl helpers */ 92 extern struct list_head net_todo_list; 93 void netdev_run_todo(void); 94 95 int netdev_queue_config_validate(struct net_device *dev, int rxq_idx, 96 struct netdev_queue_config *qcfg, 97 struct netlink_ext_ack *extack); 98 99 /* netdev management, shared between various uAPI entry points */ 100 struct netdev_name_node { 101 struct hlist_node hlist; 102 struct list_head list; 103 struct net_device *dev; 104 const char *name; 105 struct rcu_head rcu; 106 }; 107 108 int netdev_get_name(struct net *net, char *name, int ifindex); 109 int netif_change_name(struct net_device *dev, const char *newname); 110 int dev_change_name(struct net_device *dev, const char *newname); 111 112 #define netdev_for_each_altname(dev, namenode) \ 113 list_for_each_entry((namenode), &(dev)->name_node->list, list) 114 #define netdev_for_each_altname_safe(dev, namenode, next) \ 115 list_for_each_entry_safe((namenode), (next), &(dev)->name_node->list, \ 116 list) 117 118 int netdev_name_node_alt_create(struct net_device *dev, const char *name); 119 int netdev_name_node_alt_destroy(struct net_device *dev, const char *name); 120 121 int dev_validate_mtu(struct net_device *dev, int mtu, 122 struct netlink_ext_ack *extack); 123 int netif_set_mtu_ext(struct net_device *dev, int new_mtu, 124 struct netlink_ext_ack *extack); 125 126 int dev_get_phys_port_id(struct net_device *dev, 127 struct netdev_phys_item_id *ppid); 128 int dev_get_phys_port_name(struct net_device *dev, 129 char *name, size_t len); 130 131 int netif_change_proto_down(struct net_device *dev, bool proto_down); 132 int dev_change_proto_down(struct net_device *dev, bool proto_down); 133 void netdev_change_proto_down_reason_locked(struct net_device *dev, 134 unsigned long mask, u32 value); 135 136 typedef int (*bpf_op_t)(struct net_device *dev, struct netdev_bpf *bpf); 137 int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack, 138 int fd, int expected_fd, u32 flags); 139 140 int netif_change_tx_queue_len(struct net_device *dev, unsigned long new_len); 141 int dev_change_tx_queue_len(struct net_device *dev, unsigned long new_len); 142 void netif_set_group(struct net_device *dev, int new_group); 143 void dev_set_group(struct net_device *dev, int new_group); 144 int netif_change_carrier(struct net_device *dev, bool new_carrier); 145 int dev_change_carrier(struct net_device *dev, bool new_carrier); 146 147 void __dev_set_rx_mode(struct net_device *dev); 148 149 void __dev_notify_flags(struct net_device *dev, unsigned int old_flags, 150 unsigned int gchanges, u32 portid, 151 const struct nlmsghdr *nlh); 152 153 void unregister_netdevice_many_notify(struct list_head *head, 154 u32 portid, const struct nlmsghdr *nlh); 155 156 static inline void netif_set_up(struct net_device *dev, bool value) 157 { 158 if (value) 159 dev->flags |= IFF_UP; 160 else 161 dev->flags &= ~IFF_UP; 162 163 if (!netdev_need_ops_lock(dev)) 164 netdev_lock(dev); 165 dev->up = value; 166 if (!netdev_need_ops_lock(dev)) 167 netdev_unlock(dev); 168 } 169 170 static inline void netif_set_gso_max_size(struct net_device *dev, 171 unsigned int size) 172 { 173 /* dev->gso_max_size is read locklessly from sk_setup_caps() */ 174 WRITE_ONCE(dev->gso_max_size, size); 175 if (size <= GSO_LEGACY_MAX_SIZE) 176 WRITE_ONCE(dev->gso_ipv4_max_size, size); 177 } 178 179 static inline void netif_set_gso_max_segs(struct net_device *dev, 180 unsigned int segs) 181 { 182 /* dev->gso_max_segs is read locklessly from sk_setup_caps() */ 183 WRITE_ONCE(dev->gso_max_segs, segs); 184 } 185 186 static inline void netif_set_gro_max_size(struct net_device *dev, 187 unsigned int size) 188 { 189 /* This pairs with the READ_ONCE() in skb_gro_receive() */ 190 WRITE_ONCE(dev->gro_max_size, size); 191 if (size <= GRO_LEGACY_MAX_SIZE) 192 WRITE_ONCE(dev->gro_ipv4_max_size, size); 193 } 194 195 static inline void netif_set_gso_ipv4_max_size(struct net_device *dev, 196 unsigned int size) 197 { 198 /* dev->gso_ipv4_max_size is read locklessly from sk_setup_caps() */ 199 WRITE_ONCE(dev->gso_ipv4_max_size, size); 200 } 201 202 static inline void netif_set_gro_ipv4_max_size(struct net_device *dev, 203 unsigned int size) 204 { 205 /* This pairs with the READ_ONCE() in skb_gro_receive() */ 206 WRITE_ONCE(dev->gro_ipv4_max_size, size); 207 } 208 209 /** 210 * napi_get_defer_hard_irqs - get the NAPI's defer_hard_irqs 211 * @n: napi struct to get the defer_hard_irqs field from 212 * 213 * Return: the per-NAPI value of the defar_hard_irqs field. 214 */ 215 static inline u32 napi_get_defer_hard_irqs(const struct napi_struct *n) 216 { 217 return READ_ONCE(n->defer_hard_irqs); 218 } 219 220 /** 221 * napi_set_defer_hard_irqs - set the defer_hard_irqs for a napi 222 * @n: napi_struct to set the defer_hard_irqs field 223 * @defer: the value the field should be set to 224 */ 225 static inline void napi_set_defer_hard_irqs(struct napi_struct *n, u32 defer) 226 { 227 WRITE_ONCE(n->defer_hard_irqs, defer); 228 } 229 230 /** 231 * netdev_set_defer_hard_irqs - set defer_hard_irqs for all NAPIs of a netdev 232 * @netdev: the net_device for which all NAPIs will have defer_hard_irqs set 233 * @defer: the defer_hard_irqs value to set 234 */ 235 static inline void netdev_set_defer_hard_irqs(struct net_device *netdev, 236 u32 defer) 237 { 238 unsigned int count = max(netdev->num_rx_queues, 239 netdev->num_tx_queues); 240 struct napi_struct *napi; 241 int i; 242 243 WRITE_ONCE(netdev->napi_defer_hard_irqs, defer); 244 list_for_each_entry(napi, &netdev->napi_list, dev_list) 245 napi_set_defer_hard_irqs(napi, defer); 246 247 for (i = 0; i < count; i++) 248 netdev->napi_config[i].defer_hard_irqs = defer; 249 } 250 251 /** 252 * napi_get_gro_flush_timeout - get the gro_flush_timeout 253 * @n: napi struct to get the gro_flush_timeout from 254 * 255 * Return: the per-NAPI value of the gro_flush_timeout field. 256 */ 257 static inline unsigned long 258 napi_get_gro_flush_timeout(const struct napi_struct *n) 259 { 260 return READ_ONCE(n->gro_flush_timeout); 261 } 262 263 /** 264 * napi_set_gro_flush_timeout - set the gro_flush_timeout for a napi 265 * @n: napi struct to set the gro_flush_timeout 266 * @timeout: timeout value to set 267 * 268 * napi_set_gro_flush_timeout sets the per-NAPI gro_flush_timeout 269 */ 270 static inline void napi_set_gro_flush_timeout(struct napi_struct *n, 271 unsigned long timeout) 272 { 273 WRITE_ONCE(n->gro_flush_timeout, timeout); 274 } 275 276 /** 277 * netdev_set_gro_flush_timeout - set gro_flush_timeout of a netdev's NAPIs 278 * @netdev: the net_device for which all NAPIs will have gro_flush_timeout set 279 * @timeout: the timeout value to set 280 */ 281 static inline void netdev_set_gro_flush_timeout(struct net_device *netdev, 282 unsigned long timeout) 283 { 284 unsigned int count = max(netdev->num_rx_queues, 285 netdev->num_tx_queues); 286 struct napi_struct *napi; 287 int i; 288 289 WRITE_ONCE(netdev->gro_flush_timeout, timeout); 290 list_for_each_entry(napi, &netdev->napi_list, dev_list) 291 napi_set_gro_flush_timeout(napi, timeout); 292 293 for (i = 0; i < count; i++) 294 netdev->napi_config[i].gro_flush_timeout = timeout; 295 } 296 297 /** 298 * napi_get_irq_suspend_timeout - get the irq_suspend_timeout 299 * @n: napi struct to get the irq_suspend_timeout from 300 * 301 * Return: the per-NAPI value of the irq_suspend_timeout field. 302 */ 303 static inline unsigned long 304 napi_get_irq_suspend_timeout(const struct napi_struct *n) 305 { 306 return READ_ONCE(n->irq_suspend_timeout); 307 } 308 309 /** 310 * napi_set_irq_suspend_timeout - set the irq_suspend_timeout for a napi 311 * @n: napi struct to set the irq_suspend_timeout 312 * @timeout: timeout value to set 313 * 314 * napi_set_irq_suspend_timeout sets the per-NAPI irq_suspend_timeout 315 */ 316 static inline void napi_set_irq_suspend_timeout(struct napi_struct *n, 317 unsigned long timeout) 318 { 319 WRITE_ONCE(n->irq_suspend_timeout, timeout); 320 } 321 322 static inline enum netdev_napi_threaded napi_get_threaded(struct napi_struct *n) 323 { 324 if (test_bit(NAPI_STATE_THREADED_BUSY_POLL, &n->state)) 325 return NETDEV_NAPI_THREADED_BUSY_POLL; 326 327 if (test_bit(NAPI_STATE_THREADED, &n->state)) 328 return NETDEV_NAPI_THREADED_ENABLED; 329 330 return NETDEV_NAPI_THREADED_DISABLED; 331 } 332 333 static inline enum netdev_napi_threaded 334 napi_get_threaded_config(struct net_device *dev, struct napi_struct *n) 335 { 336 if (n->config) 337 return n->config->threaded; 338 return dev->threaded; 339 } 340 341 int napi_set_threaded(struct napi_struct *n, 342 enum netdev_napi_threaded threaded); 343 344 int netif_set_threaded(struct net_device *dev, 345 enum netdev_napi_threaded threaded); 346 347 int rps_cpumask_housekeeping(struct cpumask *mask); 348 349 #if defined(CONFIG_DEBUG_NET) && defined(CONFIG_BPF_SYSCALL) 350 void xdp_do_check_flushed(struct napi_struct *napi); 351 #else 352 static inline void xdp_do_check_flushed(struct napi_struct *napi) { } 353 #endif 354 355 /* Best effort check that NAPI is not idle (can't be scheduled to run) */ 356 static inline void napi_assert_will_not_race(const struct napi_struct *napi) 357 { 358 /* uninitialized instance, can't race */ 359 if (!napi->poll_list.next) 360 return; 361 362 /* SCHED bit is set on disabled instances */ 363 WARN_ON(!test_bit(NAPI_STATE_SCHED, &napi->state)); 364 WARN_ON(READ_ONCE(napi->list_owner) != -1); 365 } 366 367 void kick_defer_list_purge(unsigned int cpu); 368 369 #define XMIT_RECURSION_LIMIT 8 370 371 #ifndef CONFIG_PREEMPT_RT 372 static inline bool dev_xmit_recursion(void) 373 { 374 return unlikely(__this_cpu_read(softnet_data.xmit.recursion) > 375 XMIT_RECURSION_LIMIT); 376 } 377 378 static inline void dev_xmit_recursion_inc(void) 379 { 380 __this_cpu_inc(softnet_data.xmit.recursion); 381 } 382 383 static inline void dev_xmit_recursion_dec(void) 384 { 385 __this_cpu_dec(softnet_data.xmit.recursion); 386 } 387 #else 388 static inline bool dev_xmit_recursion(void) 389 { 390 return unlikely(current->net_xmit.recursion > XMIT_RECURSION_LIMIT); 391 } 392 393 static inline void dev_xmit_recursion_inc(void) 394 { 395 current->net_xmit.recursion++; 396 } 397 398 static inline void dev_xmit_recursion_dec(void) 399 { 400 current->net_xmit.recursion--; 401 } 402 #endif 403 404 int dev_set_hwtstamp_phylib(struct net_device *dev, 405 struct kernel_hwtstamp_config *cfg, 406 struct netlink_ext_ack *extack); 407 int dev_get_hwtstamp_phylib(struct net_device *dev, 408 struct kernel_hwtstamp_config *cfg); 409 int net_hwtstamp_validate(const struct kernel_hwtstamp_config *cfg); 410 411 #endif 412