1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 #ifndef _NET_CORE_DEV_H 3 #define _NET_CORE_DEV_H 4 5 #include <linux/cleanup.h> 6 #include <linux/types.h> 7 #include <linux/rwsem.h> 8 #include <linux/netdevice.h> 9 #include <net/netdev_lock.h> 10 11 struct net; 12 struct netlink_ext_ack; 13 struct cpumask; 14 15 /* Random bits of netdevice that don't need to be exposed */ 16 #define FLOW_LIMIT_HISTORY (1 << 7) /* must be ^2 and !overflow buckets */ 17 struct sd_flow_limit { 18 struct rcu_head rcu; 19 unsigned int count; 20 u8 log_buckets; 21 unsigned int history_head; 22 u16 history[FLOW_LIMIT_HISTORY]; 23 u8 buckets[]; 24 }; 25 26 extern int netdev_flow_limit_table_len; 27 28 struct napi_struct * 29 netdev_napi_by_id_lock(struct net *net, unsigned int napi_id); 30 struct net_device *dev_get_by_napi_id(unsigned int napi_id); 31 32 struct net_device *__netdev_put_lock(struct net_device *dev, struct net *net); 33 struct net_device *netdev_put_lock(struct net_device *dev, 34 netdevice_tracker *tracker); 35 struct net_device * 36 netdev_xa_find_lock(struct net *net, struct net_device *dev, 37 unsigned long *index); 38 39 DEFINE_FREE(netdev_unlock, struct net_device *, if (_T) netdev_unlock(_T)); 40 41 #define for_each_netdev_lock_scoped(net, var_name, ifindex) \ 42 for (struct net_device *var_name __free(netdev_unlock) = NULL; \ 43 (var_name = netdev_xa_find_lock(net, var_name, &ifindex)); \ 44 ifindex++) 45 46 struct net_device * 47 netdev_get_by_index_lock_ops_compat(struct net *net, int ifindex); 48 struct net_device * 49 netdev_xa_find_lock_ops_compat(struct net *net, struct net_device *dev, 50 unsigned long *index); 51 52 DEFINE_FREE(netdev_unlock_ops_compat, struct net_device *, 53 if (_T) netdev_unlock_ops_compat(_T)); 54 55 #define for_each_netdev_lock_ops_compat_scoped(net, var_name, ifindex) \ 56 for (struct net_device *var_name __free(netdev_unlock_ops_compat) = NULL; \ 57 (var_name = netdev_xa_find_lock_ops_compat(net, var_name, \ 58 &ifindex)); \ 59 ifindex++) 60 61 #ifdef CONFIG_PROC_FS 62 int __init dev_proc_init(void); 63 #else 64 #define dev_proc_init() 0 65 #endif 66 67 void linkwatch_init_dev(struct net_device *dev); 68 void linkwatch_run_queue(void); 69 70 void dev_addr_flush(struct net_device *dev); 71 int dev_addr_init(struct net_device *dev); 72 void dev_addr_check(struct net_device *dev); 73 74 #if IS_ENABLED(CONFIG_NET_SHAPER) 75 void net_shaper_flush_netdev(struct net_device *dev); 76 void net_shaper_set_real_num_tx_queues(struct net_device *dev, 77 unsigned int txq); 78 #else 79 static inline void net_shaper_flush_netdev(struct net_device *dev) {} 80 static inline void net_shaper_set_real_num_tx_queues(struct net_device *dev, 81 unsigned int txq) {} 82 #endif 83 84 /* sysctls not referred to from outside net/core/ */ 85 extern int netdev_unregister_timeout_secs; 86 extern int weight_p; 87 extern int dev_weight_rx_bias; 88 extern int dev_weight_tx_bias; 89 90 extern struct rw_semaphore dev_addr_sem; 91 92 /* rtnl helpers */ 93 extern struct list_head net_todo_list; 94 void netdev_run_todo(void); 95 96 /* netdev management, shared between various uAPI entry points */ 97 struct netdev_name_node { 98 struct hlist_node hlist; 99 struct list_head list; 100 struct net_device *dev; 101 const char *name; 102 struct rcu_head rcu; 103 }; 104 105 int netdev_get_name(struct net *net, char *name, int ifindex); 106 int netif_change_name(struct net_device *dev, const char *newname); 107 int dev_change_name(struct net_device *dev, const char *newname); 108 109 #define netdev_for_each_altname(dev, namenode) \ 110 list_for_each_entry((namenode), &(dev)->name_node->list, list) 111 #define netdev_for_each_altname_safe(dev, namenode, next) \ 112 list_for_each_entry_safe((namenode), (next), &(dev)->name_node->list, \ 113 list) 114 115 int netdev_name_node_alt_create(struct net_device *dev, const char *name); 116 int netdev_name_node_alt_destroy(struct net_device *dev, const char *name); 117 118 int dev_validate_mtu(struct net_device *dev, int mtu, 119 struct netlink_ext_ack *extack); 120 int netif_set_mtu_ext(struct net_device *dev, int new_mtu, 121 struct netlink_ext_ack *extack); 122 123 int dev_get_phys_port_id(struct net_device *dev, 124 struct netdev_phys_item_id *ppid); 125 int dev_get_phys_port_name(struct net_device *dev, 126 char *name, size_t len); 127 128 int netif_change_proto_down(struct net_device *dev, bool proto_down); 129 int dev_change_proto_down(struct net_device *dev, bool proto_down); 130 void netdev_change_proto_down_reason_locked(struct net_device *dev, 131 unsigned long mask, u32 value); 132 133 typedef int (*bpf_op_t)(struct net_device *dev, struct netdev_bpf *bpf); 134 int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack, 135 int fd, int expected_fd, u32 flags); 136 137 int netif_change_tx_queue_len(struct net_device *dev, unsigned long new_len); 138 int dev_change_tx_queue_len(struct net_device *dev, unsigned long new_len); 139 void netif_set_group(struct net_device *dev, int new_group); 140 void dev_set_group(struct net_device *dev, int new_group); 141 int netif_change_carrier(struct net_device *dev, bool new_carrier); 142 int dev_change_carrier(struct net_device *dev, bool new_carrier); 143 144 void __dev_set_rx_mode(struct net_device *dev); 145 146 void __dev_notify_flags(struct net_device *dev, unsigned int old_flags, 147 unsigned int gchanges, u32 portid, 148 const struct nlmsghdr *nlh); 149 150 void unregister_netdevice_many_notify(struct list_head *head, 151 u32 portid, const struct nlmsghdr *nlh); 152 153 static inline void netif_set_up(struct net_device *dev, bool value) 154 { 155 if (value) 156 dev->flags |= IFF_UP; 157 else 158 dev->flags &= ~IFF_UP; 159 160 if (!netdev_need_ops_lock(dev)) 161 netdev_lock(dev); 162 dev->up = value; 163 if (!netdev_need_ops_lock(dev)) 164 netdev_unlock(dev); 165 } 166 167 static inline void netif_set_gso_max_size(struct net_device *dev, 168 unsigned int size) 169 { 170 /* dev->gso_max_size is read locklessly from sk_setup_caps() */ 171 WRITE_ONCE(dev->gso_max_size, size); 172 if (size <= GSO_LEGACY_MAX_SIZE) 173 WRITE_ONCE(dev->gso_ipv4_max_size, size); 174 } 175 176 static inline void netif_set_gso_max_segs(struct net_device *dev, 177 unsigned int segs) 178 { 179 /* dev->gso_max_segs is read locklessly from sk_setup_caps() */ 180 WRITE_ONCE(dev->gso_max_segs, segs); 181 } 182 183 static inline void netif_set_gro_max_size(struct net_device *dev, 184 unsigned int size) 185 { 186 /* This pairs with the READ_ONCE() in skb_gro_receive() */ 187 WRITE_ONCE(dev->gro_max_size, size); 188 if (size <= GRO_LEGACY_MAX_SIZE) 189 WRITE_ONCE(dev->gro_ipv4_max_size, size); 190 } 191 192 static inline void netif_set_gso_ipv4_max_size(struct net_device *dev, 193 unsigned int size) 194 { 195 /* dev->gso_ipv4_max_size is read locklessly from sk_setup_caps() */ 196 WRITE_ONCE(dev->gso_ipv4_max_size, size); 197 } 198 199 static inline void netif_set_gro_ipv4_max_size(struct net_device *dev, 200 unsigned int size) 201 { 202 /* This pairs with the READ_ONCE() in skb_gro_receive() */ 203 WRITE_ONCE(dev->gro_ipv4_max_size, size); 204 } 205 206 /** 207 * napi_get_defer_hard_irqs - get the NAPI's defer_hard_irqs 208 * @n: napi struct to get the defer_hard_irqs field from 209 * 210 * Return: the per-NAPI value of the defar_hard_irqs field. 211 */ 212 static inline u32 napi_get_defer_hard_irqs(const struct napi_struct *n) 213 { 214 return READ_ONCE(n->defer_hard_irqs); 215 } 216 217 /** 218 * napi_set_defer_hard_irqs - set the defer_hard_irqs for a napi 219 * @n: napi_struct to set the defer_hard_irqs field 220 * @defer: the value the field should be set to 221 */ 222 static inline void napi_set_defer_hard_irqs(struct napi_struct *n, u32 defer) 223 { 224 WRITE_ONCE(n->defer_hard_irqs, defer); 225 } 226 227 /** 228 * netdev_set_defer_hard_irqs - set defer_hard_irqs for all NAPIs of a netdev 229 * @netdev: the net_device for which all NAPIs will have defer_hard_irqs set 230 * @defer: the defer_hard_irqs value to set 231 */ 232 static inline void netdev_set_defer_hard_irqs(struct net_device *netdev, 233 u32 defer) 234 { 235 unsigned int count = max(netdev->num_rx_queues, 236 netdev->num_tx_queues); 237 struct napi_struct *napi; 238 int i; 239 240 WRITE_ONCE(netdev->napi_defer_hard_irqs, defer); 241 list_for_each_entry(napi, &netdev->napi_list, dev_list) 242 napi_set_defer_hard_irqs(napi, defer); 243 244 for (i = 0; i < count; i++) 245 netdev->napi_config[i].defer_hard_irqs = defer; 246 } 247 248 /** 249 * napi_get_gro_flush_timeout - get the gro_flush_timeout 250 * @n: napi struct to get the gro_flush_timeout from 251 * 252 * Return: the per-NAPI value of the gro_flush_timeout field. 253 */ 254 static inline unsigned long 255 napi_get_gro_flush_timeout(const struct napi_struct *n) 256 { 257 return READ_ONCE(n->gro_flush_timeout); 258 } 259 260 /** 261 * napi_set_gro_flush_timeout - set the gro_flush_timeout for a napi 262 * @n: napi struct to set the gro_flush_timeout 263 * @timeout: timeout value to set 264 * 265 * napi_set_gro_flush_timeout sets the per-NAPI gro_flush_timeout 266 */ 267 static inline void napi_set_gro_flush_timeout(struct napi_struct *n, 268 unsigned long timeout) 269 { 270 WRITE_ONCE(n->gro_flush_timeout, timeout); 271 } 272 273 /** 274 * netdev_set_gro_flush_timeout - set gro_flush_timeout of a netdev's NAPIs 275 * @netdev: the net_device for which all NAPIs will have gro_flush_timeout set 276 * @timeout: the timeout value to set 277 */ 278 static inline void netdev_set_gro_flush_timeout(struct net_device *netdev, 279 unsigned long timeout) 280 { 281 unsigned int count = max(netdev->num_rx_queues, 282 netdev->num_tx_queues); 283 struct napi_struct *napi; 284 int i; 285 286 WRITE_ONCE(netdev->gro_flush_timeout, timeout); 287 list_for_each_entry(napi, &netdev->napi_list, dev_list) 288 napi_set_gro_flush_timeout(napi, timeout); 289 290 for (i = 0; i < count; i++) 291 netdev->napi_config[i].gro_flush_timeout = timeout; 292 } 293 294 /** 295 * napi_get_irq_suspend_timeout - get the irq_suspend_timeout 296 * @n: napi struct to get the irq_suspend_timeout from 297 * 298 * Return: the per-NAPI value of the irq_suspend_timeout field. 299 */ 300 static inline unsigned long 301 napi_get_irq_suspend_timeout(const struct napi_struct *n) 302 { 303 return READ_ONCE(n->irq_suspend_timeout); 304 } 305 306 /** 307 * napi_set_irq_suspend_timeout - set the irq_suspend_timeout for a napi 308 * @n: napi struct to set the irq_suspend_timeout 309 * @timeout: timeout value to set 310 * 311 * napi_set_irq_suspend_timeout sets the per-NAPI irq_suspend_timeout 312 */ 313 static inline void napi_set_irq_suspend_timeout(struct napi_struct *n, 314 unsigned long timeout) 315 { 316 WRITE_ONCE(n->irq_suspend_timeout, timeout); 317 } 318 319 static inline enum netdev_napi_threaded napi_get_threaded(struct napi_struct *n) 320 { 321 if (test_bit(NAPI_STATE_THREADED_BUSY_POLL, &n->state)) 322 return NETDEV_NAPI_THREADED_BUSY_POLL; 323 324 if (test_bit(NAPI_STATE_THREADED, &n->state)) 325 return NETDEV_NAPI_THREADED_ENABLED; 326 327 return NETDEV_NAPI_THREADED_DISABLED; 328 } 329 330 static inline enum netdev_napi_threaded 331 napi_get_threaded_config(struct net_device *dev, struct napi_struct *n) 332 { 333 if (n->config) 334 return n->config->threaded; 335 return dev->threaded; 336 } 337 338 int napi_set_threaded(struct napi_struct *n, 339 enum netdev_napi_threaded threaded); 340 341 int netif_set_threaded(struct net_device *dev, 342 enum netdev_napi_threaded threaded); 343 344 int rps_cpumask_housekeeping(struct cpumask *mask); 345 346 #if defined(CONFIG_DEBUG_NET) && defined(CONFIG_BPF_SYSCALL) 347 void xdp_do_check_flushed(struct napi_struct *napi); 348 #else 349 static inline void xdp_do_check_flushed(struct napi_struct *napi) { } 350 #endif 351 352 /* Best effort check that NAPI is not idle (can't be scheduled to run) */ 353 static inline void napi_assert_will_not_race(const struct napi_struct *napi) 354 { 355 /* uninitialized instance, can't race */ 356 if (!napi->poll_list.next) 357 return; 358 359 /* SCHED bit is set on disabled instances */ 360 WARN_ON(!test_bit(NAPI_STATE_SCHED, &napi->state)); 361 WARN_ON(READ_ONCE(napi->list_owner) != -1); 362 } 363 364 void kick_defer_list_purge(unsigned int cpu); 365 366 #define XMIT_RECURSION_LIMIT 8 367 368 #ifndef CONFIG_PREEMPT_RT 369 static inline bool dev_xmit_recursion(void) 370 { 371 return unlikely(__this_cpu_read(softnet_data.xmit.recursion) > 372 XMIT_RECURSION_LIMIT); 373 } 374 375 static inline void dev_xmit_recursion_inc(void) 376 { 377 __this_cpu_inc(softnet_data.xmit.recursion); 378 } 379 380 static inline void dev_xmit_recursion_dec(void) 381 { 382 __this_cpu_dec(softnet_data.xmit.recursion); 383 } 384 #else 385 static inline bool dev_xmit_recursion(void) 386 { 387 return unlikely(current->net_xmit.recursion > XMIT_RECURSION_LIMIT); 388 } 389 390 static inline void dev_xmit_recursion_inc(void) 391 { 392 current->net_xmit.recursion++; 393 } 394 395 static inline void dev_xmit_recursion_dec(void) 396 { 397 current->net_xmit.recursion--; 398 } 399 #endif 400 401 int dev_set_hwtstamp_phylib(struct net_device *dev, 402 struct kernel_hwtstamp_config *cfg, 403 struct netlink_ext_ack *extack); 404 int dev_get_hwtstamp_phylib(struct net_device *dev, 405 struct kernel_hwtstamp_config *cfg); 406 int net_hwtstamp_validate(const struct kernel_hwtstamp_config *cfg); 407 408 #endif 409