1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 #ifndef _NET_CORE_DEV_H 3 #define _NET_CORE_DEV_H 4 5 #include <linux/cleanup.h> 6 #include <linux/types.h> 7 #include <linux/rwsem.h> 8 #include <linux/netdevice.h> 9 #include <net/netdev_lock.h> 10 11 struct net; 12 struct netlink_ext_ack; 13 struct cpumask; 14 15 /* Random bits of netdevice that don't need to be exposed */ 16 #define FLOW_LIMIT_HISTORY (1 << 7) /* must be ^2 and !overflow buckets */ 17 struct sd_flow_limit { 18 struct rcu_head rcu; 19 unsigned int count; 20 u8 log_buckets; 21 unsigned int history_head; 22 u16 history[FLOW_LIMIT_HISTORY]; 23 u8 buckets[]; 24 }; 25 26 extern int netdev_flow_limit_table_len; 27 28 struct napi_struct * 29 netdev_napi_by_id_lock(struct net *net, unsigned int napi_id); 30 struct net_device *dev_get_by_napi_id(unsigned int napi_id); 31 32 struct net_device *__netdev_put_lock(struct net_device *dev, struct net *net); 33 struct net_device * 34 netdev_xa_find_lock(struct net *net, struct net_device *dev, 35 unsigned long *index); 36 37 DEFINE_FREE(netdev_unlock, struct net_device *, if (_T) netdev_unlock(_T)); 38 39 #define for_each_netdev_lock_scoped(net, var_name, ifindex) \ 40 for (struct net_device *var_name __free(netdev_unlock) = NULL; \ 41 (var_name = netdev_xa_find_lock(net, var_name, &ifindex)); \ 42 ifindex++) 43 44 struct net_device * 45 netdev_get_by_index_lock_ops_compat(struct net *net, int ifindex); 46 struct net_device * 47 netdev_xa_find_lock_ops_compat(struct net *net, struct net_device *dev, 48 unsigned long *index); 49 50 DEFINE_FREE(netdev_unlock_ops_compat, struct net_device *, 51 if (_T) netdev_unlock_ops_compat(_T)); 52 53 #define for_each_netdev_lock_ops_compat_scoped(net, var_name, ifindex) \ 54 for (struct net_device *var_name __free(netdev_unlock_ops_compat) = NULL; \ 55 (var_name = netdev_xa_find_lock_ops_compat(net, var_name, \ 56 &ifindex)); \ 57 ifindex++) 58 59 #ifdef CONFIG_PROC_FS 60 int __init dev_proc_init(void); 61 #else 62 #define dev_proc_init() 0 63 #endif 64 65 void linkwatch_init_dev(struct net_device *dev); 66 void linkwatch_run_queue(void); 67 68 void dev_addr_flush(struct net_device *dev); 69 int dev_addr_init(struct net_device *dev); 70 void dev_addr_check(struct net_device *dev); 71 72 #if IS_ENABLED(CONFIG_NET_SHAPER) 73 void net_shaper_flush_netdev(struct net_device *dev); 74 void net_shaper_set_real_num_tx_queues(struct net_device *dev, 75 unsigned int txq); 76 #else 77 static inline void net_shaper_flush_netdev(struct net_device *dev) {} 78 static inline void net_shaper_set_real_num_tx_queues(struct net_device *dev, 79 unsigned int txq) {} 80 #endif 81 82 /* sysctls not referred to from outside net/core/ */ 83 extern int netdev_unregister_timeout_secs; 84 extern int weight_p; 85 extern int dev_weight_rx_bias; 86 extern int dev_weight_tx_bias; 87 88 extern struct rw_semaphore dev_addr_sem; 89 90 /* rtnl helpers */ 91 extern struct list_head net_todo_list; 92 void netdev_run_todo(void); 93 94 /* netdev management, shared between various uAPI entry points */ 95 struct netdev_name_node { 96 struct hlist_node hlist; 97 struct list_head list; 98 struct net_device *dev; 99 const char *name; 100 struct rcu_head rcu; 101 }; 102 103 int netdev_get_name(struct net *net, char *name, int ifindex); 104 int netif_change_name(struct net_device *dev, const char *newname); 105 int dev_change_name(struct net_device *dev, const char *newname); 106 107 #define netdev_for_each_altname(dev, namenode) \ 108 list_for_each_entry((namenode), &(dev)->name_node->list, list) 109 #define netdev_for_each_altname_safe(dev, namenode, next) \ 110 list_for_each_entry_safe((namenode), (next), &(dev)->name_node->list, \ 111 list) 112 113 int netdev_name_node_alt_create(struct net_device *dev, const char *name); 114 int netdev_name_node_alt_destroy(struct net_device *dev, const char *name); 115 116 int dev_validate_mtu(struct net_device *dev, int mtu, 117 struct netlink_ext_ack *extack); 118 int netif_set_mtu_ext(struct net_device *dev, int new_mtu, 119 struct netlink_ext_ack *extack); 120 121 int dev_get_phys_port_id(struct net_device *dev, 122 struct netdev_phys_item_id *ppid); 123 int dev_get_phys_port_name(struct net_device *dev, 124 char *name, size_t len); 125 126 int netif_change_proto_down(struct net_device *dev, bool proto_down); 127 int dev_change_proto_down(struct net_device *dev, bool proto_down); 128 void netdev_change_proto_down_reason_locked(struct net_device *dev, 129 unsigned long mask, u32 value); 130 131 typedef int (*bpf_op_t)(struct net_device *dev, struct netdev_bpf *bpf); 132 int dev_change_xdp_fd(struct net_device *dev, struct netlink_ext_ack *extack, 133 int fd, int expected_fd, u32 flags); 134 135 int netif_change_tx_queue_len(struct net_device *dev, unsigned long new_len); 136 int dev_change_tx_queue_len(struct net_device *dev, unsigned long new_len); 137 void netif_set_group(struct net_device *dev, int new_group); 138 void dev_set_group(struct net_device *dev, int new_group); 139 int netif_change_carrier(struct net_device *dev, bool new_carrier); 140 int dev_change_carrier(struct net_device *dev, bool new_carrier); 141 142 void __dev_set_rx_mode(struct net_device *dev); 143 144 void __dev_notify_flags(struct net_device *dev, unsigned int old_flags, 145 unsigned int gchanges, u32 portid, 146 const struct nlmsghdr *nlh); 147 148 void unregister_netdevice_many_notify(struct list_head *head, 149 u32 portid, const struct nlmsghdr *nlh); 150 151 static inline void netif_set_up(struct net_device *dev, bool value) 152 { 153 if (value) 154 dev->flags |= IFF_UP; 155 else 156 dev->flags &= ~IFF_UP; 157 158 if (!netdev_need_ops_lock(dev)) 159 netdev_lock(dev); 160 dev->up = value; 161 if (!netdev_need_ops_lock(dev)) 162 netdev_unlock(dev); 163 } 164 165 static inline void netif_set_gso_max_size(struct net_device *dev, 166 unsigned int size) 167 { 168 /* dev->gso_max_size is read locklessly from sk_setup_caps() */ 169 WRITE_ONCE(dev->gso_max_size, size); 170 if (size <= GSO_LEGACY_MAX_SIZE) 171 WRITE_ONCE(dev->gso_ipv4_max_size, size); 172 } 173 174 static inline void netif_set_gso_max_segs(struct net_device *dev, 175 unsigned int segs) 176 { 177 /* dev->gso_max_segs is read locklessly from sk_setup_caps() */ 178 WRITE_ONCE(dev->gso_max_segs, segs); 179 } 180 181 static inline void netif_set_gro_max_size(struct net_device *dev, 182 unsigned int size) 183 { 184 /* This pairs with the READ_ONCE() in skb_gro_receive() */ 185 WRITE_ONCE(dev->gro_max_size, size); 186 if (size <= GRO_LEGACY_MAX_SIZE) 187 WRITE_ONCE(dev->gro_ipv4_max_size, size); 188 } 189 190 static inline void netif_set_gso_ipv4_max_size(struct net_device *dev, 191 unsigned int size) 192 { 193 /* dev->gso_ipv4_max_size is read locklessly from sk_setup_caps() */ 194 WRITE_ONCE(dev->gso_ipv4_max_size, size); 195 } 196 197 static inline void netif_set_gro_ipv4_max_size(struct net_device *dev, 198 unsigned int size) 199 { 200 /* This pairs with the READ_ONCE() in skb_gro_receive() */ 201 WRITE_ONCE(dev->gro_ipv4_max_size, size); 202 } 203 204 /** 205 * napi_get_defer_hard_irqs - get the NAPI's defer_hard_irqs 206 * @n: napi struct to get the defer_hard_irqs field from 207 * 208 * Return: the per-NAPI value of the defar_hard_irqs field. 209 */ 210 static inline u32 napi_get_defer_hard_irqs(const struct napi_struct *n) 211 { 212 return READ_ONCE(n->defer_hard_irqs); 213 } 214 215 /** 216 * napi_set_defer_hard_irqs - set the defer_hard_irqs for a napi 217 * @n: napi_struct to set the defer_hard_irqs field 218 * @defer: the value the field should be set to 219 */ 220 static inline void napi_set_defer_hard_irqs(struct napi_struct *n, u32 defer) 221 { 222 WRITE_ONCE(n->defer_hard_irqs, defer); 223 } 224 225 /** 226 * netdev_set_defer_hard_irqs - set defer_hard_irqs for all NAPIs of a netdev 227 * @netdev: the net_device for which all NAPIs will have defer_hard_irqs set 228 * @defer: the defer_hard_irqs value to set 229 */ 230 static inline void netdev_set_defer_hard_irqs(struct net_device *netdev, 231 u32 defer) 232 { 233 unsigned int count = max(netdev->num_rx_queues, 234 netdev->num_tx_queues); 235 struct napi_struct *napi; 236 int i; 237 238 WRITE_ONCE(netdev->napi_defer_hard_irqs, defer); 239 list_for_each_entry(napi, &netdev->napi_list, dev_list) 240 napi_set_defer_hard_irqs(napi, defer); 241 242 for (i = 0; i < count; i++) 243 netdev->napi_config[i].defer_hard_irqs = defer; 244 } 245 246 /** 247 * napi_get_gro_flush_timeout - get the gro_flush_timeout 248 * @n: napi struct to get the gro_flush_timeout from 249 * 250 * Return: the per-NAPI value of the gro_flush_timeout field. 251 */ 252 static inline unsigned long 253 napi_get_gro_flush_timeout(const struct napi_struct *n) 254 { 255 return READ_ONCE(n->gro_flush_timeout); 256 } 257 258 /** 259 * napi_set_gro_flush_timeout - set the gro_flush_timeout for a napi 260 * @n: napi struct to set the gro_flush_timeout 261 * @timeout: timeout value to set 262 * 263 * napi_set_gro_flush_timeout sets the per-NAPI gro_flush_timeout 264 */ 265 static inline void napi_set_gro_flush_timeout(struct napi_struct *n, 266 unsigned long timeout) 267 { 268 WRITE_ONCE(n->gro_flush_timeout, timeout); 269 } 270 271 /** 272 * netdev_set_gro_flush_timeout - set gro_flush_timeout of a netdev's NAPIs 273 * @netdev: the net_device for which all NAPIs will have gro_flush_timeout set 274 * @timeout: the timeout value to set 275 */ 276 static inline void netdev_set_gro_flush_timeout(struct net_device *netdev, 277 unsigned long timeout) 278 { 279 unsigned int count = max(netdev->num_rx_queues, 280 netdev->num_tx_queues); 281 struct napi_struct *napi; 282 int i; 283 284 WRITE_ONCE(netdev->gro_flush_timeout, timeout); 285 list_for_each_entry(napi, &netdev->napi_list, dev_list) 286 napi_set_gro_flush_timeout(napi, timeout); 287 288 for (i = 0; i < count; i++) 289 netdev->napi_config[i].gro_flush_timeout = timeout; 290 } 291 292 /** 293 * napi_get_irq_suspend_timeout - get the irq_suspend_timeout 294 * @n: napi struct to get the irq_suspend_timeout from 295 * 296 * Return: the per-NAPI value of the irq_suspend_timeout field. 297 */ 298 static inline unsigned long 299 napi_get_irq_suspend_timeout(const struct napi_struct *n) 300 { 301 return READ_ONCE(n->irq_suspend_timeout); 302 } 303 304 /** 305 * napi_set_irq_suspend_timeout - set the irq_suspend_timeout for a napi 306 * @n: napi struct to set the irq_suspend_timeout 307 * @timeout: timeout value to set 308 * 309 * napi_set_irq_suspend_timeout sets the per-NAPI irq_suspend_timeout 310 */ 311 static inline void napi_set_irq_suspend_timeout(struct napi_struct *n, 312 unsigned long timeout) 313 { 314 WRITE_ONCE(n->irq_suspend_timeout, timeout); 315 } 316 317 static inline enum netdev_napi_threaded napi_get_threaded(struct napi_struct *n) 318 { 319 if (test_bit(NAPI_STATE_THREADED_BUSY_POLL, &n->state)) 320 return NETDEV_NAPI_THREADED_BUSY_POLL; 321 322 if (test_bit(NAPI_STATE_THREADED, &n->state)) 323 return NETDEV_NAPI_THREADED_ENABLED; 324 325 return NETDEV_NAPI_THREADED_DISABLED; 326 } 327 328 static inline enum netdev_napi_threaded 329 napi_get_threaded_config(struct net_device *dev, struct napi_struct *n) 330 { 331 if (n->config) 332 return n->config->threaded; 333 return dev->threaded; 334 } 335 336 int napi_set_threaded(struct napi_struct *n, 337 enum netdev_napi_threaded threaded); 338 339 int netif_set_threaded(struct net_device *dev, 340 enum netdev_napi_threaded threaded); 341 342 int rps_cpumask_housekeeping(struct cpumask *mask); 343 344 #if defined(CONFIG_DEBUG_NET) && defined(CONFIG_BPF_SYSCALL) 345 void xdp_do_check_flushed(struct napi_struct *napi); 346 #else 347 static inline void xdp_do_check_flushed(struct napi_struct *napi) { } 348 #endif 349 350 /* Best effort check that NAPI is not idle (can't be scheduled to run) */ 351 static inline void napi_assert_will_not_race(const struct napi_struct *napi) 352 { 353 /* uninitialized instance, can't race */ 354 if (!napi->poll_list.next) 355 return; 356 357 /* SCHED bit is set on disabled instances */ 358 WARN_ON(!test_bit(NAPI_STATE_SCHED, &napi->state)); 359 WARN_ON(READ_ONCE(napi->list_owner) != -1); 360 } 361 362 void kick_defer_list_purge(unsigned int cpu); 363 364 #define XMIT_RECURSION_LIMIT 8 365 366 #ifndef CONFIG_PREEMPT_RT 367 static inline bool dev_xmit_recursion(void) 368 { 369 return unlikely(__this_cpu_read(softnet_data.xmit.recursion) > 370 XMIT_RECURSION_LIMIT); 371 } 372 373 static inline void dev_xmit_recursion_inc(void) 374 { 375 __this_cpu_inc(softnet_data.xmit.recursion); 376 } 377 378 static inline void dev_xmit_recursion_dec(void) 379 { 380 __this_cpu_dec(softnet_data.xmit.recursion); 381 } 382 #else 383 static inline bool dev_xmit_recursion(void) 384 { 385 return unlikely(current->net_xmit.recursion > XMIT_RECURSION_LIMIT); 386 } 387 388 static inline void dev_xmit_recursion_inc(void) 389 { 390 current->net_xmit.recursion++; 391 } 392 393 static inline void dev_xmit_recursion_dec(void) 394 { 395 current->net_xmit.recursion--; 396 } 397 #endif 398 399 int dev_set_hwtstamp_phylib(struct net_device *dev, 400 struct kernel_hwtstamp_config *cfg, 401 struct netlink_ext_ack *extack); 402 int dev_get_hwtstamp_phylib(struct net_device *dev, 403 struct kernel_hwtstamp_config *cfg); 404 int net_hwtstamp_validate(const struct kernel_hwtstamp_config *cfg); 405 406 #endif 407