/linux/include/net/ |
H A D | hotdata.h | 10 struct net_hotdata { struct 46 #define inet_ehash_secret net_hotdata.tcp_protocol.secret argument 47 #define udp_ehash_secret net_hotdata.udp_protocol.secret 48 #define inet6_ehash_secret net_hotdata.tcpv6_protocol.secret 49 #define tcp_ipv6_hash_secret net_hotdata.tcpv6_offload.secret 50 #define udp6_ehash_secret net_hotdata.udpv6_protocol.secret 51 #define udp_ipv6_hash_secret net_hotdata.udpv6_offload.secret 53 extern struct net_hotdata net_hotdata;
|
H A D | proto_memory.h | 68 if (unlikely(val >= READ_ONCE(net_hotdata.sysctl_mem_pcpu_rsv))) in sk_memory_allocated_add() 79 if (unlikely(val <= -READ_ONCE(net_hotdata.sysctl_mem_pcpu_rsv))) in sk_memory_allocated_sub()
|
H A D | rps.h | 72 u32 val = hash & ~net_hotdata.rps_cpu_mask; in rps_record_sock_flow() 94 sock_flow_table = rcu_dereference(net_hotdata.rps_sock_flow_table); in sock_rps_record_flow_hash()
|
H A D | gro.h | 530 if (napi->rx_count >= READ_ONCE(net_hotdata.gro_normal_batch)) in gro_normal_one()
|
/linux/net/core/ |
H A D | hotdata.c | 8 struct net_hotdata net_hotdata __cacheline_aligned = { 9 .offload_base = LIST_HEAD_INIT(net_hotdata.offload_base), 10 .ptype_all = LIST_HEAD_INIT(net_hotdata.ptype_all), 25 EXPORT_SYMBOL(net_hotdata);
|
H A D | sysctl_net_core.c | 162 net_hotdata.rps_sock_flow_table, in rps_sock_flow_sysctl() 183 net_hotdata.rps_cpu_mask = in rps_sock_flow_sysctl() 195 rcu_assign_pointer(net_hotdata.rps_sock_flow_table, in rps_sock_flow_sysctl() 326 WRITE_ONCE(net_hotdata.dev_rx_weight, weight * dev_weight_rx_bias); in proc_do_dev_weight() 327 WRITE_ONCE(net_hotdata.dev_tx_weight, weight * dev_weight_tx_bias); in proc_do_dev_weight() 404 .data = &net_hotdata.sysctl_mem_pcpu_rsv, 436 .data = &net_hotdata.max_backlog, 495 .data = &net_hotdata.tstamp_prequeue, 564 .data = &net_hotdata.netdev_budget, 578 .data = &net_hotdata.sysctl_max_skb_frags, [all …]
|
H A D | gro.c | 29 list_for_each_entry(elem, &net_hotdata.offload_base, list) { in dev_add_offload() 53 struct list_head *head = &net_hotdata.offload_base; in __dev_remove_offload() 256 struct list_head *head = &net_hotdata.offload_base; in napi_gro_complete() 465 struct list_head *head = &net_hotdata.offload_base; in dev_gro_receive() 570 struct list_head *offload_head = &net_hotdata.offload_base; in gro_find_receive_by_type() 584 struct list_head *offload_head = &net_hotdata.offload_base; in gro_find_complete_by_type()
|
H A D | net-procfs.c | 188 list_for_each_entry_rcu(pt, &net_hotdata.ptype_all, list) { in ptype_get_idx() 236 nxt = net_hotdata.ptype_all.next; in ptype_seq_next() 242 if (nxt != &net_hotdata.ptype_all) in ptype_seq_next()
|
H A D | gso.c | 20 list_for_each_entry_rcu(ptype, &net_hotdata.offload_base, list) { in skb_eth_gso_segment() 51 list_for_each_entry_rcu(ptype, &net_hotdata.offload_base, list) { in skb_mac_gso_segment()
|
H A D | gro_cells.c | 30 if (skb_queue_len(&cell->napi_skbs) > READ_ONCE(net_hotdata.max_backlog)) { in gro_cells_receive()
|
H A D | skbuff.c | 281 nc->skb_count = kmem_cache_alloc_bulk(net_hotdata.skbuff_cache, in napi_skb_cache_get() 293 kasan_mempool_unpoison_object(skb, kmem_cache_size(net_hotdata.skbuff_cache)); in napi_skb_cache_get() 351 skb = kmem_cache_alloc(net_hotdata.skbuff_cache, in slab_build_skb() 403 skb = kmem_cache_alloc(net_hotdata.skbuff_cache, in __build_skb() 515 obj = kmem_cache_alloc_node(net_hotdata.skb_small_head_cache, in kmalloc_reserve() 523 obj = kmem_cache_alloc_node(net_hotdata.skb_small_head_cache, flags, node); in kmalloc_reserve() 586 ? net_hotdata.skbuff_fclone_cache : net_hotdata.skbuff_cache; in __alloc_skb() 994 kmem_cache_free(net_hotdata.skb_small_head_cache, head); in skb_kfree_head() 1058 kmem_cache_free(net_hotdata.skbuff_cache, skb); in kfree_skbmem() 1079 kmem_cache_free(net_hotdata.skbuff_fclone_cache, fclones); in kfree_skbmem() [all …]
|
H A D | xdp.c | 623 n_skb = kmem_cache_alloc_bulk(net_hotdata.skbuff_cache, gfp, n_skb, skbs); in xdp_alloc_skb_bulk() 859 skb = kmem_cache_alloc(net_hotdata.skbuff_cache, GFP_ATOMIC); in xdp_build_skb_from_frame()
|
H A D | dev.c | 574 return pt->dev ? &pt->dev->ptype_all : &net_hotdata.ptype_all; in ptype_head() 2490 return !list_empty(&net_hotdata.ptype_all) || in dev_nit_active() 2502 struct list_head *ptype_list = &net_hotdata.ptype_all; in dev_queue_xmit_nit() 2550 if (ptype_list == &net_hotdata.ptype_all) { in dev_queue_xmit_nit() 4860 sock_flow_table = rcu_dereference(net_hotdata.rps_sock_flow_table); in get_rps_cpu() 4870 if ((ident ^ hash) & ~net_hotdata.rps_cpu_mask) in get_rps_cpu() 4873 next_cpu = ident & net_hotdata.rps_cpu_mask; in get_rps_cpu() 5044 if (qlen < (READ_ONCE(net_hotdata.max_backlog) >> 1)) in skb_flow_limit() 5095 max_backlog = READ_ONCE(net_hotdata.max_backlog); in enqueue_to_backlog() 5396 net_timestamp_check(READ_ONCE(net_hotdata.tstamp_prequeue), skb); in netif_rx_internal() [all …]
|
/linux/net/ipv6/ |
H A D | udp_offload.c | 192 net_hotdata.udpv6_offload = (struct net_offload) { in udpv6_offload_init() 199 return inet6_add_offload(&net_hotdata.udpv6_offload, IPPROTO_UDP); in udpv6_offload_init() 204 return inet6_del_offload(&net_hotdata.udpv6_offload, IPPROTO_UDP); in udpv6_offload_exit()
|
H A D | tcpv6_offload.c | 189 net_hotdata.tcpv6_offload = (struct net_offload) { in tcpv6_offload_init() 196 return inet6_add_offload(&net_hotdata.tcpv6_offload, IPPROTO_TCP); in tcpv6_offload_init()
|
H A D | ip6_offload.c | 470 net_hotdata.ipv6_packet_offload = (struct packet_offload) { in ipv6_offload_init() 478 dev_add_offload(&net_hotdata.ipv6_packet_offload); in ipv6_offload_init()
|
H A D | udp.c | 1957 net_hotdata.udpv6_protocol = (struct inet6_protocol) { in udpv6_init() 1962 ret = inet6_add_protocol(&net_hotdata.udpv6_protocol, IPPROTO_UDP); in udpv6_init() 1973 inet6_del_protocol(&net_hotdata.udpv6_protocol, IPPROTO_UDP); in udpv6_init() 1980 inet6_del_protocol(&net_hotdata.udpv6_protocol, IPPROTO_UDP); in udpv6_exit()
|
H A D | tcp_ipv6.c | 2410 net_hotdata.tcpv6_protocol = (struct inet6_protocol) { 2415 ret = inet6_add_protocol(&net_hotdata.tcpv6_protocol, IPPROTO_TCP); 2440 inet6_del_protocol(&net_hotdata.tcpv6_protocol, IPPROTO_TCP); 2448 inet6_del_protocol(&net_hotdata.tcpv6_protocol, IPPROTO_TCP);
|
/linux/net/ipv4/ |
H A D | tcp_offload.c | 492 net_hotdata.tcpv4_offload = (struct net_offload) { in tcpv4_offload_init() 499 return inet_add_offload(&net_hotdata.tcpv4_offload, IPPROTO_TCP); in tcpv4_offload_init()
|
H A D | udp_offload.c | 763 net_hotdata.udpv4_offload = (struct net_offload) { in udpv4_offload_init() 770 return inet_add_offload(&net_hotdata.udpv4_offload, IPPROTO_UDP); in udpv4_offload_init()
|
H A D | tcp.c | 1203 if (i >= READ_ONCE(net_hotdata.sysctl_max_skb_frags)) { in tcp_sendmsg_locked()
|
/linux/net/xfrm/ |
H A D | espintcp.c | 174 READ_ONCE(net_hotdata.max_backlog)) in espintcp_queue_out()
|
/linux/net/bpf/ |
H A D | test_run.c | 260 n = kmem_cache_alloc_bulk(net_hotdata.skbuff_cache, gfp, nframes, in xdp_recv_frames()
|
/linux/net/mptcp/ |
H A D | protocol.c | 1295 if (!can_coalesce && i >= READ_ONCE(net_hotdata.sysctl_max_skb_frags)) { in mptcp_sendmsg_frag()
|