| /linux/net/core/ |
| H A D | devmem.h | 108 net_devmem_iov_to_chunk_owner(const struct net_iov *niov) in net_devmem_iov_to_chunk_owner() argument
110 struct net_iov_area *owner = net_iov_owner(niov); in net_devmem_iov_to_chunk_owner()
116 net_devmem_iov_binding(const struct net_iov *niov) in net_devmem_iov_binding() argument
118 return net_devmem_iov_to_chunk_owner(niov)->binding; in net_devmem_iov_binding()
121 static inline u32 net_devmem_iov_binding_id(const struct net_iov *niov) in net_devmem_iov_binding_id() argument
123 return net_devmem_iov_binding(niov)->id; in net_devmem_iov_binding_id()
126 static inline unsigned long net_iov_virtual_addr(const struct net_iov *niov) in net_iov_virtual_addr() argument
129 net_devmem_iov_to_chunk_owner(niov); in net_iov_virtual_addr()
131 return net_iov_owner(niov)->base_virtual + in net_iov_virtual_addr()
132 ((unsigned long)net_iov_idx(niov) << co->binding->niov_shift); in net_iov_virtual_addr()
[all …]
|
| H A D | devmem.c | 43 static dma_addr_t net_devmem_get_dma_addr(const struct net_iov *niov) in net_devmem_get_dma_addr() argument
47 owner = net_devmem_iov_to_chunk_owner(niov); in net_devmem_get_dma_addr()
49 ((dma_addr_t)net_iov_idx(niov) << owner->binding->niov_shift); in net_devmem_get_dma_addr()
92 struct net_iov *niov; in net_devmem_alloc_dmabuf() local
104 niov = &owner->area.niovs[index]; in net_devmem_alloc_dmabuf()
106 niov->desc.pp_magic = 0; in net_devmem_alloc_dmabuf()
107 niov->desc.pp = NULL; in net_devmem_alloc_dmabuf()
108 atomic_long_set(&niov->desc.pp_ref_count, 0); in net_devmem_alloc_dmabuf()
110 return niov; in net_devmem_alloc_dmabuf()
113 void net_devmem_free_dmabuf(struct net_iov *niov) in net_devmem_free_dmabuf() argument
[all …]
|
| H A D | datagram.c | 713 struct net_iov *niov; in zerocopy_fill_skb_from_devmem() local
731 niov = net_devmem_get_niov_at(binding, virt_addr, &off, &size); in zerocopy_fill_skb_from_devmem()
732 if (!niov) in zerocopy_fill_skb_from_devmem()
738 get_netmem(net_iov_to_netmem(niov)); in zerocopy_fill_skb_from_devmem()
739 skb_add_rx_frag_netmem(skb, i, net_iov_to_netmem(niov), off, in zerocopy_fill_skb_from_devmem()
|
| H A D | page_pool.c | 1338 bool net_mp_niov_set_dma_addr(struct net_iov *niov, dma_addr_t addr) in net_mp_niov_set_dma_addr() argument
1340 return page_pool_set_dma_addr_netmem(net_iov_to_netmem(niov), addr); in net_mp_niov_set_dma_addr()
1343 /* Associate a niov with a page pool. Should follow with a matching
1346 void net_mp_niov_set_page_pool(struct page_pool *pool, struct net_iov *niov) in net_mp_niov_set_page_pool() argument
1348 netmem_ref netmem = net_iov_to_netmem(niov); in net_mp_niov_set_page_pool()
1356 /* Disassociate a niov from a page pool. Should only be used in the
1359 void net_mp_niov_clear_page_pool(struct net_iov *niov) in net_mp_niov_clear_page_pool() argument
1361 netmem_ref netmem = net_iov_to_netmem(niov); in net_mp_niov_clear_page_pool()
|
| H A D | skbuff.c | 7545 struct net_iov *niov = netmem_to_net_iov(netmem);
7547 if (net_is_devmem_iov(niov)) in __put_netmem()
7554 struct net_iov *niov = netmem_to_net_iov(netmem);
7556 if (net_is_devmem_iov(niov)) in __vlan_get_protocol_offset()
7539 struct net_iov *niov = netmem_to_net_iov(netmem); __get_netmem() local
7548 struct net_iov *niov = netmem_to_net_iov(netmem); __put_netmem() local
|
| H A D | dev.c | 4058 struct net_iov *niov; in validate_xmit_unreadable_skb()
4070 niov = netmem_to_net_iov(skb_frag_netmem(&shinfo->frags[0]));
4071 if (net_is_devmem_iov(niov) &&
4072 READ_ONCE(net_devmem_iov_binding(niov)->dev) != dev) in validate_xmit_skb()
4043 struct net_iov *niov; validate_xmit_unreadable_skb() local
|
| /linux/include/net/ |
| H A D | netmem.h | 110 static inline struct net_iov_area *net_iov_owner(const struct net_iov *niov) in net_iov_owner() argument
112 return niov->owner; in net_iov_owner()
115 static inline unsigned int net_iov_idx(const struct net_iov *niov) in net_iov_idx() argument
117 return niov - net_iov_owner(niov)->niovs; in net_iov_idx()
122 static inline void net_iov_init(struct net_iov *niov, in net_iov_init() argument
126 niov->owner = owner; in net_iov_init()
127 niov->type = type; in net_iov_init()
181 static inline netmem_ref net_iov_to_netmem(struct net_iov *niov) in net_iov_to_netmem() argument
183 return (__force netmem_ref)((unsigned long)niov | NET_IOV); in net_iov_to_netmem()
367 static inline bool net_is_devmem_iov(const struct net_iov *niov) in net_is_devmem_iov() argument
[all …]
|
| /linux/io_uring/ |
| H A D | zcrx.c | 58 static inline struct io_zcrx_area *io_zcrx_iov_to_area(const struct net_iov *niov) in io_zcrx_iov_to_area() argument
60 struct net_iov_area *owner = net_iov_owner(niov); in io_zcrx_iov_to_area()
76 static inline struct page *io_zcrx_iov_page(const struct net_iov *niov) in io_zcrx_iov_page() argument
78 struct io_zcrx_area *area = io_zcrx_iov_to_area(niov); in io_zcrx_iov_page()
84 return area->mem.pages[net_iov_idx(niov) << niov_pages_shift]; in io_zcrx_iov_page()
115 struct net_iov *niov = &area->nia.niovs[niov_idx]; in io_populate_area_dma() local
117 if (net_mp_niov_set_dma_addr(niov, dma)) in io_populate_area_dma()
382 static inline atomic_t *io_get_user_counter(struct net_iov *niov) in io_get_user_counter()
384 struct io_zcrx_area *area = io_zcrx_iov_to_area(niov); in io_get_user_counter()
386 return &area->user_refs[net_iov_idx(niov)]; in io_get_user_counter()
381 io_get_user_counter(struct net_iov * niov) io_get_user_counter() argument
388 io_zcrx_put_niov_uref(struct net_iov * niov,unsigned refs) io_zcrx_put_niov_uref() argument
402 io_zcrx_get_niov_uref(struct net_iov * niov) io_zcrx_get_niov_uref() argument
562 struct net_iov *niov = &area->nia.niovs[i]; __zcrx_create_area() local
686 io_zcrx_return_niov_freelist(struct net_iov * niov) io_zcrx_return_niov_freelist() argument
710 io_zcrx_return_niov(struct net_iov * niov) io_zcrx_return_niov() argument
728 struct net_iov *niov = &area->nia.niovs[i]; io_zcrx_scrub_area() local
1243 zcrx_put_refill_niov(struct net_iov * niov,struct page_pool * pp,unsigned refs) zcrx_put_refill_niov() argument
1266 struct net_iov *niov = NULL; io_zcrx_ring_refill() local
1309 struct net_iov *niov = zcrx_get_free_niov(ifq->areas[area_idx]); io_zcrx_refill_slow() local
1401 struct net_iov *niov; io_pp_zc_release_netmem() local
1490 struct net_iov *niov; zcrx_parse_rq() local
1509 struct net_iov *niov = netmem_to_net_iov(netmem); zcrx_return_buffers() local
1628 io_zcrx_queue_cqe(struct io_kiocb * req,struct net_iov * niov,struct io_zcrx_ifq * ifq,int off,int len) io_zcrx_queue_cqe() argument
1656 struct net_iov *niov = NULL; io_alloc_fallback_niov() local
1730 struct net_iov *niov; io_zcrx_copy_chunk() local
1781 struct net_iov *niov; io_zcrx_recv_frag() local
[all...] |
| /linux/drivers/target/iscsi/ |
| H A D | iscsi_target.c | 503 u32 niov = 0, tx_size = ISCSI_HDR_LEN; in iscsit_xmit_nondatain_pdu() local
507 iov[niov].iov_base = cmd->pdu; in iscsit_xmit_nondatain_pdu()
508 iov[niov++].iov_len = ISCSI_HDR_LEN; in iscsit_xmit_nondatain_pdu()
525 iov[niov].iov_base = (void *)data_buf; in iscsit_xmit_nondatain_pdu()
526 iov[niov++].iov_len = data_buf_len; in iscsit_xmit_nondatain_pdu()
530 iov[niov].iov_base = &cmd->pad_bytes; in iscsit_xmit_nondatain_pdu()
531 iov[niov++].iov_len = padding; in iscsit_xmit_nondatain_pdu()
541 iov[niov].iov_base = &cmd->data_crc; in iscsit_xmit_nondatain_pdu()
542 iov[niov++].iov_len = ISCSI_CRC_LEN; in iscsit_xmit_nondatain_pdu()
550 cmd->iov_misc_count = niov; in iscsit_xmit_nondatain_pdu()
[all …]
|
| /linux/net/ipv4/ |
| H A D | tcp.c | 2560 struct net_iov *niov; in tcp_recvmsg_dmabuf()
2577 niov = skb_frag_net_iov(frag); in tcp_recvmsg_dmabuf()
2578 if (!net_is_devmem_iov(niov)) { in tcp_recvmsg_dmabuf()
2589 frag_offset = net_iov_virtual_addr(niov) + in tcp_recvmsg_dmabuf()
2601 dmabuf_cmsg.dmabuf_id = net_devmem_iov_binding_id(niov); in tcp_recvmsg_dmabuf()
2613 atomic_long_inc(&niov->desc.pp_ref_count); in tcp_recvmsg_dmabuf()
2558 struct net_iov *niov; tcp_recvmsg_dmabuf() local
|