| /linux/net/xdp/ |
| H A D | xdp_umem.c | 24 static void xdp_umem_unpin_pages(struct xdp_umem *umem) in xdp_umem_unpin_pages() argument
26 unpin_user_pages_dirty_lock(umem->pgs, umem->npgs, true); in xdp_umem_unpin_pages()
28 kvfree(umem->pgs); in xdp_umem_unpin_pages()
29 umem->pgs = NULL; in xdp_umem_unpin_pages()
32 static void xdp_umem_unaccount_pages(struct xdp_umem *umem) in xdp_umem_unaccount_pages() argument
34 if (umem->user) { in xdp_umem_unaccount_pages()
35 atomic_long_sub(umem->npgs, &umem->user->locked_vm); in xdp_umem_unaccount_pages()
36 free_uid(umem->user); in xdp_umem_unaccount_pages()
40 static void xdp_umem_addr_unmap(struct xdp_umem *umem) in xdp_umem_addr_unmap() argument
42 vunmap(umem->addrs); in xdp_umem_addr_unmap()
[all …]
|
| H A D | xsk_buff_pool.c | 55 struct xdp_umem *umem) in xp_create_and_assign_umem() argument
57 bool unaligned = umem->flags & XDP_UMEM_UNALIGNED_CHUNK_FLAG; in xp_create_and_assign_umem()
62 entries = unaligned ? umem->chunks : 0; in xp_create_and_assign_umem()
67 pool->heads = kvzalloc_objs(*pool->heads, umem->chunks); in xp_create_and_assign_umem()
75 pool->chunk_mask = ~((u64)umem->chunk_size - 1); in xp_create_and_assign_umem()
76 pool->addrs_cnt = umem->size; in xp_create_and_assign_umem()
77 pool->heads_cnt = umem->chunks; in xp_create_and_assign_umem()
78 pool->free_heads_cnt = umem->chunks; in xp_create_and_assign_umem()
79 pool->headroom = umem->headroom; in xp_create_and_assign_umem()
80 pool->chunk_size = umem in xp_create_and_assign_umem()
272 struct xdp_umem *umem = umem_xs->umem; xp_assign_dev_shared() local
355 xp_create_dma_map(struct device * dev,struct net_device * netdev,u32 nr_pages,struct xdp_umem * umem) xp_create_dma_map() argument
[all...] |
| H A D | xsk_diag.c | 50 struct xdp_umem *umem = xs->umem; in xsk_diag_put_umem() local
54 if (!umem) in xsk_diag_put_umem()
57 du.id = umem->id; in xsk_diag_put_umem()
58 du.size = umem->size; in xsk_diag_put_umem()
59 du.num_pages = umem->npgs; in xsk_diag_put_umem()
60 du.chunk_size = umem->chunk_size; in xsk_diag_put_umem()
61 du.headroom = umem->headroom; in xsk_diag_put_umem()
65 if (umem->zc) in xsk_diag_put_umem()
67 du.refs = refcount_read(&umem in xsk_diag_put_umem()
[all...] |
| H A D | xdp_umem.h | 11 void xdp_get_umem(struct xdp_umem *umem);
12 void xdp_put_umem(struct xdp_umem *umem, bool defer_cleanup);
|
| /linux/drivers/gpu/drm/nouveau/nvkm/subdev/mmu/ |
| H A D | umem.c | 40 struct nvkm_umem *umem; in nvkm_umem_search() local
46 list_for_each_entry(umem, &master->umem, head) { in nvkm_umem_search()
47 if (umem->object.object == handle) { in nvkm_umem_search()
48 memory = nvkm_memory_ref(umem->memory); in nvkm_umem_search()
55 umem = nvkm_umem(object); in nvkm_umem_search()
56 memory = nvkm_memory_ref(umem->memory); in nvkm_umem_search()
65 struct nvkm_umem *umem = nvkm_umem(object); in nvkm_umem_unmap() local
67 if (!umem->map) in nvkm_umem_unmap()
70 if (umem->io) { in nvkm_umem_unmap()
71 if (!IS_ERR(umem->bar)) { in nvkm_umem_unmap()
[all …]
|
| /linux/drivers/infiniband/core/ |
| H A D | umem.c | 50 static void __ib_umem_release(struct ib_device *dev, struct ib_umem *umem, int dirty) in __ib_umem_release() argument
52 bool make_dirty = umem->writable && dirty; in __ib_umem_release()
57 ib_dma_unmap_sgtable_attrs(dev, &umem->sgt_append.sgt, in __ib_umem_release()
58 DMA_BIDIRECTIONAL, umem->dma_attrs); in __ib_umem_release()
60 for_each_sgtable_sg(&umem->sgt_append.sgt, sg, i) { in __ib_umem_release()
68 sg_free_append_table(&umem->sgt_append); in __ib_umem_release()
74 * @umem: umem struct
81 * Returns 0 if the umem requires page sizes not supported by
85 unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem, in ib_umem_find_best_pgsz() argument
167 struct ib_umem *umem; ib_umem_get() local
284 ib_umem_release(struct ib_umem * umem) ib_umem_release() argument
311 ib_umem_copy_from(void * dst,struct ib_umem * umem,size_t offset,size_t length) ib_umem_copy_from() argument
[all...] |
| H A D | umem_odp.c | 54 umem_odp->umem.is_odp = 1; in ib_init_umem_implicit_odp()
61 struct ib_device *dev = umem_odp->umem.ibdev; in ib_init_umem_odp()
69 umem_odp->umem.is_odp = 1; in ib_init_umem_odp()
72 start = ALIGN_DOWN(umem_odp->umem.address, page_size); in ib_init_umem_odp()
73 if (check_add_overflow(umem_odp->umem.address, in ib_init_umem_odp()
74 (unsigned long)umem_odp->umem.length, &end)) in ib_init_umem_odp()
109 umem_odp->umem.owning_mm, start, in ib_init_umem_odp()
137 struct ib_umem *umem; in ib_umem_odp_alloc_implicit() local
146 umem = &umem_odp->umem; in ib_umem_odp_alloc_implicit()
147 umem->ibdev = device; in ib_umem_odp_alloc_implicit()
[all …]
|
| H A D | umem_dmabuf.c | 39 start = ALIGN_DOWN(umem_dmabuf->umem.address, PAGE_SIZE); in ib_umem_dmabuf_map_pages()
40 end = ALIGN(umem_dmabuf->umem.address + umem_dmabuf->umem.length, in ib_umem_dmabuf_map_pages()
65 umem_dmabuf->umem.sgt_append.sgt.sgl = umem_dmabuf->first_sg; in ib_umem_dmabuf_map_pages()
66 umem_dmabuf->umem.sgt_append.sgt.nents = nmap; in ib_umem_dmabuf_map_pages()
125 struct ib_umem *umem; in ib_umem_dmabuf_get_with_dma_device() local
145 umem = &umem_dmabuf->umem; in ib_umem_dmabuf_get_with_dma_device()
146 umem->ibdev = device; in ib_umem_dmabuf_get_with_dma_device()
147 umem->length = size; in ib_umem_dmabuf_get_with_dma_device()
148 umem->address = offset; in ib_umem_dmabuf_get_with_dma_device()
149 umem->writable = ib_access_writable(access); in ib_umem_dmabuf_get_with_dma_device()
[all …]
|
| /linux/tools/testing/selftests/bpf/ |
| H A D | xsk.c | 72 struct xsk_umem *umem; member
92 int xsk_umem__fd(const struct xsk_umem *umem) in xsk_umem__fd() argument
94 return umem ? umem->fd : -EINVAL; in xsk_umem__fd()
163 static int xsk_create_umem_rings(struct xsk_umem *umem, int fd, in xsk_create_umem_rings() argument
172 &umem->config.fill_size, in xsk_create_umem_rings()
173 sizeof(umem->config.fill_size)); in xsk_create_umem_rings()
178 &umem->config.comp_size, in xsk_create_umem_rings()
179 sizeof(umem->config.comp_size)); in xsk_create_umem_rings()
187 map = mmap(NULL, off.fr.desc + umem->config.fill_size * sizeof(__u64), in xsk_create_umem_rings()
193 fill->mask = umem->config.fill_size - 1; in xsk_create_umem_rings()
[all …]
|
| H A D | xskxceiver.c | 125 struct xsk_umem_info *umem; in ifobj_zc_avail() local
134 umem = calloc(1, sizeof(struct xsk_umem_info)); in ifobj_zc_avail()
135 if (!umem) { in ifobj_zc_avail()
139 umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE; in ifobj_zc_avail()
140 ret = xsk_configure_umem(ifobject, umem, bufs, umem_sz); in ifobj_zc_avail()
150 ret = xsk_configure_socket(xsk, umem, ifobject, false); in ifobj_zc_avail()
157 munmap(umem->buffer, umem_sz); in ifobj_zc_avail()
158 xsk_umem__delete(umem->umem); in ifobj_zc_avail()
159 free(umem); in ifobj_zc_avail()
|
| H A D | xsk.h | 187 int xsk_umem__fd(const struct xsk_umem *umem);
219 int xsk_umem__create(struct xsk_umem **umem,
226 struct xsk_umem *umem,
232 __u32 queue_id, struct xsk_umem *umem,
239 /* Returns 0 for success and -EBUSY if the umem is still in use. */
240 int xsk_umem__delete(struct xsk_umem *umem);
|
| /linux/tools/testing/selftests/bpf/prog_tests/ |
| H A D | test_xsk.c | 72 static u64 umem_size(struct xsk_umem_info *umem)
74 return umem->num_frames * umem->frame_size; in mode_to_xdp_flags()
77 int xsk_configure_umem(struct ifobject *ifobj, struct xsk_umem_info *umem, void *buffer,
83 .frame_size = umem->frame_size, in xsk_configure_umem() argument
84 .frame_headroom = umem->frame_headroom, in xsk_configure_umem()
89 if (umem->fill_size) in xsk_configure_umem()
90 cfg.fill_size = umem->fill_size; in xsk_configure_umem()
92 if (umem->comp_size) in xsk_configure_umem()
93 cfg.comp_size = umem in xsk_configure_umem()
78 umem_size(struct xsk_umem_info * umem) umem_size() argument
118 umem_alloc_buffer(struct xsk_umem_info * umem) umem_alloc_buffer() argument
130 umem_reset_alloc(struct xsk_umem_info * umem) umem_reset_alloc() argument
157 xsk_configure_socket(struct xsk_socket_info * xsk,struct xsk_umem_info * umem,struct ifobject * ifobject,bool shared) xsk_configure_socket() argument
451 pkt_get_buffer_len(struct xsk_umem_info * umem,u32 len) pkt_get_buffer_len() argument
602 pkt_get_addr(struct pkt * pkt,struct xsk_umem_info * umem) pkt_get_addr() argument
614 pkt_generate(struct xsk_socket_info * xsk,struct xsk_umem_info * umem,u64 addr,u32 len,u32 pkt_nb,u32 bytes_written) pkt_generate() argument
748 is_offset_correct(struct xsk_umem_info * umem,struct pkt * pkt,u64 addr) is_offset_correct() argument
808 is_frag_valid(struct xsk_umem_info * umem,u64 addr,u32 len,u32 expected_pkt_nb,u32 bytes_processed) is_frag_valid() argument
975 struct xsk_umem_info *umem = xsk->umem; __receive_pkts() local
1154 struct xsk_umem_info *umem = ifobject->umem; __send_pkts() local
1460 xsk_configure(struct test_spec * test,struct ifobject * ifobject,struct xsk_umem_info * umem,bool tx) xsk_configure() argument
1503 xsk_populate_fill_ring(struct xsk_umem_info * umem,struct pkt_stream * pkt_stream,bool fill_up) xsk_populate_fill_ring() argument
2067 struct xsk_umem_info *umem = test->ifobj_tx->umem; testapp_invalid_desc_mb() local
2108 struct xsk_umem_info *umem = test->ifobj_tx->umem; testapp_invalid_desc() local
[all...] |
| /linux/drivers/infiniband/sw/siw/ |
| H A D | siw_mem.c | 42 void siw_umem_release(struct siw_umem *umem) in siw_umem_release() argument
44 unsigned int i, num_chunks = umem->num_chunks; in siw_umem_release()
46 if (umem->base_mem) in siw_umem_release()
47 ib_umem_release(umem->base_mem); in siw_umem_release()
50 kfree(umem->page_chunk[i].plist); in siw_umem_release()
52 kfree(umem); in siw_umem_release()
114 siw_umem_release(mem->umem); in siw_free_mem()
334 struct siw_umem *umem; in siw_umem_get()
349 umem = kzalloc_flex(*umem, page_chun in siw_umem_get()
336 struct siw_umem *umem; siw_umem_get() local
[all...] |
| H A D | siw_mem.h | 11 void siw_umem_release(struct siw_umem *umem);
52 * Get page pointer for address on given umem.
54 * @umem: two dimensional list of page pointers
57 static inline struct page *siw_get_upage(struct siw_umem *umem, u64 addr)
59 unsigned int page_idx = (addr - umem->fp_addr) >> PAGE_SHIFT, in siw_get_upage()
63 if (page_idx < umem->num_pages) in siw_get_upage()
64 return umem->page_chunk[chunk_idx].plist[page_in_chunk]; in siw_get_upage()
58 siw_get_upage(struct siw_umem * umem,u64 addr) siw_get_upage() argument
|
| /linux/drivers/infiniband/hw/mlx4/ |
| H A D | doorbell.c | 40 struct ib_umem *umem; member
67 page->umem = ib_umem_get_va(context->ibucontext.device, in mlx4_ib_db_map_user()
69 if (IS_ERR(page->umem)) { in mlx4_ib_db_map_user()
70 err = PTR_ERR(page->umem); in mlx4_ib_db_map_user()
78 db->dma = sg_dma_address(page->umem->sgt_append.sgt.sgl) + in mlx4_ib_db_map_user()
95 ib_umem_release(db->u.user_page->umem); in mlx4_ib_db_unmap_user()
|
| /linux/drivers/infiniband/hw/mlx5/ |
| H A D | doorbell.c | 44 struct ib_umem *umem;
55 struct ib_umem *umem; in mlx5_ib_db_map_user()
61 * For VA descriptors the umem is normalized to a single PAGE_SIZE in mlx5_ib_db_map_user()
73 * doorbell umem; FD-typed descriptors are pinned individually. in mlx5_ib_db_map_user()
94 * multiple DBRs that fall in the same user page share one umem. in mlx5_ib_db_unmap_user()
101 umem = ib_umem_get_desc(context->ibucontext.device, &page->desc, 0); in mlx5_ib_db_unmap_user()
102 if (IS_ERR(umem)) { in mlx5_ib_db_unmap_user()
103 err = PTR_ERR(umem); in mlx5_ib_db_unmap_user()
112 if (!ib_umem_is_contiguous(umem)) {
113 ib_umem_release(umem);
42 struct ib_umem *umem; global() member
[all...] |
| /linux/drivers/infiniband/hw/vmw_pvrdma/ |
| H A D | pvrdma_mr.c | 119 struct ib_umem *umem; in pvrdma_reg_user_mr() local
134 umem = ib_umem_get_va(pd->device, start, length, access_flags); in pvrdma_reg_user_mr()
135 if (IS_ERR(umem)) { in pvrdma_reg_user_mr()
137 "could not get umem for mem region\n"); in pvrdma_reg_user_mr()
138 return ERR_CAST(umem); in pvrdma_reg_user_mr()
141 npages = ib_umem_num_dma_blocks(umem, PAGE_SIZE); in pvrdma_reg_user_mr()
157 mr->umem = umem; in pvrdma_reg_user_mr()
166 ret = pvrdma_page_dir_insert_umem(&mr->pdir, mr->umem, 0); in pvrdma_reg_user_mr()
195 ib_umem_release(umem); in pvrdma_reg_user_mr()
[all...] |
| /linux/include/rdma/ |
| H A D | ib_umem_odp.h | 14 struct ib_umem umem; member
41 static inline struct ib_umem_odp *to_ib_umem_odp(struct ib_umem *umem) in to_ib_umem_odp() argument
43 return container_of(umem, struct ib_umem_odp, umem); in to_ib_umem_odp()
|
| /linux/drivers/infiniband/hw/hns/ |
| H A D | hns_roce_mr.c | 303 ret = ib_umem_check_rereg(mr->pbl_mtr.umem, flags, mr_access_flags); in hns_roce_rereg_user_mr()
575 if (mtr->umem) { in mtr_free_bufs()
576 ib_umem_release(mtr->umem); in mtr_free_bufs()
577 mtr->umem = NULL; in mtr_free_bufs()
598 mtr->umem = ib_umem_get_va(ibdev, user_addr, total_size, in mtr_alloc_bufs()
600 if (IS_ERR(mtr->umem)) { in mtr_alloc_bufs()
601 ibdev_err(ibdev, "failed to get umem, ret = %pe.\n", in mtr_alloc_bufs()
602 mtr->umem); in mtr_alloc_bufs()
606 mtr->umem = NULL; in mtr_alloc_bufs()
661 if (mtr->umem) in mtr_map_bufs()
[all...] |
| H A D | hns_roce_db.c | 32 page->umem = ib_umem_get_va(context->ibucontext.device, page_addr, in hns_roce_db_map_user()
34 if (IS_ERR(page->umem)) { in hns_roce_db_map_user()
35 ret = PTR_ERR(page->umem); in hns_roce_db_map_user()
44 db->dma = sg_dma_address(page->umem->sgt_append.sgt.sgl) + offset; in hns_roce_db_map_user()
45 db->virt_addr = sg_virt(page->umem->sgt_append.sgt.sgl) + offset; in hns_roce_db_map_user()
63 ib_umem_release(db->u.user_page->umem); in hns_roce_db_unmap_user()
|
| /linux/drivers/vdpa/vdpa_user/ |
| H A D | vduse_dev.c | 102 struct vduse_umem *umem; member
1210 if (!dev->as[asid].umem) in vduse_dev_dereg_umem()
1217 if (dev->as[asid].umem->iova != iova || in vduse_dev_reg_umem()
1222 unpin_user_pages_dirty_lock(dev->as[asid].umem->pages, in vduse_dev_reg_umem()
1223 dev->as[asid].umem->npages, true); in vduse_dev_reg_umem()
1224 atomic64_sub(dev->as[asid].umem->npages, &dev->as[asid].umem->mm->pinned_vm); in vduse_dev_reg_umem()
1225 mmdrop(dev->as[asid].umem->mm); in vduse_dev_reg_umem()
1226 vfree(dev->as[asid].umem->pages); in vduse_dev_reg_umem()
1227 kfree(dev->as[asid].umem); in vduse_dev_reg_umem()
1216 struct vduse_umem *umem = NULL; vduse_dev_reg_umem() local
1532 struct vduse_iova_umem umem; vduse_dev_ioctl() local
1554 struct vduse_iova_umem umem; vduse_dev_ioctl() local
[all...] |
| /linux/drivers/infiniband/hw/mana/ |
| H A D | main.c | 258 struct ib_umem *umem; in mana_ib_create_queue()
261 queue->umem = NULL;
265 umem = ib_umem_get_va(&mdev->ib_dev, addr, size, IB_ACCESS_LOCAL_WRITE); in mana_ib_destroy_queue()
266 if (IS_ERR(umem)) { in mana_ib_destroy_queue()
267 ibdev_dbg(&mdev->ib_dev, "Failed to get umem, %pe\n", umem); in mana_ib_destroy_queue()
268 return PTR_ERR(umem); in mana_ib_destroy_queue()
271 err = mana_ib_create_zero_offset_dma_region(mdev, umem, &queue->gdma_region); in mana_ib_destroy_queue()
276 queue->umem = umem; in mana_ib_gd_first_dma_region()
234 struct ib_umem *umem; mana_ib_create_queue() local
337 mana_ib_gd_create_dma_region(struct mana_ib_dev * dev,struct ib_umem * umem,mana_handle_t * gdma_region,unsigned long page_sz) mana_ib_gd_create_dma_region() argument
443 mana_ib_create_dma_region(struct mana_ib_dev * dev,struct ib_umem * umem,mana_handle_t * gdma_region,u64 virt) mana_ib_create_dma_region() argument
457 mana_ib_create_zero_offset_dma_region(struct mana_ib_dev * dev,struct ib_umem * umem,mana_handle_t * gdma_region) mana_ib_create_zero_offset_dma_region() argument
[all...] |
| /linux/include/net/ |
| H A D | xdp_sock_drv.h | 62 struct xdp_umem *umem = pool->umem; in xsk_pool_get_rx_frame_size() local
69 mbuf = pool->dev && (umem->flags & XDP_UMEM_SG_FLAG); in xsk_pool_get_rx_frame_size()
101 struct xdp_umem *umem = pool->umem; in xsk_pool_dma_map() local
103 return xp_dma_map(pool, dev, attrs, umem->pgs, umem->npgs); in xsk_pool_dma_map()
|
| /linux/drivers/infiniband/sw/rxe/ |
| H A D | rxe_mr.c | 195 struct ib_umem *umem; in rxe_mr_init_user() local
200 umem = ib_umem_get_va(&rxe->ib_dev, start, length, access); in rxe_mr_init_user()
201 if (IS_ERR(umem)) { in rxe_mr_init_user()
203 (int)PTR_ERR(umem)); in rxe_mr_init_user()
204 return PTR_ERR(umem); in rxe_mr_init_user()
207 err = alloc_mr_page_info(mr, ib_umem_num_pages(umem)); in rxe_mr_init_user()
211 err = rxe_mr_fill_pages_from_sgt(mr, &umem->sgt_append.sgt); in rxe_mr_init_user()
215 mr->umem = umem; in rxe_mr_init_user()
223 ib_umem_release(umem); in rxe_mr_init_user()
[all...] |
| /linux/drivers/infiniband/sw/rdmavt/ |
| H A D | mr.c | 343 struct ib_umem *umem; in rvt_reg_user_mr() local
354 umem = ib_umem_get_va(pd->device, start, length, mr_access_flags); in rvt_reg_user_mr()
355 if (IS_ERR(umem)) in rvt_reg_user_mr()
356 return ERR_CAST(umem); in rvt_reg_user_mr()
358 n = ib_umem_num_pages(umem); in rvt_reg_user_mr()
369 mr->mr.offset = ib_umem_offset(umem); in rvt_reg_user_mr()
371 mr->umem = umem; in rvt_reg_user_mr()
376 for_each_sgtable_page (&umem->sgt_append.sgt, &sg_iter, 0) { in rvt_reg_user_mr()
398 ib_umem_release(umem); in rvt_reg_user_mr()
[all...] |