1 // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) 2 /* 3 * Copyright (c) 2020 Intel Corporation. All rights reserved. 4 */ 5 6 #include <linux/dma-buf.h> 7 #include <linux/dma-resv.h> 8 #include <linux/dma-mapping.h> 9 #include <linux/module.h> 10 11 #include "uverbs.h" 12 13 MODULE_IMPORT_NS(DMA_BUF); 14 15 int ib_umem_dmabuf_map_pages(struct ib_umem_dmabuf *umem_dmabuf) 16 { 17 struct sg_table *sgt; 18 struct scatterlist *sg; 19 unsigned long start, end, cur = 0; 20 unsigned int nmap = 0; 21 long ret; 22 int i; 23 24 dma_resv_assert_held(umem_dmabuf->attach->dmabuf->resv); 25 26 if (umem_dmabuf->sgt) 27 goto wait_fence; 28 29 sgt = dma_buf_map_attachment_unlocked(umem_dmabuf->attach, 30 DMA_BIDIRECTIONAL); 31 if (IS_ERR(sgt)) 32 return PTR_ERR(sgt); 33 34 /* modify the sg list in-place to match umem address and length */ 35 36 start = ALIGN_DOWN(umem_dmabuf->umem.address, PAGE_SIZE); 37 end = ALIGN(umem_dmabuf->umem.address + umem_dmabuf->umem.length, 38 PAGE_SIZE); 39 for_each_sgtable_dma_sg(sgt, sg, i) { 40 if (start < cur + sg_dma_len(sg) && cur < end) 41 nmap++; 42 if (cur <= start && start < cur + sg_dma_len(sg)) { 43 unsigned long offset = start - cur; 44 45 umem_dmabuf->first_sg = sg; 46 umem_dmabuf->first_sg_offset = offset; 47 sg_dma_address(sg) += offset; 48 sg_dma_len(sg) -= offset; 49 cur += offset; 50 } 51 if (cur < end && end <= cur + sg_dma_len(sg)) { 52 unsigned long trim = cur + sg_dma_len(sg) - end; 53 54 umem_dmabuf->last_sg = sg; 55 umem_dmabuf->last_sg_trim = trim; 56 sg_dma_len(sg) -= trim; 57 break; 58 } 59 cur += sg_dma_len(sg); 60 } 61 62 umem_dmabuf->umem.sgt_append.sgt.sgl = umem_dmabuf->first_sg; 63 umem_dmabuf->umem.sgt_append.sgt.nents = nmap; 64 umem_dmabuf->sgt = sgt; 65 66 wait_fence: 67 /* 68 * Although the sg list is valid now, the content of the pages 69 * may be not up-to-date. Wait for the exporter to finish 70 * the migration. 71 */ 72 ret = dma_resv_wait_timeout(umem_dmabuf->attach->dmabuf->resv, 73 DMA_RESV_USAGE_KERNEL, 74 false, MAX_SCHEDULE_TIMEOUT); 75 if (ret < 0) 76 return ret; 77 if (ret == 0) 78 return -ETIMEDOUT; 79 return 0; 80 } 81 EXPORT_SYMBOL(ib_umem_dmabuf_map_pages); 82 83 void ib_umem_dmabuf_unmap_pages(struct ib_umem_dmabuf *umem_dmabuf) 84 { 85 dma_resv_assert_held(umem_dmabuf->attach->dmabuf->resv); 86 87 if (!umem_dmabuf->sgt) 88 return; 89 90 /* retore the original sg list */ 91 if (umem_dmabuf->first_sg) { 92 sg_dma_address(umem_dmabuf->first_sg) -= 93 umem_dmabuf->first_sg_offset; 94 sg_dma_len(umem_dmabuf->first_sg) += 95 umem_dmabuf->first_sg_offset; 96 umem_dmabuf->first_sg = NULL; 97 umem_dmabuf->first_sg_offset = 0; 98 } 99 if (umem_dmabuf->last_sg) { 100 sg_dma_len(umem_dmabuf->last_sg) += 101 umem_dmabuf->last_sg_trim; 102 umem_dmabuf->last_sg = NULL; 103 umem_dmabuf->last_sg_trim = 0; 104 } 105 106 dma_buf_unmap_attachment_unlocked(umem_dmabuf->attach, umem_dmabuf->sgt, 107 DMA_BIDIRECTIONAL); 108 109 umem_dmabuf->sgt = NULL; 110 } 111 EXPORT_SYMBOL(ib_umem_dmabuf_unmap_pages); 112 113 struct ib_umem_dmabuf *ib_umem_dmabuf_get(struct ib_device *device, 114 unsigned long offset, size_t size, 115 int fd, int access, 116 const struct dma_buf_attach_ops *ops) 117 { 118 struct dma_buf *dmabuf; 119 struct ib_umem_dmabuf *umem_dmabuf; 120 struct ib_umem *umem; 121 unsigned long end; 122 struct ib_umem_dmabuf *ret = ERR_PTR(-EINVAL); 123 124 if (check_add_overflow(offset, (unsigned long)size, &end)) 125 return ret; 126 127 if (unlikely(!ops || !ops->move_notify)) 128 return ret; 129 130 dmabuf = dma_buf_get(fd); 131 if (IS_ERR(dmabuf)) 132 return ERR_CAST(dmabuf); 133 134 if (dmabuf->size < end) 135 goto out_release_dmabuf; 136 137 umem_dmabuf = kzalloc(sizeof(*umem_dmabuf), GFP_KERNEL); 138 if (!umem_dmabuf) { 139 ret = ERR_PTR(-ENOMEM); 140 goto out_release_dmabuf; 141 } 142 143 umem = &umem_dmabuf->umem; 144 umem->ibdev = device; 145 umem->length = size; 146 umem->address = offset; 147 umem->writable = ib_access_writable(access); 148 umem->is_dmabuf = 1; 149 150 if (!ib_umem_num_pages(umem)) 151 goto out_free_umem; 152 153 umem_dmabuf->attach = dma_buf_dynamic_attach( 154 dmabuf, 155 device->dma_device, 156 ops, 157 umem_dmabuf); 158 if (IS_ERR(umem_dmabuf->attach)) { 159 ret = ERR_CAST(umem_dmabuf->attach); 160 goto out_free_umem; 161 } 162 return umem_dmabuf; 163 164 out_free_umem: 165 kfree(umem_dmabuf); 166 167 out_release_dmabuf: 168 dma_buf_put(dmabuf); 169 return ret; 170 } 171 EXPORT_SYMBOL(ib_umem_dmabuf_get); 172 173 static void 174 ib_umem_dmabuf_unsupported_move_notify(struct dma_buf_attachment *attach) 175 { 176 struct ib_umem_dmabuf *umem_dmabuf = attach->importer_priv; 177 178 ibdev_warn_ratelimited(umem_dmabuf->umem.ibdev, 179 "Invalidate callback should not be called when memory is pinned\n"); 180 } 181 182 static struct dma_buf_attach_ops ib_umem_dmabuf_attach_pinned_ops = { 183 .allow_peer2peer = true, 184 .move_notify = ib_umem_dmabuf_unsupported_move_notify, 185 }; 186 187 struct ib_umem_dmabuf *ib_umem_dmabuf_get_pinned(struct ib_device *device, 188 unsigned long offset, 189 size_t size, int fd, 190 int access) 191 { 192 struct ib_umem_dmabuf *umem_dmabuf; 193 int err; 194 195 umem_dmabuf = ib_umem_dmabuf_get(device, offset, size, fd, access, 196 &ib_umem_dmabuf_attach_pinned_ops); 197 if (IS_ERR(umem_dmabuf)) 198 return umem_dmabuf; 199 200 dma_resv_lock(umem_dmabuf->attach->dmabuf->resv, NULL); 201 err = dma_buf_pin(umem_dmabuf->attach); 202 if (err) 203 goto err_release; 204 umem_dmabuf->pinned = 1; 205 206 err = ib_umem_dmabuf_map_pages(umem_dmabuf); 207 if (err) 208 goto err_unpin; 209 dma_resv_unlock(umem_dmabuf->attach->dmabuf->resv); 210 211 return umem_dmabuf; 212 213 err_unpin: 214 dma_buf_unpin(umem_dmabuf->attach); 215 err_release: 216 dma_resv_unlock(umem_dmabuf->attach->dmabuf->resv); 217 ib_umem_release(&umem_dmabuf->umem); 218 return ERR_PTR(err); 219 } 220 EXPORT_SYMBOL(ib_umem_dmabuf_get_pinned); 221 222 void ib_umem_dmabuf_release(struct ib_umem_dmabuf *umem_dmabuf) 223 { 224 struct dma_buf *dmabuf = umem_dmabuf->attach->dmabuf; 225 226 dma_resv_lock(dmabuf->resv, NULL); 227 ib_umem_dmabuf_unmap_pages(umem_dmabuf); 228 if (umem_dmabuf->pinned) 229 dma_buf_unpin(umem_dmabuf->attach); 230 dma_resv_unlock(dmabuf->resv); 231 232 dma_buf_detach(dmabuf, umem_dmabuf->attach); 233 dma_buf_put(dmabuf); 234 kfree(umem_dmabuf); 235 } 236