1 // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) 2 /* 3 * Copyright (c) 2020 Intel Corporation. All rights reserved. 4 */ 5 6 #include <linux/dma-buf.h> 7 #include <linux/dma-resv.h> 8 #include <linux/dma-mapping.h> 9 #include <linux/module.h> 10 11 #include "uverbs.h" 12 13 MODULE_IMPORT_NS("DMA_BUF"); 14 15 int ib_umem_dmabuf_map_pages(struct ib_umem_dmabuf *umem_dmabuf) 16 { 17 struct sg_table *sgt; 18 struct scatterlist *sg; 19 unsigned long start, end, cur = 0; 20 unsigned int nmap = 0; 21 long ret; 22 int i; 23 24 dma_resv_assert_held(umem_dmabuf->attach->dmabuf->resv); 25 26 if (umem_dmabuf->revoked) 27 return -EINVAL; 28 29 if (umem_dmabuf->sgt) 30 goto wait_fence; 31 32 sgt = dma_buf_map_attachment(umem_dmabuf->attach, 33 DMA_BIDIRECTIONAL); 34 if (IS_ERR(sgt)) 35 return PTR_ERR(sgt); 36 37 /* modify the sg list in-place to match umem address and length */ 38 39 start = ALIGN_DOWN(umem_dmabuf->umem.address, PAGE_SIZE); 40 end = ALIGN(umem_dmabuf->umem.address + umem_dmabuf->umem.length, 41 PAGE_SIZE); 42 for_each_sgtable_dma_sg(sgt, sg, i) { 43 if (start < cur + sg_dma_len(sg) && cur < end) 44 nmap++; 45 if (cur <= start && start < cur + sg_dma_len(sg)) { 46 unsigned long offset = start - cur; 47 48 umem_dmabuf->first_sg = sg; 49 umem_dmabuf->first_sg_offset = offset; 50 sg_dma_address(sg) += offset; 51 sg_dma_len(sg) -= offset; 52 cur += offset; 53 } 54 if (cur < end && end <= cur + sg_dma_len(sg)) { 55 unsigned long trim = cur + sg_dma_len(sg) - end; 56 57 umem_dmabuf->last_sg = sg; 58 umem_dmabuf->last_sg_trim = trim; 59 sg_dma_len(sg) -= trim; 60 break; 61 } 62 cur += sg_dma_len(sg); 63 } 64 65 umem_dmabuf->umem.sgt_append.sgt.sgl = umem_dmabuf->first_sg; 66 umem_dmabuf->umem.sgt_append.sgt.nents = nmap; 67 umem_dmabuf->sgt = sgt; 68 69 wait_fence: 70 /* 71 * Although the sg list is valid now, the content of the pages 72 * may be not up-to-date. Wait for the exporter to finish 73 * the migration. 74 */ 75 ret = dma_resv_wait_timeout(umem_dmabuf->attach->dmabuf->resv, 76 DMA_RESV_USAGE_KERNEL, 77 false, MAX_SCHEDULE_TIMEOUT); 78 if (ret < 0) 79 return ret; 80 if (ret == 0) 81 return -ETIMEDOUT; 82 return 0; 83 } 84 EXPORT_SYMBOL(ib_umem_dmabuf_map_pages); 85 86 void ib_umem_dmabuf_unmap_pages(struct ib_umem_dmabuf *umem_dmabuf) 87 { 88 dma_resv_assert_held(umem_dmabuf->attach->dmabuf->resv); 89 90 if (!umem_dmabuf->sgt) 91 return; 92 93 /* retore the original sg list */ 94 if (umem_dmabuf->first_sg) { 95 sg_dma_address(umem_dmabuf->first_sg) -= 96 umem_dmabuf->first_sg_offset; 97 sg_dma_len(umem_dmabuf->first_sg) += 98 umem_dmabuf->first_sg_offset; 99 umem_dmabuf->first_sg = NULL; 100 umem_dmabuf->first_sg_offset = 0; 101 } 102 if (umem_dmabuf->last_sg) { 103 sg_dma_len(umem_dmabuf->last_sg) += 104 umem_dmabuf->last_sg_trim; 105 umem_dmabuf->last_sg = NULL; 106 umem_dmabuf->last_sg_trim = 0; 107 } 108 109 dma_buf_unmap_attachment(umem_dmabuf->attach, umem_dmabuf->sgt, 110 DMA_BIDIRECTIONAL); 111 112 umem_dmabuf->sgt = NULL; 113 } 114 EXPORT_SYMBOL(ib_umem_dmabuf_unmap_pages); 115 116 static struct ib_umem_dmabuf * 117 ib_umem_dmabuf_get_with_dma_device(struct ib_device *device, 118 struct device *dma_device, 119 unsigned long offset, size_t size, 120 int fd, int access, 121 const struct dma_buf_attach_ops *ops) 122 { 123 struct dma_buf *dmabuf; 124 struct ib_umem_dmabuf *umem_dmabuf; 125 struct ib_umem *umem; 126 unsigned long end; 127 struct ib_umem_dmabuf *ret = ERR_PTR(-EINVAL); 128 129 if (check_add_overflow(offset, (unsigned long)size, &end)) 130 return ret; 131 132 dmabuf = dma_buf_get(fd); 133 if (IS_ERR(dmabuf)) 134 return ERR_CAST(dmabuf); 135 136 if (dmabuf->size < end) 137 goto out_release_dmabuf; 138 139 umem_dmabuf = kzalloc_obj(*umem_dmabuf); 140 if (!umem_dmabuf) { 141 ret = ERR_PTR(-ENOMEM); 142 goto out_release_dmabuf; 143 } 144 145 umem = &umem_dmabuf->umem; 146 umem->ibdev = device; 147 umem->length = size; 148 umem->address = offset; 149 umem->writable = ib_access_writable(access); 150 umem->is_dmabuf = 1; 151 152 if (!ib_umem_num_pages(umem)) 153 goto out_free_umem; 154 155 umem_dmabuf->attach = dma_buf_dynamic_attach( 156 dmabuf, 157 dma_device, 158 ops, 159 umem_dmabuf); 160 if (IS_ERR(umem_dmabuf->attach)) { 161 ret = ERR_CAST(umem_dmabuf->attach); 162 goto out_free_umem; 163 } 164 return umem_dmabuf; 165 166 out_free_umem: 167 kfree(umem_dmabuf); 168 169 out_release_dmabuf: 170 dma_buf_put(dmabuf); 171 return ret; 172 } 173 174 struct ib_umem_dmabuf *ib_umem_dmabuf_get(struct ib_device *device, 175 unsigned long offset, size_t size, 176 int fd, int access, 177 const struct dma_buf_attach_ops *ops) 178 { 179 return ib_umem_dmabuf_get_with_dma_device(device, device->dma_device, 180 offset, size, fd, access, ops); 181 } 182 EXPORT_SYMBOL(ib_umem_dmabuf_get); 183 184 static struct dma_buf_attach_ops ib_umem_dmabuf_attach_pinned_ops = { 185 .allow_peer2peer = true, 186 }; 187 188 static void ib_umem_dmabuf_revoke_locked(struct dma_buf_attachment *attach) 189 { 190 struct ib_umem_dmabuf *umem_dmabuf = attach->importer_priv; 191 192 dma_resv_assert_held(attach->dmabuf->resv); 193 194 if (umem_dmabuf->revoked) 195 return; 196 197 if (umem_dmabuf->pinned_revoke) 198 umem_dmabuf->pinned_revoke(umem_dmabuf->private); 199 200 ib_umem_dmabuf_unmap_pages(umem_dmabuf); 201 if (umem_dmabuf->pinned) { 202 dma_buf_unpin(umem_dmabuf->attach); 203 umem_dmabuf->pinned = 0; 204 } 205 umem_dmabuf->revoked = 1; 206 } 207 208 static struct dma_buf_attach_ops ib_umem_dmabuf_attach_pinned_revocable_ops = { 209 .allow_peer2peer = true, 210 .invalidate_mappings = ib_umem_dmabuf_revoke_locked, 211 }; 212 213 static struct ib_umem_dmabuf * 214 ib_umem_dmabuf_get_pinned_and_lock(struct ib_device *device, 215 struct device *dma_device, 216 unsigned long offset, 217 size_t size, int fd, int access, 218 const struct dma_buf_attach_ops *ops) 219 { 220 struct ib_umem_dmabuf *umem_dmabuf; 221 int err; 222 223 umem_dmabuf = 224 ib_umem_dmabuf_get_with_dma_device(device, dma_device, offset, 225 size, fd, access, ops); 226 if (IS_ERR(umem_dmabuf)) 227 return umem_dmabuf; 228 229 dma_resv_lock(umem_dmabuf->attach->dmabuf->resv, NULL); 230 err = dma_buf_pin(umem_dmabuf->attach); 231 if (err) 232 goto err_release; 233 umem_dmabuf->pinned = 1; 234 235 err = ib_umem_dmabuf_map_pages(umem_dmabuf); 236 if (err) 237 goto err_release; 238 239 return umem_dmabuf; 240 241 err_release: 242 dma_resv_unlock(umem_dmabuf->attach->dmabuf->resv); 243 ib_umem_release(&umem_dmabuf->umem); 244 return ERR_PTR(err); 245 } 246 247 struct ib_umem_dmabuf * 248 ib_umem_dmabuf_get_pinned_with_dma_device(struct ib_device *device, 249 struct device *dma_device, 250 unsigned long offset, size_t size, 251 int fd, int access) 252 { 253 struct ib_umem_dmabuf *umem_dmabuf = 254 ib_umem_dmabuf_get_pinned_and_lock(device, dma_device, offset, 255 size, fd, access, 256 &ib_umem_dmabuf_attach_pinned_ops); 257 if (IS_ERR(umem_dmabuf)) 258 return umem_dmabuf; 259 260 dma_resv_unlock(umem_dmabuf->attach->dmabuf->resv); 261 return umem_dmabuf; 262 } 263 EXPORT_SYMBOL(ib_umem_dmabuf_get_pinned_with_dma_device); 264 265 /** 266 * ib_umem_dmabuf_get_pinned_revocable_and_lock - Map & pin a revocable dmabuf 267 * @device: IB device. 268 * @offset: Start offset. 269 * @size: Length. 270 * @fd: dmabuf fd. 271 * @access: Access flags. 272 * 273 * Obtains a umem from a dmabuf for drivers/devices that can support revocation. 274 * 275 * Returns with dma_resv_lock held upon success. The driver must set the revoke 276 * callback prior to unlock by calling ib_umem_dmabuf_set_revoke_locked(). 277 * 278 * When a revocation occurs, the revoke callback will be called. The driver must 279 * ensure that the region is no longer accessed when the callback returns. Any 280 * subsequent access attempts should also probably cause an AE for MRs. 281 * 282 * If the umem is used for an MR, the driver must ensure that the key remains in 283 * use such that it cannot be obtained by a new region until this region is 284 * fully deregistered (i.e., ibv_dereg_mr). If a driver needs to serialize with 285 * revoke calls, it can use dma_resv_lock. 286 * 287 * If successful, then the revoke callback may be called at any time and will 288 * also be called automatically upon ib_umem_release (serialized). The revoke 289 * callback will be called one time at most. 290 * 291 * Return: A pointer to ib_umem_dmabuf on success, or an ERR_PTR on failure. 292 */ 293 struct ib_umem_dmabuf * 294 ib_umem_dmabuf_get_pinned_revocable_and_lock(struct ib_device *device, 295 unsigned long offset, size_t size, 296 int fd, int access) 297 { 298 const struct dma_buf_attach_ops *ops = 299 &ib_umem_dmabuf_attach_pinned_revocable_ops; 300 301 return ib_umem_dmabuf_get_pinned_and_lock(device, device->dma_device, 302 offset, size, fd, access, 303 ops); 304 } 305 EXPORT_SYMBOL(ib_umem_dmabuf_get_pinned_revocable_and_lock); 306 307 void ib_umem_dmabuf_set_revoke_locked(struct ib_umem_dmabuf *umem_dmabuf, 308 void (*revoke)(void *priv), void *priv) 309 { 310 dma_resv_assert_held(umem_dmabuf->attach->dmabuf->resv); 311 312 umem_dmabuf->pinned_revoke = revoke; 313 umem_dmabuf->private = priv; 314 } 315 EXPORT_SYMBOL(ib_umem_dmabuf_set_revoke_locked); 316 317 struct ib_umem_dmabuf *ib_umem_dmabuf_get_pinned(struct ib_device *device, 318 unsigned long offset, 319 size_t size, int fd, 320 int access) 321 { 322 return ib_umem_dmabuf_get_pinned_with_dma_device(device, device->dma_device, 323 offset, size, fd, access); 324 } 325 EXPORT_SYMBOL(ib_umem_dmabuf_get_pinned); 326 327 void ib_umem_dmabuf_revoke_lock(struct ib_umem_dmabuf *umem_dmabuf) 328 { 329 struct dma_buf *dmabuf = umem_dmabuf->attach->dmabuf; 330 331 dma_resv_lock(dmabuf->resv, NULL); 332 } 333 EXPORT_SYMBOL(ib_umem_dmabuf_revoke_lock); 334 335 void ib_umem_dmabuf_revoke_unlock(struct ib_umem_dmabuf *umem_dmabuf) 336 { 337 struct dma_buf *dmabuf = umem_dmabuf->attach->dmabuf; 338 339 dma_resv_unlock(dmabuf->resv); 340 } 341 EXPORT_SYMBOL(ib_umem_dmabuf_revoke_unlock); 342 343 void ib_umem_dmabuf_revoke(struct ib_umem_dmabuf *umem_dmabuf) 344 { 345 struct dma_buf *dmabuf = umem_dmabuf->attach->dmabuf; 346 347 dma_resv_lock(dmabuf->resv, NULL); 348 ib_umem_dmabuf_revoke_locked(umem_dmabuf->attach); 349 dma_resv_unlock(dmabuf->resv); 350 } 351 EXPORT_SYMBOL(ib_umem_dmabuf_revoke); 352 353 void ib_umem_dmabuf_release(struct ib_umem_dmabuf *umem_dmabuf) 354 { 355 struct dma_buf *dmabuf = umem_dmabuf->attach->dmabuf; 356 357 ib_umem_dmabuf_revoke(umem_dmabuf); 358 359 dma_buf_detach(dmabuf, umem_dmabuf->attach); 360 dma_buf_put(dmabuf); 361 kfree(umem_dmabuf); 362 } 363